1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::SPLAT_VECTOR:
67 case ISD::BUILD_VECTOR:
68 R = ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(N);
69 break;
70 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
71 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
72 case ISD::CONVERT_FROM_ARBITRARY_FP:
73 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
74 break;
75 case ISD::CONVERT_TO_ARBITRARY_FP:
76 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
77 break;
78 case ISD::AssertZext:
79 case ISD::AssertSext:
80 case ISD::FPOWI:
81 case ISD::AssertNoFPClass:
82 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
83 break;
84 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
85 case ISD::ATOMIC_LOAD:
86 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
87 break;
88 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
89 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
90 case ISD::VECTOR_DEINTERLEAVE:
91 case ISD::VECTOR_INTERLEAVE:
92 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
93 break;
94 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
95 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
96 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
97 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
98 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
99 case ISD::VECTOR_MATCH:
100 R = ScalarizeVecRes_VECTOR_MATCH(N);
101 break;
102 case ISD::POISON:
103 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
104 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
105 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
106 case ISD::ANY_EXTEND_VECTOR_INREG:
107 case ISD::SIGN_EXTEND_VECTOR_INREG:
108 case ISD::ZERO_EXTEND_VECTOR_INREG:
109 R = ScalarizeVecRes_VecInregOp(N);
110 break;
111 case ISD::ABS:
112 case ISD::ABS_MIN_POISON:
113 case ISD::ANY_EXTEND:
114 case ISD::BITREVERSE:
115 case ISD::BSWAP:
116 case ISD::CTLZ:
117 case ISD::CTLZ_ZERO_POISON:
118 case ISD::CTPOP:
119 case ISD::CTTZ:
120 case ISD::CTTZ_ZERO_POISON:
121 case ISD::FABS:
122 case ISD::FACOS:
123 case ISD::FASIN:
124 case ISD::FATAN:
125 case ISD::FCEIL:
126 case ISD::FCOS:
127 case ISD::FCOSH:
128 case ISD::FEXP:
129 case ISD::FEXP2:
130 case ISD::FEXP10:
131 case ISD::FFLOOR:
132 case ISD::FLOG:
133 case ISD::FLOG10:
134 case ISD::FLOG2:
135 case ISD::FNEARBYINT:
136 case ISD::FNEG:
137 case ISD::FREEZE:
138 case ISD::ARITH_FENCE:
139 case ISD::FP_EXTEND:
140 case ISD::FP_TO_SINT:
141 case ISD::FP_TO_UINT:
142 case ISD::FRINT:
143 case ISD::LRINT:
144 case ISD::LLRINT:
145 case ISD::FROUND:
146 case ISD::FROUNDEVEN:
147 case ISD::LROUND:
148 case ISD::LLROUND:
149 case ISD::FSIN:
150 case ISD::FSINH:
151 case ISD::FSQRT:
152 case ISD::FTAN:
153 case ISD::FTANH:
154 case ISD::FTRUNC:
155 case ISD::SIGN_EXTEND:
156 case ISD::SINT_TO_FP:
157 case ISD::TRUNCATE:
158 case ISD::UINT_TO_FP:
159 case ISD::ZERO_EXTEND:
160 case ISD::FCANONICALIZE:
161 R = ScalarizeVecRes_UnaryOp(N);
162 break;
163 case ISD::ADDRSPACECAST:
164 R = ScalarizeVecRes_ADDRSPACECAST(N);
165 break;
166 case ISD::FMODF:
167 case ISD::FFREXP:
168 case ISD::FSINCOS:
169 case ISD::FSINCOSPI:
170 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
171 break;
172 case ISD::ADD:
173 case ISD::AND:
174 case ISD::AVGCEILS:
175 case ISD::AVGCEILU:
176 case ISD::AVGFLOORS:
177 case ISD::AVGFLOORU:
178 case ISD::FADD:
179 case ISD::FCOPYSIGN:
180 case ISD::FDIV:
181 case ISD::FMUL:
182 case ISD::FMINNUM:
183 case ISD::FMAXNUM:
184 case ISD::FMINNUM_IEEE:
185 case ISD::FMAXNUM_IEEE:
186 case ISD::FMINIMUM:
187 case ISD::FMAXIMUM:
188 case ISD::FMINIMUMNUM:
189 case ISD::FMAXIMUMNUM:
190 case ISD::ABDS:
191 case ISD::ABDU:
192 case ISD::SMIN:
193 case ISD::SMAX:
194 case ISD::UMIN:
195 case ISD::UMAX:
196
197 case ISD::SADDSAT:
198 case ISD::UADDSAT:
199 case ISD::SSUBSAT:
200 case ISD::USUBSAT:
201 case ISD::SSHLSAT:
202 case ISD::USHLSAT:
203
204 case ISD::FPOW:
205 case ISD::FATAN2:
206 case ISD::FREM:
207 case ISD::FSUB:
208 case ISD::MUL:
209 case ISD::MULHS:
210 case ISD::MULHU:
211 case ISD::OR:
212 case ISD::SDIV:
213 case ISD::SREM:
214 case ISD::SUB:
215 case ISD::UDIV:
216 case ISD::UREM:
217 case ISD::XOR:
218 case ISD::SHL:
219 case ISD::SRA:
220 case ISD::SRL:
221 case ISD::ROTL:
222 case ISD::ROTR:
223 case ISD::CLMUL:
224 case ISD::CLMULR:
225 case ISD::CLMULH:
226 case ISD::PEXT:
227 case ISD::PDEP:
228 R = ScalarizeVecRes_BinOp(N);
229 break;
230
231 case ISD::MASKED_UDIV:
232 case ISD::MASKED_SDIV:
233 case ISD::MASKED_UREM:
234 case ISD::MASKED_SREM:
235 R = ScalarizeVecRes_MaskedBinOp(N);
236 break;
237
238 case ISD::FLDEXP:
239 R = ScalarizeVecRes_FPOp_MultiType(N);
240 break;
241
242 case ISD::SCMP:
243 case ISD::UCMP:
244 R = ScalarizeVecRes_CMP(N);
245 break;
246
247 case ISD::FMA:
248 case ISD::FSHL:
249 case ISD::FSHR:
250 R = ScalarizeVecRes_TernaryOp(N);
251 break;
252
253#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
254 case ISD::STRICT_##DAGN:
255#include "llvm/IR/ConstrainedOps.def"
256 R = ScalarizeVecRes_StrictFPOp(N);
257 break;
258
259 case ISD::FP_TO_UINT_SAT:
260 case ISD::FP_TO_SINT_SAT:
261 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
262 break;
263
264 case ISD::UADDO:
265 case ISD::SADDO:
266 case ISD::USUBO:
267 case ISD::SSUBO:
268 case ISD::UMULO:
269 case ISD::SMULO:
270 R = ScalarizeVecRes_OverflowOp(N, ResNo);
271 break;
272 case ISD::SMULFIX:
273 case ISD::SMULFIXSAT:
274 case ISD::UMULFIX:
275 case ISD::UMULFIXSAT:
276 case ISD::SDIVFIX:
277 case ISD::SDIVFIXSAT:
278 case ISD::UDIVFIX:
279 case ISD::UDIVFIXSAT:
280 R = ScalarizeVecRes_FIX(N);
281 break;
282 }
283
284 // If R is null, the sub-method took care of registering the result.
285 if (R.getNode())
286 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
287}
288
289SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
290 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
291 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
292 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
293 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
294}
295
296SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
297 SDLoc DL(N);
298 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
299 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
300 SDValue Mask = N->getOperand(Num: 2);
301 EVT MaskVT = Mask.getValueType();
302 // The vselect result and input vectors need scalarizing, but it's
303 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
304 // See the similar logic in ScalarizeVecRes_SETCC.
305 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
306 Mask = GetScalarizedVector(Op: Mask);
307 else
308 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
309 // Vectors may have a different boolean contents to scalars, so truncate to i1
310 // and let type legalization promote appropriately.
311 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
312 // Masked binary ops don't have UB on disabled lanes but produce poison, so
313 // use 1 as the divisor to avoid division by zero and overflow.
314 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
315 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
316 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
317 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
318}
319
320SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
321 SDLoc DL(N);
322
323 SDValue LHS = N->getOperand(Num: 0);
324 SDValue RHS = N->getOperand(Num: 1);
325 if (getTypeAction(VT: LHS.getValueType()) ==
326 TargetLowering::TypeScalarizeVector) {
327 LHS = GetScalarizedVector(Op: LHS);
328 RHS = GetScalarizedVector(Op: RHS);
329 } else {
330 EVT VT = LHS.getValueType().getVectorElementType();
331 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
332 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
333 }
334
335 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
336 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
337}
338
339SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
340 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
341 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
342 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
343 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
344 N3: Op2, Flags: N->getFlags());
345}
346
347SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
348 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
349 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
350 SDValue Op2 = N->getOperand(Num: 2);
351 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
352 N3: Op2, Flags: N->getFlags());
353}
354
355SDValue
356DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
357 unsigned ResNo) {
358 assert(N->getValueType(0).getVectorNumElements() == 1 &&
359 "Unexpected vector type!");
360 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
361
362 EVT VT0 = N->getValueType(ResNo: 0);
363 EVT VT1 = N->getValueType(ResNo: 1);
364 SDLoc dl(N);
365
366 SDNode *ScalarNode =
367 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
368 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
369 .getNode();
370
371 // Replace the other vector result not being explicitly scalarized here.
372 unsigned OtherNo = 1 - ResNo;
373 EVT OtherVT = N->getValueType(ResNo: OtherNo);
374 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
375 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
376 } else {
377 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
378 Operand: SDValue(ScalarNode, OtherNo));
379 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
380 }
381
382 return SDValue(ScalarNode, ResNo);
383}
384
385SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
386 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
387 unsigned NumOpers = N->getNumOperands();
388 SDValue Chain = N->getOperand(Num: 0);
389 EVT ValueVTs[] = {VT, MVT::Other};
390 SDLoc dl(N);
391
392 SmallVector<SDValue, 4> Opers(NumOpers);
393
394 // The Chain is the first operand.
395 Opers[0] = Chain;
396
397 // Now process the remaining operands.
398 for (unsigned i = 1; i < NumOpers; ++i) {
399 SDValue Oper = N->getOperand(Num: i);
400 EVT OperVT = Oper.getValueType();
401
402 if (OperVT.isVector()) {
403 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
404 Oper = GetScalarizedVector(Op: Oper);
405 else
406 Oper =
407 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
408 }
409
410 Opers[i] = Oper;
411 }
412
413 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
414 Ops: Opers, Flags: N->getFlags());
415
416 // Legalize the chain result - switch anything that used the old chain to
417 // use the new one.
418 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
419 return Result;
420}
421
422SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
423 unsigned ResNo) {
424 SDLoc DL(N);
425 EVT ResVT = N->getValueType(ResNo: 0);
426 EVT OvVT = N->getValueType(ResNo: 1);
427
428 SDValue ScalarLHS, ScalarRHS;
429 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
430 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
431 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
432 } else {
433 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
434 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
435 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
436 ScalarLHS = ElemsLHS[0];
437 ScalarRHS = ElemsRHS[0];
438 }
439
440 SDVTList ScalarVTs = DAG.getVTList(
441 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
442 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
443 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
444 .getNode();
445
446 // Replace the other vector result not being explicitly scalarized here.
447 unsigned OtherNo = 1 - ResNo;
448 EVT OtherVT = N->getValueType(ResNo: OtherNo);
449 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
450 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
451 } else {
452 SDValue OtherVal = DAG.getNode(
453 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
454 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
455 }
456
457 return SDValue(ScalarNode, ResNo);
458}
459
460SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
461 unsigned ResNo) {
462 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
463 return GetScalarizedVector(Op);
464}
465
466SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
467 SDLoc DL(N);
468 // Reuse the expansion (which should scalarize).
469 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
470 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
471 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
472 N2: DAG.getVectorIdxConstant(Val: 0, DL));
473}
474
475SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
476 SDValue Op = N->getOperand(Num: 0);
477 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
478 Op = GetScalarizedVector(Op);
479 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
480 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
481 VT: NewVT, Operand: Op);
482}
483
484SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(SDNode *N) {
485 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
486 SDValue InOp = N->getOperand(Num: 0);
487 // The BUILD_VECTOR / SPLAT operands may be of wider element types and
488 // we may need to truncate them back to the requested return type.
489 if (EltVT.isInteger())
490 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
491 return InOp;
492}
493
494SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
495 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
496 VT: N->getValueType(ResNo: 0).getVectorElementType(),
497 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
498}
499
500SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
501 SDLoc DL(N);
502 SDValue Op = N->getOperand(Num: 0);
503 EVT OpVT = Op.getValueType();
504 // The result needs scalarizing, but it's not a given that the source does.
505 // See similar logic in ScalarizeVecRes_UnaryOp.
506 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
507 Op = GetScalarizedVector(Op);
508 } else {
509 EVT VT = OpVT.getVectorElementType();
510 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
511 }
512 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
513 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
514 N2: N->getOperand(Num: 1));
515}
516
517SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
518 SDLoc DL(N);
519 SDValue Op = N->getOperand(Num: 0);
520 EVT OpVT = Op.getValueType();
521 // The result needs scalarizing, but it's not a given that the source does.
522 // See similar logic in ScalarizeVecRes_UnaryOp.
523 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
524 Op = GetScalarizedVector(Op);
525 } else {
526 EVT VT = OpVT.getVectorElementType();
527 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
528 }
529 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
530 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
531 N2: N->getOperand(Num: 1));
532}
533
534SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
535 SDLoc DL(N);
536 SDValue Op = N->getOperand(Num: 0);
537 EVT OpVT = Op.getValueType();
538 // The result needs scalarizing, but it's not a given that the source does.
539 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
540 Op = GetScalarizedVector(Op);
541 } else {
542 EVT VT = OpVT.getVectorElementType();
543 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
544 }
545 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
546 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
547 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
548}
549
550SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
551 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
552 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
553 N2: N->getOperand(Num: 1));
554}
555
556SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOp_MultiType(SDNode *N) {
557 SDLoc DL(N);
558 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
559 SDValue RHS = N->getOperand(Num: 1);
560 EVT RHSVT = RHS.getValueType();
561 // The exponent has its own type action and may not have been scalarized:
562 // v1i1 is legal on AVX-512, v1i32 is widened on AArch64.
563 if (RHSVT.isVector()) {
564 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeScalarizeVector)
565 RHS = GetScalarizedVector(Op: RHS);
566 else
567 RHS = DAG.getExtractVectorElt(DL, VT: RHSVT.getVectorElementType(), Vec: RHS, Idx: 0);
568 }
569 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHS.getValueType(), N1: LHS, N2: RHS,
570 Flags: N->getFlags());
571}
572
573SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
574 // The value to insert may have a wider type than the vector element type,
575 // so be sure to truncate it to the element type if necessary.
576 SDValue Op = N->getOperand(Num: 1);
577 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
578 if (Op.getValueType() != EltVT)
579 // FIXME: Can this happen for floating point types?
580 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
581 return Op;
582}
583
584SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
585 SDValue Result = DAG.getAtomicLoad(
586 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
587 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
588 MMO: N->getMemOperand());
589
590 // Legalize the chain result - switch anything that used the old chain to
591 // use the new one.
592 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
593 return Result;
594}
595
596SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
597 assert(N->isUnindexed() && "Indexed vector load?");
598
599 SDValue Result = DAG.getLoad(
600 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
601 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
602 Ptr: N->getBasePtr(), Offset: DAG.getPOISON(VT: N->getBasePtr().getValueType()),
603 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
604 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(),
605 Metadata: N->getMMOMetadataForSubAccess());
606
607 // Legalize the chain result - switch anything that used the old chain to
608 // use the new one.
609 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
610 return Result;
611}
612
613SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
614 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
615 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
616 SDValue Op = N->getOperand(Num: 0);
617 EVT OpVT = Op.getValueType();
618 SDLoc DL(N);
619 // The result needs scalarizing, but it's not a given that the source does.
620 // This is a workaround for targets where it's impossible to scalarize the
621 // result of a conversion, because the source type is legal.
622 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
623 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
624 // legal and was not scalarized.
625 // See the similar logic in ScalarizeVecRes_SETCC
626 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
627 Op = GetScalarizedVector(Op);
628 } else {
629 EVT VT = OpVT.getVectorElementType();
630 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
631 }
632 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
633}
634
635SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
636 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
637 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
638 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
639 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
640 N1: LHS, N2: DAG.getValueType(ExtVT));
641}
642
643SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
644 SDLoc DL(N);
645 SDValue Op = N->getOperand(Num: 0);
646
647 EVT OpVT = Op.getValueType();
648 EVT OpEltVT = OpVT.getVectorElementType();
649 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
650
651 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
652 Op = GetScalarizedVector(Op);
653 } else {
654 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
655 }
656
657 switch (N->getOpcode()) {
658 case ISD::ANY_EXTEND_VECTOR_INREG:
659 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
660 case ISD::SIGN_EXTEND_VECTOR_INREG:
661 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
662 case ISD::ZERO_EXTEND_VECTOR_INREG:
663 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
664 }
665
666 llvm_unreachable("Illegal extend_vector_inreg opcode");
667}
668
669SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
670 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
671 SDValue Op = N->getOperand(Num: 0);
672 EVT OpVT = Op.getValueType();
673 SDLoc DL(N);
674 // The result needs scalarizing, but it's not a given that the source does.
675 // This is a workaround for targets where it's impossible to scalarize the
676 // result of a conversion, because the source type is legal.
677 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
678 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
679 // legal and was not scalarized.
680 // See the similar logic in ScalarizeVecRes_SETCC
681 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
682 Op = GetScalarizedVector(Op);
683 } else {
684 EVT VT = OpVT.getVectorElementType();
685 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
686 }
687 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
688 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
689 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
690 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS,
691 Flags: AddrSpaceCastN->getFlags());
692}
693
694SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
695 // If the operand is wider than the vector element type then it is implicitly
696 // truncated. Make that explicit here.
697 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
698 SDValue InOp = N->getOperand(Num: 0);
699 if (InOp.getValueType() != EltVT)
700 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
701 return InOp;
702}
703
704SDValue
705DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
706 assert(N->getNumValues() == N->getNumOperands() &&
707 "Expected one result per operand");
708
709 // Interleaving or deinterleaving one-element vectors leaves each result
710 // equal to the corresponding operand.
711 for (unsigned I = 0; I != N->getNumValues(); ++I)
712 SetScalarizedVector(Op: SDValue(N, I), Result: GetScalarizedVector(Op: N->getOperand(Num: I)));
713 return SDValue();
714}
715
716SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
717 SDValue Cond = N->getOperand(Num: 0);
718 EVT OpVT = Cond.getValueType();
719 SDLoc DL(N);
720 // The vselect result and true/value operands needs scalarizing, but it's
721 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
722 // See the similar logic in ScalarizeVecRes_SETCC
723 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
724 Cond = GetScalarizedVector(Op: Cond);
725 } else {
726 EVT VT = OpVT.getVectorElementType();
727 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
728 }
729
730 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
731 TargetLowering::BooleanContent ScalarBool =
732 TLI.getBooleanContents(isVec: false, isFloat: false);
733 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
734
735 // If integer and float booleans have different contents then we can't
736 // reliably optimize in all cases. There is a full explanation for this in
737 // DAGCombiner::visitSELECT() where the same issue affects folding
738 // (select C, 0, 1) to (xor C, 1).
739 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
740 TLI.getBooleanContents(isVec: false, isFloat: true)) {
741 // At least try the common case where the boolean is generated by a
742 // comparison.
743 if (Cond->getOpcode() == ISD::SETCC) {
744 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
745 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
746 VecBool = TLI.getBooleanContents(Type: OpVT);
747 } else
748 ScalarBool = TargetLowering::UndefinedBooleanContent;
749 }
750
751 EVT CondVT = Cond.getValueType();
752 if (ScalarBool != VecBool) {
753 switch (ScalarBool) {
754 case TargetLowering::UndefinedBooleanContent:
755 break;
756 case TargetLowering::ZeroOrOneBooleanContent:
757 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
758 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
759 // Vector read from all ones, scalar expects a single 1 so mask.
760 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
761 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
762 break;
763 case TargetLowering::ZeroOrNegativeOneBooleanContent:
764 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
765 VecBool == TargetLowering::ZeroOrOneBooleanContent);
766 // Vector reads from a one, scalar from all ones so sign extend.
767 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
768 N1: Cond, N2: DAG.getValueType(MVT::i1));
769 break;
770 }
771 }
772
773 // Truncate the condition if needed
774 auto BoolVT = getSetCCResultType(VT: CondVT);
775 if (BoolVT.bitsLT(VT: CondVT))
776 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
777
778 return DAG.getSelect(DL: SDLoc(N), VT: LHS.getValueType(), Cond, LHS,
779 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)), Flags: N->getFlags());
780}
781
782SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
783 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
784 return DAG.getSelect(DL: SDLoc(N),
785 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
786 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
787}
788
789SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
790 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
791 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
792 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
793 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
794 N5: N->getOperand(Num: 4));
795}
796
797SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
798 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
799}
800
801SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
802 // Figure out if the scalar is the LHS or RHS and return it.
803 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
804 if (Arg.isUndef())
805 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
806 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
807 return GetScalarizedVector(Op: N->getOperand(Num: Op));
808}
809
810SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
811 SDValue Src = N->getOperand(Num: 0);
812 EVT SrcVT = Src.getValueType();
813 SDLoc dl(N);
814
815 // Handle case where result is scalarized but operand is not
816 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
817 Src = GetScalarizedVector(Op: Src);
818 else
819 Src = DAG.getNode(
820 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
821 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
822
823 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
824 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
825}
826
827SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
828 assert(N->getValueType(0).isVector() &&
829 N->getOperand(0).getValueType().isVector() &&
830 "Operand types must be vectors");
831 SDValue LHS = N->getOperand(Num: 0);
832 SDValue RHS = N->getOperand(Num: 1);
833 EVT OpVT = LHS.getValueType();
834 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
835 SDLoc DL(N);
836
837 // The result needs scalarizing, but it's not a given that the source does.
838 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
839 LHS = GetScalarizedVector(Op: LHS);
840 RHS = GetScalarizedVector(Op: RHS);
841 } else {
842 EVT VT = OpVT.getVectorElementType();
843 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
844 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
845 }
846
847 // Turn it into a scalar SETCC.
848 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
849 N3: N->getOperand(Num: 2));
850 // Vectors may have a different boolean contents to scalars. Promote the
851 // value appropriately.
852 ISD::NodeType ExtendCode =
853 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
854 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
855}
856
857SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
858 SDLoc DL(N);
859 SDValue Arg = N->getOperand(Num: 0);
860 SDValue Test = N->getOperand(Num: 1);
861 EVT ArgVT = Arg.getValueType();
862 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
863
864 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
865 Arg = GetScalarizedVector(Op: Arg);
866 } else {
867 EVT VT = ArgVT.getVectorElementType();
868 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
869 }
870
871 SDValue Res =
872 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
873 // Vectors may have a different boolean contents to scalars. Promote the
874 // value appropriately.
875 ISD::NodeType ExtendCode =
876 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
877 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
878}
879
880//===----------------------------------------------------------------------===//
881// Operand Vector Scalarization <1 x ty> -> ty.
882//===----------------------------------------------------------------------===//
883
884bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
885 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
886 N->dump(&DAG));
887 SDValue Res = SDValue();
888
889 // See if the target wants to custom scalarize this node.
890 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
891 return false;
892
893 switch (N->getOpcode()) {
894 default:
895#ifndef NDEBUG
896 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
897 N->dump(&DAG);
898 dbgs() << "\n";
899#endif
900 report_fatal_error(reason: "Do not know how to scalarize this operator's "
901 "operand!\n");
902 case ISD::BITCAST:
903 Res = ScalarizeVecOp_BITCAST(N);
904 break;
905 case ISD::FAKE_USE:
906 Res = ScalarizeVecOp_FAKE_USE(N);
907 break;
908 case ISD::ANY_EXTEND:
909 case ISD::ZERO_EXTEND:
910 case ISD::SIGN_EXTEND:
911 case ISD::TRUNCATE:
912 case ISD::FP_TO_SINT:
913 case ISD::FP_TO_UINT:
914 case ISD::SINT_TO_FP:
915 case ISD::UINT_TO_FP:
916 case ISD::LROUND:
917 case ISD::LLROUND:
918 case ISD::LRINT:
919 case ISD::LLRINT:
920 Res = ScalarizeVecOp_UnaryOp(N);
921 break;
922 case ISD::FP_TO_SINT_SAT:
923 case ISD::FP_TO_UINT_SAT:
924 case ISD::CONVERT_FROM_ARBITRARY_FP:
925 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
926 break;
927 case ISD::CONVERT_TO_ARBITRARY_FP: {
928 assert(N->getValueType(0).getVectorNumElements() == 1 &&
929 "Unexpected vector type!");
930 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
931 SDValue Op = DAG.getNode(
932 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
933 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
934 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
935 break;
936 }
937 case ISD::STRICT_SINT_TO_FP:
938 case ISD::STRICT_UINT_TO_FP:
939 case ISD::STRICT_FP_TO_SINT:
940 case ISD::STRICT_FP_TO_UINT:
941 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
942 break;
943 case ISD::CONCAT_VECTORS:
944 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
945 break;
946 case ISD::INSERT_SUBVECTOR:
947 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
948 break;
949 case ISD::EXTRACT_VECTOR_ELT:
950 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
951 break;
952 case ISD::VSELECT:
953 Res = ScalarizeVecOp_VSELECT(N);
954 break;
955 case ISD::SETCC:
956 Res = ScalarizeVecOp_VSETCC(N);
957 break;
958 case ISD::STRICT_FSETCC:
959 case ISD::STRICT_FSETCCS:
960 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
961 break;
962 case ISD::STORE:
963 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
964 break;
965 case ISD::ATOMIC_STORE:
966 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
967 break;
968 case ISD::STRICT_FP_ROUND:
969 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
970 break;
971 case ISD::FP_ROUND:
972 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
973 break;
974 case ISD::STRICT_FP_EXTEND:
975 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
976 break;
977 case ISD::FP_EXTEND:
978 Res = ScalarizeVecOp_FP_EXTEND(N);
979 break;
980 case ISD::VECREDUCE_FADD:
981 case ISD::VECREDUCE_FMUL:
982 case ISD::VECREDUCE_ADD:
983 case ISD::VECREDUCE_MUL:
984 case ISD::VECREDUCE_AND:
985 case ISD::VECREDUCE_OR:
986 case ISD::VECREDUCE_XOR:
987 case ISD::VECREDUCE_SMAX:
988 case ISD::VECREDUCE_SMIN:
989 case ISD::VECREDUCE_UMAX:
990 case ISD::VECREDUCE_UMIN:
991 case ISD::VECREDUCE_FMAX:
992 case ISD::VECREDUCE_FMIN:
993 case ISD::VECREDUCE_FMAXIMUM:
994 case ISD::VECREDUCE_FMINIMUM:
995 case ISD::VECREDUCE_FMAXIMUMNUM:
996 case ISD::VECREDUCE_FMINIMUMNUM:
997 Res = ScalarizeVecOp_VECREDUCE(N);
998 break;
999 case ISD::VECREDUCE_SEQ_FADD:
1000 case ISD::VECREDUCE_SEQ_FMUL:
1001 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
1002 break;
1003 case ISD::SCMP:
1004 case ISD::UCMP:
1005 Res = ScalarizeVecOp_CMP(N);
1006 break;
1007 case ISD::VECTOR_FIND_LAST_ACTIVE:
1008 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
1009 break;
1010 case ISD::CTTZ_ELTS:
1011 case ISD::CTTZ_ELTS_ZERO_POISON:
1012 Res = ScalarizeVecOp_CTTZ_ELTS(N);
1013 break;
1014 case ISD::VECTOR_MATCH:
1015 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
1016 break;
1017 case ISD::MASKED_UDIV:
1018 case ISD::MASKED_SDIV:
1019 case ISD::MASKED_UREM:
1020 case ISD::MASKED_SREM:
1021 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
1022 break;
1023 }
1024
1025 // If the result is null, the sub-method took care of registering results etc.
1026 if (!Res.getNode()) return false;
1027
1028 // If the result is N, the sub-method updated N in place. Tell the legalizer
1029 // core about this.
1030 if (Res.getNode() == N)
1031 return true;
1032
1033 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1034 "Invalid operand expansion");
1035
1036 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1037 return false;
1038}
1039
1040/// If the value to convert is a vector that needs to be scalarized, it must be
1041/// <1 x ty>. Convert the element instead.
1042SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1043 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1044 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1045 VT: N->getValueType(ResNo: 0), Operand: Elt);
1046}
1047
1048// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1049SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1050 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1051 "Fake Use: Unexpected vector type!");
1052 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1053 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1054}
1055
1056/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1057/// Do the operation on the element instead.
1058SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1059 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1060 "Unexpected vector type!");
1061 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1062 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1063 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1064 // Revectorize the result so the types line up with what the uses of this
1065 // expression expect.
1066 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1067}
1068
1069/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1070/// typesize).
1071SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1072 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1073 "Unexpected vector type!");
1074 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1075 SDValue Op =
1076 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1077 N1: Elt, N2: N->getOperand(Num: 1));
1078 // Revectorize the result so the types line up with what the uses of this
1079 // expression expect.
1080 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1081}
1082
1083/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1084/// Do the strict FP operation on the element instead.
1085SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1086 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1087 "Unexpected vector type!");
1088 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1089 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1090 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1091 Ops: { N->getOperand(Num: 0), Elt });
1092 // Legalize the chain result - switch anything that used the old chain to
1093 // use the new one.
1094 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1095 // Revectorize the result so the types line up with what the uses of this
1096 // expression expect.
1097 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1098
1099 // Do our own replacement and return SDValue() to tell the caller that we
1100 // handled all replacements since caller can only handle a single result.
1101 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1102 return SDValue();
1103}
1104
1105/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1106SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1107 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1108 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1109 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1110 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1111}
1112
1113/// The inserted subvector is to be scalarized - use insert vector element
1114/// instead.
1115SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1116 unsigned OpNo) {
1117 // We should not be attempting to scalarize the containing vector
1118 assert(OpNo == 1);
1119 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1120 SDValue ContainingVec = N->getOperand(Num: 0);
1121 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1122 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1123 N3: N->getOperand(Num: 2));
1124}
1125
1126/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1127/// so just return the element, ignoring the index.
1128SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1129 EVT VT = N->getValueType(ResNo: 0);
1130 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1131 if (Res.getValueType() != VT)
1132 Res = VT.isFloatingPoint()
1133 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1134 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1135 return Res;
1136}
1137
1138/// If the input condition is a vector that needs to be scalarized, it must be
1139/// <1 x i1>, so just convert to a normal ISD::SELECT
1140/// (still with vector output type since that was acceptable if we got here).
1141SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1142 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1143 EVT VT = N->getValueType(ResNo: 0);
1144
1145 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1146 N3: N->getOperand(Num: 2));
1147}
1148
1149/// If the operand is a vector that needs to be scalarized then the
1150/// result must be a single-element vector, so just convert to a scalar
1151/// SETCC and wrap with a scalar_to_vector since the res type is legal
1152/// if we got here
1153SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1154 assert(N->getValueType(0).isVector() &&
1155 N->getOperand(0).getValueType().isVector() &&
1156 "Operand types must be vectors");
1157 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1158 "Expected single-element vector type");
1159
1160 EVT VT = N->getValueType(ResNo: 0);
1161 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1162 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1163
1164 EVT OpVT = N->getOperand(Num: 0).getValueType();
1165 EVT NVT = VT.getVectorElementType();
1166 SDLoc DL(N);
1167 // Turn it into a scalar SETCC.
1168 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1169 N3: N->getOperand(Num: 2));
1170
1171 // Vectors may have a different boolean contents to scalars. Promote the
1172 // value appropriately.
1173 ISD::NodeType ExtendCode =
1174 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1175
1176 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1177
1178 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1179}
1180
1181// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1182SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1183 unsigned OpNo) {
1184 assert(OpNo == 1 && "Wrong operand for scalarization!");
1185 assert(N->getValueType(0).isVector() &&
1186 N->getOperand(1).getValueType().isVector() &&
1187 "Operand types must be vectors");
1188 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1189 "Expected single-element vector type");
1190
1191 EVT VT = N->getValueType(ResNo: 0);
1192 SDValue Ch = N->getOperand(Num: 0);
1193 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1194 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1195 SDValue CC = N->getOperand(Num: 3);
1196
1197 EVT OpVT = N->getOperand(Num: 1).getValueType();
1198 EVT NVT = VT.getVectorElementType();
1199 SDLoc DL(N);
1200 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1201 Ops: {Ch, LHS, RHS, CC});
1202
1203 // Legalize the chain result - switch anything that used the old chain to
1204 // use the new one.
1205 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1206
1207 ISD::NodeType ExtendCode =
1208 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1209
1210 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1211 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1212
1213 // Do our own replacement and return SDValue() to tell the caller that we
1214 // handled all replacements since caller can only handle a single result.
1215 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1216 return SDValue();
1217}
1218
1219/// If the value to store is a vector that needs to be scalarized, it must be
1220/// <1 x ty>. Just store the element.
1221SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1222 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1223 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1224 SDLoc dl(N);
1225
1226 if (N->isTruncatingStore())
1227 return DAG.getTruncStore(
1228 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1229 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1230 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1231 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getMMOMetadataForSubAccess());
1232
1233 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1234 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1235 MMOFlags: N->getMemOperand()->getFlags(),
1236 Metadata: N->getMMOMetadataForSubAccess());
1237}
1238
1239/// If the value to store is a vector that needs to be scalarized, it must be
1240/// <1 x ty>. Just store the element.
1241SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1242 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1243 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1244 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1245 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1246}
1247
1248/// If the value to round is a vector that needs to be scalarized, it must be
1249/// <1 x ty>. Convert the element instead.
1250SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1251 assert(OpNo == 0 && "Wrong operand for scalarization!");
1252 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1253 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1254 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1255 N2: N->getOperand(Num: 1));
1256 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1257}
1258
1259SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1260 unsigned OpNo) {
1261 assert(OpNo == 1 && "Wrong operand for scalarization!");
1262 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1263 SDValue Res =
1264 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1265 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1266 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1267 // Legalize the chain result - switch anything that used the old chain to
1268 // use the new one.
1269 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1270
1271 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1272
1273 // Do our own replacement and return SDValue() to tell the caller that we
1274 // handled all replacements since caller can only handle a single result.
1275 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1276 return SDValue();
1277}
1278
1279/// If the value to extend is a vector that needs to be scalarized, it must be
1280/// <1 x ty>. Convert the element instead.
1281SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1282 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1283 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1284 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1285 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1286}
1287
1288/// If the value to extend is a vector that needs to be scalarized, it must be
1289/// <1 x ty>. Convert the element instead.
1290SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1291 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1292 SDValue Res =
1293 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1294 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1295 Ops: {N->getOperand(Num: 0), Elt});
1296 // Legalize the chain result - switch anything that used the old chain to
1297 // use the new one.
1298 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1299
1300 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1301
1302 // Do our own replacement and return SDValue() to tell the caller that we
1303 // handled all replacements since caller can only handle a single result.
1304 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1305 return SDValue();
1306}
1307
1308SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1309 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1310 // Result type may be wider than element type.
1311 if (Res.getValueType() != N->getValueType(ResNo: 0))
1312 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1313 return Res;
1314}
1315
1316SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1317 SDValue AccOp = N->getOperand(Num: 0);
1318 SDValue VecOp = N->getOperand(Num: 1);
1319
1320 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1321
1322 SDValue Op = GetScalarizedVector(Op: VecOp);
1323 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1324 N1: AccOp, N2: Op, Flags: N->getFlags());
1325}
1326
1327SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1328 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1329 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1330
1331 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1332 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1333 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1334}
1335
1336SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1337 // Since there is no "none-active" result, the only valid return for <1 x ty>
1338 // is 0. Note: Since we check the high mask during splitting this is safe.
1339 // As e.g., a <2 x ty> operation would split to:
1340 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1341 // : `last_active(%lo_mask)`
1342 // Which then scalarizes to:
1343 // %mask[1] ? 1 : 0
1344 EVT VT = N->getValueType(ResNo: 0);
1345 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1346}
1347
1348SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1349 // The number of trailing zero elements is 1 if the element is 0, and 0
1350 // otherwise.
1351 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1352 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1353 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1354 SDValue SetCC =
1355 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1356 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1357 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1358}
1359
1360SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1361 SDLoc DL(N);
1362 // Reuse the expansion (which should scalarize).
1363 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1364 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
1365 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
1366 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1367}
1368
1369SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1370 unsigned OpNo) {
1371 return TLI.expandVectorMatch(N, DAG);
1372}
1373
1374SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1375 assert(OpNo == 2 && "Can only scalarize mask operand");
1376 SDLoc DL(N);
1377 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1378 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1379 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1380 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1381 // Vectors may have a different boolean contents to scalars, so truncate to i1
1382 // and let type legalization promote appropriately.
1383 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1384 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1385 // use 1 as the divisor to avoid division by zero and overflow.
1386 SDValue BinOp =
1387 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1388 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1389 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1390}
1391
1392//===----------------------------------------------------------------------===//
1393// Result Vector Splitting
1394//===----------------------------------------------------------------------===//
1395
1396/// This method is called when the specified result of the specified node is
1397/// found to need vector splitting. At this point, the node may also have
1398/// invalid operands or may have other results that need legalization, we just
1399/// know that (at least) one result needs vector splitting.
1400void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1401 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1402 SDValue Lo, Hi;
1403
1404 // See if the target wants to custom expand this node.
1405 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1406 return;
1407
1408 switch (N->getOpcode()) {
1409 default:
1410#ifndef NDEBUG
1411 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1412 N->dump(&DAG);
1413 dbgs() << "\n";
1414#endif
1415 report_fatal_error(reason: "Do not know how to split the result of this "
1416 "operator!\n");
1417
1418 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1419 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1420 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1421 break;
1422 case ISD::MASK_BEFOREFIRST:
1423 SplitVecRes_MASK_BEFOREFIRST(N, Lo, Hi);
1424 break;
1425 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1426 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1427 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1428 case ISD::VSELECT:
1429 case ISD::SELECT:
1430 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1431 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1432 case ISD::POISON:
1433 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1434 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1435 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1436 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1437 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1438 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1439 case ISD::FPOWI:
1440 case ISD::FLDEXP:
1441 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1442 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1443 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1444 case ISD::SPLAT_VECTOR:
1445 case ISD::SCALAR_TO_VECTOR:
1446 SplitVecRes_ScalarOp(N, Lo, Hi);
1447 break;
1448 case ISD::STEP_VECTOR:
1449 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1450 break;
1451 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1452 case ISD::ATOMIC_LOAD:
1453 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1454 break;
1455 case ISD::LOAD:
1456 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1457 break;
1458 case ISD::VP_LOAD:
1459 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1460 break;
1461 case ISD::VP_LOAD_FF:
1462 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1463 break;
1464 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1465 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1466 break;
1467 case ISD::MLOAD:
1468 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1469 break;
1470 case ISD::MGATHER:
1471 case ISD::VP_GATHER:
1472 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1473 break;
1474 case ISD::VECTOR_COMPRESS:
1475 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1476 break;
1477 case ISD::SETCC:
1478 SplitVecRes_SETCC(N, Lo, Hi);
1479 break;
1480 case ISD::VECTOR_REPEAT:
1481 SplitVecRes_VECTOR_REPEAT(N, Lo, Hi);
1482 break;
1483 case ISD::VECTOR_REVERSE:
1484 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1485 break;
1486 case ISD::VECTOR_SHUFFLE:
1487 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1488 break;
1489 case ISD::VECTOR_SPLICE_LEFT:
1490 case ISD::VECTOR_SPLICE_RIGHT:
1491 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1492 break;
1493 case ISD::VECTOR_DEINTERLEAVE:
1494 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1495 return;
1496 case ISD::VECTOR_INTERLEAVE:
1497 SplitVecRes_VECTOR_INTERLEAVE(N);
1498 return;
1499 case ISD::VAARG:
1500 SplitVecRes_VAARG(N, Lo, Hi);
1501 break;
1502
1503 case ISD::ANY_EXTEND_VECTOR_INREG:
1504 case ISD::SIGN_EXTEND_VECTOR_INREG:
1505 case ISD::ZERO_EXTEND_VECTOR_INREG:
1506 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1507 break;
1508
1509 case ISD::ABS:
1510 case ISD::ABS_MIN_POISON:
1511 case ISD::BITREVERSE:
1512 case ISD::BSWAP:
1513 case ISD::CTLZ:
1514 case ISD::CTTZ:
1515 case ISD::CTLZ_ZERO_POISON:
1516 case ISD::CTTZ_ZERO_POISON:
1517 case ISD::CTPOP:
1518 case ISD::FABS:
1519 case ISD::FACOS:
1520 case ISD::FASIN:
1521 case ISD::FATAN:
1522 case ISD::FCEIL:
1523 case ISD::FCOS:
1524 case ISD::FCOSH:
1525 case ISD::FEXP:
1526 case ISD::FEXP2:
1527 case ISD::FEXP10:
1528 case ISD::FFLOOR:
1529 case ISD::FLOG:
1530 case ISD::FLOG10:
1531 case ISD::FLOG2:
1532 case ISD::FNEARBYINT:
1533 case ISD::FNEG:
1534 case ISD::FREEZE:
1535 case ISD::ARITH_FENCE:
1536 case ISD::FP_EXTEND:
1537 case ISD::FP_ROUND:
1538 case ISD::FP_TO_SINT:
1539 case ISD::FP_TO_UINT:
1540 case ISD::FRINT:
1541 case ISD::LRINT:
1542 case ISD::LLRINT:
1543 case ISD::FROUND:
1544 case ISD::FROUNDEVEN:
1545 case ISD::LROUND:
1546 case ISD::LLROUND:
1547 case ISD::FSIN:
1548 case ISD::FSINH:
1549 case ISD::FSQRT:
1550 case ISD::FTAN:
1551 case ISD::FTANH:
1552 case ISD::FTRUNC:
1553 case ISD::SINT_TO_FP:
1554 case ISD::TRUNCATE:
1555 case ISD::UINT_TO_FP:
1556 case ISD::FCANONICALIZE:
1557 case ISD::AssertNoFPClass:
1558 case ISD::CONVERT_FROM_ARBITRARY_FP:
1559 case ISD::CONVERT_TO_ARBITRARY_FP:
1560 SplitVecRes_UnaryOp(N, Lo, Hi);
1561 break;
1562 case ISD::ADDRSPACECAST:
1563 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1564 break;
1565 case ISD::FMODF:
1566 case ISD::FFREXP:
1567 case ISD::FSINCOS:
1568 case ISD::FSINCOSPI:
1569 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1570 break;
1571
1572 case ISD::ANY_EXTEND:
1573 case ISD::SIGN_EXTEND:
1574 case ISD::ZERO_EXTEND:
1575 SplitVecRes_ExtendOp(N, Lo, Hi);
1576 break;
1577
1578 case ISD::ADD:
1579 case ISD::SUB:
1580 case ISD::MUL:
1581 case ISD::CLMUL:
1582 case ISD::CLMULR:
1583 case ISD::CLMULH:
1584 case ISD::PEXT:
1585 case ISD::PDEP:
1586 case ISD::MULHS:
1587 case ISD::MULHU:
1588 case ISD::ABDS:
1589 case ISD::ABDU:
1590 case ISD::AVGCEILS:
1591 case ISD::AVGCEILU:
1592 case ISD::AVGFLOORS:
1593 case ISD::AVGFLOORU:
1594 case ISD::FADD:
1595 case ISD::FSUB:
1596 case ISD::FMUL:
1597 case ISD::FMINNUM:
1598 case ISD::FMINNUM_IEEE:
1599 case ISD::FMAXNUM:
1600 case ISD::FMAXNUM_IEEE:
1601 case ISD::FMINIMUM:
1602 case ISD::FMAXIMUM:
1603 case ISD::FMINIMUMNUM:
1604 case ISD::FMAXIMUMNUM:
1605 case ISD::SDIV: case ISD::VP_SDIV:
1606 case ISD::UDIV: case ISD::VP_UDIV:
1607 case ISD::FDIV:
1608 case ISD::FPOW:
1609 case ISD::FATAN2:
1610 case ISD::AND:
1611 case ISD::OR:
1612 case ISD::XOR:
1613 case ISD::SHL:
1614 case ISD::SRA:
1615 case ISD::SRL:
1616 case ISD::UREM: case ISD::VP_UREM:
1617 case ISD::SREM: case ISD::VP_SREM:
1618 case ISD::FREM:
1619 case ISD::SMIN:
1620 case ISD::SMAX:
1621 case ISD::UMIN:
1622 case ISD::UMAX:
1623 case ISD::SADDSAT:
1624 case ISD::UADDSAT:
1625 case ISD::SSUBSAT:
1626 case ISD::USUBSAT:
1627 case ISD::SSHLSAT:
1628 case ISD::USHLSAT:
1629 case ISD::ROTL:
1630 case ISD::ROTR:
1631 SplitVecRes_BinOp(N, Lo, Hi);
1632 break;
1633 case ISD::MASKED_UDIV:
1634 case ISD::MASKED_SDIV:
1635 case ISD::MASKED_UREM:
1636 case ISD::MASKED_SREM:
1637 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1638 break;
1639 case ISD::FMA:
1640 case ISD::FSHL:
1641 case ISD::FSHR:
1642 SplitVecRes_TernaryOp(N, Lo, Hi);
1643 break;
1644
1645 case ISD::SCMP: case ISD::UCMP:
1646 SplitVecRes_CMP(N, Lo, Hi);
1647 break;
1648
1649#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1650 case ISD::STRICT_##DAGN:
1651#include "llvm/IR/ConstrainedOps.def"
1652 SplitVecRes_StrictFPOp(N, Lo, Hi);
1653 break;
1654
1655 case ISD::FP_TO_UINT_SAT:
1656 case ISD::FP_TO_SINT_SAT:
1657 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1658 break;
1659
1660 case ISD::UADDO:
1661 case ISD::SADDO:
1662 case ISD::USUBO:
1663 case ISD::SSUBO:
1664 case ISD::UMULO:
1665 case ISD::SMULO:
1666 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1667 break;
1668 case ISD::SMULFIX:
1669 case ISD::SMULFIXSAT:
1670 case ISD::UMULFIX:
1671 case ISD::UMULFIXSAT:
1672 case ISD::SDIVFIX:
1673 case ISD::SDIVFIXSAT:
1674 case ISD::UDIVFIX:
1675 case ISD::UDIVFIXSAT:
1676 SplitVecRes_FIX(N, Lo, Hi);
1677 break;
1678 case ISD::EXPERIMENTAL_VP_SPLICE:
1679 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1680 break;
1681 case ISD::EXPERIMENTAL_VP_REVERSE:
1682 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1683 break;
1684 case ISD::PARTIAL_REDUCE_UMLA:
1685 case ISD::PARTIAL_REDUCE_SMLA:
1686 case ISD::PARTIAL_REDUCE_SUMLA:
1687 case ISD::PARTIAL_REDUCE_FMLA:
1688 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1689 break;
1690 case ISD::GET_ACTIVE_LANE_MASK:
1691 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1692 break;
1693 case ISD::VECTOR_MATCH:
1694 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1695 break;
1696 }
1697
1698 // If Lo/Hi is null, the sub-method took care of registering results etc.
1699 if (Lo.getNode())
1700 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1701}
1702
1703void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1704 MachinePointerInfo &MPI, SDValue &Ptr,
1705 uint64_t *ScaledOffset) {
1706 SDLoc DL(N);
1707 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1708
1709 if (MemVT.isScalableVector()) {
1710 SDValue BytesIncrement = DAG.getVScale(
1711 DL, VT: Ptr.getValueType(),
1712 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1713 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1714 if (ScaledOffset)
1715 *ScaledOffset += IncrementSize;
1716 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1717 Flags: SDNodeFlags::NoUnsignedWrap);
1718 } else {
1719 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1720 // Increment the pointer to the other half.
1721 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1722 }
1723}
1724
1725std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1726 return SplitMask(Mask, DL: SDLoc(Mask));
1727}
1728
1729std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1730 const SDLoc &DL) {
1731 SDValue MaskLo, MaskHi;
1732 EVT MaskVT = Mask.getValueType();
1733 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1734 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1735 else
1736 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1737 return std::make_pair(x&: MaskLo, y&: MaskHi);
1738}
1739
1740void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1741 SDValue LHSLo, LHSHi;
1742 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1743 SDValue RHSLo, RHSHi;
1744 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1745 SDLoc dl(N);
1746
1747 const SDNodeFlags Flags = N->getFlags();
1748 unsigned Opcode = N->getOpcode();
1749 if (N->getNumOperands() == 2) {
1750 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1751 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1752 return;
1753 }
1754
1755 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1756 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1757 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1758 "Expected VP opcode");
1759
1760 SDValue MaskLo, MaskHi;
1761 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1762
1763 SDValue EVLLo, EVLHi;
1764 std::tie(args&: EVLLo, args&: EVLHi) =
1765 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1766
1767 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1768 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1769 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1770 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1771}
1772
1773void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1774 SDValue &Hi) {
1775 SDValue LHSLo, LHSHi;
1776 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1777 SDValue RHSLo, RHSHi;
1778 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1779
1780 SDValue MaskLo, MaskHi, Mask = N->getOperand(Num: 2);
1781 if (Mask.getOpcode() == ISD::SETCC)
1782 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1783 else
1784 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask);
1785
1786 SDLoc dl(N);
1787
1788 const SDNodeFlags Flags = N->getFlags();
1789 unsigned Opcode = N->getOpcode();
1790 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1791 Flags);
1792 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1793 Flags);
1794}
1795
1796void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1797 SDValue &Hi) {
1798 SDValue Op0Lo, Op0Hi;
1799 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1800 SDValue Op1Lo, Op1Hi;
1801 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1802 SDValue Op2Lo, Op2Hi;
1803 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1804 SDLoc dl(N);
1805
1806 const SDNodeFlags Flags = N->getFlags();
1807 unsigned Opcode = N->getOpcode();
1808 Lo =
1809 DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1810 Hi =
1811 DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1812}
1813
1814void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1815 LLVMContext &Ctxt = *DAG.getContext();
1816 SDLoc dl(N);
1817
1818 SDValue LHS = N->getOperand(Num: 0);
1819 SDValue RHS = N->getOperand(Num: 1);
1820
1821 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1822 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1823 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1824 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1825 } else {
1826 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1827 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1828 }
1829
1830 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1831 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1832 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1833}
1834
1835void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1836 SDValue LHSLo, LHSHi;
1837 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1838 SDValue RHSLo, RHSHi;
1839 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1840 SDLoc dl(N);
1841 SDValue Op2 = N->getOperand(Num: 2);
1842
1843 unsigned Opcode = N->getOpcode();
1844 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1845 Flags: N->getFlags());
1846 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1847 Flags: N->getFlags());
1848}
1849
1850void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1851 SDValue &Hi) {
1852 // We know the result is a vector. The input may be either a vector or a
1853 // scalar value.
1854 EVT LoVT, HiVT;
1855 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1856 SDLoc dl(N);
1857
1858 SDValue InOp = N->getOperand(Num: 0);
1859 EVT InVT = InOp.getValueType();
1860
1861 // Handle some special cases efficiently.
1862 switch (getTypeAction(VT: InVT)) {
1863 case TargetLowering::TypeLegal:
1864 case TargetLowering::TypePromoteInteger:
1865 case TargetLowering::TypeSoftPromoteHalf:
1866 case TargetLowering::TypeSoftenFloat:
1867 case TargetLowering::TypeScalarizeVector:
1868 case TargetLowering::TypeWidenVector:
1869 break;
1870 case TargetLowering::TypeExpandInteger:
1871 case TargetLowering::TypeExpandFloat:
1872 // A scalar to vector conversion, where the scalar needs expansion.
1873 // If the vector is being split in two then we can just convert the
1874 // expanded pieces.
1875 if (LoVT == HiVT) {
1876 GetExpandedOp(Op: InOp, Lo, Hi);
1877 if (DAG.getDataLayout().isBigEndian())
1878 std::swap(a&: Lo, b&: Hi);
1879 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1880 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1881 return;
1882 }
1883 break;
1884 case TargetLowering::TypeSplitVector:
1885 // If the input is a vector that needs to be split, convert each split
1886 // piece of the input now.
1887 GetSplitVector(Op: InOp, Lo, Hi);
1888 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1889 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1890 return;
1891 case TargetLowering::TypeScalarizeScalableVector:
1892 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
1893 }
1894
1895 if (LoVT.isScalableVector()) {
1896 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1897 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1898 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1899 return;
1900 }
1901
1902 // In the general case, convert the input to an integer and split it by hand.
1903 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1904 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1905 if (DAG.getDataLayout().isBigEndian())
1906 std::swap(a&: LoIntVT, b&: HiIntVT);
1907
1908 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1909
1910 if (DAG.getDataLayout().isBigEndian())
1911 std::swap(a&: Lo, b&: Hi);
1912 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1913 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1914}
1915
1916void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1917 SDValue &Hi) {
1918 SDLoc DL(N);
1919 EVT LoVT, HiVT;
1920 SDValue PtrA = N->getOperand(Num: 0);
1921 SDValue PtrB = N->getOperand(Num: 1);
1922 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1923
1924 // The lane offset for the "Lo" half of the mask is unchanged.
1925 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1926 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1927 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1928 // The lane offset for the "Hi" half of the mask is incremented by the number
1929 // of elements in the "Lo" half.
1930 unsigned LaneOffset =
1931 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1932 // Note: The lane offset is implicitly scalable for scalable masks.
1933 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1934 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1935 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1936}
1937
1938void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
1939 SDValue &Hi) {
1940 SDLoc DL(N);
1941 SDValue InLo, InHi;
1942 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
1943 EVT VT = InLo.getValueType();
1944 Lo = DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InLo);
1945
1946 // hi = AnyLoActive ? all-zeros : (mask_beforefirst hi)
1947 SDValue AnyLoActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: InLo);
1948 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyLoActive, SL: DL,
1949 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
1950 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond, N2: DAG.getConstant(Val: 0, DL, VT),
1951 N3: DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InHi));
1952}
1953
1954void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1955 SDValue &Hi) {
1956 EVT LoVT, HiVT;
1957 SDLoc dl(N);
1958 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1959 unsigned LoNumElts = LoVT.getVectorNumElements();
1960 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1961 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1962
1963 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1964 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1965}
1966
1967void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1968 SDValue &Hi) {
1969 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1970 SDLoc dl(N);
1971 unsigned NumSubvectors = N->getNumOperands() / 2;
1972 if (NumSubvectors == 1) {
1973 Lo = N->getOperand(Num: 0);
1974 Hi = N->getOperand(Num: 1);
1975 return;
1976 }
1977
1978 EVT LoVT, HiVT;
1979 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1980
1981 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1982 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1983
1984 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1985 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1986}
1987
1988void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1989 SDValue &Hi) {
1990 SDValue Vec = N->getOperand(Num: 0);
1991 SDValue Idx = N->getOperand(Num: 1);
1992 SDLoc dl(N);
1993
1994 EVT LoVT, HiVT;
1995 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1996
1997 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1998 uint64_t IdxVal = Idx->getAsZExtVal();
1999 Hi = DAG.getNode(
2000 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
2001 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
2002}
2003
2004void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
2005 SDValue &Hi) {
2006 SDValue Vec = N->getOperand(Num: 0);
2007 SDValue SubVec = N->getOperand(Num: 1);
2008 SDValue Idx = N->getOperand(Num: 2);
2009 SDLoc dl(N);
2010 GetSplitVector(Op: Vec, Lo, Hi);
2011
2012 EVT VecVT = Vec.getValueType();
2013 EVT LoVT = Lo.getValueType();
2014 EVT SubVecVT = SubVec.getValueType();
2015 unsigned VecElems = VecVT.getVectorMinNumElements();
2016 unsigned SubElems = SubVecVT.getVectorMinNumElements();
2017 unsigned LoElems = LoVT.getVectorMinNumElements();
2018
2019 // If we know the index is in the first half, and we know the subvector
2020 // doesn't cross the boundary between the halves, we can avoid spilling the
2021 // vector, and insert into the lower half of the split vector directly.
2022 unsigned IdxVal = Idx->getAsZExtVal();
2023 if (IdxVal + SubElems <= LoElems) {
2024 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
2025 return;
2026 }
2027 // Similarly if the subvector is fully in the high half, but mind that we
2028 // can't tell whether a fixed-length subvector is fully within the high half
2029 // of a scalable vector.
2030 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
2031 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
2032 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
2033 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
2034 return;
2035 }
2036
2037 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
2038 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
2039 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
2040 if (WideSubVec.getValueType() == VecVT) {
2041 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
2042 return;
2043 }
2044 }
2045
2046 // Spill the vector to the stack.
2047 // In cases where the vector is illegal it will be broken down into parts
2048 // and stored in parts - we should use the alignment for the smallest part.
2049 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2050 SDValue StackPtr =
2051 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2052 auto &MF = DAG.getMachineFunction();
2053 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2054 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2055
2056 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2057 Alignment: SmallestAlign);
2058
2059 // Store the new subvector into the specified index.
2060 SDValue SubVecPtr =
2061 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2062 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2063 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2064
2065 // Load the Lo part from the stack slot.
2066 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2067 Alignment: SmallestAlign);
2068
2069 // Increment the pointer to the other part.
2070 auto *Load = cast<LoadSDNode>(Val&: Lo);
2071 MachinePointerInfo MPI = Load->getPointerInfo();
2072 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2073
2074 // Load the Hi part from the stack slot.
2075 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2076}
2077
2078// Handle splitting an FP where the second operand does not match the first
2079// type. The second operand may be a scalar, or a vector that has exactly as
2080// many elements as the first
2081void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2082 SDValue &Hi) {
2083 SDValue LHSLo, LHSHi;
2084 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2085 SDLoc DL(N);
2086
2087 SDValue RHSLo, RHSHi;
2088 SDValue RHS = N->getOperand(Num: 1);
2089 EVT RHSVT = RHS.getValueType();
2090 if (RHSVT.isVector()) {
2091 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2092 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2093 else
2094 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2095
2096 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2097 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2098 } else {
2099 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2100 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2101 }
2102}
2103
2104void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2105 SDValue &Hi) {
2106 SDLoc DL(N);
2107 SDValue ArgLo, ArgHi;
2108 SDValue Test = N->getOperand(Num: 1);
2109 SDValue FpValue = N->getOperand(Num: 0);
2110 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2111 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2112 else
2113 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2114 EVT LoVT, HiVT;
2115 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2116
2117 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2118 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2119}
2120
2121void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2122 SDValue &Hi) {
2123 SDValue LHSLo, LHSHi;
2124 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2125 SDLoc dl(N);
2126
2127 EVT LoVT, HiVT;
2128 std::tie(args&: LoVT, args&: HiVT) =
2129 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2130
2131 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2132 N2: DAG.getValueType(LoVT));
2133 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2134 N2: DAG.getValueType(HiVT));
2135}
2136
2137void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2138 SDValue &Hi) {
2139 unsigned Opcode = N->getOpcode();
2140 SDValue N0 = N->getOperand(Num: 0);
2141
2142 SDLoc dl(N);
2143 SDValue InLo, InHi;
2144
2145 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2146 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2147 else
2148 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2149
2150 EVT InLoVT = InLo.getValueType();
2151 unsigned InNumElements = InLoVT.getVectorNumElements();
2152
2153 EVT OutLoVT, OutHiVT;
2154 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2155 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2156 assert((2 * OutNumElements) <= InNumElements &&
2157 "Illegal extend vector in reg split");
2158
2159 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2160 // input vector (i.e. we only use InLo):
2161 // OutLo will extend the first OutNumElements from InLo.
2162 // OutHi will extend the next OutNumElements from InLo.
2163
2164 // Shuffle the elements from InLo for OutHi into the bottom elements to
2165 // create a 'fake' InHi.
2166 SmallVector<int, 8> SplitHi(InNumElements, -1);
2167 for (unsigned i = 0; i != OutNumElements; ++i)
2168 SplitHi[i] = i + OutNumElements;
2169 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2170
2171 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2172 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2173}
2174
2175void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2176 SDValue &Hi) {
2177 unsigned NumOps = N->getNumOperands();
2178 SDValue Chain = N->getOperand(Num: 0);
2179 EVT LoVT, HiVT;
2180 SDLoc dl(N);
2181 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2182
2183 SmallVector<SDValue, 4> OpsLo(NumOps);
2184 SmallVector<SDValue, 4> OpsHi(NumOps);
2185
2186 // The Chain is the first operand.
2187 OpsLo[0] = Chain;
2188 OpsHi[0] = Chain;
2189
2190 // Now process the remaining operands.
2191 for (unsigned i = 1; i < NumOps; ++i) {
2192 SDValue Op = N->getOperand(Num: i);
2193 SDValue OpLo = Op;
2194 SDValue OpHi = Op;
2195
2196 EVT InVT = Op.getValueType();
2197 if (InVT.isVector()) {
2198 // If the input also splits, handle it directly for a
2199 // compile time speedup. Otherwise split it by hand.
2200 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2201 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2202 else
2203 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2204 }
2205
2206 OpsLo[i] = OpLo;
2207 OpsHi[i] = OpHi;
2208 }
2209
2210 EVT LoValueVTs[] = {LoVT, MVT::Other};
2211 EVT HiValueVTs[] = {HiVT, MVT::Other};
2212 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2213 Flags: N->getFlags());
2214 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2215 Flags: N->getFlags());
2216
2217 // Build a factor node to remember that this Op is independent of the
2218 // other one.
2219 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2220 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2221
2222 // Legalize the chain result - switch anything that used the old chain to
2223 // use the new one.
2224 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2225}
2226
2227SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2228 SDValue Chain = N->getOperand(Num: 0);
2229 EVT VT = N->getValueType(ResNo: 0);
2230 unsigned NE = VT.getVectorNumElements();
2231 EVT EltVT = VT.getVectorElementType();
2232 SDLoc dl(N);
2233
2234 SmallVector<SDValue, 8> Scalars;
2235 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2236
2237 // If ResNE is 0, fully unroll the vector op.
2238 if (ResNE == 0)
2239 ResNE = NE;
2240 else if (NE > ResNE)
2241 NE = ResNE;
2242
2243 //The results of each unrolled operation, including the chain.
2244 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2245 SmallVector<SDValue, 8> Chains;
2246
2247 unsigned i;
2248 for (i = 0; i != NE; ++i) {
2249 Operands[0] = Chain;
2250 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2251 SDValue Operand = N->getOperand(Num: j);
2252 EVT OperandVT = Operand.getValueType();
2253 if (OperandVT.isVector()) {
2254 EVT OperandEltVT = OperandVT.getVectorElementType();
2255 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2256 } else {
2257 Operands[j] = Operand;
2258 }
2259 }
2260 SDValue Scalar =
2261 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2262
2263 //Add in the scalar as well as its chain value to the
2264 //result vectors.
2265 Scalars.push_back(Elt: Scalar);
2266 Chains.push_back(Elt: Scalar.getValue(R: 1));
2267 }
2268
2269 for (; i < ResNE; ++i)
2270 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2271
2272 // Build a new factor node to connect the chain back together.
2273 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2274 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2275
2276 // Create a new BUILD_VECTOR node
2277 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2278 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2279}
2280
2281void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2282 SDValue &Lo, SDValue &Hi) {
2283 SDLoc dl(N);
2284 EVT ResVT = N->getValueType(ResNo: 0);
2285 EVT OvVT = N->getValueType(ResNo: 1);
2286 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2287 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2288 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2289
2290 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2291 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2292 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2293 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2294 } else {
2295 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2296 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2297 }
2298
2299 unsigned Opcode = N->getOpcode();
2300 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2301 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2302 SDNode *LoNode =
2303 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2304 SDNode *HiNode =
2305 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2306
2307 Lo = SDValue(LoNode, ResNo);
2308 Hi = SDValue(HiNode, ResNo);
2309
2310 // Replace the other vector result not being explicitly split here.
2311 unsigned OtherNo = 1 - ResNo;
2312 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2313 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2314 SetSplitVector(Op: SDValue(N, OtherNo),
2315 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2316 } else {
2317 SDValue OtherVal = DAG.getNode(
2318 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2319 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2320 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2321 }
2322}
2323
2324void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2325 SDValue &Hi) {
2326 SDValue Vec = N->getOperand(Num: 0);
2327 SDValue Elt = N->getOperand(Num: 1);
2328 SDValue Idx = N->getOperand(Num: 2);
2329 SDLoc dl(N);
2330 GetSplitVector(Op: Vec, Lo, Hi);
2331
2332 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2333 unsigned IdxVal = CIdx->getZExtValue();
2334 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2335 if (IdxVal < LoNumElts) {
2336 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2337 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2338 return;
2339 } else if (!Vec.getValueType().isScalableVector()) {
2340 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2341 return;
2342 }
2343 }
2344
2345 // Make the vector elements byte-addressable if they aren't already.
2346 EVT VecVT = Vec.getValueType();
2347 EVT EltVT = VecVT.getVectorElementType();
2348 if (!EltVT.isByteSized()) {
2349 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2350 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2351 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2352 // Extend the element type to match if needed.
2353 if (EltVT.bitsGT(VT: Elt.getValueType()))
2354 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2355 }
2356
2357 // Spill the vector to the stack.
2358 // In cases where the vector is illegal it will be broken down into parts
2359 // and stored in parts - we should use the alignment for the smallest part.
2360 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2361 SDValue StackPtr =
2362 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2363 auto &MF = DAG.getMachineFunction();
2364 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2365 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2366
2367 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2368 Alignment: SmallestAlign);
2369
2370 // Store the new element. This may be larger than the vector element type,
2371 // so use a truncating store.
2372 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2373 Store = DAG.getTruncStore(
2374 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2375 Alignment: commonAlignment(A: SmallestAlign,
2376 Offset: EltVT.getFixedSizeInBits() / 8));
2377
2378 EVT LoVT, HiVT;
2379 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2380
2381 // Load the Lo part from the stack slot.
2382 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2383
2384 // Increment the pointer to the other part.
2385 auto Load = cast<LoadSDNode>(Val&: Lo);
2386 MachinePointerInfo MPI = Load->getPointerInfo();
2387 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2388
2389 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2390
2391 // If we adjusted the original type, we need to truncate the results.
2392 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2393 if (LoVT != Lo.getValueType())
2394 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2395 if (HiVT != Hi.getValueType())
2396 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2397}
2398
2399void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2400 SDValue &Hi) {
2401 EVT LoVT, HiVT;
2402 SDLoc dl(N);
2403 assert(N->getValueType(0).isScalableVector() &&
2404 "Only scalable vectors are supported for STEP_VECTOR");
2405 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2406 SDValue Step = N->getOperand(Num: 0);
2407
2408 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2409
2410 // Hi = Lo + (EltCnt * Step)
2411 EVT EltVT = Step.getValueType();
2412 APInt StepVal = Step->getAsAPIntVal();
2413 SDValue StartOfHi =
2414 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2415 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2416 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2417
2418 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2419 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2420}
2421
2422void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2423 SDValue &Hi) {
2424 EVT LoVT, HiVT;
2425 SDLoc dl(N);
2426 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2427 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2428 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2429 Hi = DAG.getPOISON(VT: HiVT);
2430 } else {
2431 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2432 Hi = Lo;
2433 }
2434}
2435
2436void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2437 SDValue &Hi) {
2438 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2439 "Extended load during type legalization!");
2440 SDLoc dl(LD);
2441 EVT VT = LD->getValueType(ResNo: 0);
2442 EVT LoVT, HiVT;
2443 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2444
2445 SDValue Ch = LD->getChain();
2446 SDValue Ptr = LD->getBasePtr();
2447
2448 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2449 EVT MemIntVT =
2450 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2451 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2452 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2453
2454 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2455 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2456 SDValue ExtractLo, ExtractHi;
2457 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2458
2459 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2460 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2461
2462 // Legalize the chain result - switch anything that used the old chain to
2463 // use the new one.
2464 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2465}
2466
2467void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2468 SDValue &Hi) {
2469 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2470 EVT LoVT, HiVT;
2471 SDLoc dl(LD);
2472 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2473
2474 ISD::LoadExtType ExtType = LD->getExtensionType();
2475 SDValue Ch = LD->getChain();
2476 SDValue Ptr = LD->getBasePtr();
2477 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
2478 EVT MemoryVT = LD->getMemoryVT();
2479 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2480 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
2481
2482 EVT LoMemVT, HiMemVT;
2483 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2484
2485 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2486 SDValue Value, NewChain;
2487 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2488 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2489 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2490 return;
2491 }
2492
2493 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2494 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2495 Metadata);
2496
2497 MachinePointerInfo MPI;
2498 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2499
2500 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2501 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
2502
2503 // Build a factor node to remember that this load is independent of the
2504 // other one.
2505 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2506 N2: Hi.getValue(R: 1));
2507
2508 // Legalize the chain result - switch anything that used the old chain to
2509 // use the new one.
2510 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2511}
2512
2513void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2514 SDValue &Hi) {
2515 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2516 EVT LoVT, HiVT;
2517 SDLoc dl(LD);
2518 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2519
2520 ISD::LoadExtType ExtType = LD->getExtensionType();
2521 SDValue Ch = LD->getChain();
2522 SDValue Ptr = LD->getBasePtr();
2523 SDValue Offset = LD->getOffset();
2524 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2525 Align Alignment = LD->getBaseAlign();
2526 SDValue Mask = LD->getMask();
2527 SDValue EVL = LD->getVectorLength();
2528 EVT MemoryVT = LD->getMemoryVT();
2529
2530 EVT LoMemVT, HiMemVT;
2531 bool HiIsEmpty = false;
2532 std::tie(args&: LoMemVT, args&: HiMemVT) =
2533 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2534
2535 // Split Mask operand
2536 SDValue MaskLo, MaskHi;
2537 if (Mask.getOpcode() == ISD::SETCC) {
2538 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2539 } else {
2540 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2541 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2542 else
2543 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2544 }
2545
2546 // Split EVL operand
2547 SDValue EVLLo, EVLHi;
2548 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2549
2550 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2551 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2552 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2553 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2554
2555 Lo =
2556 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2557 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2558
2559 if (HiIsEmpty) {
2560 // The hi vp_load has zero storage size. We therefore simply set it to
2561 // the low vp_load and rely on subsequent removal from the chain.
2562 Hi = Lo;
2563 } else {
2564 // Generate hi vp_load.
2565 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2566 IsCompressedMemory: LD->isExpandingLoad());
2567
2568 MachinePointerInfo MPI;
2569 if (LoMemVT.isScalableVector())
2570 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2571 else
2572 MPI = LD->getPointerInfo().getWithOffset(
2573 O: LoMemVT.getStoreSize().getFixedValue());
2574
2575 MMO = DAG.getMachineFunction().getMachineMemOperand(
2576 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2577 BaseAlignment: Alignment, Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2578
2579 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2580 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2581 IsExpanding: LD->isExpandingLoad());
2582 }
2583
2584 // Build a factor node to remember that this load is independent of the
2585 // other one.
2586 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2587 N2: Hi.getValue(R: 1));
2588
2589 // Legalize the chain result - switch anything that used the old chain to
2590 // use the new one.
2591 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2592}
2593
2594void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2595 SDValue &Hi) {
2596 SDLoc dl(LD);
2597 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2598
2599 SDValue Ch = LD->getChain();
2600 SDValue Ptr = LD->getBasePtr();
2601 Align Alignment = LD->getBaseAlign();
2602 SDValue Mask = LD->getMask();
2603 SDValue EVL = LD->getVectorLength();
2604
2605 // Split Mask operand
2606 SDValue MaskLo, MaskHi;
2607 if (Mask.getOpcode() == ISD::SETCC) {
2608 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2609 } else {
2610 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2611 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2612 else
2613 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2614 }
2615
2616 // Split EVL operand
2617 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2618
2619 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2620 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2621 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2622 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2623
2624 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2625
2626 // Fill the upper half with poison.
2627 Hi = DAG.getPOISON(VT: HiVT);
2628
2629 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2630 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2631}
2632
2633void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2634 SDValue &Lo, SDValue &Hi) {
2635 assert(SLD->isUnindexed() &&
2636 "Indexed VP strided load during type legalization!");
2637 assert(SLD->getOffset().isUndef() &&
2638 "Unexpected indexed variable-length load offset");
2639
2640 SDLoc DL(SLD);
2641
2642 EVT LoVT, HiVT;
2643 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2644
2645 EVT LoMemVT, HiMemVT;
2646 bool HiIsEmpty = false;
2647 std::tie(args&: LoMemVT, args&: HiMemVT) =
2648 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2649
2650 SDValue Mask = SLD->getMask();
2651 SDValue LoMask, HiMask;
2652 if (Mask.getOpcode() == ISD::SETCC) {
2653 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2654 } else {
2655 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2656 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2657 else
2658 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2659 }
2660
2661 SDValue LoEVL, HiEVL;
2662 std::tie(args&: LoEVL, args&: HiEVL) =
2663 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2664
2665 // Generate the low vp_strided_load
2666 Lo = DAG.getStridedLoadVP(
2667 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2668 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2669 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2670
2671 if (HiIsEmpty) {
2672 // The high vp_strided_load has zero storage size. We therefore simply set
2673 // it to the low vp_strided_load and rely on subsequent removal from the
2674 // chain.
2675 Hi = Lo;
2676 } else {
2677 // Generate the high vp_strided_load.
2678 // To calculate the high base address, we need to sum to the low base
2679 // address stride number of bytes for each element already loaded by low,
2680 // that is: Ptr = Ptr + (LoEVL * Stride)
2681 EVT PtrVT = SLD->getBasePtr().getValueType();
2682 SDValue Increment =
2683 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2684 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2685 SDValue Ptr =
2686 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2687
2688 Align Alignment = SLD->getBaseAlign();
2689 if (LoMemVT.isScalableVector())
2690 Alignment = commonAlignment(
2691 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2692
2693 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2694 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2695 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2696 BaseAlignment: Alignment, Metadata: MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2697
2698 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2699 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2700 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2701 IsExpanding: SLD->isExpandingLoad());
2702 }
2703
2704 // Build a factor node to remember that this load is independent of the
2705 // other one.
2706 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2707 N2: Hi.getValue(R: 1));
2708
2709 // Legalize the chain result - switch anything that used the old chain to
2710 // use the new one.
2711 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2712}
2713
2714void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2715 SDValue &Lo, SDValue &Hi) {
2716 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2717 EVT LoVT, HiVT;
2718 SDLoc dl(MLD);
2719 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2720
2721 SDValue Ch = MLD->getChain();
2722 SDValue Ptr = MLD->getBasePtr();
2723 SDValue Offset = MLD->getOffset();
2724 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2725 SDValue Mask = MLD->getMask();
2726 SDValue PassThru = MLD->getPassThru();
2727 Align Alignment = MLD->getBaseAlign();
2728 ISD::LoadExtType ExtType = MLD->getExtensionType();
2729 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2730
2731 // Split Mask operand
2732 SDValue MaskLo, MaskHi;
2733 if (Mask.getOpcode() == ISD::SETCC) {
2734 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2735 } else {
2736 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2737 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2738 else
2739 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2740 }
2741
2742 EVT MemoryVT = MLD->getMemoryVT();
2743 EVT LoMemVT, HiMemVT;
2744 bool HiIsEmpty = false;
2745 std::tie(args&: LoMemVT, args&: HiMemVT) =
2746 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2747
2748 SDValue PassThruLo, PassThruHi;
2749 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2750 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2751 else
2752 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2753
2754 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2755 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2756 BaseAlignment: Alignment,
2757 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(), MLD->getMemCacheHint()));
2758
2759 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2760 MMO, AM: MLD->getAddressingMode(), ExtType,
2761 IsExpanding: MLD->isExpandingLoad());
2762
2763 if (HiIsEmpty) {
2764 // The hi masked load has zero storage size. We therefore simply set it to
2765 // the low masked load and rely on subsequent removal from the chain.
2766 Hi = Lo;
2767 } else {
2768 // Generate hi masked load.
2769 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2770 IsCompressedMemory: MLD->isExpandingLoad());
2771
2772 MachinePointerInfo MPI;
2773 if (LoMemVT.isScalableVector())
2774 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2775 else
2776 MPI = MLD->getPointerInfo().getWithOffset(
2777 O: LoMemVT.getStoreSize().getFixedValue());
2778
2779 MMO = DAG.getMachineFunction().getMachineMemOperand(
2780 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2781 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
2782 MLD->getMemCacheHint()));
2783
2784 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2785 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2786 IsExpanding: MLD->isExpandingLoad());
2787 }
2788
2789 // Build a factor node to remember that this load is independent of the
2790 // other one.
2791 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2792 N2: Hi.getValue(R: 1));
2793
2794 // Legalize the chain result - switch anything that used the old chain to
2795 // use the new one.
2796 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2797
2798}
2799
2800void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2801 SDValue &Hi, bool SplitSETCC) {
2802 EVT LoVT, HiVT;
2803 SDLoc dl(N);
2804 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2805
2806 SDValue Ch = N->getChain();
2807 SDValue Ptr = N->getBasePtr();
2808 struct Operands {
2809 SDValue Mask;
2810 SDValue Index;
2811 SDValue Scale;
2812 } Ops = [&]() -> Operands {
2813 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2814 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2815 }
2816 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2817 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2818 }();
2819
2820 EVT MemoryVT = N->getMemoryVT();
2821 Align Alignment = N->getBaseAlign();
2822
2823 // Split Mask operand
2824 SDValue MaskLo, MaskHi;
2825 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2826 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2827 } else {
2828 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2829 }
2830
2831 EVT LoMemVT, HiMemVT;
2832 // Split MemoryVT
2833 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2834
2835 SDValue IndexHi, IndexLo;
2836 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2837 TargetLowering::TypeSplitVector)
2838 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2839 else
2840 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2841
2842 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2843 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2844 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2845 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
2846
2847 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2848 SDValue PassThru = MGT->getPassThru();
2849 SDValue PassThruLo, PassThruHi;
2850 if (getTypeAction(VT: PassThru.getValueType()) ==
2851 TargetLowering::TypeSplitVector)
2852 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2853 else
2854 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2855
2856 ISD::LoadExtType ExtType = MGT->getExtensionType();
2857 ISD::MemIndexType IndexTy = MGT->getIndexType();
2858
2859 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2860 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2861 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2862
2863 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2864 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2865 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2866 } else {
2867 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2868 SDValue EVLLo, EVLHi;
2869 std::tie(args&: EVLLo, args&: EVLHi) =
2870 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2871
2872 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2873 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2874 MMO, IndexType: VPGT->getIndexType());
2875
2876 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2877 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2878 MMO, IndexType: VPGT->getIndexType());
2879 }
2880
2881 // Build a factor node to remember that this load is independent of the
2882 // other one.
2883 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2884 N2: Hi.getValue(R: 1));
2885
2886 // Legalize the chain result - switch anything that used the old chain to
2887 // use the new one.
2888 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2889}
2890
2891void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2892 SDValue &Hi) {
2893 // This is not "trivial", as there is a dependency between the two subvectors.
2894 // Depending on the number of 1s in the mask, the elements from the Hi vector
2895 // need to be moved to the Lo vector. Passthru values make this even harder.
2896 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2897 // smaller types and passthru is undef, as it is most likely faster than the
2898 // fully expand path. Otherwise, just do the full expansion as one "big"
2899 // operation and then extract the Lo and Hi vectors from that. This gets
2900 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2901 SDLoc DL(N);
2902 EVT VecVT = N->getValueType(ResNo: 0);
2903
2904 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2905 bool HasCustomLowering = false;
2906 EVT CheckVT = LoVT;
2907 while (CheckVT.getVectorMinNumElements() > 1) {
2908 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2909 // custom lowering for illegal types. So we do the checks separately.
2910 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2911 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2912 HasCustomLowering = true;
2913 break;
2914 }
2915 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2916 }
2917
2918 SDValue Passthru = N->getOperand(Num: 2);
2919 if (!HasCustomLowering) {
2920 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2921 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2922 return;
2923 }
2924
2925 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2926 SDValue Mask = N->getOperand(Num: 1);
2927 SDValue LoMask, HiMask;
2928 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2929 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2930
2931 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2932 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2933 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2934
2935 SDValue StackPtr = DAG.CreateStackTemporary(
2936 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2937 MachineFunction &MF = DAG.getMachineFunction();
2938 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2939 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2940
2941 EVT LoMaskVT = LoMask.getValueType();
2942 assert(LoMaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2943
2944 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2945 EVT WideLoMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2946 EC: LoMaskVT.getVectorElementCount());
2947 SDValue WideLoMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideLoMaskVT, Operand: LoMask);
2948 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideLoMask);
2949 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2950
2951 SDValue Chain = DAG.getEntryNode();
2952 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2953 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2954 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2955
2956 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2957 if (!Passthru.isUndef()) {
2958 // Compress the input mask so only inactive lanes of the result are replaced
2959 // by their passthrough value.
2960 EVT MaskVT = Mask.getValueType();
2961 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2962 EC: MaskVT.getVectorElementCount());
2963 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: Mask);
2964 SDValue NumActiveElts =
2965 DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2966
2967 SDValue StepVector = DAG.getStepVector(DL, ResVT: WideMaskVT);
2968 SDValue SplatNumActiveElts = DAG.getSplat(VT: WideMaskVT, DL, Op: NumActiveElts);
2969 SDValue CompressedMask =
2970 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVector, RHS: SplatNumActiveElts, Cond: ISD::SETULT);
2971
2972 Compressed = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: CompressedMask,
2973 N2: Compressed, N3: Passthru);
2974 }
2975 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2976}
2977
2978void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2979 assert(N->getValueType(0).isVector() &&
2980 N->getOperand(0).getValueType().isVector() &&
2981 "Operand types must be vectors");
2982
2983 EVT LoVT, HiVT;
2984 SDLoc DL(N);
2985 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2986
2987 // If the input also splits, handle it directly. Otherwise split it by hand.
2988 SDValue LL, LH, RL, RH;
2989 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2990 TargetLowering::TypeSplitVector)
2991 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2992 else
2993 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2994
2995 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2996 TargetLowering::TypeSplitVector)
2997 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2998 else
2999 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
3000
3001 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
3002 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
3003}
3004
3005void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
3006 SDValue &Hi) {
3007 // Get the dest types - they may not match the input types, e.g. int_to_fp.
3008 EVT LoVT, HiVT;
3009 SDLoc dl(N);
3010 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3011
3012 // If the input also splits, handle it directly for a compile time speedup.
3013 // Otherwise split it by hand.
3014 EVT InVT = N->getOperand(Num: 0).getValueType();
3015 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3016 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3017 else
3018 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3019
3020 const SDNodeFlags Flags = N->getFlags();
3021 unsigned Opcode = N->getOpcode();
3022 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
3023 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3024 N4: N->getOperand(Num: 3), Flags);
3025 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3026 N4: N->getOperand(Num: 3), Flags);
3027 return;
3028 }
3029
3030 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
3031 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
3032 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
3033 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
3034 } else {
3035 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
3036 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
3037 }
3038}
3039
3040void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3041 SDValue &Hi) {
3042 SDLoc dl(N);
3043 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3044
3045 // If the input also splits, handle it directly for a compile time speedup.
3046 // Otherwise split it by hand.
3047 EVT InVT = N->getOperand(Num: 0).getValueType();
3048 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3049 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3050 else
3051 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3052
3053 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3054 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3055 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3056 SDNodeFlags Flags = AddrSpaceCastN->getFlags();
3057 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS, Flags);
3058 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS, Flags);
3059}
3060
3061void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3062 unsigned ResNo,
3063 SDValue &Lo,
3064 SDValue &Hi) {
3065 SDLoc dl(N);
3066 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3067 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3068
3069 // If the input also splits, handle it directly for a compile time speedup.
3070 // Otherwise split it by hand.
3071 EVT InVT = N->getOperand(Num: 0).getValueType();
3072 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3073 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3074 else
3075 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3076
3077 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3078 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3079
3080 SDNode *HiNode = Hi.getNode();
3081 SDNode *LoNode = Lo.getNode();
3082
3083 // Replace the other vector result not being explicitly split here.
3084 unsigned OtherNo = 1 - ResNo;
3085 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3086 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3087 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3088 Hi: SDValue(HiNode, OtherNo));
3089 } else {
3090 SDValue OtherVal =
3091 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3092 N2: SDValue(HiNode, OtherNo));
3093 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3094 }
3095}
3096
3097void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3098 SDValue &Hi) {
3099 SDLoc dl(N);
3100 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3101 EVT DestVT = N->getValueType(ResNo: 0);
3102 EVT LoVT, HiVT;
3103 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3104
3105 // We can do better than a generic split operation if the extend is doing
3106 // more than just doubling the width of the elements and the following are
3107 // true:
3108 // - The number of vector elements is even,
3109 // - the source type is legal,
3110 // - the type of a split source is illegal,
3111 // - the type of an extended (by doubling element size) source is legal, and
3112 // - the type of that extended source when split is legal.
3113 //
3114 // This won't necessarily completely legalize the operation, but it will
3115 // more effectively move in the right direction and prevent falling down
3116 // to scalarization in many cases due to the input vector being split too
3117 // far.
3118 if (SrcVT.getVectorElementCount().isKnownEven() &&
3119 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3120 LLVMContext &Ctx = *DAG.getContext();
3121 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3122 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3123
3124 EVT SplitLoVT, SplitHiVT;
3125 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3126 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3127 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3128 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3129 N->dump(&DAG); dbgs() << "\n");
3130 // Extend the source vector by one step.
3131 SDValue NewSrc =
3132 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3133 // Get the low and high halves of the new, extended one step, vector.
3134 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3135 // Extend those vector halves the rest of the way.
3136 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3137 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3138 return;
3139 }
3140 }
3141 // Fall back to the generic unary operator splitting otherwise.
3142 SplitVecRes_UnaryOp(N, Lo, Hi);
3143}
3144
3145void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3146 SDValue &Lo, SDValue &Hi) {
3147 // The low and high parts of the original input give four input vectors.
3148 SDValue Inputs[4];
3149 SDLoc DL(N);
3150 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3151 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3152 EVT NewVT = Inputs[0].getValueType();
3153 unsigned NewElts = NewVT.getVectorNumElements();
3154
3155 auto &&IsConstant = [](const SDValue &N) {
3156 APInt SplatValue;
3157 return N.getResNo() == 0 &&
3158 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3159 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3160 };
3161 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3162 SDValue &Input2,
3163 ArrayRef<int> Mask) {
3164 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3165 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3166 "Expected build vector node.");
3167 EVT EltVT = NewVT.getVectorElementType();
3168 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3169 for (unsigned I = 0; I < NewElts; ++I) {
3170 if (Mask[I] == PoisonMaskElem)
3171 continue;
3172 unsigned Idx = Mask[I];
3173 if (Idx >= NewElts)
3174 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3175 else
3176 Ops[I] = Input1.getOperand(i: Idx);
3177 // Make the type of all elements the same as the element type.
3178 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3179 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3180 }
3181 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3182 };
3183
3184 // If Lo or Hi uses elements from at most two of the four input vectors, then
3185 // express it as a vector shuffle of those two inputs. Otherwise extract the
3186 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3187 SmallVector<int> OrigMask(N->getMask());
3188 // Try to pack incoming shuffles/inputs.
3189 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3190 &DL](SmallVectorImpl<int> &Mask) {
3191 // Check if all inputs are shuffles of the same operands or non-shuffles.
3192 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3193 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3194 SDValue Input = Inputs[Idx];
3195 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3196 if (!Shuffle ||
3197 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3198 continue;
3199 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3200 .push_back(Elt: Idx);
3201 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3202 .push_back(Elt: Idx);
3203 }
3204 for (auto &P : ShufflesIdxs) {
3205 if (P.second.size() < 2)
3206 continue;
3207 // Use shuffles operands instead of shuffles themselves.
3208 // 1. Adjust mask.
3209 for (int &Idx : Mask) {
3210 if (Idx == PoisonMaskElem)
3211 continue;
3212 unsigned SrcRegIdx = Idx / NewElts;
3213 if (Inputs[SrcRegIdx].isUndef()) {
3214 Idx = PoisonMaskElem;
3215 continue;
3216 }
3217 auto *Shuffle =
3218 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3219 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3220 continue;
3221 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3222 if (MaskElt == PoisonMaskElem) {
3223 Idx = PoisonMaskElem;
3224 continue;
3225 }
3226 Idx = MaskElt % NewElts +
3227 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3228 ? 0
3229 : 1] *
3230 NewElts;
3231 }
3232 // 2. Update inputs.
3233 Inputs[P.second[0]] = P.first.first;
3234 Inputs[P.second[1]] = P.first.second;
3235 // Clear the pair data.
3236 P.second.clear();
3237 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3238 }
3239 // Check if any concat_vectors can be simplified.
3240 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3241 for (int &Idx : Mask) {
3242 if (Idx == PoisonMaskElem)
3243 continue;
3244 unsigned SrcRegIdx = Idx / NewElts;
3245 if (Inputs[SrcRegIdx].isUndef()) {
3246 Idx = PoisonMaskElem;
3247 continue;
3248 }
3249 TargetLowering::LegalizeTypeAction TypeAction =
3250 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3251 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3252 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3253 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3254 (TypeAction == TargetLowering::TypeLegal ||
3255 TypeAction == TargetLowering::TypeWidenVector))
3256 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3257 }
3258 if (UsedSubVector.count() > 1) {
3259 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3260 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3261 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3262 continue;
3263 if (Pairs.empty() || Pairs.back().size() == 2)
3264 Pairs.emplace_back();
3265 if (UsedSubVector.test(Idx: 2 * I)) {
3266 Pairs.back().emplace_back(Args&: I, Args: 0);
3267 } else {
3268 assert(UsedSubVector.test(2 * I + 1) &&
3269 "Expected to be used one of the subvectors.");
3270 Pairs.back().emplace_back(Args&: I, Args: 1);
3271 }
3272 }
3273 if (!Pairs.empty() && Pairs.front().size() > 1) {
3274 // Adjust mask.
3275 for (int &Idx : Mask) {
3276 if (Idx == PoisonMaskElem)
3277 continue;
3278 unsigned SrcRegIdx = Idx / NewElts;
3279 auto *It = find_if(
3280 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3281 return Idxs.front().first == SrcRegIdx ||
3282 Idxs.back().first == SrcRegIdx;
3283 });
3284 if (It == Pairs.end())
3285 continue;
3286 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3287 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3288 }
3289 // Adjust inputs.
3290 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3291 Inputs[Idxs.front().first] = DAG.getNode(
3292 Opcode: ISD::CONCAT_VECTORS, DL,
3293 VT: Inputs[Idxs.front().first].getValueType(),
3294 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3295 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3296 }
3297 }
3298 }
3299 bool Changed;
3300 do {
3301 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3302 // reused operands.
3303 Changed = false;
3304 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3305 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3306 if (!Shuffle)
3307 continue;
3308 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3309 continue;
3310 int Op = -1;
3311 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3312 !Shuffle->isSplat()) {
3313 Op = 0;
3314 } else if (!Inputs[I].hasOneUse() &&
3315 !Shuffle->getOperand(Num: 1).isUndef()) {
3316 // Find the only used operand, if possible.
3317 for (int &Idx : Mask) {
3318 if (Idx == PoisonMaskElem)
3319 continue;
3320 unsigned SrcRegIdx = Idx / NewElts;
3321 if (SrcRegIdx != I)
3322 continue;
3323 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3324 if (MaskElt == PoisonMaskElem) {
3325 Idx = PoisonMaskElem;
3326 continue;
3327 }
3328 int OpIdx = MaskElt / NewElts;
3329 if (Op == -1) {
3330 Op = OpIdx;
3331 continue;
3332 }
3333 if (Op != OpIdx) {
3334 Op = -1;
3335 break;
3336 }
3337 }
3338 }
3339 if (Op < 0) {
3340 // Try to check if one of the shuffle operands is used already.
3341 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3342 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3343 continue;
3344 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3345 if (It == std::end(arr&: Inputs))
3346 continue;
3347 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3348 // Found that operand is used already.
3349 // 1. Fix the mask for the reused operand.
3350 for (int &Idx : Mask) {
3351 if (Idx == PoisonMaskElem)
3352 continue;
3353 unsigned SrcRegIdx = Idx / NewElts;
3354 if (SrcRegIdx != I)
3355 continue;
3356 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3357 if (MaskElt == PoisonMaskElem) {
3358 Idx = PoisonMaskElem;
3359 continue;
3360 }
3361 int MaskIdx = MaskElt / NewElts;
3362 if (OpIdx == MaskIdx)
3363 Idx = MaskElt % NewElts + FoundOp * NewElts;
3364 }
3365 // 2. Set Op to the unused OpIdx.
3366 Op = (OpIdx + 1) % 2;
3367 break;
3368 }
3369 }
3370 if (Op >= 0) {
3371 Changed = true;
3372 Inputs[I] = Shuffle->getOperand(Num: Op);
3373 // Adjust mask.
3374 for (int &Idx : Mask) {
3375 if (Idx == PoisonMaskElem)
3376 continue;
3377 unsigned SrcRegIdx = Idx / NewElts;
3378 if (SrcRegIdx != I)
3379 continue;
3380 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3381 int OpIdx = MaskElt / NewElts;
3382 if (OpIdx != Op)
3383 continue;
3384 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3385 }
3386 }
3387 }
3388 } while (Changed);
3389 };
3390 TryPeekThroughShufflesInputs(OrigMask);
3391 // Proces unique inputs.
3392 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3393 NewElts](SmallVectorImpl<int> &Mask) {
3394 SetVector<SDValue> UniqueInputs;
3395 SetVector<SDValue> UniqueConstantInputs;
3396 for (const auto &I : Inputs) {
3397 if (IsConstant(I))
3398 UniqueConstantInputs.insert(X: I);
3399 else if (!I.isUndef())
3400 UniqueInputs.insert(X: I);
3401 }
3402 // Adjust mask in case of reused inputs. Also, need to insert constant
3403 // inputs at first, otherwise it affects the final outcome.
3404 if (UniqueInputs.size() != std::size(Inputs)) {
3405 auto &&UniqueVec = UniqueInputs.takeVector();
3406 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3407 unsigned ConstNum = UniqueConstantVec.size();
3408 for (int &Idx : Mask) {
3409 if (Idx == PoisonMaskElem)
3410 continue;
3411 unsigned SrcRegIdx = Idx / NewElts;
3412 if (Inputs[SrcRegIdx].isUndef()) {
3413 Idx = PoisonMaskElem;
3414 continue;
3415 }
3416 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3417 if (It != UniqueConstantVec.end()) {
3418 Idx = (Idx % NewElts) +
3419 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3420 assert(Idx >= 0 && "Expected defined mask idx.");
3421 continue;
3422 }
3423 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3424 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3425 Idx = (Idx % NewElts) +
3426 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3427 assert(Idx >= 0 && "Expected defined mask idx.");
3428 }
3429 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3430 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3431 }
3432 };
3433 MakeUniqueInputs(OrigMask);
3434 SDValue OrigInputs[4];
3435 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3436 for (unsigned High = 0; High < 2; ++High) {
3437 SDValue &Output = High ? Hi : Lo;
3438
3439 // Build a shuffle mask for the output, discovering on the fly which
3440 // input vectors to use as shuffle operands.
3441 unsigned FirstMaskIdx = High * NewElts;
3442 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3443 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3444 assert(!Output && "Expected default initialized initial value.");
3445 TryPeekThroughShufflesInputs(Mask);
3446 MakeUniqueInputs(Mask);
3447 SDValue TmpInputs[4];
3448 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3449 // Track changes in the output registers.
3450 int UsedIdx = -1;
3451 bool SecondIteration = false;
3452 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3453 if (UsedIdx < 0) {
3454 UsedIdx = Idx;
3455 return false;
3456 }
3457 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3458 SecondIteration = true;
3459 return SecondIteration;
3460 };
3461 processShuffleMasks(
3462 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3463 /*NumOfUsedRegs=*/1,
3464 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3465 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3466 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3467 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3468 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3469 else
3470 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3471 N2: DAG.getPOISON(VT: NewVT), Mask);
3472 Inputs[Idx] = Output;
3473 },
3474 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3475 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3476 unsigned Idx2, bool /*Unused*/) {
3477 if (AccumulateResults(Idx1)) {
3478 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3479 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3480 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3481 else
3482 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3483 N2: Inputs[Idx2], Mask);
3484 } else {
3485 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3486 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3487 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3488 else
3489 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3490 N2: TmpInputs[Idx2], Mask);
3491 }
3492 Inputs[Idx1] = Output;
3493 });
3494 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3495 }
3496}
3497
3498void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3499 EVT OVT = N->getValueType(ResNo: 0);
3500 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3501 SDValue Chain = N->getOperand(Num: 0);
3502 SDValue Ptr = N->getOperand(Num: 1);
3503 SDValue SV = N->getOperand(Num: 2);
3504 SDLoc dl(N);
3505
3506 const Align Alignment =
3507 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3508
3509 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3510 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3511 Chain = Hi.getValue(R: 1);
3512
3513 // Modified the chain - switch anything that used the old chain to use
3514 // the new one.
3515 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3516}
3517
3518void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3519 SDValue &Hi) {
3520 EVT DstVTLo, DstVTHi;
3521 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3522 SDLoc dl(N);
3523
3524 SDValue SrcLo, SrcHi;
3525 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3526 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3527 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3528 else
3529 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3530
3531 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3532 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3533}
3534
3535void DAGTypeLegalizer::SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo,
3536 SDValue &Hi) {
3537 EVT VT = N->getValueType(ResNo: 0);
3538 SDValue Src = N->getOperand(Num: 0);
3539 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
3540 assert(LoVT == HiVT && "Expected equal split types");
3541
3542 // Use smaller even/odd source vectors so their broadcasts can be
3543 // reinterleaved in the original lane order for every value of vscale.
3544 SDLoc DL(N);
3545 auto [SrcLo, SrcHi] = DAG.SplitVector(N: Src, DL);
3546 EVT SplitSrcVT = SrcLo.getValueType();
3547 SDValue Deinterleaved =
3548 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
3549 VTList: DAG.getVTList(VT1: SplitSrcVT, VT2: SplitSrcVT), N1: SrcLo, N2: SrcHi);
3550 SDValue Even =
3551 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 0));
3552 SDValue Odd =
3553 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 1));
3554 SDValue Interleaved = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
3555 VTList: DAG.getVTList(VT1: LoVT, VT2: LoVT), N1: Even, N2: Odd);
3556 Lo = Interleaved.getValue(R: 0);
3557 Hi = Interleaved.getValue(R: 1);
3558}
3559
3560void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3561 SDValue &Hi) {
3562 SDValue InLo, InHi;
3563 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3564 SDLoc DL(N);
3565
3566 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3567 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3568}
3569
3570void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3571 SDValue &Hi) {
3572 SDLoc DL(N);
3573
3574 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3575 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3576}
3577
3578void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3579 SDValue &Hi) {
3580 EVT VT = N->getValueType(ResNo: 0);
3581 SDValue Val = N->getOperand(Num: 0);
3582 SDValue Mask = N->getOperand(Num: 1);
3583 SDValue EVL = N->getOperand(Num: 2);
3584 SDLoc DL(N);
3585
3586 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3587 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3588 // then truncate back.
3589 EVT OrigVT = VT;
3590 if (!VT.getVectorElementType().isByteSized()) {
3591 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3592 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3593 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3594 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3595 }
3596
3597 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3598 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3599
3600 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3601 EC: VT.getVectorElementCount());
3602 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3603 EVT PtrVT = StackPtr.getValueType();
3604 auto &MF = DAG.getMachineFunction();
3605 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3606 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3607
3608 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3609 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3610 BaseAlignment: Alignment);
3611 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3612 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3613 BaseAlignment: Alignment);
3614
3615 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3616 SDValue NumElemMinus1 =
3617 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3618 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3619 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3620 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3621 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3622 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3623
3624 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3625 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3626 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3627 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3628
3629 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3630
3631 // Truncate back if we widened above.
3632 if (OrigVT != VT)
3633 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3634
3635 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3636}
3637
3638void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3639 SDValue &Hi) {
3640 EVT VT = N->getValueType(ResNo: 0);
3641 SDValue V1 = N->getOperand(Num: 0);
3642 SDValue V2 = N->getOperand(Num: 1);
3643 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3644 SDValue Mask = N->getOperand(Num: 3);
3645 SDValue EVL1 = N->getOperand(Num: 4);
3646 SDValue EVL2 = N->getOperand(Num: 5);
3647 SDLoc DL(N);
3648
3649 // Since EVL2 is considered the real VL it gets promoted during
3650 // SelectionDAGBuilder. Promote EVL1 here if needed.
3651 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3652 EVL1 = ZExtPromotedInteger(Op: EVL1);
3653
3654 // The stack splice addresses elements by byte offset/stride, which breaks for
3655 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3656 // stride is 0. Widen to a byte integer, splice, then truncate back.
3657 EVT OrigVT = VT;
3658 if (!VT.getVectorElementType().isByteSized()) {
3659 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3660 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3661 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3662 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3663 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3664 }
3665
3666 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3667
3668 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3669 EC: VT.getVectorElementCount() * 2);
3670 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3671 EVT PtrVT = StackPtr.getValueType();
3672 auto &MF = DAG.getMachineFunction();
3673 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3674 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3675
3676 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3677 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3678 BaseAlignment: Alignment);
3679 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3680 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3681 BaseAlignment: Alignment);
3682
3683 SDValue EltByteSize =
3684 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3685 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3686 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3687 // Clip EVL1Bytes to make sure we stay within the stack object.
3688 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3689 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3690 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3691 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3692
3693 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3694 SDValue StoreV1 =
3695 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3696 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3697
3698 SDValue StoreV2 =
3699 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3700 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3701
3702 SDValue Load;
3703 if (Imm >= 0) {
3704 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3705 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3706 } else {
3707 uint64_t TrailingElts = -Imm;
3708 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3709 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3710
3711 // Make sure TrailingBytes doesn't exceed the size of vec1.
3712 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3713 TrailingBytes =
3714 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3715
3716 // Calculate the start address of the spliced result.
3717 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3718 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3719 }
3720
3721 // Truncate back if we widened above.
3722 if (OrigVT != VT)
3723 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3724
3725 EVT LoVT, HiVT;
3726 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3727 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3728 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3729 Hi =
3730 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3731 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3732}
3733
3734void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3735 SDValue &Hi) {
3736 SDLoc DL(N);
3737 SDValue Acc = N->getOperand(Num: 0);
3738 SDValue Input1 = N->getOperand(Num: 1);
3739 SDValue Input2 = N->getOperand(Num: 2);
3740
3741 SDValue AccLo, AccHi;
3742 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3743 unsigned Opcode = N->getOpcode();
3744
3745 // If the input types don't need splitting, just accumulate into the
3746 // low part of the accumulator.
3747 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3748 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3749 Hi = AccHi;
3750 return;
3751 }
3752
3753 SDValue Input1Lo, Input1Hi;
3754 SDValue Input2Lo, Input2Hi;
3755 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3756 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3757 EVT ResultVT = AccLo.getValueType();
3758
3759 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3760 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3761}
3762
3763void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3764 SDValue &Hi) {
3765 SDLoc DL(N);
3766 SDValue Op0 = N->getOperand(Num: 0);
3767 SDValue Op1 = N->getOperand(Num: 1);
3768 EVT OpVT = Op0.getValueType();
3769
3770 EVT LoVT, HiVT;
3771 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3772
3773 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3774 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3775 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3776 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3777}
3778
3779void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3780 SDValue &Hi) {
3781 SDValue SourceLo, SourceHi;
3782 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SourceLo, Hi&: SourceHi);
3783 SDValue MaskLo, MaskHi;
3784 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
3785 SDLoc DL(N);
3786
3787 Lo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskLo.getValueType(), N1: SourceLo,
3788 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
3789 Hi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskHi.getValueType(), N1: SourceHi,
3790 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
3791}
3792
3793void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3794 unsigned Factor = N->getNumOperands();
3795
3796 SmallVector<SDValue, 8> Ops(Factor * 2);
3797 for (unsigned i = 0; i != Factor; ++i) {
3798 SDValue OpLo, OpHi;
3799 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3800 Ops[i * 2] = OpLo;
3801 Ops[i * 2 + 1] = OpHi;
3802 }
3803
3804 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3805
3806 SDLoc DL(N);
3807 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3808 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3809 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3810 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3811
3812 for (unsigned i = 0; i != Factor; ++i)
3813 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3814}
3815
3816void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3817 unsigned Factor = N->getNumOperands();
3818
3819 SmallVector<SDValue, 8> Ops(Factor * 2);
3820 for (unsigned i = 0; i != Factor; ++i) {
3821 SDValue OpLo, OpHi;
3822 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3823 Ops[i] = OpLo;
3824 Ops[i + Factor] = OpHi;
3825 }
3826
3827 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3828
3829 SDLoc DL(N);
3830 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3831 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3832 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3833 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3834
3835 for (unsigned i = 0; i != Factor; ++i) {
3836 unsigned IdxLo = 2 * i;
3837 unsigned IdxHi = 2 * i + 1;
3838 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3839 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3840 }
3841}
3842
3843//===----------------------------------------------------------------------===//
3844// Operand Vector Splitting
3845//===----------------------------------------------------------------------===//
3846
3847/// This method is called when the specified operand of the specified node is
3848/// found to need vector splitting. At this point, all of the result types of
3849/// the node are known to be legal, but other operands of the node may need
3850/// legalization as well as the specified one.
3851bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3852 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3853 SDValue Res = SDValue();
3854
3855 // See if the target wants to custom split this node.
3856 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3857 return false;
3858
3859 switch (N->getOpcode()) {
3860 default:
3861#ifndef NDEBUG
3862 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3863 N->dump(&DAG);
3864 dbgs() << "\n";
3865#endif
3866 report_fatal_error(reason: "Do not know how to split this operator's "
3867 "operand!\n");
3868
3869 case ISD::STRICT_FSETCC:
3870 case ISD::STRICT_FSETCCS:
3871 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3872 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3873 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3874 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3875 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3876 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3877 case ISD::VECTOR_FIND_LAST_ACTIVE:
3878 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3879 break;
3880 case ISD::TRUNCATE:
3881 Res = SplitVecOp_TruncateHelper(N);
3882 break;
3883 case ISD::STRICT_FP_ROUND:
3884 case ISD::FP_ROUND:
3885 case ISD::CONVERT_FROM_ARBITRARY_FP:
3886 case ISD::CONVERT_TO_ARBITRARY_FP:
3887 Res = SplitVecOp_FP_ROUND(N);
3888 break;
3889 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3890 case ISD::STORE:
3891 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3892 break;
3893 case ISD::ATOMIC_STORE:
3894 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3895 break;
3896 case ISD::VP_STORE:
3897 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3898 break;
3899 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3900 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3901 break;
3902 case ISD::MSTORE:
3903 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3904 break;
3905 case ISD::MSCATTER:
3906 case ISD::VP_SCATTER:
3907 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3908 break;
3909 case ISD::MGATHER:
3910 case ISD::VP_GATHER:
3911 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3912 break;
3913 case ISD::VSELECT:
3914 Res = SplitVecOp_VSELECT(N, OpNo);
3915 break;
3916 case ISD::MASKED_UDIV:
3917 case ISD::MASKED_SDIV:
3918 case ISD::MASKED_UREM:
3919 case ISD::MASKED_SREM:
3920 Res = SplitVecOp_MaskedBinOp(N, OpNo);
3921 break;
3922 case ISD::VECTOR_COMPRESS:
3923 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3924 break;
3925 case ISD::STRICT_SINT_TO_FP:
3926 case ISD::STRICT_UINT_TO_FP:
3927 case ISD::SINT_TO_FP:
3928 case ISD::UINT_TO_FP:
3929 if (N->getValueType(ResNo: 0).bitsLT(
3930 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3931 Res = SplitVecOp_TruncateHelper(N);
3932 else
3933 Res = SplitVecOp_UnaryOp(N);
3934 break;
3935 case ISD::FP_TO_SINT_SAT:
3936 case ISD::FP_TO_UINT_SAT:
3937 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3938 break;
3939 case ISD::FP_TO_SINT:
3940 case ISD::FP_TO_UINT:
3941 case ISD::STRICT_FP_TO_SINT:
3942 case ISD::STRICT_FP_TO_UINT:
3943 case ISD::STRICT_FP_EXTEND:
3944 case ISD::FP_EXTEND:
3945 case ISD::SIGN_EXTEND:
3946 case ISD::ZERO_EXTEND:
3947 case ISD::ANY_EXTEND:
3948 case ISD::FTRUNC:
3949 case ISD::LROUND:
3950 case ISD::LLROUND:
3951 case ISD::LRINT:
3952 case ISD::LLRINT:
3953 Res = SplitVecOp_UnaryOp(N);
3954 break;
3955 case ISD::FLDEXP:
3956 Res = SplitVecOp_FPOpDifferentTypes(N);
3957 break;
3958
3959 case ISD::SCMP:
3960 case ISD::UCMP:
3961 Res = SplitVecOp_CMP(N);
3962 break;
3963
3964 case ISD::FAKE_USE:
3965 Res = SplitVecOp_FAKE_USE(N);
3966 break;
3967 case ISD::ANY_EXTEND_VECTOR_INREG:
3968 case ISD::SIGN_EXTEND_VECTOR_INREG:
3969 case ISD::ZERO_EXTEND_VECTOR_INREG:
3970 Res = SplitVecOp_ExtVecInRegOp(N);
3971 break;
3972
3973 case ISD::VECREDUCE_FADD:
3974 case ISD::VECREDUCE_FMUL:
3975 case ISD::VECREDUCE_ADD:
3976 case ISD::VECREDUCE_MUL:
3977 case ISD::VECREDUCE_AND:
3978 case ISD::VECREDUCE_OR:
3979 case ISD::VECREDUCE_XOR:
3980 case ISD::VECREDUCE_SMAX:
3981 case ISD::VECREDUCE_SMIN:
3982 case ISD::VECREDUCE_UMAX:
3983 case ISD::VECREDUCE_UMIN:
3984 case ISD::VECREDUCE_FMAX:
3985 case ISD::VECREDUCE_FMIN:
3986 case ISD::VECREDUCE_FMAXIMUM:
3987 case ISD::VECREDUCE_FMINIMUM:
3988 case ISD::VECREDUCE_FMAXIMUMNUM:
3989 case ISD::VECREDUCE_FMINIMUMNUM:
3990 Res = SplitVecOp_VECREDUCE(N, OpNo);
3991 break;
3992 case ISD::VECREDUCE_SEQ_FADD:
3993 case ISD::VECREDUCE_SEQ_FMUL:
3994 Res = SplitVecOp_VECREDUCE_SEQ(N);
3995 break;
3996 case ISD::VP_REDUCE_FADD:
3997 case ISD::VP_REDUCE_SEQ_FADD:
3998 case ISD::VP_REDUCE_FMUL:
3999 case ISD::VP_REDUCE_SEQ_FMUL:
4000 case ISD::VP_REDUCE_ADD:
4001 case ISD::VP_REDUCE_MUL:
4002 case ISD::VP_REDUCE_AND:
4003 case ISD::VP_REDUCE_OR:
4004 case ISD::VP_REDUCE_XOR:
4005 case ISD::VP_REDUCE_SMAX:
4006 case ISD::VP_REDUCE_SMIN:
4007 case ISD::VP_REDUCE_UMAX:
4008 case ISD::VP_REDUCE_UMIN:
4009 case ISD::VP_REDUCE_FMAX:
4010 case ISD::VP_REDUCE_FMIN:
4011 case ISD::VP_REDUCE_FMAXIMUM:
4012 case ISD::VP_REDUCE_FMINIMUM:
4013 Res = SplitVecOp_VP_REDUCE(N, OpNo);
4014 break;
4015 case ISD::CTTZ_ELTS:
4016 case ISD::CTTZ_ELTS_ZERO_POISON:
4017 Res = SplitVecOp_CttzElts(N);
4018 break;
4019 case ISD::VP_CTTZ_ELTS:
4020 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4021 Res = SplitVecOp_VP_CttzElements(N);
4022 break;
4023 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
4024 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
4025 break;
4026 case ISD::PARTIAL_REDUCE_UMLA:
4027 case ISD::PARTIAL_REDUCE_SMLA:
4028 case ISD::PARTIAL_REDUCE_SUMLA:
4029 case ISD::PARTIAL_REDUCE_FMLA:
4030 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
4031 break;
4032 case ISD::VECTOR_MATCH:
4033 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
4034 break;
4035 }
4036
4037 // If the result is null, the sub-method took care of registering results etc.
4038 if (!Res.getNode()) return false;
4039
4040 // If the result is N, the sub-method updated N in place. Tell the legalizer
4041 // core about this.
4042 if (Res.getNode() == N)
4043 return true;
4044
4045 if (N->isStrictFPOpcode())
4046 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4047 "Invalid operand expansion");
4048 else
4049 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4050 "Invalid operand expansion");
4051
4052 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4053 return false;
4054}
4055
4056SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
4057 SDLoc DL(N);
4058
4059 SDValue LoMask, HiMask;
4060 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
4061
4062 EVT VT = N->getValueType(ResNo: 0);
4063 EVT SplitVT = LoMask.getValueType();
4064 ElementCount SplitEC = SplitVT.getVectorElementCount();
4065
4066 // Find the last active in both the low and the high masks.
4067 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
4068 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
4069
4070 // Check if any lane is active in the high mask.
4071 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4072 // sentinel value for "none active".
4073 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4074 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4075 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4076
4077 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4078 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4079 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4080 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4081 N3: LoFind);
4082}
4083
4084SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4085 // The only possibility for an illegal operand is the mask, since result type
4086 // legalization would have handled this node already otherwise.
4087 assert(OpNo == 0 && "Illegal operand must be mask");
4088
4089 SDValue Mask = N->getOperand(Num: 0);
4090 SDValue Src0 = N->getOperand(Num: 1);
4091 SDValue Src1 = N->getOperand(Num: 2);
4092 EVT Src0VT = Src0.getValueType();
4093 SDLoc DL(N);
4094 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4095
4096 SDValue Lo, Hi;
4097 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4098 assert(Lo.getValueType() == Hi.getValueType() &&
4099 "Lo and Hi have differing types");
4100
4101 EVT LoOpVT, HiOpVT;
4102 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4103 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4104
4105 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4106 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4107 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4108 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4109
4110 SDValue LoSelect =
4111 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4112 SDValue HiSelect =
4113 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4114
4115 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4116}
4117
4118SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4119 assert(OpNo == 2 && "Illegal operand must be mask");
4120
4121 SDLoc DL(N);
4122 auto [LHSLo, LHSHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
4123 auto [RHSLo, RHSHi] = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4124 SDValue MaskLo, MaskHi;
4125 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
4126
4127 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo,
4128 N2: RHSLo, N3: MaskLo, Flags: N->getFlags());
4129 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi,
4130 N2: RHSHi, N3: MaskHi, Flags: N->getFlags());
4131 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4132}
4133
4134SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4135 // The only possibility for an illegal operand is the mask, since result type
4136 // legalization would have handled this node already otherwise.
4137 assert(OpNo == 1 && "Illegal operand must be mask");
4138
4139 // To split the mask, we need to split the result type too, so we can just
4140 // reuse that logic here.
4141 SDValue Lo, Hi;
4142 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4143
4144 EVT VecVT = N->getValueType(ResNo: 0);
4145 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4146}
4147
4148SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4149 EVT ResVT = N->getValueType(ResNo: 0);
4150 SDValue Lo, Hi;
4151 SDLoc dl(N);
4152
4153 SDValue VecOp = N->getOperand(Num: OpNo);
4154 EVT VecVT = VecOp.getValueType();
4155 assert(VecVT.isVector() && "Can only split reduce vector operand");
4156 GetSplitVector(Op: VecOp, Lo, Hi);
4157 EVT LoOpVT, HiOpVT;
4158 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4159
4160 // Use the appropriate scalar instruction on the split subvectors before
4161 // reducing the now partially reduced smaller vector.
4162 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4163 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4164 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4165}
4166
4167SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4168 EVT ResVT = N->getValueType(ResNo: 0);
4169 SDValue Lo, Hi;
4170 SDLoc dl(N);
4171
4172 SDValue AccOp = N->getOperand(Num: 0);
4173 SDValue VecOp = N->getOperand(Num: 1);
4174 SDNodeFlags Flags = N->getFlags();
4175
4176 EVT VecVT = VecOp.getValueType();
4177 assert(VecVT.isVector() && "Can only split reduce vector operand");
4178 GetSplitVector(Op: VecOp, Lo, Hi);
4179 EVT LoOpVT, HiOpVT;
4180 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4181
4182 // Reduce low half.
4183 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4184
4185 // Reduce high half, using low half result as initial value.
4186 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4187}
4188
4189SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4190 assert(N->isVPOpcode() && "Expected VP opcode");
4191 assert(OpNo == 1 && "Can only split reduce vector operand");
4192
4193 unsigned Opc = N->getOpcode();
4194 EVT ResVT = N->getValueType(ResNo: 0);
4195 SDValue Lo, Hi;
4196 SDLoc dl(N);
4197
4198 SDValue VecOp = N->getOperand(Num: OpNo);
4199 EVT VecVT = VecOp.getValueType();
4200 assert(VecVT.isVector() && "Can only split reduce vector operand");
4201 GetSplitVector(Op: VecOp, Lo, Hi);
4202
4203 SDValue MaskLo, MaskHi;
4204 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4205
4206 SDValue EVLLo, EVLHi;
4207 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4208
4209 const SDNodeFlags Flags = N->getFlags();
4210
4211 SDValue ResLo =
4212 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4213 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4214}
4215
4216SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4217 // The result has a legal vector type, but the input needs splitting.
4218 EVT ResVT = N->getValueType(ResNo: 0);
4219 SDValue Lo, Hi;
4220 SDLoc dl(N);
4221 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4222 EVT InVT = Lo.getValueType();
4223
4224 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4225 EC: InVT.getVectorElementCount());
4226
4227 if (N->isStrictFPOpcode()) {
4228 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4229 Ops: {N->getOperand(Num: 0), Lo});
4230 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4231 Ops: {N->getOperand(Num: 0), Hi});
4232
4233 // Build a factor node to remember that this operation is independent
4234 // of the other one.
4235 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4236 N2: Hi.getValue(R: 1));
4237
4238 // Legalize the chain result - switch anything that used the old chain to
4239 // use the new one.
4240 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4241 } else {
4242 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4243 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4244 }
4245
4246 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4247}
4248
4249// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4250SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4251 SDValue Lo, Hi;
4252 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4253 SDValue Chain =
4254 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4255 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4256}
4257
4258SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4259 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4260 // end up being split all the way down to individual components. Convert the
4261 // split pieces into integers and reassemble.
4262 EVT ResVT = N->getValueType(ResNo: 0);
4263 SDValue Lo, Hi;
4264 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4265 SDLoc dl(N);
4266
4267 if (ResVT.isScalableVector()) {
4268 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4269 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4270 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4271 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4272 }
4273
4274 Lo = BitConvertToInteger(Op: Lo);
4275 Hi = BitConvertToInteger(Op: Hi);
4276
4277 if (DAG.getDataLayout().isBigEndian())
4278 std::swap(a&: Lo, b&: Hi);
4279
4280 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4281}
4282
4283SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4284 unsigned OpNo) {
4285 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4286 // We know that the result type is legal.
4287 EVT ResVT = N->getValueType(ResNo: 0);
4288
4289 SDValue Vec = N->getOperand(Num: 0);
4290 SDValue SubVec = N->getOperand(Num: 1);
4291 SDValue Idx = N->getOperand(Num: 2);
4292 SDLoc dl(N);
4293
4294 SDValue Lo, Hi;
4295 GetSplitVector(Op: SubVec, Lo, Hi);
4296
4297 uint64_t IdxVal = Idx->getAsZExtVal();
4298 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4299
4300 SDValue FirstInsertion =
4301 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4302 SDValue SecondInsertion =
4303 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4304 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4305
4306 return SecondInsertion;
4307}
4308
4309SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4310 // We know that the extracted result type is legal.
4311 EVT SubVT = N->getValueType(ResNo: 0);
4312 SDValue Idx = N->getOperand(Num: 1);
4313 SDLoc dl(N);
4314 SDValue Lo, Hi;
4315
4316 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4317
4318 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4319 // Note: For scalable vectors, the index is scaled by vscale.
4320 ElementCount IdxVal =
4321 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4322 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4323
4324 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4325 ElementCount NumResultElts = SubVT.getVectorElementCount();
4326
4327 // If the extracted elements are all in the low half, do a simple extract.
4328 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4329 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4330
4331 unsigned LoEltsMin = LoElts.getKnownMinValue();
4332 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4333 SrcVT.isFixedLengthVector()) {
4334 // Extracted subvector crosses vector split, so we need to blend the two
4335 // halves.
4336 // TODO: May be able to emit partial extract_subvector.
4337 SmallVector<SDValue, 8> Elts;
4338 Elts.reserve(N: NumResultElts.getFixedValue());
4339
4340 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4341 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4342 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4343 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4344 /*Count=*/LoEltsMin - IdxValMin);
4345 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4346 /*Count=*/SubVT.getVectorNumElements() -
4347 Elts.size());
4348 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4349 }
4350
4351 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4352 ElementCount ExtractIdx = IdxVal - LoElts;
4353 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4354 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4355 Idx: ExtractIdx.getKnownMinValue());
4356
4357 EVT HiVT = Hi.getValueType();
4358 assert(HiVT.isFixedLengthVector() &&
4359 "Only fixed-vector extracts are supported in this case");
4360
4361 // We cannot create an extract_subvector that isn't a multiple of the
4362 // result size, which may go out of bounds for the last elements. Shuffle
4363 // the desired elements down to 0 and do a simple 0 extract.
4364 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4365 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4366 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4367
4368 SDValue Shuffle =
4369 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4370 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4371 }
4372
4373 // After this point the DAG node only permits extracting fixed-width
4374 // subvectors from scalable vectors.
4375 assert(SubVT.isFixedLengthVector() &&
4376 "Extracting scalable subvector from fixed-width unsupported");
4377
4378 // If the element type is i1 and we're not promoting the result, then we may
4379 // end up loading the wrong data since the bits are packed tightly into
4380 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4381 // type at index 4, then we will load a byte starting at index 0.
4382 if (SubVT.getScalarType() == MVT::i1)
4383 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4384 "subvector from a scalable predicate vector");
4385
4386 // Spill the vector to the stack. We should use the alignment for
4387 // the smallest part.
4388 SDValue Vec = N->getOperand(Num: 0);
4389 EVT VecVT = Vec.getValueType();
4390 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4391 SDValue StackPtr =
4392 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4393 auto &MF = DAG.getMachineFunction();
4394 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4395 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4396
4397 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4398 Alignment: SmallestAlign);
4399
4400 // Extract the subvector by loading the correct part.
4401 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4402
4403 return DAG.getLoad(
4404 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4405 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4406}
4407
4408SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4409 SDValue Vec = N->getOperand(Num: 0);
4410 SDValue Idx = N->getOperand(Num: 1);
4411 EVT VecVT = Vec.getValueType();
4412
4413 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4414 uint64_t IdxVal = Index->getZExtValue();
4415
4416 SDValue Lo, Hi;
4417 GetSplitVector(Op: Vec, Lo, Hi);
4418
4419 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4420
4421 if (IdxVal < LoElts)
4422 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4423 else if (!Vec.getValueType().isScalableVector())
4424 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4425 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4426 VT: Idx.getValueType())), 0);
4427 }
4428
4429 // See if the target wants to custom expand this node.
4430 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4431 return SDValue();
4432
4433 // Make the vector elements byte-addressable if they aren't already.
4434 SDLoc dl(N);
4435 EVT EltVT = VecVT.getVectorElementType();
4436 if (!EltVT.isByteSized()) {
4437 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4438 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4439 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4440 SDValue NewExtract =
4441 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4442 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4443 }
4444
4445 // Store the vector to the stack.
4446 // In cases where the vector is illegal it will be broken down into parts
4447 // and stored in parts - we should use the alignment for the smallest part.
4448 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4449 SDValue StackPtr =
4450 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4451 auto &MF = DAG.getMachineFunction();
4452 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4453 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4454 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4455 Alignment: SmallestAlign);
4456
4457 // Load back the required element.
4458 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4459
4460 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4461 // type, leaving the high bits undefined. But it can't truncate.
4462 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4463
4464 return DAG.getExtLoad(
4465 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4466 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4467 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4468}
4469
4470SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4471 SDValue Lo, Hi;
4472
4473 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4474 // splitting the result has the same effect as splitting the input operand.
4475 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4476
4477 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4478}
4479
4480SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4481 (void)OpNo;
4482 SDValue Lo, Hi;
4483 SplitVecRes_Gather(N, Lo, Hi);
4484
4485 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4486 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4487 return SDValue();
4488}
4489
4490SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4491 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4492 SDValue Ch = N->getChain();
4493 SDValue Ptr = N->getBasePtr();
4494 SDValue Offset = N->getOffset();
4495 assert(Offset.isUndef() && "Unexpected VP store offset");
4496 SDValue Mask = N->getMask();
4497 SDValue EVL = N->getVectorLength();
4498 SDValue Data = N->getValue();
4499 Align Alignment = N->getBaseAlign();
4500 SDLoc DL(N);
4501
4502 SDValue DataLo, DataHi;
4503 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4504 // Split Data operand
4505 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4506 else
4507 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4508
4509 // Split Mask operand
4510 SDValue MaskLo, MaskHi;
4511 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4512 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4513 } else {
4514 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4515 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4516 else
4517 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4518 }
4519
4520 EVT MemoryVT = N->getMemoryVT();
4521 EVT LoMemVT, HiMemVT;
4522 bool HiIsEmpty = false;
4523 std::tie(args&: LoMemVT, args&: HiMemVT) =
4524 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4525
4526 // Split EVL
4527 SDValue EVLLo, EVLHi;
4528 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4529
4530 SDValue Lo, Hi;
4531 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4532 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4533 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4534 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4535
4536 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4537 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4538 IsCompressing: N->isCompressingStore());
4539
4540 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4541 if (HiIsEmpty)
4542 return Lo;
4543
4544 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4545 IsCompressedMemory: N->isCompressingStore());
4546
4547 MachinePointerInfo MPI;
4548 if (LoMemVT.isScalableVector()) {
4549 Alignment = commonAlignment(A: Alignment,
4550 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4551 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4552 } else
4553 MPI = N->getPointerInfo().getWithOffset(
4554 O: LoMemVT.getStoreSize().getFixedValue());
4555
4556 MMO = DAG.getMachineFunction().getMachineMemOperand(
4557 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4558 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4559
4560 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4561 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4562 IsCompressing: N->isCompressingStore());
4563
4564 // Build a factor node to remember that this store is independent of the
4565 // other one.
4566 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4567}
4568
4569SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4570 unsigned OpNo) {
4571 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4572 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4573
4574 SDLoc DL(N);
4575
4576 SDValue Data = N->getValue();
4577 SDValue LoData, HiData;
4578 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4579 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4580 else
4581 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4582
4583 EVT LoMemVT, HiMemVT;
4584 bool HiIsEmpty = false;
4585 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4586 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4587
4588 SDValue Mask = N->getMask();
4589 SDValue LoMask, HiMask;
4590 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4591 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4592 else if (getTypeAction(VT: Mask.getValueType()) ==
4593 TargetLowering::TypeSplitVector)
4594 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4595 else
4596 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4597
4598 SDValue LoEVL, HiEVL;
4599 std::tie(args&: LoEVL, args&: HiEVL) =
4600 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4601
4602 // Generate the low vp_strided_store
4603 SDValue Lo = DAG.getStridedStoreVP(
4604 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4605 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4606 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4607
4608 // If the high vp_strided_store has zero storage size, only the low
4609 // vp_strided_store is needed.
4610 if (HiIsEmpty)
4611 return Lo;
4612
4613 // Generate the high vp_strided_store.
4614 // To calculate the high base address, we need to sum to the low base
4615 // address stride number of bytes for each element already stored by low,
4616 // that is: Ptr = Ptr + (LoEVL * Stride)
4617 EVT PtrVT = N->getBasePtr().getValueType();
4618 SDValue Increment =
4619 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4620 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4621 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4622
4623 Align Alignment = N->getBaseAlign();
4624 if (LoMemVT.isScalableVector())
4625 Alignment = commonAlignment(A: Alignment,
4626 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4627
4628 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4629 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4630 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4631 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4632
4633 SDValue Hi = DAG.getStridedStoreVP(
4634 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4635 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4636 IsCompressing: N->isCompressingStore());
4637
4638 // Build a factor node to remember that this store is independent of the
4639 // other one.
4640 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4641}
4642
4643SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4644 unsigned OpNo) {
4645 assert(N->isUnindexed() && "Indexed masked store of vector?");
4646 SDValue Ch = N->getChain();
4647 SDValue Ptr = N->getBasePtr();
4648 SDValue Offset = N->getOffset();
4649 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4650 SDValue Mask = N->getMask();
4651 SDValue Data = N->getValue();
4652 Align Alignment = N->getBaseAlign();
4653 SDLoc DL(N);
4654
4655 SDValue DataLo, DataHi;
4656 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4657 // Split Data operand
4658 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4659 else
4660 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4661
4662 // Split Mask operand
4663 SDValue MaskLo, MaskHi;
4664 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4665 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4666 } else {
4667 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4668 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4669 else
4670 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4671 }
4672
4673 EVT MemoryVT = N->getMemoryVT();
4674 EVT LoMemVT, HiMemVT;
4675 bool HiIsEmpty = false;
4676 std::tie(args&: LoMemVT, args&: HiMemVT) =
4677 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4678
4679 SDValue Lo, Hi, Res;
4680 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4681 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4682 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4683 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4684
4685 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4686 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4687 IsCompressing: N->isCompressingStore());
4688
4689 if (HiIsEmpty) {
4690 // The hi masked store has zero storage size.
4691 // Only the lo masked store is needed.
4692 Res = Lo;
4693 } else {
4694
4695 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4696 IsCompressedMemory: N->isCompressingStore());
4697
4698 MachinePointerInfo MPI;
4699 if (LoMemVT.isScalableVector()) {
4700 Alignment = commonAlignment(
4701 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4702 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4703 } else
4704 MPI = N->getPointerInfo().getWithOffset(
4705 O: LoMemVT.getStoreSize().getFixedValue());
4706
4707 MMO = DAG.getMachineFunction().getMachineMemOperand(
4708 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4709 BaseAlignment: Alignment,
4710 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4711
4712 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4713 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4714 IsCompressing: N->isCompressingStore());
4715
4716 // Build a factor node to remember that this store is independent of the
4717 // other one.
4718 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4719 }
4720
4721 return Res;
4722}
4723
4724SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4725 SDValue Ch = N->getChain();
4726 SDValue Ptr = N->getBasePtr();
4727 EVT MemoryVT = N->getMemoryVT();
4728 Align Alignment = N->getBaseAlign();
4729 SDLoc DL(N);
4730 struct Operands {
4731 SDValue Mask;
4732 SDValue Index;
4733 SDValue Scale;
4734 SDValue Data;
4735 } Ops = [&]() -> Operands {
4736 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4737 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4738 .Data: MSC->getValue()};
4739 }
4740 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4741 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4742 .Data: VPSC->getValue()};
4743 }();
4744 // Split all operands
4745
4746 EVT LoMemVT, HiMemVT;
4747 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4748
4749 SDValue DataLo, DataHi;
4750 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4751 // Split Data operand
4752 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4753 else
4754 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4755
4756 // Split Mask operand
4757 SDValue MaskLo, MaskHi;
4758 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4759 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4760 } else {
4761 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4762 }
4763
4764 SDValue IndexHi, IndexLo;
4765 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4766 TargetLowering::TypeSplitVector)
4767 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4768 else
4769 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4770
4771 SDValue Lo;
4772 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4773 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4774 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4775 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4776
4777 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4778 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4779 Lo =
4780 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4781 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4782
4783 // The order of the Scatter operation after split is well defined. The "Hi"
4784 // part comes after the "Lo". So these two operations should be chained one
4785 // after another.
4786 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4787 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4788 MMO, IndexType: MSC->getIndexType(),
4789 IsTruncating: MSC->isTruncatingStore());
4790 }
4791 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4792 SDValue EVLLo, EVLHi;
4793 std::tie(args&: EVLLo, args&: EVLHi) =
4794 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4795
4796 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4797 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4798 IndexType: VPSC->getIndexType());
4799
4800 // The order of the Scatter operation after split is well defined. The "Hi"
4801 // part comes after the "Lo". So these two operations should be chained one
4802 // after another.
4803 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4804 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4805 IndexType: VPSC->getIndexType());
4806}
4807
4808SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4809 assert(N->isUnindexed() && "Indexed store of vector?");
4810 assert(OpNo == 1 && "Can only split the stored value");
4811 SDLoc DL(N);
4812
4813 bool isTruncating = N->isTruncatingStore();
4814 SDValue Ch = N->getChain();
4815 SDValue Ptr = N->getBasePtr();
4816 EVT MemoryVT = N->getMemoryVT();
4817 Align Alignment = N->getBaseAlign();
4818 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4819 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
4820 SDValue Lo, Hi;
4821 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4822
4823 EVT LoMemVT, HiMemVT;
4824 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4825
4826 // Scalarize if the split halves are not byte-sized.
4827 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4828 return TLI.scalarizeVectorStore(ST: N, DAG);
4829
4830 if (isTruncating)
4831 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4832 Alignment, MMOFlags, Metadata);
4833 else
4834 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4835 Metadata);
4836
4837 MachinePointerInfo MPI;
4838 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4839
4840 if (isTruncating)
4841 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, SVT: HiMemVT, Alignment, MMOFlags,
4842 Metadata);
4843 else
4844 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, Metadata);
4845
4846 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4847}
4848
4849SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4850 SDLoc DL(N);
4851 LLVMContext &Ctx = *DAG.getContext();
4852 SDValue StVal = N->getVal();
4853 EVT VT = StVal.getValueType();
4854 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4855
4856 // The store needs a single value spanning the full memory width. If the
4857 // value can be held in a legal vector register, keep it there and extract
4858 // the low integer element of the memory width. This lets the store be issued
4859 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4860 // bitcasting the split vector straight to a scalar integer, which would
4861 // reassemble the value element by element in GPRs.
4862 //
4863 // Reinterpret the value as a same-shaped integer vector first: an FP element
4864 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4865 // integer-of-element-size form does. Ask the target which legal vector type
4866 // it widens to.
4867 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4868 EVT IntEltVT = IntVecVT.getVectorElementType();
4869 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4870 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4871 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4872 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4873 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4874 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4875 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4876 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4877 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4878 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4879 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4880 Val: N->getBasePtr(), MMO: N->getMemOperand());
4881 }
4882
4883 // Otherwise issue a single atomic store of an integer that spans the full
4884 // memory width. Bitcasting the (illegal) vector value to that integer lets
4885 // the type legalizer further legalize the BITCAST input as needed, while the
4886 // ATOMIC_STORE itself uses only the legal integer type.
4887 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4888 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4889 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4890 Val: N->getBasePtr(), MMO: N->getMemOperand());
4891}
4892
4893SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4894 SDLoc DL(N);
4895
4896 // The input operands all must have the same type, and we know the result
4897 // type is valid. Convert this to a buildvector which extracts all the
4898 // input elements.
4899 // TODO: If the input elements are power-two vectors, we could convert this to
4900 // a new CONCAT_VECTORS node with elements that are half-wide.
4901 SmallVector<SDValue, 32> Elts;
4902 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4903 for (const SDValue &Op : N->op_values()) {
4904 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4905 i != e; ++i) {
4906 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4907 }
4908 }
4909
4910 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4911}
4912
4913SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4914 // The result type is legal, but the input type is illegal. If splitting
4915 // ends up with the result type of each half still being legal, just
4916 // do that. If, however, that would result in an illegal result type,
4917 // we can try to get more clever with power-two vectors. Specifically,
4918 // split the input type, but also widen the result element size, then
4919 // concatenate the halves and truncate again. For example, consider a target
4920 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4921 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4922 // %inlo = v4i32 extract_subvector %in, 0
4923 // %inhi = v4i32 extract_subvector %in, 4
4924 // %lo16 = v4i16 trunc v4i32 %inlo
4925 // %hi16 = v4i16 trunc v4i32 %inhi
4926 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4927 // %res = v8i8 trunc v8i16 %in16
4928 //
4929 // Without this transform, the original truncate would end up being
4930 // scalarized, which is pretty much always a last resort.
4931 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4932 SDValue InVec = N->getOperand(Num: OpNo);
4933 EVT InVT = InVec->getValueType(ResNo: 0);
4934 EVT OutVT = N->getValueType(ResNo: 0);
4935 ElementCount NumElements = OutVT.getVectorElementCount();
4936 bool IsFloat = OutVT.isFloatingPoint();
4937
4938 unsigned InElementSize = InVT.getScalarSizeInBits();
4939 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4940
4941 // Determine the split output VT. If its legal we can just split dirctly.
4942 EVT LoOutVT, HiOutVT;
4943 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4944 assert(LoOutVT == HiOutVT && "Unequal split?");
4945
4946 // If the input elements are only 1/2 the width of the result elements,
4947 // just use the normal splitting. Our trick only work if there's room
4948 // to split more than once.
4949 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4950 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4951 return SplitVecOp_UnaryOp(N);
4952 SDLoc DL(N);
4953
4954 // Don't touch if this will be scalarized.
4955 EVT FinalVT = InVT;
4956 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4957 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4958
4959 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4960 return SplitVecOp_UnaryOp(N);
4961
4962 // Get the split input vector.
4963 SDValue InLoVec, InHiVec;
4964 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4965
4966 // Truncate them to 1/2 the element size.
4967 //
4968 // This assumes the number of elements is a power of two; any vector that
4969 // isn't should be widened, not split.
4970 EVT HalfElementVT = IsFloat ?
4971 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4972 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4973 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4974 EC: NumElements.divideCoefficientBy(RHS: 2));
4975
4976 SDValue HalfLo;
4977 SDValue HalfHi;
4978 SDValue Chain;
4979 if (N->isStrictFPOpcode()) {
4980 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4981 Ops: {N->getOperand(Num: 0), InLoVec});
4982 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4983 Ops: {N->getOperand(Num: 0), InHiVec});
4984 // Legalize the chain result - switch anything that used the old chain to
4985 // use the new one.
4986 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4987 N2: HalfHi.getValue(R: 1));
4988 } else {
4989 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4990 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4991 }
4992
4993 // Concatenate them to get the full intermediate truncation result.
4994 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4995 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4996 N2: HalfHi);
4997 // Now finish up by truncating all the way down to the original result
4998 // type. This should normally be something that ends up being legal directly,
4999 // but in theory if a target has very wide vectors and an annoyingly
5000 // restricted set of legal types, this split can chain to build things up.
5001
5002 if (N->isStrictFPOpcode()) {
5003 SDValue Res = DAG.getNode(
5004 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
5005 Ops: {Chain, InterVec,
5006 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
5007 // Relink the chain
5008 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
5009 return Res;
5010 }
5011
5012 return IsFloat
5013 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
5014 N2: DAG.getTargetConstant(
5015 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
5016 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
5017}
5018
5019SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
5020 unsigned Opc = N->getOpcode();
5021 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
5022 assert(N->getValueType(0).isVector() &&
5023 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
5024 "Operand types must be vectors");
5025 // The result has a legal vector type, but the input needs splitting.
5026 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
5027 SDLoc DL(N);
5028 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
5029 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
5030
5031 EVT VT = N->getValueType(ResNo: 0);
5032 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
5033
5034 if (Opc == ISD::SETCC) {
5035 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
5036 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
5037 } else {
5038 assert(isStrict && "unexpected node");
5039 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5040 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
5041 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5042 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
5043 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5044 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
5045 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5046 }
5047
5048 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
5049 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
5050 if (VT == ConcatVT)
5051 return Con;
5052
5053 EVT OpVT = N->getOperand(Num: 0).getValueType();
5054 ISD::NodeType ExtendCode =
5055 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
5056 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
5057}
5058
5059
5060SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5061 // The result has a legal vector type, but the input needs splitting.
5062 EVT ResVT = N->getValueType(ResNo: 0);
5063 SDValue Lo, Hi;
5064 SDLoc DL(N);
5065 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5066 EVT InVT = Lo.getValueType();
5067
5068 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5069 EC: InVT.getVectorElementCount());
5070
5071 if (N->isStrictFPOpcode()) {
5072 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5073 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5074 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5075 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5076 // Legalize the chain result - switch anything that used the old chain to
5077 // use the new one.
5078 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5079 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5080 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5081 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5082 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5083 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5084 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5085 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5086 } else {
5087 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5088 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5089 }
5090
5091 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5092}
5093
5094// Split a vector type in an FP binary operation where the second operand has a
5095// different type from the first.
5096//
5097// The result (and the first input) has a legal vector type, but the second
5098// input needs splitting.
5099SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5100 SDLoc DL(N);
5101
5102 EVT LHSLoVT, LHSHiVT;
5103 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5104
5105 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5106 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5107
5108 SDValue LHSLo, LHSHi;
5109 std::tie(args&: LHSLo, args&: LHSHi) =
5110 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5111
5112 SDValue RHSLo, RHSHi;
5113 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5114
5115 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5116 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5117
5118 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5119}
5120
5121SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5122 LLVMContext &Ctxt = *DAG.getContext();
5123 SDLoc dl(N);
5124
5125 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5126 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5127 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5128
5129 EVT ResVT = N->getValueType(ResNo: 0);
5130 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5131 EVT NewResVT =
5132 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5133
5134 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5135 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5136
5137 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5138}
5139
5140SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5141 EVT ResVT = N->getValueType(ResNo: 0);
5142 SDValue Lo, Hi;
5143 SDLoc dl(N);
5144 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5145 EVT InVT = Lo.getValueType();
5146
5147 EVT NewResVT =
5148 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5149 EC: InVT.getVectorElementCount());
5150
5151 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5152 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5153
5154 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5155}
5156
5157SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5158 SDLoc DL(N);
5159 EVT ResVT = N->getValueType(ResNo: 0);
5160
5161 SDValue Lo, Hi;
5162 SDValue VecOp = N->getOperand(Num: 0);
5163 GetSplitVector(Op: VecOp, Lo, Hi);
5164
5165 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5166 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5167 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5168 SDValue VL =
5169 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5170 SDValue ResLoNotVL =
5171 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5172 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5173 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5174 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5175}
5176
5177SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5178 SDLoc DL(N);
5179 EVT ResVT = N->getValueType(ResNo: 0);
5180
5181 SDValue Lo, Hi;
5182 SDValue VecOp = N->getOperand(Num: 0);
5183 GetSplitVector(Op: VecOp, Lo, Hi);
5184
5185 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5186 auto [EVLLo, EVLHi] =
5187 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5188 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5189
5190 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5191 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5192 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5193 SDValue ResLoNotEVL =
5194 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5195 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5196 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5197 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5198}
5199
5200SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5201 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5202 SDLoc DL(HG);
5203 SDValue Inc = HG->getInc();
5204 SDValue Ptr = HG->getBasePtr();
5205 SDValue Scale = HG->getScale();
5206 SDValue IntID = HG->getIntID();
5207 EVT MemVT = HG->getMemoryVT();
5208 MachineMemOperand *MMO = HG->getMemOperand();
5209 ISD::MemIndexType IndexType = HG->getIndexType();
5210
5211 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5212 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5213 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5214 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5215 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5216 Ops: OpsLo, MMO, IndexType);
5217 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5218 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5219 MMO, IndexType);
5220}
5221
5222SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5223 SDLoc DL(N);
5224
5225 if (OpNo == 0) {
5226 EVT LoResVT, HiResVT;
5227 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5228 SDValue SourceLo, SourceHi;
5229 std::tie(args&: SourceLo, args&: SourceHi) = DAG.SplitVectorOperand(N, OpNo: 0);
5230 SDValue MaskLo, MaskHi;
5231 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVectorOperand(N, OpNo: 2);
5232
5233 SDValue MatchLo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: LoResVT, N1: SourceLo,
5234 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
5235 SDValue MatchHi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: HiResVT, N1: SourceHi,
5236 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
5237 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo,
5238 N2: MatchHi);
5239 }
5240
5241 // Note: The Mask (OpNo == 2) should be widened with the result.
5242 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5243
5244 SDValue NeedleLo, NeedleHi;
5245 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: NeedleLo, Hi&: NeedleHi);
5246
5247 SDValue MatchLo =
5248 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5249 N2: NeedleLo, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5250 SDValue MatchHi =
5251 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5252 N2: NeedleHi, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5253 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo, N2: MatchHi);
5254}
5255
5256SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5257 SDValue Acc = N->getOperand(Num: 0);
5258 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5259 "Accumulator should already be a legal type, and shouldn't need "
5260 "further splitting");
5261
5262 SDLoc DL(N);
5263 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5264 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5265 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5266 unsigned Opcode = N->getOpcode();
5267 EVT ResultVT = Acc.getValueType();
5268
5269 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5270 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5271}
5272
5273//===----------------------------------------------------------------------===//
5274// Result Vector Widening
5275//===----------------------------------------------------------------------===//
5276
5277void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5278 unsigned WidenResNo) {
5279 unsigned NumResults = N->getNumValues();
5280 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5281 if (ResNo == WidenResNo)
5282 continue;
5283 EVT ResVT = N->getValueType(ResNo);
5284 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5285 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5286 } else {
5287 SDLoc DL(N);
5288 SDValue ResVal =
5289 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5290 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5291 }
5292 }
5293}
5294
5295void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5296 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5297
5298 // See if the target wants to custom widen this node.
5299 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5300 return;
5301
5302 SDValue Res = SDValue();
5303
5304 auto unrollExpandedOp = [&]() {
5305 // We're going to widen this vector op to a legal type by padding with undef
5306 // elements. If the wide vector op is eventually going to be expanded to
5307 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5308 // libcalls on the undef elements.
5309 EVT ResVT = N->getValueType(ResNo);
5310 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5311 EVT VT0 = N->getValueType(ResNo: 0);
5312 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5313 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT0.getScalarType())) {
5314 SDValue Unrolled =
5315 DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5316 Res = Unrolled.getValue(R: ResNo);
5317 if (N->getNumValues() > 1)
5318 ReplaceOtherWidenResults(N, WidenNode: Unrolled.getNode(), WidenResNo: ResNo);
5319 return true;
5320 }
5321 return false;
5322 };
5323
5324 switch (N->getOpcode()) {
5325 default:
5326#ifndef NDEBUG
5327 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5328 N->dump(&DAG);
5329 dbgs() << "\n";
5330#endif
5331 report_fatal_error(reason: "do not know how to widen the result of this operator!");
5332
5333 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5334 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5335 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5336 break;
5337 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5338 case ISD::ADDRSPACECAST:
5339 Res = WidenVecRes_ADDRSPACECAST(N);
5340 break;
5341 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5342 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5343 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5344 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5345 case ISD::INSERT_SUBVECTOR:
5346 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5347 break;
5348 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5349 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5350 case ISD::ATOMIC_LOAD:
5351 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5352 break;
5353 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5354 case ISD::STEP_VECTOR:
5355 case ISD::SPLAT_VECTOR:
5356 case ISD::SCALAR_TO_VECTOR:
5357 Res = WidenVecRes_ScalarOp(N);
5358 break;
5359 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5360 case ISD::VSELECT:
5361 case ISD::SELECT:
5362 case ISD::VP_MERGE:
5363 Res = WidenVecRes_Select(N);
5364 break;
5365 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5366 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5367 case ISD::POISON:
5368 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5369 case ISD::VECTOR_SHUFFLE:
5370 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5371 break;
5372 case ISD::VP_LOAD:
5373 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5374 break;
5375 case ISD::VP_LOAD_FF:
5376 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5377 break;
5378 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5379 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5380 break;
5381 case ISD::VECTOR_COMPRESS:
5382 Res = WidenVecRes_VECTOR_COMPRESS(N);
5383 break;
5384 case ISD::MLOAD:
5385 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5386 break;
5387 case ISD::MGATHER:
5388 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5389 break;
5390 case ISD::VP_GATHER:
5391 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5392 break;
5393 case ISD::VECTOR_REVERSE:
5394 Res = WidenVecRes_VECTOR_REVERSE(N);
5395 break;
5396 case ISD::GET_ACTIVE_LANE_MASK:
5397 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5398 break;
5399 case ISD::VECTOR_INTERLEAVE:
5400 WidenVecRes_VECTOR_INTERLEAVE(N);
5401 break;
5402 case ISD::VECTOR_MATCH:
5403 Res = WidenVecRes_VECTOR_MATCH(N);
5404 break;
5405 case ISD::VECTOR_DEINTERLEAVE:
5406 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5407 break;
5408
5409 case ISD::ADD:
5410 case ISD::AND:
5411 case ISD::MUL:
5412 case ISD::MULHS:
5413 case ISD::MULHU:
5414 case ISD::ABDS:
5415 case ISD::ABDU:
5416 case ISD::OR:
5417 case ISD::SUB:
5418 case ISD::XOR:
5419 case ISD::SHL:
5420 case ISD::SRA:
5421 case ISD::SRL:
5422 case ISD::CLMUL:
5423 case ISD::CLMULR:
5424 case ISD::CLMULH:
5425 case ISD::PEXT:
5426 case ISD::PDEP:
5427 case ISD::FMINNUM:
5428 case ISD::FMINNUM_IEEE:
5429 case ISD::FMAXNUM:
5430 case ISD::FMAXNUM_IEEE:
5431 case ISD::FMINIMUM:
5432 case ISD::FMAXIMUM:
5433 case ISD::FMINIMUMNUM:
5434 case ISD::FMAXIMUMNUM:
5435 case ISD::SMIN:
5436 case ISD::SMAX:
5437 case ISD::UMIN:
5438 case ISD::UMAX:
5439 case ISD::UADDSAT:
5440 case ISD::SADDSAT:
5441 case ISD::USUBSAT:
5442 case ISD::SSUBSAT:
5443 case ISD::SSHLSAT:
5444 case ISD::USHLSAT:
5445 case ISD::ROTL:
5446 case ISD::ROTR:
5447 case ISD::AVGFLOORS:
5448 case ISD::AVGFLOORU:
5449 case ISD::AVGCEILS:
5450 case ISD::AVGCEILU:
5451 // Vector-predicated binary op widening. Note that -- unlike the
5452 // unpredicated versions -- we don't have to worry about trapping on
5453 // operations like UDIV, FADD, etc., as we pass on the original vector
5454 // length parameter. This means the widened elements containing garbage
5455 // aren't active.
5456 case ISD::VP_SDIV:
5457 case ISD::VP_UDIV:
5458 case ISD::VP_SREM:
5459 case ISD::VP_UREM:
5460 Res = WidenVecRes_Binary(N);
5461 break;
5462
5463 case ISD::MASKED_UDIV:
5464 case ISD::MASKED_SDIV:
5465 case ISD::MASKED_UREM:
5466 case ISD::MASKED_SREM:
5467 Res = WidenVecRes_MaskedBinary(N);
5468 break;
5469
5470 case ISD::SCMP:
5471 case ISD::UCMP:
5472 Res = WidenVecRes_CMP(N);
5473 break;
5474
5475 case ISD::FPOW:
5476 case ISD::FATAN2:
5477 case ISD::FREM:
5478 if (unrollExpandedOp())
5479 break;
5480 // If the target has custom/legal support for the scalar FP intrinsic ops
5481 // (they are probably not destined to become libcalls), then widen those
5482 // like any other binary ops.
5483 [[fallthrough]];
5484
5485 case ISD::FADD:
5486 case ISD::FMUL:
5487 case ISD::FSUB:
5488 case ISD::FDIV:
5489 case ISD::SDIV:
5490 case ISD::UDIV:
5491 case ISD::SREM:
5492 case ISD::UREM:
5493 Res = WidenVecRes_BinaryCanTrap(N);
5494 break;
5495
5496 case ISD::SMULFIX:
5497 case ISD::SMULFIXSAT:
5498 case ISD::UMULFIX:
5499 case ISD::UMULFIXSAT:
5500 // These are binary operations, but with an extra operand that shouldn't
5501 // be widened (the scale).
5502 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5503 break;
5504
5505#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5506 case ISD::STRICT_##DAGN:
5507#include "llvm/IR/ConstrainedOps.def"
5508 Res = WidenVecRes_StrictFP(N);
5509 break;
5510
5511 case ISD::UADDO:
5512 case ISD::SADDO:
5513 case ISD::USUBO:
5514 case ISD::SSUBO:
5515 case ISD::UMULO:
5516 case ISD::SMULO:
5517 Res = WidenVecRes_OverflowOp(N, ResNo);
5518 break;
5519
5520 case ISD::FCOPYSIGN:
5521 Res = WidenVecRes_FCOPYSIGN(N);
5522 break;
5523
5524 case ISD::IS_FPCLASS:
5525 case ISD::FPTRUNC_ROUND:
5526 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5527 break;
5528
5529 case ISD::FLDEXP:
5530 case ISD::FPOWI:
5531 if (!unrollExpandedOp())
5532 Res = WidenVecRes_ExpOp(N);
5533 break;
5534
5535 case ISD::ANY_EXTEND_VECTOR_INREG:
5536 case ISD::SIGN_EXTEND_VECTOR_INREG:
5537 case ISD::ZERO_EXTEND_VECTOR_INREG:
5538 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5539 break;
5540
5541 case ISD::ANY_EXTEND:
5542 case ISD::FP_EXTEND:
5543 case ISD::FP_ROUND:
5544 case ISD::FP_TO_SINT:
5545 case ISD::FP_TO_UINT:
5546 case ISD::SIGN_EXTEND:
5547 case ISD::SINT_TO_FP:
5548 case ISD::TRUNCATE:
5549 case ISD::UINT_TO_FP:
5550 case ISD::ZERO_EXTEND:
5551 case ISD::CONVERT_FROM_ARBITRARY_FP:
5552 case ISD::CONVERT_TO_ARBITRARY_FP:
5553 Res = WidenVecRes_Convert(N);
5554 break;
5555
5556 case ISD::FP_TO_SINT_SAT:
5557 case ISD::FP_TO_UINT_SAT:
5558 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5559 break;
5560
5561 case ISD::LRINT:
5562 case ISD::LLRINT:
5563 case ISD::LROUND:
5564 case ISD::LLROUND:
5565 Res = WidenVecRes_XROUND(N);
5566 break;
5567
5568 case ISD::FACOS:
5569 case ISD::FASIN:
5570 case ISD::FATAN:
5571 case ISD::FCEIL:
5572 case ISD::FCOS:
5573 case ISD::FCOSH:
5574 case ISD::FEXP:
5575 case ISD::FEXP2:
5576 case ISD::FEXP10:
5577 case ISD::FFLOOR:
5578 case ISD::FLOG:
5579 case ISD::FLOG10:
5580 case ISD::FLOG2:
5581 case ISD::FNEARBYINT:
5582 case ISD::FRINT:
5583 case ISD::FROUND:
5584 case ISD::FROUNDEVEN:
5585 case ISD::FSIN:
5586 case ISD::FSINH:
5587 case ISD::FSQRT:
5588 case ISD::FTAN:
5589 case ISD::FTANH:
5590 case ISD::FTRUNC:
5591 if (unrollExpandedOp())
5592 break;
5593 // If the target has custom/legal support for the scalar FP intrinsic ops
5594 // (they are probably not destined to become libcalls), then widen those
5595 // like any other unary ops.
5596 [[fallthrough]];
5597
5598 case ISD::ABS:
5599 case ISD::ABS_MIN_POISON:
5600 case ISD::BITREVERSE:
5601 case ISD::BSWAP:
5602 case ISD::CTLZ:
5603 case ISD::CTLZ_ZERO_POISON:
5604 case ISD::CTPOP:
5605 case ISD::CTTZ:
5606 case ISD::CTTZ_ZERO_POISON:
5607 case ISD::FNEG:
5608 case ISD::FABS:
5609 case ISD::FREEZE:
5610 case ISD::ARITH_FENCE:
5611 case ISD::FCANONICALIZE:
5612 case ISD::AssertNoFPClass:
5613 case ISD::MASK_BEFOREFIRST:
5614 Res = WidenVecRes_Unary(N);
5615 break;
5616 case ISD::FMA:
5617 case ISD::FSHL:
5618 case ISD::FSHR:
5619 Res = WidenVecRes_Ternary(N);
5620 break;
5621 case ISD::FMODF:
5622 case ISD::FFREXP:
5623 case ISD::FSINCOS:
5624 case ISD::FSINCOSPI: {
5625 if (!unrollExpandedOp())
5626 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5627 break;
5628 }
5629 case ISD::PARTIAL_REDUCE_UMLA:
5630 case ISD::PARTIAL_REDUCE_SMLA:
5631 case ISD::PARTIAL_REDUCE_SUMLA:
5632 case ISD::PARTIAL_REDUCE_FMLA:
5633 Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
5634 break;
5635 }
5636
5637 // If Res is null, the sub-method took care of registering the result.
5638 if (Res.getNode())
5639 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5640}
5641
5642SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5643 // Ternary op widening.
5644 SDLoc dl(N);
5645 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5646 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5647 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5648 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5649 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5650}
5651
5652SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5653 // Binary op widening.
5654 SDLoc dl(N);
5655 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5656 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5657 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5658 if (N->getNumOperands() == 2)
5659 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5660 Flags: N->getFlags());
5661
5662 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5663 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5664 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5665 "Expected VP opcode");
5666
5667 SDValue Mask =
5668 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5669 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5670 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5671}
5672
5673SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5674 SDLoc dl(N);
5675 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5676 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5677 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5678 SDValue Mask = N->getOperand(Num: 2);
5679 EVT WideMaskVT = WidenVT.changeVectorElementType(
5680 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5681 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5682 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5683 Flags: N->getFlags());
5684}
5685
5686SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5687 LLVMContext &Ctxt = *DAG.getContext();
5688 SDLoc dl(N);
5689
5690 SDValue LHS = N->getOperand(Num: 0);
5691 SDValue RHS = N->getOperand(Num: 1);
5692 EVT OpVT = LHS.getValueType();
5693 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5694 LHS = GetWidenedVector(Op: LHS);
5695 RHS = GetWidenedVector(Op: RHS);
5696 OpVT = LHS.getValueType();
5697 }
5698
5699 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5700 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5701 if (WidenResEC == OpVT.getVectorElementCount()) {
5702 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5703 }
5704
5705 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5706}
5707
5708SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5709 // Binary op widening, but with an extra operand that shouldn't be widened.
5710 SDLoc dl(N);
5711 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5712 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5713 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5714 SDValue InOp3 = N->getOperand(Num: 2);
5715 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5716 Flags: N->getFlags());
5717}
5718
5719// Given a vector of operations that have been broken up to widen, see
5720// if we can collect them together into the next widest legal VT. This
5721// implementation is trap-safe.
5722static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5723 SmallVectorImpl<SDValue> &ConcatOps,
5724 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5725 EVT WidenVT) {
5726 // Check to see if we have a single operation with the widen type.
5727 if (ConcatEnd == 1) {
5728 VT = ConcatOps[0].getValueType();
5729 if (VT == WidenVT)
5730 return ConcatOps[0];
5731 }
5732
5733 SDLoc dl(ConcatOps[0]);
5734 EVT WidenEltVT = WidenVT.getVectorElementType();
5735
5736 // while (Some element of ConcatOps is not of type MaxVT) {
5737 // From the end of ConcatOps, collect elements of the same type and put
5738 // them into an op of the next larger supported type
5739 // }
5740 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5741 int Idx = ConcatEnd - 1;
5742 VT = ConcatOps[Idx--].getValueType();
5743 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5744 Idx--;
5745
5746 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5747 EVT NextVT;
5748 do {
5749 NextSize *= 2;
5750 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5751 } while (!TLI.isTypeLegal(VT: NextVT));
5752
5753 if (!VT.isVector()) {
5754 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5755 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5756 unsigned NumToInsert = ConcatEnd - Idx - 1;
5757 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5758 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5759 ConcatOps[Idx+1] = VecOp;
5760 ConcatEnd = Idx + 2;
5761 } else {
5762 // Vector type, create a CONCAT_VECTORS of type NextVT
5763 SDValue undefVec = DAG.getPOISON(VT);
5764 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5765 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5766 unsigned RealVals = ConcatEnd - Idx - 1;
5767 unsigned SubConcatEnd = 0;
5768 unsigned SubConcatIdx = Idx + 1;
5769 while (SubConcatEnd < RealVals)
5770 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5771 while (SubConcatEnd < OpsToConcat)
5772 SubConcatOps[SubConcatEnd++] = undefVec;
5773 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5774 VT: NextVT, Ops: SubConcatOps);
5775 ConcatEnd = SubConcatIdx + 1;
5776 }
5777 }
5778
5779 // Check to see if we have a single operation with the widen type.
5780 if (ConcatEnd == 1) {
5781 VT = ConcatOps[0].getValueType();
5782 if (VT == WidenVT)
5783 return ConcatOps[0];
5784 }
5785
5786 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5787 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5788 if (NumOps != ConcatEnd ) {
5789 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5790 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5791 ConcatOps[j] = UndefVal;
5792 }
5793 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5794 Ops: ArrayRef(ConcatOps.data(), NumOps));
5795}
5796
5797SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5798 // Binary op widening for operations that can trap.
5799 unsigned Opcode = N->getOpcode();
5800 SDLoc dl(N);
5801 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5802 EVT WidenEltVT = WidenVT.getVectorElementType();
5803 EVT VT = WidenVT;
5804 unsigned NumElts = VT.getVectorMinNumElements();
5805 const SDNodeFlags Flags = N->getFlags();
5806 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5807 NumElts = NumElts / 2;
5808 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5809 }
5810
5811 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5812 // Operation doesn't trap so just widen as normal.
5813 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5814 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5815 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5816 }
5817
5818 // Generate a vp.op if it is custom/legal for the target. This avoids need
5819 // to split and tile the subvectors (below), because the inactive lanes can
5820 // simply be disabled. To avoid possible recursion, only do this if the
5821 // widened mask type is legal.
5822 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5823 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5824 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5825 EC: WidenVT.getVectorElementCount());
5826 TLI.isTypeLegal(VT: WideMaskVT)) {
5827 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5828 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5829 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5830 SDValue EVL =
5831 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5832 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5833 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5834 Flags);
5835 }
5836 }
5837
5838 // FIXME: Improve support for scalable vectors.
5839 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5840
5841 // No legal vector version so unroll the vector operation and then widen.
5842 if (NumElts == 1)
5843 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5844
5845 // Since the operation can trap, apply operation on the original vector.
5846 EVT MaxVT = VT;
5847 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5848 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5849 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5850
5851 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5852 unsigned ConcatEnd = 0; // Current ConcatOps index.
5853 int Idx = 0; // Current Idx into input vectors.
5854
5855 // NumElts := greatest legal vector size (at most WidenVT)
5856 // while (orig. vector has unhandled elements) {
5857 // take munches of size NumElts from the beginning and add to ConcatOps
5858 // NumElts := next smaller supported vector size or 1
5859 // }
5860 while (CurNumElts != 0) {
5861 while (CurNumElts >= NumElts) {
5862 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5863 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5864 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5865 Idx += NumElts;
5866 CurNumElts -= NumElts;
5867 }
5868 do {
5869 NumElts = NumElts / 2;
5870 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5871 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5872
5873 if (NumElts == 1) {
5874 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5875 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5876 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5877 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5878 N1: EOp1, N2: EOp2, Flags);
5879 }
5880 CurNumElts = 0;
5881 }
5882 }
5883
5884 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5885}
5886
5887SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5888 switch (N->getOpcode()) {
5889 case ISD::STRICT_FSETCC:
5890 case ISD::STRICT_FSETCCS:
5891 return WidenVecRes_STRICT_FSETCC(N);
5892 case ISD::STRICT_FP_EXTEND:
5893 case ISD::STRICT_FP_ROUND:
5894 case ISD::STRICT_FP_TO_SINT:
5895 case ISD::STRICT_FP_TO_UINT:
5896 case ISD::STRICT_SINT_TO_FP:
5897 case ISD::STRICT_UINT_TO_FP:
5898 return WidenVecRes_Convert_StrictFP(N);
5899 default:
5900 break;
5901 }
5902
5903 // StrictFP op widening for operations that can trap.
5904 unsigned NumOpers = N->getNumOperands();
5905 unsigned Opcode = N->getOpcode();
5906 SDLoc dl(N);
5907 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5908 EVT WidenEltVT = WidenVT.getVectorElementType();
5909 EVT VT = WidenVT;
5910 unsigned NumElts = VT.getVectorNumElements();
5911 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5912 NumElts = NumElts / 2;
5913 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5914 }
5915
5916 // No legal vector version so unroll the vector operation and then widen.
5917 if (NumElts == 1)
5918 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5919
5920 // Since the operation can trap, apply operation on the original vector.
5921 EVT MaxVT = VT;
5922 SmallVector<SDValue, 4> InOps;
5923 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5924
5925 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5926 SmallVector<SDValue, 16> Chains;
5927 unsigned ConcatEnd = 0; // Current ConcatOps index.
5928 int Idx = 0; // Current Idx into input vectors.
5929
5930 // The Chain is the first operand.
5931 InOps.push_back(Elt: N->getOperand(Num: 0));
5932
5933 // Now process the remaining operands.
5934 for (unsigned i = 1; i < NumOpers; ++i) {
5935 SDValue Oper = N->getOperand(Num: i);
5936
5937 EVT OpVT = Oper.getValueType();
5938 if (OpVT.isVector()) {
5939 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5940 Oper = GetWidenedVector(Op: Oper);
5941 else {
5942 EVT WideOpVT =
5943 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5944 EC: WidenVT.getVectorElementCount());
5945 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5946 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5947 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5948 }
5949 }
5950
5951 InOps.push_back(Elt: Oper);
5952 }
5953
5954 // NumElts := greatest legal vector size (at most WidenVT)
5955 // while (orig. vector has unhandled elements) {
5956 // take munches of size NumElts from the beginning and add to ConcatOps
5957 // NumElts := next smaller supported vector size or 1
5958 // }
5959 while (CurNumElts != 0) {
5960 while (CurNumElts >= NumElts) {
5961 SmallVector<SDValue, 4> EOps;
5962
5963 for (unsigned i = 0; i < NumOpers; ++i) {
5964 SDValue Op = InOps[i];
5965
5966 EVT OpVT = Op.getValueType();
5967 if (OpVT.isVector()) {
5968 EVT OpExtractVT =
5969 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5970 EC: VT.getVectorElementCount());
5971 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5972 }
5973
5974 EOps.push_back(Elt: Op);
5975 }
5976
5977 EVT OperVT[] = {VT, MVT::Other};
5978 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5979 ConcatOps[ConcatEnd++] = Oper;
5980 Chains.push_back(Elt: Oper.getValue(R: 1));
5981 Idx += NumElts;
5982 CurNumElts -= NumElts;
5983 }
5984 do {
5985 NumElts = NumElts / 2;
5986 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5987 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5988
5989 if (NumElts == 1) {
5990 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5991 SmallVector<SDValue, 4> EOps;
5992
5993 for (unsigned i = 0; i < NumOpers; ++i) {
5994 SDValue Op = InOps[i];
5995
5996 EVT OpVT = Op.getValueType();
5997 if (OpVT.isVector())
5998 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5999 Idx);
6000
6001 EOps.push_back(Elt: Op);
6002 }
6003
6004 EVT WidenVT[] = {WidenEltVT, MVT::Other};
6005 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
6006 ConcatOps[ConcatEnd++] = Oper;
6007 Chains.push_back(Elt: Oper.getValue(R: 1));
6008 }
6009 CurNumElts = 0;
6010 }
6011 }
6012
6013 // Build a factor node to remember all the Ops that have been created.
6014 SDValue NewChain;
6015 if (Chains.size() == 1)
6016 NewChain = Chains[0];
6017 else
6018 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
6019 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6020
6021 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6022}
6023
6024SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
6025 SDLoc DL(N);
6026 EVT ResVT = N->getValueType(ResNo: 0);
6027 EVT OvVT = N->getValueType(ResNo: 1);
6028 EVT WideResVT, WideOvVT;
6029 SDValue WideLHS, WideRHS;
6030
6031 // TODO: This might result in a widen/split loop.
6032 if (ResNo == 0) {
6033 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
6034 WideOvVT = EVT::getVectorVT(
6035 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
6036 NumElements: WideResVT.getVectorNumElements());
6037
6038 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6039 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
6040 } else {
6041 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
6042 WideResVT = EVT::getVectorVT(
6043 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
6044 NumElements: WideOvVT.getVectorNumElements());
6045
6046 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6047 SDValue Poison = DAG.getPOISON(VT: WideResVT);
6048
6049 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6050 N2: N->getOperand(Num: 0), N3: Zero);
6051 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6052 N2: N->getOperand(Num: 1), N3: Zero);
6053 }
6054
6055 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
6056 SDNode *WideNode = DAG.getNode(
6057 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
6058
6059 // Replace the other vector result not being explicitly widened here.
6060 unsigned OtherNo = 1 - ResNo;
6061 EVT OtherVT = N->getValueType(ResNo: OtherNo);
6062 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
6063 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
6064 } else {
6065 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6066 SDValue OtherVal = DAG.getNode(
6067 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
6068 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
6069 }
6070
6071 return SDValue(WideNode, ResNo);
6072}
6073
6074SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6075 LLVMContext &Ctx = *DAG.getContext();
6076 SDValue InOp = N->getOperand(Num: 0);
6077 SDLoc DL(N);
6078
6079 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6080 ElementCount WidenEC = WidenVT.getVectorElementCount();
6081
6082 EVT InVT = InOp.getValueType();
6083
6084 unsigned Opcode = N->getOpcode();
6085 const SDNodeFlags Flags = N->getFlags();
6086
6087 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6088 // not equal that of WidenVT.
6089 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6090 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6091 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6092 WidenVT.getScalarSizeInBits()) {
6093 InOp = ZExtPromotedInteger(Op: InOp);
6094 InVT = InOp.getValueType();
6095 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6096 Opcode = ISD::TRUNCATE;
6097 }
6098
6099 EVT InEltVT = InVT.getVectorElementType();
6100 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6101 ElementCount InVTEC = InVT.getVectorElementCount();
6102
6103 // Helper to build node with all scalar trailing operands.
6104 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6105 if (N->getNumOperands() == 1)
6106 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6107 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6108 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6109 N4: N->getOperand(Num: 3), Flags);
6110 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6111 };
6112
6113 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6114 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6115 InVT = InOp.getValueType();
6116 InVTEC = InVT.getVectorElementCount();
6117 if (InVTEC == WidenEC)
6118 return MakeConvertNode(WidenVT, InOp);
6119 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6120 // If both input and result vector types are of same width, extend
6121 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6122 // accepts fewer elements in the result than in the input.
6123 if (Opcode == ISD::ANY_EXTEND)
6124 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6125 if (Opcode == ISD::SIGN_EXTEND)
6126 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6127 if (Opcode == ISD::ZERO_EXTEND)
6128 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6129 }
6130
6131 // For TRUNCATE, try to widen using the legal EC of the input type instead
6132 // if the legalisation action for that intermediate type is not widening.
6133 // E.g. for trunc nxv1i64 -> nxv1i8 where
6134 // - nxv1i64 input gets widened to nxv2i64
6135 // - nxv1i8 output gets widened to nxv16i8
6136 // Then one can try widening the result to nxv2i8 (instead of going all the
6137 // way to nxv16i8) if this later allows type promotion.
6138 EVT MidResVT =
6139 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6140 if (N->getOpcode() == ISD::TRUNCATE &&
6141 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6142 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6143 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6144 }
6145 }
6146
6147 if (TLI.isTypeLegal(VT: InWidenVT)) {
6148 // Because the result and the input are different vector types, widening
6149 // the result could create a legal type but widening the input might make
6150 // it an illegal type that might lead to repeatedly splitting the input
6151 // and then widening it. To avoid this, we widen the input only if
6152 // it results in a legal type.
6153 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6154 // Widen the input and call convert on the widened input vector.
6155 unsigned NumConcat =
6156 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6157 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6158 Ops[0] = InOp;
6159 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6160 return MakeConvertNode(WidenVT, InVec);
6161 }
6162
6163 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6164 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6165 // Extract the input and convert the shorten input vector.
6166 return MakeConvertNode(WidenVT, InVal);
6167 }
6168 }
6169
6170 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6171 EVT EltVT = WidenVT.getVectorElementType();
6172 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6173 // Use the original element count so we don't do more scalar opts than
6174 // necessary.
6175 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6176 for (unsigned i=0; i < MinElts; ++i) {
6177 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6178 Ops[i] = MakeConvertNode(EltVT, Val);
6179 }
6180
6181 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6182}
6183
6184SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6185 SDLoc dl(N);
6186 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6187 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6188
6189 SDValue Src = N->getOperand(Num: 0);
6190 EVT SrcVT = Src.getValueType();
6191
6192 // Also widen the input.
6193 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6194 Src = GetWidenedVector(Op: Src);
6195 SrcVT = Src.getValueType();
6196 }
6197
6198 // Input and output not widened to the same size, give up.
6199 if (WidenNumElts != SrcVT.getVectorElementCount())
6200 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6201
6202 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6203}
6204
6205SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6206 SDLoc dl(N);
6207 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6208 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6209
6210 SDValue Src = N->getOperand(Num: 0);
6211 EVT SrcVT = Src.getValueType();
6212
6213 // Also widen the input.
6214 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6215 Src = GetWidenedVector(Op: Src);
6216 SrcVT = Src.getValueType();
6217 }
6218
6219 // Input and output not widened to the same size, give up.
6220 if (WidenNumElts != SrcVT.getVectorElementCount())
6221 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6222
6223 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6224}
6225
6226SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6227 SDValue InOp = N->getOperand(Num: 1);
6228 SDLoc DL(N);
6229 SmallVector<SDValue, 4> NewOps(N->ops());
6230
6231 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6232 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6233
6234 EVT InVT = InOp.getValueType();
6235 EVT InEltVT = InVT.getVectorElementType();
6236
6237 unsigned Opcode = N->getOpcode();
6238
6239 // FIXME: Optimizations need to be implemented here.
6240
6241 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6242 EVT EltVT = WidenVT.getVectorElementType();
6243 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6244 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6245 SmallVector<SDValue, 32> OpChains;
6246 // Use the original element count so we don't do more scalar opts than
6247 // necessary.
6248 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6249 for (unsigned i=0; i < MinElts; ++i) {
6250 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6251 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6252 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6253 }
6254 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6255 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6256
6257 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6258}
6259
6260SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6261 unsigned Opcode = N->getOpcode();
6262 SDValue InOp = N->getOperand(Num: 0);
6263 SDLoc DL(N);
6264
6265 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6266 EVT WidenSVT = WidenVT.getVectorElementType();
6267 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6268
6269 EVT InVT = InOp.getValueType();
6270 EVT InSVT = InVT.getVectorElementType();
6271 unsigned InVTNumElts = InVT.getVectorNumElements();
6272
6273 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6274 InOp = GetWidenedVector(Op: InOp);
6275 InVT = InOp.getValueType();
6276 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6277 switch (Opcode) {
6278 case ISD::ANY_EXTEND_VECTOR_INREG:
6279 case ISD::SIGN_EXTEND_VECTOR_INREG:
6280 case ISD::ZERO_EXTEND_VECTOR_INREG:
6281 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6282 }
6283 }
6284 }
6285
6286 // Unroll, extend the scalars and rebuild the vector.
6287 SmallVector<SDValue, 16> Ops;
6288 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6289 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6290 switch (Opcode) {
6291 case ISD::ANY_EXTEND_VECTOR_INREG:
6292 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6293 break;
6294 case ISD::SIGN_EXTEND_VECTOR_INREG:
6295 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6296 break;
6297 case ISD::ZERO_EXTEND_VECTOR_INREG:
6298 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6299 break;
6300 default:
6301 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6302 }
6303 Ops.push_back(Elt: Val);
6304 }
6305
6306 while (Ops.size() != WidenNumElts)
6307 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6308
6309 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6310}
6311
6312SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6313 // If this is an FCOPYSIGN with same input types, we can treat it as a
6314 // normal (can trap) binary op.
6315 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6316 return WidenVecRes_BinaryCanTrap(N);
6317
6318 // If the types are different, fall back to unrolling.
6319 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6320 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6321}
6322
6323/// Result and first source operand are different scalar types, but must have
6324/// the same number of elements. There is an additional control argument which
6325/// should be passed through unchanged.
6326SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6327 SDValue FpValue = N->getOperand(Num: 0);
6328 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6329 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6330 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6331 SDValue Arg = GetWidenedVector(Op: FpValue);
6332 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6333 Flags: N->getFlags());
6334}
6335
6336SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6337 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6338 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6339 SDValue RHS = N->getOperand(Num: 1);
6340 EVT ExpVT = RHS.getValueType();
6341 SDValue ExpOp = RHS;
6342 if (ExpVT.isVector()) {
6343 EVT WideExpVT = WidenVT.changeVectorElementType(
6344 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6345 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6346 }
6347
6348 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6349}
6350
6351SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6352 // Unary op widening.
6353 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6354 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6355 if (N->getNumOperands() == 1)
6356 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6357 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6358 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1),
6359 Flags: N->getFlags());
6360}
6361
6362SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6363 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6364 EVT ExtVT = EVT::getVectorVT(
6365 Context&: *DAG.getContext(),
6366 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6367 EC: WidenVT.getVectorElementCount());
6368 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6369 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6370 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6371}
6372
6373SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6374 unsigned ResNo) {
6375 EVT VT0 = N->getValueType(ResNo: 0);
6376 EVT VT1 = N->getValueType(ResNo: 1);
6377
6378 assert(VT0.isVector() && VT1.isVector() &&
6379 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6380 "expected both results to be vectors of matching element count");
6381
6382 LLVMContext &Ctx = *DAG.getContext();
6383 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6384
6385 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6386 ElementCount WidenEC = WidenVT.getVectorElementCount();
6387
6388 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6389 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6390
6391 SDNode *WidenNode =
6392 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6393 .getNode();
6394
6395 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6396 return SDValue(WidenNode, ResNo);
6397}
6398
6399SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6400 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6401 return GetWidenedVector(Op: WidenVec);
6402}
6403
6404SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6405 SDLoc DL(N);
6406 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6407 ElementCount WidenEC = WidenVT.getVectorElementCount();
6408 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6409
6410 // The source has the same number of elements as the result, so widen it to
6411 // match WidenVT. It only lives in the widened-vector map if it is itself
6412 // widened; otherwise pad it up to the widened element count
6413 // when it is illegal.
6414 SDValue InOp = N->getOperand(Num: 0);
6415 EVT InVT = InOp.getValueType();
6416 TargetLowering::LegalizeTypeAction InAction = getTypeAction(VT: InVT);
6417 if (InAction == TargetLowering::TypeWidenVector) {
6418 InOp = GetWidenedVector(Op: InOp);
6419 } else if (InAction != TargetLowering::TypeLegal) {
6420 EVT InWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6421 VT: InVT.getVectorElementType(), EC: WidenEC);
6422 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InWidenVT), SubVec: InOp, Idx: 0);
6423 }
6424
6425 return DAG.getAddrSpaceCast(
6426 dl: DL, VT: WidenVT, Ptr: InOp, SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6427 DestAS: AddrSpaceCastN->getDestAddressSpace(), Flags: AddrSpaceCastN->getFlags());
6428}
6429
6430SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6431 SDValue InOp = N->getOperand(Num: 0);
6432 EVT InVT = InOp.getValueType();
6433 EVT VT = N->getValueType(ResNo: 0);
6434 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6435 SDLoc dl(N);
6436
6437 switch (getTypeAction(VT: InVT)) {
6438 case TargetLowering::TypeLegal:
6439 break;
6440 case TargetLowering::TypeScalarizeScalableVector:
6441 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
6442 case TargetLowering::TypePromoteInteger: {
6443 // If the incoming type is a vector that is being promoted, then
6444 // we know that the elements are arranged differently and that we
6445 // must perform the conversion using a stack slot.
6446 if (InVT.isVector())
6447 break;
6448
6449 // If the InOp is promoted to the same size, convert it. Otherwise,
6450 // fall out of the switch and widen the promoted input.
6451 SDValue NInOp = GetPromotedInteger(Op: InOp);
6452 EVT NInVT = NInOp.getValueType();
6453 if (WidenVT.bitsEq(VT: NInVT)) {
6454 // For big endian targets we need to shift the input integer or the
6455 // interesting bits will end up at the wrong place.
6456 if (DAG.getDataLayout().isBigEndian()) {
6457 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6458 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6459 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6460 }
6461 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6462 }
6463 InOp = NInOp;
6464 InVT = NInVT;
6465 break;
6466 }
6467 case TargetLowering::TypeSoftenFloat:
6468 case TargetLowering::TypeSoftPromoteHalf:
6469 case TargetLowering::TypeExpandInteger:
6470 case TargetLowering::TypeExpandFloat:
6471 case TargetLowering::TypeScalarizeVector:
6472 case TargetLowering::TypeSplitVector:
6473 break;
6474 case TargetLowering::TypeWidenVector:
6475 // If the InOp is widened to the same size, convert it. Otherwise, fall
6476 // out of the switch and widen the widened input.
6477 InOp = GetWidenedVector(Op: InOp);
6478 InVT = InOp.getValueType();
6479 if (WidenVT.bitsEq(VT: InVT))
6480 // The input widens to the same size. Convert to the widen value.
6481 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6482 break;
6483 }
6484
6485 unsigned WidenSize = WidenVT.getSizeInBits();
6486 unsigned InSize = InVT.getSizeInBits();
6487 unsigned InScalarSize = InVT.getScalarSizeInBits();
6488 // x86mmx is not an acceptable vector element type, so don't try.
6489 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6490 // Determine new input vector type. The new input vector type will use
6491 // the same element type (if its a vector) or use the input type as a
6492 // vector. It is the same size as the type to widen to.
6493 EVT NewInVT;
6494 unsigned NewNumParts = WidenSize / InSize;
6495 if (InVT.isVector()) {
6496 EVT InEltVT = InVT.getVectorElementType();
6497 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6498 NumElements: WidenSize / InEltVT.getSizeInBits());
6499 } else {
6500 // For big endian systems, using the promoted input scalar type
6501 // to produce the scalar_to_vector would put the desired bits into
6502 // the least significant byte(s) of the wider element zero. This
6503 // will mean that the users of the result vector are using incorrect
6504 // bits. Use the original input type instead. Although either input
6505 // type can be used on little endian systems, for consistency we
6506 // use the original type there as well.
6507 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6508 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6509 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6510 }
6511
6512 if (TLI.isTypeLegal(VT: NewInVT)) {
6513 SDValue NewVec;
6514 if (InVT.isVector()) {
6515 // Because the result and the input are different vector types, widening
6516 // the result could create a legal type but widening the input might
6517 // make it an illegal type that might lead to repeatedly splitting the
6518 // input and then widening it. To avoid this, we widen the input only if
6519 // it results in a legal type.
6520 if (WidenSize % InSize == 0) {
6521 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6522 Ops[0] = InOp;
6523
6524 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6525 } else {
6526 SmallVector<SDValue, 16> Ops;
6527 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6528 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6529 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6530
6531 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6532 }
6533 } else {
6534 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6535 }
6536 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6537 }
6538 }
6539
6540 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6541}
6542
6543SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6544 return DAG.getNode(
6545 Opcode: N->getOpcode(), DL: SDLoc(N),
6546 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6547 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6548}
6549
6550SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6551 SDLoc dl(N);
6552 // Build a vector with poison for the new nodes.
6553 EVT VT = N->getValueType(ResNo: 0);
6554
6555 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6556 // type. The POISONs need to have the same type as the existing operands.
6557 EVT EltVT = N->getOperand(Num: 0).getValueType();
6558 unsigned NumElts = VT.getVectorNumElements();
6559
6560 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6561 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6562
6563 SmallVector<SDValue, 16> NewOps(N->ops());
6564 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6565 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6566
6567 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6568}
6569
6570SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6571 EVT InVT = N->getOperand(Num: 0).getValueType();
6572 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6573 SDLoc dl(N);
6574 unsigned NumOperands = N->getNumOperands();
6575
6576 bool InputWidened = false; // Indicates we need to widen the input.
6577 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6578 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6579 unsigned NumInElts = InVT.getVectorMinNumElements();
6580 if (WidenNumElts % NumInElts == 0) {
6581 // Add undef vectors to widen to correct length.
6582 unsigned NumConcat = WidenNumElts / NumInElts;
6583 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6584 SmallVector<SDValue, 16> Ops(NumConcat);
6585 for (unsigned i=0; i < NumOperands; ++i)
6586 Ops[i] = N->getOperand(Num: i);
6587 for (unsigned i = NumOperands; i != NumConcat; ++i)
6588 Ops[i] = UndefVal;
6589 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6590 }
6591 } else {
6592 InputWidened = true;
6593 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6594 // The inputs and the result are widen to the same value.
6595 unsigned i;
6596 for (i=1; i < NumOperands; ++i)
6597 if (!N->getOperand(Num: i).isUndef())
6598 break;
6599
6600 if (i == NumOperands)
6601 // Everything but the first operand is an UNDEF so just return the
6602 // widened first operand.
6603 return GetWidenedVector(Op: N->getOperand(Num: 0));
6604
6605 if (NumOperands == 2) {
6606 assert(!WidenVT.isScalableVector() &&
6607 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6608 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6609 unsigned NumInElts = InVT.getVectorNumElements();
6610
6611 // Replace concat of two operands with a shuffle.
6612 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6613 for (unsigned i = 0; i < NumInElts; ++i) {
6614 MaskOps[i] = i;
6615 MaskOps[i + NumInElts] = i + WidenNumElts;
6616 }
6617 return DAG.getVectorShuffle(VT: WidenVT, dl,
6618 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6619 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6620 Mask: MaskOps);
6621 }
6622 }
6623 }
6624
6625 if (WidenVT.isScalableVector()) {
6626 SDValue WideVec = DAG.getPOISON(VT: WidenVT);
6627 unsigned NumInElts = InVT.getVectorMinNumElements();
6628 for (unsigned I = 0; I < NumOperands; ++I)
6629 WideVec =
6630 DAG.getInsertSubvector(DL: dl, Vec: WideVec, SubVec: N->getOperand(Num: I), Idx: I * NumInElts);
6631 return WideVec;
6632 }
6633
6634 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6635 unsigned NumInElts = InVT.getVectorNumElements();
6636
6637 // Fall back to use extracts and build vector.
6638 EVT EltVT = WidenVT.getVectorElementType();
6639 SmallVector<SDValue, 16> Ops(WidenNumElts);
6640 unsigned Idx = 0;
6641 for (unsigned i=0; i < NumOperands; ++i) {
6642 SDValue InOp = N->getOperand(Num: i);
6643 if (InputWidened)
6644 InOp = GetWidenedVector(Op: InOp);
6645 for (unsigned j = 0; j < NumInElts; ++j)
6646 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6647 }
6648 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6649 for (; Idx < WidenNumElts; ++Idx)
6650 Ops[Idx] = UndefVal;
6651 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6652}
6653
6654SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6655 EVT VT = N->getValueType(ResNo: 0);
6656 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6657 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6658 SDValue InOp2 = N->getOperand(Num: 1);
6659 SDValue Idx = N->getOperand(Num: 2);
6660 SDLoc dl(N);
6661 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6662}
6663
6664SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6665 EVT VT = N->getValueType(ResNo: 0);
6666 EVT EltVT = VT.getVectorElementType();
6667 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6668 SDValue InOp = N->getOperand(Num: 0);
6669 SDValue Idx = N->getOperand(Num: 1);
6670 SDLoc dl(N);
6671
6672 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6673 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6674 InOp = GetWidenedVector(Op: InOp);
6675
6676 EVT InVT = InOp.getValueType();
6677
6678 // Check if we can just return the input vector after widening.
6679 uint64_t IdxVal = Idx->getAsZExtVal();
6680 if (IdxVal == 0 && InVT == WidenVT)
6681 return InOp;
6682
6683 // Check if we can extract from the vector.
6684 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6685 unsigned InNumElts = InVT.getVectorMinNumElements();
6686 unsigned VTNumElts = VT.getVectorMinNumElements();
6687 assert(IdxVal % VTNumElts == 0 &&
6688 "Expected Idx to be a multiple of subvector minimum vector length");
6689 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6690 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6691
6692 if (VT.isScalableVector()) {
6693 // Try to split the operation up into smaller extracts and concat the
6694 // results together, e.g.
6695 // nxv6i64 extract_subvector(nxv12i64, 6)
6696 // <->
6697 // nxv8i64 concat(
6698 // nxv2i64 extract_subvector(nxv16i64, 6)
6699 // nxv2i64 extract_subvector(nxv16i64, 8)
6700 // nxv2i64 extract_subvector(nxv16i64, 10)
6701 // undef)
6702 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6703 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6704 "down type's element count");
6705 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6706 EC: ElementCount::getScalable(MinVal: GCD));
6707 // Avoid recursion around e.g. nxv1i8.
6708 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6709 SmallVector<SDValue> Parts;
6710 unsigned I = 0;
6711 for (; I < VTNumElts / GCD; ++I)
6712 Parts.push_back(
6713 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6714 for (; I < WidenNumElts / GCD; ++I)
6715 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6716
6717 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6718 }
6719
6720 // Fallback to extracting through memory.
6721
6722 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6723 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6724 MachineFunction &MF = DAG.getMachineFunction();
6725 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6726 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6727
6728 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6729 PtrInfo, F: MachineMemOperand::MOStore,
6730 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6731 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6732 PtrInfo, F: MachineMemOperand::MOLoad,
6733 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6734
6735 // Write out the input vector.
6736 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6737
6738 // Build a mask to match the length of the non-widened result.
6739 SDValue Mask =
6740 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6741
6742 // Read back the sub-vector setting the remaining lanes to poison.
6743 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6744 return DAG.getMaskedLoad(
6745 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6746 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6747 }
6748
6749 // We could try widening the input to the right length but for now, extract
6750 // the original elements, fill the rest with undefs and build a vector.
6751 SmallVector<SDValue, 16> Ops(WidenNumElts);
6752 unsigned i;
6753 for (i = 0; i < VTNumElts; ++i)
6754 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6755
6756 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6757 for (; i < WidenNumElts; ++i)
6758 Ops[i] = UndefVal;
6759 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6760}
6761
6762SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6763 SDValue InOp = ModifyToType(
6764 InOp: N->getOperand(Num: 0),
6765 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6766 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6767 N2: N->getOperand(Num: 1));
6768}
6769
6770SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6771 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6772 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6773 VT: InOp.getValueType(), N1: InOp,
6774 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6775}
6776
6777/// Either return the same load or provide appropriate casts
6778/// from the load and return that.
6779static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6780 TypeSize LdWidth, TypeSize FirstVTWidth,
6781 SDLoc dl, SelectionDAG &DAG) {
6782 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6783 "Load width must be less than or equal to first value type width");
6784 TypeSize WidenWidth = WidenVT.getSizeInBits();
6785 if (!FirstVT.isVector()) {
6786 unsigned NumElts =
6787 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6788 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6789 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6790 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6791 }
6792 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6793 return LdOp;
6794}
6795
6796/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6797/// widened value so it can be issued in a single atomic store.
6798static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6799 TypeSize FirstVTWidth, const SDLoc &dl,
6800 SelectionDAG &DAG) {
6801 TypeSize WidenWidth = WidenVT.getSizeInBits();
6802 if (!FirstVT.isVector()) {
6803 unsigned NumElts =
6804 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6805 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6806 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6807 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6808 }
6809 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6810 return StVal;
6811}
6812
6813static std::optional<EVT> findMemType(SelectionDAG &DAG,
6814 const TargetLowering &TLI, unsigned Width,
6815 EVT WidenVT, unsigned Align,
6816 unsigned WidenEx);
6817
6818SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6819 EVT WidenVT =
6820 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6821 EVT LdVT = LD->getMemoryVT();
6822 SDLoc dl(LD);
6823
6824 // Load information
6825 SDValue Chain = LD->getChain();
6826 SDValue BasePtr = LD->getBasePtr();
6827
6828 TypeSize LdWidth = LdVT.getSizeInBits();
6829 TypeSize WidenWidth = WidenVT.getSizeInBits();
6830 TypeSize WidthDiff = WidenWidth - LdWidth;
6831
6832 // Find the vector type that can load from.
6833 std::optional<EVT> FirstVT =
6834 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6835 WidenEx: WidthDiff.getKnownMinValue());
6836
6837 if (!FirstVT)
6838 return SDValue();
6839
6840 SmallVector<EVT, 8> MemVTs;
6841 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6842
6843 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6844 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6845
6846 // Load the element with one instruction.
6847 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6848 FirstVTWidth, dl, DAG);
6849
6850 // Modified the chain - switch anything that used the old chain to use
6851 // the new one.
6852 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6853 return Result;
6854}
6855
6856SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6857 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6858 ISD::LoadExtType ExtType = LD->getExtensionType();
6859
6860 // A vector must always be stored in memory as-is, i.e. without any padding
6861 // between the elements, since various code depend on it, e.g. in the
6862 // handling of a bitcast of a vector type to int, which may be done with a
6863 // vector store followed by an integer load. A vector that does not have
6864 // elements that are byte-sized must therefore be stored as an integer
6865 // built out of the extracted vector elements.
6866 if (!LD->getMemoryVT().isByteSized()) {
6867 SDValue Value, NewChain;
6868 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6869 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6870 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6871 return SDValue();
6872 }
6873
6874 // Generate a vector-predicated load if it is custom/legal on the target. To
6875 // avoid possible recursion, only do this if the widened mask type is legal.
6876 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6877 // removed from the IR by the ExpandVectorPredication pass but we're
6878 // reintroducing them here.
6879 EVT VT = LD->getValueType(ResNo: 0);
6880 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6881 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6882
6883 if (ExtType == ISD::NON_EXTLOAD &&
6884 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6885 TLI.isTypeLegal(VT: WideMaskVT)) {
6886 SDLoc DL(N);
6887 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6888 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6889 EC: VT.getVectorElementCount());
6890 SDValue NewLoad =
6891 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6892 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6893 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6894
6895 // Modified the chain - switch anything that used the old chain to use
6896 // the new one.
6897 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6898
6899 return NewLoad;
6900 }
6901
6902 SDValue Result;
6903 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6904 if (ExtType != ISD::NON_EXTLOAD)
6905 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6906 else
6907 Result = GenWidenVectorLoads(LdChain, LD);
6908
6909 if (Result) {
6910 // If we generate a single load, we can use that for the chain. Otherwise,
6911 // build a factor node to remember the multiple loads are independent and
6912 // chain to that.
6913 SDValue NewChain;
6914 if (LdChain.size() == 1)
6915 NewChain = LdChain[0];
6916 else
6917 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6918
6919 // Modified the chain - switch anything that used the old chain to use
6920 // the new one.
6921 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6922
6923 return Result;
6924 }
6925
6926 if (VT.isVector()) {
6927 // If all else fails replace the load with a wide masked load.
6928 SDLoc DL(N);
6929 SDValue Mask =
6930 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6931
6932 SDValue NewLoad = DAG.getMaskedLoad(
6933 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6934 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6935 AM: LD->getAddressingMode(), LD->getExtensionType());
6936
6937 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6938 return NewLoad;
6939 }
6940
6941 report_fatal_error(reason: "Unable to widen vector load");
6942}
6943
6944SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6945 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6946 SDValue Mask = N->getMask();
6947 SDValue EVL = N->getVectorLength();
6948 ISD::LoadExtType ExtType = N->getExtensionType();
6949 SDLoc dl(N);
6950
6951 // The mask should be widened as well
6952 assert(getTypeAction(Mask.getValueType()) ==
6953 TargetLowering::TypeWidenVector &&
6954 "Unable to widen binary VP op");
6955 Mask = GetWidenedVector(Op: Mask);
6956 assert(Mask.getValueType().getVectorElementCount() ==
6957 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6958 .getVectorElementCount() &&
6959 "Unable to widen vector load");
6960
6961 SDValue Res =
6962 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6963 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6964 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6965 // Legalize the chain result - switch anything that used the old chain to
6966 // use the new one.
6967 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6968 return Res;
6969}
6970
6971SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6972 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6973 SDValue Mask = N->getMask();
6974 SDValue EVL = N->getVectorLength();
6975 SDLoc dl(N);
6976
6977 // The mask should be widened as well
6978 assert(getTypeAction(Mask.getValueType()) ==
6979 TargetLowering::TypeWidenVector &&
6980 "Unable to widen binary VP op");
6981 Mask = GetWidenedVector(Op: Mask);
6982 assert(Mask.getValueType().getVectorElementCount() ==
6983 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6984 .getVectorElementCount() &&
6985 "Unable to widen vector load");
6986
6987 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6988 Mask, EVL, MMO: N->getMemOperand());
6989 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6990 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6991 return Res;
6992}
6993
6994SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6995 SDLoc DL(N);
6996
6997 // The mask should be widened as well
6998 SDValue Mask = N->getMask();
6999 assert(getTypeAction(Mask.getValueType()) ==
7000 TargetLowering::TypeWidenVector &&
7001 "Unable to widen VP strided load");
7002 Mask = GetWidenedVector(Op: Mask);
7003
7004 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7005 assert(Mask.getValueType().getVectorElementCount() ==
7006 WidenVT.getVectorElementCount() &&
7007 "Data and mask vectors should have the same number of elements");
7008
7009 SDValue Res = DAG.getStridedLoadVP(
7010 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
7011 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
7012 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
7013 IsExpanding: N->isExpandingLoad());
7014
7015 // Legalize the chain result - switch anything that used the old chain to
7016 // use the new one.
7017 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7018 return Res;
7019}
7020
7021SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
7022 SDValue Vec = N->getOperand(Num: 0);
7023 SDValue Mask = N->getOperand(Num: 1);
7024 SDValue Passthru = N->getOperand(Num: 2);
7025 EVT WideVecVT =
7026 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
7027 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7028 VT: Mask.getValueType().getVectorElementType(),
7029 EC: WideVecVT.getVectorElementCount());
7030
7031 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
7032 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
7033 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
7034 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
7035 N2: WideMask, N3: WidePassthru);
7036}
7037
7038SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
7039 EVT VT = N->getValueType(ResNo: 0);
7040 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7041 SDValue Mask = N->getMask();
7042 EVT MaskVT = Mask.getValueType();
7043 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7044 ISD::LoadExtType ExtType = N->getExtensionType();
7045 SDLoc dl(N);
7046
7047 EVT WideMaskVT =
7048 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7049 EC: WidenVT.getVectorElementCount());
7050
7051 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
7052 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
7053 TLI.isTypeLegal(VT: WideMaskVT) &&
7054 // If there is a passthru, we shouldn't use vp.load. However,
7055 // type legalizer will struggle on masked.load with
7056 // scalable vectors, so for scalable vectors, we still use vp.load
7057 // but manually merge the load result with the passthru using vp.select.
7058 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7059 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7060 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7061 EC: VT.getVectorElementCount());
7062 SDValue NewLoad =
7063 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
7064 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7065 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
7066 SDValue NewVal = NewLoad;
7067
7068 // Manually merge with vselect
7069 if (!N->getPassThru()->isUndef()) {
7070 assert(WidenVT.isScalableVector());
7071 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
7072 // The lanes past EVL are poison.
7073 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
7074 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
7075 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
7076 }
7077
7078 // Modified the chain - switch anything that used the old chain to use
7079 // the new one.
7080 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7081
7082 return NewVal;
7083 }
7084
7085 // The mask should be widened as well
7086 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7087
7088 SDValue Res = DAG.getMaskedLoad(
7089 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7090 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7091 ExtType, IsExpanding: N->isExpandingLoad());
7092 // Legalize the chain result - switch anything that used the old chain to
7093 // use the new one.
7094 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7095 return Res;
7096}
7097
7098SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7099
7100 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7101 SDValue Mask = N->getMask();
7102 EVT MaskVT = Mask.getValueType();
7103 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7104 SDValue Scale = N->getScale();
7105 ElementCount WideEC = WideVT.getVectorElementCount();
7106 SDLoc dl(N);
7107
7108 // The mask should be widened as well
7109 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7110 VT: MaskVT.getVectorElementType(), EC: WideEC);
7111 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7112
7113 // Widen the Index operand
7114 SDValue Index = N->getIndex();
7115 EVT WideIndexVT = EVT::getVectorVT(
7116 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7117 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7118 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7119 Scale };
7120
7121 // Widen the MemoryType
7122 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7123 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7124 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7125 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7126 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7127
7128 // Legalize the chain result - switch anything that used the old chain to
7129 // use the new one.
7130 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7131 return Res;
7132}
7133
7134SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7135 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7136 SDValue Mask = N->getMask();
7137 SDValue Scale = N->getScale();
7138 ElementCount WideEC = WideVT.getVectorElementCount();
7139 SDLoc dl(N);
7140
7141 SDValue Index = GetWidenedVector(Op: N->getIndex());
7142 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7143 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7144 Mask = GetWidenedMask(Mask, EC: WideEC);
7145
7146 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7147 Mask, N->getVectorLength()};
7148 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7149 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7150
7151 // Legalize the chain result - switch anything that used the old chain to
7152 // use the new one.
7153 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7154 return Res;
7155}
7156
7157SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7158 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7159 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7160}
7161
7162// Return true is this is a SETCC node or a strict version of it.
7163static inline bool isSETCCOp(unsigned Opcode) {
7164 switch (Opcode) {
7165 case ISD::SETCC:
7166 case ISD::STRICT_FSETCC:
7167 case ISD::STRICT_FSETCCS:
7168 return true;
7169 }
7170 return false;
7171}
7172
7173// Return true if this is a node that could have two SETCCs as operands.
7174static inline bool isLogicalMaskOp(unsigned Opcode) {
7175 switch (Opcode) {
7176 case ISD::AND:
7177 case ISD::OR:
7178 case ISD::XOR:
7179 return true;
7180 }
7181 return false;
7182}
7183
7184// If N is a SETCC or a strict variant of it, return the type
7185// of the compare operands.
7186static inline EVT getSETCCOperandType(SDValue N) {
7187 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7188 return N->getOperand(Num: OpNo).getValueType();
7189}
7190
7191// This is used just for the assert in convertMask(). Check that this either
7192// a SETCC or a previously handled SETCC by convertMask().
7193#ifndef NDEBUG
7194static inline bool isSETCCorConvertedSETCC(SDValue N) {
7195 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7196 N = N.getOperand(0);
7197 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7198 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7199 if (!N->getOperand(i)->isUndef())
7200 return false;
7201 N = N.getOperand(0);
7202 }
7203
7204 if (N.getOpcode() == ISD::TRUNCATE)
7205 N = N.getOperand(0);
7206 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7207 N = N.getOperand(0);
7208
7209 if (isLogicalMaskOp(N.getOpcode()))
7210 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7211 isSETCCorConvertedSETCC(N.getOperand(1));
7212
7213 return (isSETCCOp(N.getOpcode()) ||
7214 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7215}
7216#endif
7217
7218// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7219// to ToMaskVT if needed with vector extension or truncation.
7220SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7221 EVT ToMaskVT) {
7222 // Called from convertMaskTree for SETCC leaf nodes. Re-creates the SETCC with
7223 // result type MaskVT, then sign-extends/truncates and pads to ToMaskVT.
7224 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7225
7226 // Make a new Mask node, with a legal result VT.
7227 SDValue Mask;
7228 SmallVector<SDValue, 4> Ops;
7229 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7230 Ops.push_back(Elt: InMask->getOperand(Num: i));
7231 if (InMask->isStrictFPOpcode()) {
7232 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7233 ResultTys: { MaskVT, MVT::Other }, Ops);
7234 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7235 }
7236 else
7237 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7238 Flags: InMask->getFlags());
7239
7240 return adjustMaskToType(Mask, ToMaskVT);
7241}
7242
7243// Adjust element width (sign-extend/truncate) and element count
7244// (extract/concat) of Mask to match ToMaskVT.
7245SDValue DAGTypeLegalizer::adjustMaskToType(SDValue Mask, EVT ToMaskVT) {
7246 LLVMContext &Ctx = *DAG.getContext();
7247 EVT MaskVT = Mask.getValueType();
7248 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7249 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7250 if (MaskScalarBits < ToMaskScalBits) {
7251 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7252 NumElements: MaskVT.getVectorNumElements());
7253 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7254 } else if (MaskScalarBits > ToMaskScalBits) {
7255 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7256 NumElements: MaskVT.getVectorNumElements());
7257 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7258 }
7259
7260 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7261 ToMaskVT.getScalarSizeInBits() &&
7262 "Mask should have the right element size by now.");
7263
7264 // Adjust Mask to the right number of elements.
7265 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7266 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7267 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7268 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7269 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7270 EVT SubVT = Mask->getValueType(ResNo: 0);
7271 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7272 SubOps[0] = Mask;
7273 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7274 }
7275
7276 assert((Mask->getValueType(0) == ToMaskVT) &&
7277 "A mask of ToMaskVT should have been produced by now.");
7278
7279 return Mask;
7280}
7281
7282// Adjust both operands to a common intermediate mask type, picking a scalar
7283// width that minimizes extend/truncate overhead given the final target ToVT.
7284EVT DAGTypeLegalizer::unifyMaskTypes(SDValue &Op0, bool IsOpLenient0,
7285 SDValue &Op1, bool IsOpLenient1,
7286 EVT ToVT) {
7287 assert(Op0.getValueType().getVectorNumElements() ==
7288 Op1.getValueType().getVectorNumElements() &&
7289 "unifyMaskTypes only handles scalar width differences");
7290
7291 // If only one of the operands lenient type-wise, we can simply
7292 // adjust its type to the other operand's type assuming that this
7293 // adjustment can be folded away.
7294 //
7295 // NOTE: We essentially rely on the fact that further optimizations
7296 // do spot redundant casts in "lenient" cases. If at some
7297 // point we decide that we want to do better here, we can
7298 // postpone converting lenient sub-trees right away and postpone
7299 // it to the moment when we know the best fitting integer type
7300 // to materialize them, and do it there.
7301 if (IsOpLenient0 != IsOpLenient1) {
7302 SDValue *LenientOp, *NonLenientOp;
7303 if (IsOpLenient0) {
7304 LenientOp = &Op0;
7305 NonLenientOp = &Op1;
7306 } else {
7307 LenientOp = &Op1;
7308 NonLenientOp = &Op0;
7309 }
7310 EVT OpVT = NonLenientOp->getValueType();
7311 *LenientOp = adjustMaskToType(Mask: *LenientOp, ToMaskVT: OpVT);
7312 return OpVT;
7313 }
7314
7315 unsigned Bits0 = Op0.getScalarValueSizeInBits();
7316 unsigned Bits1 = Op1.getScalarValueSizeInBits();
7317 unsigned NarrowBits = std::min(a: Bits0, b: Bits1);
7318 unsigned WideBits = std::max(a: Bits0, b: Bits1);
7319 unsigned ToBits = ToVT.getScalarSizeInBits();
7320 unsigned IntBits = NarrowBits == WideBits ? NarrowBits
7321 : ToBits >= WideBits ? WideBits
7322 : ToBits <= NarrowBits ? NarrowBits
7323 : ToBits;
7324 EVT OpVT = Op0.getValueType().changeVectorElementType(
7325 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntBits));
7326 Op0 = adjustMaskToType(Mask: Op0, ToMaskVT: OpVT);
7327 Op1 = adjustMaskToType(Mask: Op1, ToMaskVT: OpVT);
7328 return OpVT;
7329}
7330
7331std::pair<SDValue, bool>
7332DAGTypeLegalizer::convertMaskTreeImpl(SDValue V, EVT ToVT, unsigned Depth) {
7333 // The main idea is to recursively traverse VSELECT's mask that needs
7334 // widening to see if we can avoid unnecessary casts. The problem usually
7335 // stems from a simple fact that SETCC might naturally produce results
7336 // not in i1 (as we model it in LLVM IR) and we can continue using that
7337 // type until we have to switch it up. Another important aspect is that
7338 // "all ones" and "all zeros" constants can be materialized at any type,
7339 // so we can try to utilize that to keep SETCC results at their natural
7340 // types as much as possible.
7341 //
7342 // The algorithm traverses the mask-producing tree of operations that
7343 // retain "mask-vector"-ness of the input (i.e. it remains a vector of
7344 // -1s and 0s).
7345 //
7346 // SETCC1 SETCC2 CONST1 SETCC3 CONST2 CONST3
7347 // | / | / | /
7348 // | / | / | /
7349 // |_____/ |______/ |______/
7350 // | * choose the | * choose SETCC3 | * keep it as final type
7351 // | most fitting | type | but consider it subject to
7352 // | type | / change
7353 // | | /
7354 // | |______________/
7355 // | | * choose SETCC3 type
7356 // | /
7357 // | /
7358 // |_____________/
7359 // | * choose the most fitting type
7360 // | and then cast to the final desired type
7361 // |
7362 // VSELECT
7363 //
7364 if (Depth >= DAG.MaxRecursionDepth)
7365 return {};
7366
7367 // Bail out when encounter the vector element count mismatch.
7368 // It potentially can be just an assertion, but we deliberately try to
7369 // be overly conservative here.
7370 if (V.getValueType().getVectorNumElements() != ToVT.getVectorNumElements())
7371 return {};
7372
7373 unsigned Opcode = V.getOpcode();
7374
7375 // Base case: SETCC produces the mask at its natural type.
7376 if (isSETCCOp(Opcode)) {
7377 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: V));
7378 return {convertMask(InMask: V, MaskVT, ToMaskVT: MaskVT), /*IsTypeLenient=*/false};
7379 }
7380
7381 SDLoc DL(V);
7382
7383 // Base case: all-zeros or all-ones BUILD_VECTOR. Type-lenient since these are
7384 // invariant under sign-extend/truncate.
7385 if (ISD::isBuildVectorAllZeros(N: V.getNode()))
7386 return {DAG.getConstant(Val: 0, DL, VT: ToVT), /*IsTypeLenient=*/true};
7387 if (ISD::isBuildVectorAllOnes(N: V.getNode()))
7388 return {DAG.getAllOnesConstant(DL, VT: ToVT), /*IsTypeLenient=*/true};
7389
7390 // Logical operations (AND/OR/XOR): try picking the best fitting width out
7391 // of children's element widths.
7392 if (isLogicalMaskOp(Opcode)) {
7393 auto [Op0, IsLenientOp0] =
7394 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7395 if (!Op0)
7396 return {};
7397 auto [Op1, IsLenientOp1] =
7398 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7399 if (!Op1)
7400 return {};
7401 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7402 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Op0, N2: Op1),
7403 IsLenientOp0 && IsLenientOp1};
7404 }
7405
7406 // FREEZE: widen the operand and re-wrap.
7407 if (Opcode == ISD::FREEZE) {
7408 auto [Inner, IsTypeLenient] =
7409 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7410 if (!Inner)
7411 return {};
7412 return {DAG.getNode(Opcode: ISD::FREEZE, DL, VT: Inner.getValueType(), Operand: Inner),
7413 IsTypeLenient};
7414 }
7415
7416 // Vector shuffle: try inferring the best fitting width from operands.
7417 if (Opcode == ISD::VECTOR_SHUFFLE) {
7418 auto *Shuf = cast<ShuffleVectorSDNode>(Val&: V);
7419 auto [Op0, IsLenientOp0] =
7420 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7421 if (!Op0)
7422 return {};
7423 if (V.getOperand(i: 1).isUndef()) {
7424 EVT OpVT = Op0.getValueType();
7425 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: DAG.getUNDEF(VT: OpVT),
7426 Mask: Shuf->getMask()),
7427 IsLenientOp0};
7428 }
7429 auto [Op1, IsLenientOp1] =
7430 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7431 if (!Op1)
7432 return {};
7433 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7434 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: Op1, Mask: Shuf->getMask()),
7435 IsLenientOp0 && IsLenientOp1};
7436 }
7437
7438 // SELECT/VSELECT: try inferring the best fitting width from operands.
7439 if (Opcode == ISD::SELECT || Opcode == ISD::VSELECT) {
7440 auto [Op1, IsLenientOp1] =
7441 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7442 if (!Op1)
7443 return {};
7444 auto [Op2, IsLenientOp2] =
7445 convertMaskTreeImpl(V: V.getOperand(i: 2), ToVT, Depth: Depth + 1);
7446 if (!Op2)
7447 return {};
7448 EVT OpVT = unifyMaskTypes(Op0&: Op1, IsOpLenient0: IsLenientOp1, Op1&: Op2, IsOpLenient1: IsLenientOp2, ToVT);
7449
7450 // We deliberately skip traversing/modifying VSELECT's mask because
7451 //
7452 // a. We only change bitwidth of the operands and it shouldn't affect
7453 // condition on its own.
7454 //
7455 // b. This VSELECT's mask can be widened in an independent traversal
7456 // if needed.
7457 SDValue Cond = V.getOperand(i: 0);
7458 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Cond, N2: Op1, N3: Op2),
7459 IsLenientOp1 && IsLenientOp2};
7460 }
7461
7462 return {};
7463}
7464
7465SDValue DAGTypeLegalizer::convertMaskTree(SDValue V, EVT ToVT) {
7466 // In general, we are converting from <N x i1> into <M x iW>.
7467 // This would mean that during the tree traversal we need to pay
7468 // attention to both bitwidth and element count, which can be error-prone.
7469 //
7470 // Instead, we split the task in two, we first widen the type of the tree
7471 // and then change the element count.
7472 EVT MaskTreeVT = ToVT.changeVectorElementCount(
7473 Context&: *DAG.getContext(), EC: V.getValueType().getVectorElementCount());
7474 auto [Result, _] = convertMaskTreeImpl(V, ToVT: MaskTreeVT);
7475 if (!Result)
7476 return Result;
7477 return adjustMaskToType(Mask: Result, ToMaskVT: ToVT);
7478}
7479
7480// This method tries to handle some special cases for the vselect mask
7481// and if needed adjusting the mask vector type to match that of the VSELECT.
7482// Without it, many cases end up with scalarization of the SETCC, with many
7483// unnecessary instructions.
7484SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7485 LLVMContext &Ctx = *DAG.getContext();
7486 SDValue Cond = N->getOperand(Num: 0);
7487
7488 if (N->getOpcode() != ISD::VSELECT)
7489 return SDValue();
7490
7491 // If this is a splitted VSELECT that was previously already handled, do
7492 // nothing.
7493 EVT CondVT = Cond->getValueType(ResNo: 0);
7494 if (CondVT.getScalarSizeInBits() != 1)
7495 return SDValue();
7496
7497 EVT VSelVT = N->getValueType(ResNo: 0);
7498
7499 // This method can't handle scalable vector types.
7500 // FIXME: This support could be added in the future.
7501 if (VSelVT.isScalableVector())
7502 return SDValue();
7503
7504 // Only handle vector types which are a power of 2.
7505 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7506 return SDValue();
7507
7508 // Don't touch if this will be scalarized.
7509 EVT FinalVT = VSelVT;
7510 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7511 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7512
7513 if (FinalVT.getVectorNumElements() == 1)
7514 return SDValue();
7515
7516 // If there is support for an i1 vector mask, don't touch.
7517 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7518 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7519 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7520 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7521 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7522 if (SetCCResVT.getScalarSizeInBits() == 1)
7523 return SDValue();
7524 } else if (CondVT.getScalarType() == MVT::i1) {
7525 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7526 // don't touch.
7527 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7528 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7529
7530 if (CondVT.getScalarType() == MVT::i1)
7531 return SDValue();
7532 }
7533
7534 // Widen the vselect result type if needed.
7535 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7536 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7537
7538 // The mask of the VSELECT should have integer elements.
7539 EVT ToMaskVT = VSelVT;
7540 if (!ToMaskVT.getScalarType().isInteger())
7541 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7542
7543 // Try to recursively widen the mask expression tree to the target type.
7544 return convertMaskTree(V: Cond, ToVT: ToMaskVT);
7545}
7546
7547SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7548 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7549 ElementCount WidenEC = WidenVT.getVectorElementCount();
7550
7551 SDValue Cond1 = N->getOperand(Num: 0);
7552 EVT CondVT = Cond1.getValueType();
7553 unsigned Opcode = N->getOpcode();
7554 if (CondVT.isVector()) {
7555 if (SDValue WideCond = WidenVSELECTMask(N)) {
7556 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7557 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7558 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7559 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7560 }
7561
7562 EVT CondEltVT = CondVT.getVectorElementType();
7563 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7564 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7565 Cond1 = GetWidenedVector(Op: Cond1);
7566
7567 // If we have to split the condition there is no point in widening the
7568 // select. This would result in an cycle of widening the select ->
7569 // widening the condition operand -> splitting the condition operand ->
7570 // splitting the select -> widening the select. Instead split this select
7571 // further and widen the resulting type.
7572 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7573 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7574 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7575 return Res;
7576 }
7577
7578 if (Cond1.getValueType() != CondWidenVT)
7579 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7580 }
7581
7582 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7583 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7584 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7585 if (Opcode == ISD::VP_MERGE)
7586 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7587 N4: N->getOperand(Num: 3));
7588 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7589}
7590
7591SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7592 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7593 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7594 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7595 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7596 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7597}
7598
7599SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7600 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7601 return DAG.getUNDEF(VT: WidenVT);
7602}
7603
7604SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7605 EVT VT = N->getValueType(ResNo: 0);
7606 SDLoc dl(N);
7607
7608 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7609 unsigned NumElts = VT.getVectorNumElements();
7610 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7611
7612 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7613 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7614
7615 // Adjust mask based on new input vector length.
7616 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7617 for (unsigned i = 0; i != NumElts; ++i) {
7618 int Idx = N->getMaskElt(Idx: i);
7619 if (Idx < (int)NumElts)
7620 NewMask[i] = Idx;
7621 else
7622 NewMask[i] = Idx - NumElts + WidenNumElts;
7623 }
7624 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7625}
7626
7627SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7628 EVT VT = N->getValueType(ResNo: 0);
7629 EVT EltVT = VT.getVectorElementType();
7630 SDLoc dl(N);
7631
7632 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7633 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7634 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7635
7636 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7637 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7638 unsigned VTNumElts = VT.getVectorMinNumElements();
7639 unsigned IdxVal = WidenNumElts - VTNumElts;
7640
7641 if (VT.isScalableVector()) {
7642 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7643 // results together, e.g.(nxv6i64 -> nxv8i64)
7644 // nxv8i64 vector_reverse
7645 // <->
7646 // nxv8i64 concat(
7647 // nxv2i64 extract_subvector(nxv8i64, 2)
7648 // nxv2i64 extract_subvector(nxv8i64, 4)
7649 // nxv2i64 extract_subvector(nxv8i64, 6)
7650 // nxv2i64 undef)
7651
7652 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7653 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7654 EC: ElementCount::getScalable(MinVal: GCD));
7655 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7656 "down type's element count");
7657 SmallVector<SDValue> Parts;
7658 unsigned i = 0;
7659 for (; i < VTNumElts / GCD; ++i)
7660 Parts.push_back(
7661 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7662 for (; i < WidenNumElts / GCD; ++i)
7663 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7664
7665 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7666 }
7667
7668 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7669 // fixed-vectors.
7670 SmallVector<int, 16> Mask(WidenNumElts, -1);
7671 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7672
7673 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7674 Mask);
7675}
7676
7677SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7678 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7679 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7680}
7681
7682void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7683 EVT VT = N->getValueType(ResNo: 0);
7684 EVT EltVT = VT.getVectorElementType();
7685 ElementCount OrigEC = VT.getVectorElementCount();
7686 unsigned Factor = N->getNumOperands();
7687 SDLoc DL(N);
7688
7689 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7690 ElementCount WidenEC = WidenVT.getVectorElementCount();
7691
7692 SmallVector<SDValue, 8> WidenOps(Factor);
7693 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7694 WidenOps[Idx] = GetWidenedVector(Op: N->getOperand(Num: Idx));
7695
7696 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7697 SDValue Interleaved =
7698 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: WidenVTs, Ops: WidenOps);
7699
7700 EVT PackedWidenVT =
7701 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7702 SmallVector<SDValue, 8> Slices(Factor);
7703 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7704 Slices[Idx] = Interleaved.getValue(R: Idx);
7705
7706 SDValue Packed = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedWidenVT, Ops: Slices);
7707
7708 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7709 SDValue Narrow = DAG.getExtractSubvector(DL, VT, Vec: Packed,
7710 Idx: OrigEC.getKnownMinValue() * Idx);
7711 SDValue Wide =
7712 DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: Narrow, /*Idx=*/0U);
7713 SetWidenedVector(Op: SDValue(N, Idx), Result: Wide);
7714 }
7715}
7716
7717SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7718 SDLoc DL(N);
7719 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7720 EVT SourceVT = N->getOperand(Num: 0).getValueType();
7721 EVT WideSourceVT =
7722 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceVT.getVectorElementType(),
7723 EC: WidenVT.getVectorElementCount());
7724
7725 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
7726 SubVec: N->getOperand(Num: 0), Idx: 0);
7727 SDValue WideMask = DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT),
7728 SubVec: N->getOperand(Num: 2), Idx: 0);
7729 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
7730 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
7731}
7732
7733void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7734 EVT VT = N->getValueType(ResNo: 0);
7735 EVT EltVT = VT.getVectorElementType();
7736 ElementCount OrigEC = VT.getVectorElementCount();
7737 unsigned Factor = N->getNumOperands();
7738 SDLoc DL(N);
7739
7740 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7741 ElementCount WidenEC = WidenVT.getVectorElementCount();
7742 // We cannot just use the widened operands directly: since they might be
7743 // individually widened, using them directly will result in de-interleaving
7744 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7745 // not concat the widened operands but the original ones to effectively
7746 // generate a "packed" concated and widened vector, before extracting new
7747 // operand vectors with the widened type.
7748 EVT PackedWidenVT =
7749 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7750 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: OrigEC * Factor);
7751 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7752 SDValue PackedWidenVec = DAG.getInsertSubvector(
7753 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7754
7755 // Extract the new widened operand vectors.
7756 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7757 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7758 NewOps[Idx] = DAG.getExtractSubvector(DL, VT: WidenVT, Vec: PackedWidenVec,
7759 Idx: WidenEC.getKnownMinValue() * Idx);
7760 }
7761
7762 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7763 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7764 // Set the widened results manually.
7765 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7766 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7767}
7768
7769SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7770 assert(N->getValueType(0).isVector() &&
7771 N->getOperand(0).getValueType().isVector() &&
7772 "Operands must be vectors");
7773 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7774 ElementCount WidenEC = WidenVT.getVectorElementCount();
7775
7776 SDValue InOp1 = N->getOperand(Num: 0);
7777 EVT InVT = InOp1.getValueType();
7778 assert(InVT.isVector() && "can not widen non-vector type");
7779 EVT WidenInVT =
7780 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7781
7782 // The input and output types often differ here, and it could be that while
7783 // we'd prefer to widen the result type, the input operands have been split.
7784 // In this case, we also need to split the result of this node as well.
7785 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7786 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7787 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7788 return Res;
7789 }
7790
7791 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7792 SDValue InOp2 = N->getOperand(Num: 1);
7793 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7794 InOp1 = GetWidenedVector(Op: InOp1);
7795 InOp2 = GetWidenedVector(Op: InOp2);
7796 } else {
7797 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7798 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7799 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7800 N2: InOp1, N3: ZeroIdx);
7801 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7802 N2: InOp2, N3: ZeroIdx);
7803 }
7804
7805 // Assume that the input and output will be widen appropriately. If not,
7806 // we will have to unroll it at some point.
7807 assert(InOp1.getValueType() == WidenInVT &&
7808 InOp2.getValueType() == WidenInVT &&
7809 "Input not widened to expected type!");
7810 (void)WidenInVT;
7811 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7812 N3: N->getOperand(Num: 2));
7813}
7814
7815SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7816 assert(N->getValueType(0).isVector() &&
7817 N->getOperand(1).getValueType().isVector() &&
7818 "Operands must be vectors");
7819 EVT VT = N->getValueType(ResNo: 0);
7820 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7821 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7822 unsigned NumElts = VT.getVectorNumElements();
7823 EVT EltVT = VT.getVectorElementType();
7824
7825 SDLoc dl(N);
7826 SDValue Chain = N->getOperand(Num: 0);
7827 SDValue LHS = N->getOperand(Num: 1);
7828 SDValue RHS = N->getOperand(Num: 2);
7829 SDValue CC = N->getOperand(Num: 3);
7830 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7831
7832 // Fully unroll and reassemble.
7833 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7834 SmallVector<SDValue, 8> Chains(NumElts);
7835 for (unsigned i = 0; i != NumElts; ++i) {
7836 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7837 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7838
7839 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7840 Ops: {Chain, LHSElem, RHSElem, CC});
7841 Chains[i] = Scalars[i].getValue(R: 1);
7842 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7843 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7844 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7845 }
7846
7847 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7848 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7849
7850 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7851}
7852
7853SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
7854 SDLoc DL(N);
7855 EVT VT = N->getValueType(ResNo: 0);
7856
7857 // Expand, then widen the result.
7858 SDValue Expanded = TLI.expandPartialReduceMLA(Node: N, DAG);
7859 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7860 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WideVT), SubVec: Expanded, Idx: 0);
7861}
7862
7863//===----------------------------------------------------------------------===//
7864// Widen Vector Operand
7865//===----------------------------------------------------------------------===//
7866bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7867 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7868 SDValue Res = SDValue();
7869
7870 // See if the target wants to custom widen this node.
7871 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7872 return false;
7873
7874 switch (N->getOpcode()) {
7875 default:
7876#ifndef NDEBUG
7877 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7878 N->dump(&DAG);
7879 dbgs() << "\n";
7880#endif
7881 report_fatal_error(reason: "do not know how to widen this operator's operand!");
7882
7883 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7884 case ISD::FAKE_USE:
7885 Res = WidenVecOp_FAKE_USE(N);
7886 break;
7887 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7888 case ISD::VECTOR_REPEAT:
7889 Res = WidenVecOp_VECTOR_REPEAT(N);
7890 break;
7891 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7892 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7893 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7894 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7895 case ISD::ATOMIC_STORE:
7896 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7897 break;
7898 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7899 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7900 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7901 break;
7902 case ISD::ANY_EXTEND_VECTOR_INREG:
7903 case ISD::SIGN_EXTEND_VECTOR_INREG:
7904 case ISD::ZERO_EXTEND_VECTOR_INREG:
7905 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7906 break;
7907 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7908 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7909 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7910 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7911 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7912 case ISD::STRICT_FSETCC:
7913 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7914 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7915 case ISD::FLDEXP:
7916 case ISD::FCOPYSIGN:
7917 case ISD::LROUND:
7918 case ISD::LLROUND:
7919 case ISD::LRINT:
7920 case ISD::LLRINT:
7921 Res = WidenVecOp_UnrollVectorOp(N);
7922 break;
7923 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7924
7925 case ISD::ANY_EXTEND:
7926 case ISD::SIGN_EXTEND:
7927 case ISD::ZERO_EXTEND:
7928 Res = WidenVecOp_EXTEND(N);
7929 break;
7930
7931 case ISD::SCMP:
7932 case ISD::UCMP:
7933 Res = WidenVecOp_CMP(N);
7934 break;
7935
7936 case ISD::FP_EXTEND:
7937 case ISD::STRICT_FP_EXTEND:
7938 case ISD::FP_ROUND:
7939 case ISD::STRICT_FP_ROUND:
7940 case ISD::FP_TO_SINT:
7941 case ISD::STRICT_FP_TO_SINT:
7942 case ISD::FP_TO_UINT:
7943 case ISD::STRICT_FP_TO_UINT:
7944 case ISD::SINT_TO_FP:
7945 case ISD::STRICT_SINT_TO_FP:
7946 case ISD::UINT_TO_FP:
7947 case ISD::STRICT_UINT_TO_FP:
7948 case ISD::TRUNCATE:
7949 case ISD::CONVERT_FROM_ARBITRARY_FP:
7950 case ISD::CONVERT_TO_ARBITRARY_FP:
7951 Res = WidenVecOp_Convert(N);
7952 break;
7953
7954 case ISD::FP_TO_SINT_SAT:
7955 case ISD::FP_TO_UINT_SAT:
7956 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7957 break;
7958
7959 case ISD::VECREDUCE_FADD:
7960 case ISD::VECREDUCE_FMUL:
7961 case ISD::VECREDUCE_ADD:
7962 case ISD::VECREDUCE_MUL:
7963 case ISD::VECREDUCE_AND:
7964 case ISD::VECREDUCE_OR:
7965 case ISD::VECREDUCE_XOR:
7966 case ISD::VECREDUCE_SMAX:
7967 case ISD::VECREDUCE_SMIN:
7968 case ISD::VECREDUCE_UMAX:
7969 case ISD::VECREDUCE_UMIN:
7970 case ISD::VECREDUCE_FMAX:
7971 case ISD::VECREDUCE_FMIN:
7972 case ISD::VECREDUCE_FMAXIMUM:
7973 case ISD::VECREDUCE_FMINIMUM:
7974 case ISD::VECREDUCE_FMAXIMUMNUM:
7975 case ISD::VECREDUCE_FMINIMUMNUM:
7976 Res = WidenVecOp_VECREDUCE(N);
7977 break;
7978 case ISD::VECREDUCE_SEQ_FADD:
7979 case ISD::VECREDUCE_SEQ_FMUL:
7980 Res = WidenVecOp_VECREDUCE_SEQ(N);
7981 break;
7982 case ISD::VP_REDUCE_FADD:
7983 case ISD::VP_REDUCE_SEQ_FADD:
7984 case ISD::VP_REDUCE_FMUL:
7985 case ISD::VP_REDUCE_SEQ_FMUL:
7986 case ISD::VP_REDUCE_ADD:
7987 case ISD::VP_REDUCE_MUL:
7988 case ISD::VP_REDUCE_AND:
7989 case ISD::VP_REDUCE_OR:
7990 case ISD::VP_REDUCE_XOR:
7991 case ISD::VP_REDUCE_SMAX:
7992 case ISD::VP_REDUCE_SMIN:
7993 case ISD::VP_REDUCE_UMAX:
7994 case ISD::VP_REDUCE_UMIN:
7995 case ISD::VP_REDUCE_FMAX:
7996 case ISD::VP_REDUCE_FMIN:
7997 case ISD::VP_REDUCE_FMAXIMUM:
7998 case ISD::VP_REDUCE_FMINIMUM:
7999 Res = WidenVecOp_VP_REDUCE(N);
8000 break;
8001 case ISD::CTTZ_ELTS:
8002 case ISD::CTTZ_ELTS_ZERO_POISON:
8003 Res = WidenVecOp_CttzElements(N);
8004 break;
8005 case ISD::VP_CTTZ_ELTS:
8006 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8007 Res = WidenVecOp_VP_CttzElements(N);
8008 break;
8009 case ISD::VECTOR_FIND_LAST_ACTIVE:
8010 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
8011 break;
8012 case ISD::VECTOR_MATCH:
8013 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
8014 break;
8015 }
8016
8017 // If Res is null, the sub-method took care of registering the result.
8018 if (!Res.getNode()) return false;
8019
8020 // If the result is N, the sub-method updated N in place. Tell the legalizer
8021 // core about this.
8022 if (Res.getNode() == N)
8023 return true;
8024
8025
8026 if (N->isStrictFPOpcode())
8027 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
8028 "Invalid operand expansion");
8029 else
8030 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
8031 "Invalid operand expansion");
8032
8033 ReplaceValueWith(From: SDValue(N, 0), To: Res);
8034 return false;
8035}
8036
8037SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
8038 SDLoc DL(N);
8039 EVT VT = N->getValueType(ResNo: 0);
8040
8041 SDValue InOp = N->getOperand(Num: 0);
8042 assert(getTypeAction(InOp.getValueType()) ==
8043 TargetLowering::TypeWidenVector &&
8044 "Unexpected type action");
8045 InOp = GetWidenedVector(Op: InOp);
8046 assert(VT.getVectorNumElements() <
8047 InOp.getValueType().getVectorNumElements() &&
8048 "Input wasn't widened!");
8049
8050 // We may need to further widen the operand until it has the same total
8051 // vector size as the result.
8052 EVT InVT = InOp.getValueType();
8053 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
8054 EVT InEltVT = InVT.getVectorElementType();
8055 for (EVT FixedVT : MVT::vector_valuetypes()) {
8056 EVT FixedEltVT = FixedVT.getVectorElementType();
8057 if (TLI.isTypeLegal(VT: FixedVT) &&
8058 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
8059 FixedEltVT == InEltVT) {
8060 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
8061 "Not enough elements in the fixed type for the operand!");
8062 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
8063 "We can't have the same type as we started with!");
8064 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
8065 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
8066 else
8067 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
8068 break;
8069 }
8070 }
8071 InVT = InOp.getValueType();
8072 if (InVT.getSizeInBits() != VT.getSizeInBits())
8073 // We couldn't find a legal vector type that was a widening of the input
8074 // and could be extended in-register to the result type, so we have to
8075 // scalarize.
8076 return WidenVecOp_Convert(N);
8077 }
8078
8079 // Use special DAG nodes to represent the operation of extending the
8080 // low lanes.
8081 switch (N->getOpcode()) {
8082 default:
8083 llvm_unreachable("Extend legalization on extend operation!");
8084 case ISD::ANY_EXTEND:
8085 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8086 case ISD::SIGN_EXTEND:
8087 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8088 case ISD::ZERO_EXTEND:
8089 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8090 }
8091}
8092
8093SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
8094 SDLoc dl(N);
8095
8096 EVT OpVT = N->getOperand(Num: 0).getValueType();
8097 EVT ResVT = N->getValueType(ResNo: 0);
8098 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
8099 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8100
8101 // 1. EXTRACT_SUBVECTOR
8102 // 2. SIGN_EXTEND/ZERO_EXTEND
8103 // 3. CMP
8104 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
8105 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
8106
8107 // At this point the result type is guaranteed to be valid, so we can use it
8108 // as the operand type by extending it appropriately
8109 ISD::NodeType ExtendOpcode =
8110 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8111 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
8112 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
8113
8114 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
8115}
8116
8117SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
8118 // The result (and first input) is legal, but the second input is illegal.
8119 // We can't do much to fix that, so just unroll and let the extracts off of
8120 // the second input be widened as needed later.
8121 return DAG.UnrollVectorOp(N);
8122}
8123
8124SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
8125 SDLoc DL(N);
8126 EVT ResultVT = N->getValueType(ResNo: 0);
8127 SDValue Test = N->getOperand(Num: 1);
8128 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
8129
8130 // Process this node similarly to SETCC.
8131 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
8132 if (ResultVT.getScalarType() == MVT::i1)
8133 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8134 NumElements: WideResultVT.getVectorNumElements());
8135
8136 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
8137 Ops: {WideArg, Test}, Flags: N->getFlags());
8138
8139 // Extract the needed results from the result vector.
8140 EVT ResVT =
8141 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
8142 NumElements: ResultVT.getVectorNumElements());
8143 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
8144
8145 EVT OpVT = N->getOperand(Num: 0).getValueType();
8146 ISD::NodeType ExtendCode =
8147 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8148 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
8149}
8150
8151SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
8152 // Since the result is legal and the input is illegal.
8153 EVT VT = N->getValueType(ResNo: 0);
8154 EVT EltVT = VT.getVectorElementType();
8155 SDLoc dl(N);
8156 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
8157 assert(getTypeAction(InOp.getValueType()) ==
8158 TargetLowering::TypeWidenVector &&
8159 "Unexpected type action");
8160 InOp = GetWidenedVector(Op: InOp);
8161 EVT InVT = InOp.getValueType();
8162 unsigned Opcode = N->getOpcode();
8163
8164 // Helper to build a convert node with all scalar trailing operands.
8165 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
8166 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
8167 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
8168 N4: N->getOperand(Num: 3));
8169 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
8170 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
8171 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
8172 };
8173
8174 // See if a widened result type would be legal, if so widen the node.
8175 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
8176 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
8177 EC: InVT.getVectorElementCount());
8178 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
8179 SDValue Res;
8180 if (N->isStrictFPOpcode()) {
8181 if (Opcode == ISD::STRICT_FP_ROUND)
8182 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8183 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
8184 else
8185 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8186 Ops: { N->getOperand(Num: 0), InOp });
8187 // Legalize the chain result - switch anything that used the old chain to
8188 // use the new one.
8189 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8190 } else {
8191 Res = MakeConvertNode(WideVT, InOp);
8192 }
8193 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
8194 }
8195
8196 EVT InEltVT = InVT.getVectorElementType();
8197
8198 // Unroll the convert into some scalar code and create a nasty build vector.
8199 unsigned NumElts = VT.getVectorNumElements();
8200 SmallVector<SDValue, 16> Ops(NumElts);
8201 if (N->isStrictFPOpcode()) {
8202 SmallVector<SDValue, 4> NewOps(N->ops());
8203 SmallVector<SDValue, 32> OpChains;
8204 for (unsigned i=0; i < NumElts; ++i) {
8205 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8206 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
8207 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
8208 }
8209 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
8210 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8211 } else {
8212 for (unsigned i = 0; i < NumElts; ++i) {
8213 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8214 Ops[i] = MakeConvertNode(EltVT, Elt);
8215 }
8216 }
8217
8218 return DAG.getBuildVector(VT, DL: dl, Ops);
8219}
8220
8221SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
8222 EVT DstVT = N->getValueType(ResNo: 0);
8223 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
8224 EVT SrcVT = Src.getValueType();
8225 ElementCount WideNumElts = SrcVT.getVectorElementCount();
8226 SDLoc dl(N);
8227
8228 // See if a widened result type would be legal, if so widen the node.
8229 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8230 VT: DstVT.getVectorElementType(), EC: WideNumElts);
8231 if (TLI.isTypeLegal(VT: WideDstVT)) {
8232 SDValue Res =
8233 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
8234 return DAG.getNode(
8235 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
8236 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
8237 }
8238
8239 // Give up and unroll.
8240 return DAG.UnrollVectorOp(N);
8241}
8242
8243SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
8244 EVT VT = N->getValueType(ResNo: 0);
8245 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8246 EVT InWidenVT = InOp.getValueType();
8247 SDLoc dl(N);
8248
8249 // Check if we can convert between two legal vector types and extract.
8250 TypeSize InWidenSize = InWidenVT.getSizeInBits();
8251 TypeSize Size = VT.getSizeInBits();
8252 // x86mmx is not an acceptable vector element type, so don't try.
8253 if (!VT.isVector() && VT != MVT::x86mmx &&
8254 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
8255 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
8256 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
8257 if (TLI.isTypeLegal(VT: NewVT)) {
8258 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8259 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
8260 }
8261 }
8262
8263 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8264 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8265 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8266 // having to copy via memory.
8267 if (VT.isVector()) {
8268 EVT EltVT = VT.getVectorElementType();
8269 unsigned EltSize = EltVT.getFixedSizeInBits();
8270 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
8271 ElementCount NewNumElts =
8272 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8273 .divideCoefficientBy(RHS: EltSize);
8274 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8275 if (TLI.isTypeLegal(VT: NewVT)) {
8276 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8277 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8278 }
8279 }
8280 }
8281
8282 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8283}
8284
8285// Vectors with sizes that are not powers of 2 need to be widened to the
8286// next largest power of 2. For example, we may get a vector of 3 32-bit
8287// integers or of 6 16-bit integers, both of which have to be widened to a
8288// 128-bit vector.
8289SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8290 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8291 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8292 N2: WidenedOp);
8293}
8294
8295SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8296 EVT VT = N->getValueType(ResNo: 0);
8297 EVT EltVT = VT.getVectorElementType();
8298 EVT InVT = N->getOperand(Num: 0).getValueType();
8299 SDLoc dl(N);
8300
8301 // If the widen width for this operand is the same as the width of the concat
8302 // and all but the first operand is undef, just use the widened operand.
8303 unsigned NumOperands = N->getNumOperands();
8304 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8305 unsigned i;
8306 for (i = 1; i < NumOperands; ++i)
8307 if (!N->getOperand(Num: i).isUndef())
8308 break;
8309
8310 if (i == NumOperands)
8311 return GetWidenedVector(Op: N->getOperand(Num: 0));
8312 }
8313
8314 if (VT.isScalableVector()) {
8315 SDValue Result = DAG.getPOISON(VT);
8316 unsigned NumInElts = InVT.getVectorMinNumElements();
8317 for (unsigned i = 0; i < NumOperands; ++i) {
8318 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: i));
8319 if (InOp.getValueType() != InVT)
8320 InOp = DAG.getExtractSubvector(DL: dl, VT: InVT, Vec: InOp, Idx: 0);
8321 Result = DAG.getInsertSubvector(DL: dl, Vec: Result, SubVec: InOp, Idx: i * NumInElts);
8322 }
8323 return Result;
8324 }
8325
8326 // Otherwise, fall back to a nasty build vector.
8327 unsigned NumElts = VT.getVectorNumElements();
8328 SmallVector<SDValue, 16> Ops(NumElts);
8329
8330 unsigned NumInElts = InVT.getVectorNumElements();
8331
8332 unsigned Idx = 0;
8333 for (unsigned i=0; i < NumOperands; ++i) {
8334 SDValue InOp = N->getOperand(Num: i);
8335 assert(getTypeAction(InOp.getValueType()) ==
8336 TargetLowering::TypeWidenVector &&
8337 "Unexpected type action");
8338 InOp = GetWidenedVector(Op: InOp);
8339 for (unsigned j = 0; j < NumInElts; ++j)
8340 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8341 }
8342 return DAG.getBuildVector(VT, DL: dl, Ops);
8343}
8344
8345SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_REPEAT(SDNode *N) {
8346 SDLoc DL(N);
8347 EVT VT = N->getValueType(ResNo: 0);
8348 SDValue Src = N->getOperand(Num: 0);
8349 EVT SrcVT = Src.getValueType();
8350 EVT WidenedSrcVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SrcVT);
8351
8352 if (!WidenedSrcVT.getVectorElementCount().hasKnownScalarFactor(
8353 RHS: SrcVT.getVectorElementCount()))
8354 report_fatal_error(
8355 reason: "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
8356 "a known scalar multiple of the input ElementCount.");
8357
8358 // Repeat the original source because the extra lanes of its widened value
8359 // are unspecified.
8360 unsigned NumConcat =
8361 WidenedSrcVT.getVectorNumElements() / SrcVT.getVectorNumElements();
8362 SmallVector<SDValue, 8> Ops(NumConcat, Src);
8363 SDValue WidenedSrc = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WidenedSrcVT, Ops);
8364 EVT WidenedVT = VT.changeVectorElementCount(
8365 Context&: *DAG.getContext(),
8366 EC: ElementCount::getScalable(MinVal: WidenedSrcVT.getVectorNumElements()));
8367 SDValue Widened = DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: WidenedVT, Operand: WidenedSrc);
8368 return DAG.getExtractSubvector(DL, VT, Vec: Widened, Idx: 0);
8369}
8370
8371SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8372 EVT VT = N->getValueType(ResNo: 0);
8373 SDValue SubVec = N->getOperand(Num: 1);
8374 SDValue InVec = N->getOperand(Num: 0);
8375
8376 EVT OrigVT = SubVec.getValueType();
8377 SubVec = GetWidenedVector(Op: SubVec);
8378 EVT SubVT = SubVec.getValueType();
8379
8380 // Whether or not all the elements of the widened SubVec will be inserted into
8381 // valid indices of VT.
8382 bool IndicesValid = false;
8383 // If we statically know that VT can fit SubVT, the indices are valid.
8384 if (VT.knownBitsGE(VT: SubVT))
8385 IndicesValid = true;
8386 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8387 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8388 // we know the minimum vscale we can work out if it's valid ourselves.
8389 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8390 Kind: Attribute::VScaleRange);
8391 if (Attr.isValid()) {
8392 unsigned VScaleMin = Attr.getVScaleRangeMin();
8393 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8394 SubVT.getFixedSizeInBits())
8395 IndicesValid = true;
8396 }
8397 }
8398
8399 if (!IndicesValid)
8400 report_fatal_error(
8401 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8402
8403 SDLoc DL(N);
8404
8405 // We need to make sure that the indices are still valid, otherwise we might
8406 // widen what was previously well-defined to something undefined.
8407 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8408 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8409 N3: N->getOperand(Num: 2));
8410
8411 if (OrigVT.isScalableVector()) {
8412 // When the widened types match, overwriting the start of a vector is
8413 // effectively a merge operation that can be implement as a vselect.
8414 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8415 SDValue Mask =
8416 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8417 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8418 }
8419
8420 // Fallback to inserting through memory.
8421 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8422 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8423 MachineFunction &MF = DAG.getMachineFunction();
8424 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8425 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8426
8427 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8428 PtrInfo, F: MachineMemOperand::MOStore,
8429 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8430 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8431 PtrInfo, F: MachineMemOperand::MOLoad,
8432 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8433
8434 // Write out the vector being inserting into.
8435 SDValue Ch =
8436 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8437
8438 // Build a mask to match the length of the sub-vector.
8439 SDValue Mask =
8440 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8441
8442 // Overwrite the sub-vector at the required offset.
8443 SDValue SubVecPtr =
8444 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8445 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8446 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8447 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8448
8449 // Read back the result.
8450 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8451 }
8452
8453 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8454 // with a series of INSERT_VECTOR_ELT
8455 unsigned Idx = N->getConstantOperandVal(Num: 2);
8456
8457 SDValue InsertElt = InVec;
8458 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8459 SDValue ExtractElt =
8460 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8461 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8462 }
8463
8464 return InsertElt;
8465}
8466
8467SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8468 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8469 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8470 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8471}
8472
8473SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8474 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8475 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8476 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8477}
8478
8479SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8480 SDLoc DL(N);
8481 EVT ResVT = N->getValueType(ResNo: 0);
8482
8483 // Widen the input as requested by the legalizer.
8484 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8485 EVT WideInVT = WideInOp.getValueType();
8486
8487 // Simple case: if widened input is still smaller than or equal to result,
8488 // just use it directly.
8489 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8490 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8491
8492 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8493 // If widening makes the input larger than the original result, widen the
8494 // result to match, then extract back down.
8495 EVT ResEltVT = ResVT.getVectorElementType();
8496 unsigned EltBits = ResEltVT.getSizeInBits();
8497 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8498 "Widened input size must be a multiple of result element size");
8499
8500 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8501 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8502
8503 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8504 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8505}
8506
8507SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8508 // We have to widen the value, but we want only to store the original
8509 // vector type.
8510 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8511
8512 if (!ST->getMemoryVT().getScalarType().isByteSized())
8513 return TLI.scalarizeVectorStore(ST, DAG);
8514
8515 if (ST->isTruncatingStore())
8516 return TLI.scalarizeVectorStore(ST, DAG);
8517
8518 // Generate a vector-predicated store if it is custom/legal on the target.
8519 // To avoid possible recursion, only do this if the widened mask type is
8520 // legal.
8521 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8522 // removed from the IR by the ExpandVectorPredication pass but we're
8523 // reintroducing them here.
8524 SDValue StVal = ST->getValue();
8525 EVT StVT = StVal.getValueType();
8526 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8527 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8528
8529 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8530 TLI.isTypeLegal(VT: WideMaskVT)) {
8531 // Widen the value.
8532 SDLoc DL(N);
8533 StVal = GetWidenedVector(Op: StVal);
8534 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8535 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8536 EC: StVT.getVectorElementCount());
8537 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8538 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8539 AM: ST->getAddressingMode());
8540 }
8541
8542 SmallVector<SDValue, 16> StChain;
8543 if (GenWidenVectorStores(StChain, ST)) {
8544 if (StChain.size() == 1)
8545 return StChain[0];
8546
8547 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8548 }
8549
8550 if (StVT.isVector()) {
8551 // If all else fails replace the store with a wide masked store.
8552 SDLoc DL(N);
8553 SDValue WideStVal = GetWidenedVector(Op: StVal);
8554 SDValue Mask =
8555 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8556
8557 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8558 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8559 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8560 IsTruncating: ST->isTruncatingStore());
8561 }
8562
8563 report_fatal_error(reason: "Unable to widen vector store");
8564}
8565
8566SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8567 EVT StVT = ST->getMemoryVT();
8568 SDLoc dl(ST);
8569
8570 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8571 EVT WidenVT = StVal.getValueType();
8572
8573 TypeSize StWidth = StVT.getSizeInBits();
8574 TypeSize WidenWidth = WidenVT.getSizeInBits();
8575 TypeSize WidthDiff = WidenWidth - StWidth;
8576
8577 // Find the vector type that can store the original memory width in one
8578 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8579 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8580 // implies a 4-byte movl, writing undef bytes past its object).
8581 std::optional<EVT> FirstVT =
8582 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8583 WidenEx: WidthDiff.getKnownMinValue());
8584 if (!FirstVT)
8585 return SDValue();
8586
8587 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8588
8589 SDValue StOp =
8590 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8591
8592 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8593 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8594}
8595
8596SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8597 assert((OpNo == 1 || OpNo == 3) &&
8598 "Can widen only data or mask operand of vp_store");
8599 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8600 SDValue Mask = ST->getMask();
8601 SDValue StVal = ST->getValue();
8602 SDLoc dl(N);
8603
8604 if (OpNo == 1) {
8605 // Widen the value.
8606 StVal = GetWidenedVector(Op: StVal);
8607
8608 // We only handle the case where the mask needs widening to an
8609 // identically-sized type as the vector inputs.
8610 assert(getTypeAction(Mask.getValueType()) ==
8611 TargetLowering::TypeWidenVector &&
8612 "Unable to widen VP store");
8613 Mask = GetWidenedVector(Op: Mask);
8614 } else {
8615 Mask = GetWidenedVector(Op: Mask);
8616
8617 // We only handle the case where the stored value needs widening to an
8618 // identically-sized type as the mask.
8619 assert(getTypeAction(StVal.getValueType()) ==
8620 TargetLowering::TypeWidenVector &&
8621 "Unable to widen VP store");
8622 StVal = GetWidenedVector(Op: StVal);
8623 }
8624
8625 assert(Mask.getValueType().getVectorElementCount() ==
8626 StVal.getValueType().getVectorElementCount() &&
8627 "Mask and data vectors should have the same number of elements");
8628 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8629 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8630 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8631 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8632 IsCompressing: ST->isCompressingStore());
8633}
8634
8635SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8636 unsigned OpNo) {
8637 assert((OpNo == 1 || OpNo == 4) &&
8638 "Can widen only data or mask operand of vp_strided_store");
8639 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8640 SDValue Mask = SST->getMask();
8641 SDValue StVal = SST->getValue();
8642 SDLoc DL(N);
8643
8644 if (OpNo == 1)
8645 assert(getTypeAction(Mask.getValueType()) ==
8646 TargetLowering::TypeWidenVector &&
8647 "Unable to widen VP strided store");
8648 else
8649 assert(getTypeAction(StVal.getValueType()) ==
8650 TargetLowering::TypeWidenVector &&
8651 "Unable to widen VP strided store");
8652
8653 StVal = GetWidenedVector(Op: StVal);
8654 Mask = GetWidenedVector(Op: Mask);
8655
8656 assert(StVal.getValueType().getVectorElementCount() ==
8657 Mask.getValueType().getVectorElementCount() &&
8658 "Data and mask vectors should have the same number of elements");
8659
8660 return DAG.getStridedStoreVP(
8661 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8662 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8663 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8664 IsCompressing: SST->isCompressingStore());
8665}
8666
8667SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8668 assert((OpNo == 1 || OpNo == 4) &&
8669 "Can widen only data or mask operand of mstore");
8670 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8671 SDValue Mask = MST->getMask();
8672 EVT MaskVT = Mask.getValueType();
8673 SDValue StVal = MST->getValue();
8674 EVT VT = StVal.getValueType();
8675 SDLoc dl(N);
8676
8677 EVT WideVT, WideMaskVT;
8678 if (OpNo == 1) {
8679 // Widen the value.
8680 StVal = GetWidenedVector(Op: StVal);
8681
8682 WideVT = StVal.getValueType();
8683 WideMaskVT =
8684 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8685 EC: WideVT.getVectorElementCount());
8686 } else {
8687 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8688
8689 EVT ValueVT = StVal.getValueType();
8690 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8691 EC: WideMaskVT.getVectorElementCount());
8692 }
8693
8694 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8695 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8696 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8697 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8698 EC: VT.getVectorElementCount());
8699 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8700 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8701 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8702 }
8703
8704 if (OpNo == 1) {
8705 // The mask should be widened as well.
8706 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8707 } else {
8708 // Widen the mask.
8709 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8710
8711 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8712 }
8713
8714 assert(Mask.getValueType().getVectorElementCount() ==
8715 StVal.getValueType().getVectorElementCount() &&
8716 "Mask and data vectors should have the same number of elements");
8717 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8718 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8719 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8720 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8721}
8722
8723SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8724 assert(OpNo == 4 && "Can widen only the index of mgather");
8725 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8726 SDValue DataOp = MG->getPassThru();
8727 SDValue Mask = MG->getMask();
8728 SDValue Scale = MG->getScale();
8729
8730 // Just widen the index. It's allowed to have extra elements.
8731 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8732
8733 SDLoc dl(N);
8734 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8735 Scale};
8736 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8737 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8738 ExtTy: MG->getExtensionType());
8739 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8740 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8741 return SDValue();
8742}
8743
8744SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8745 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8746 SDValue DataOp = MSC->getValue();
8747 SDValue Mask = MSC->getMask();
8748 SDValue Index = MSC->getIndex();
8749 SDValue Scale = MSC->getScale();
8750 EVT WideMemVT = MSC->getMemoryVT();
8751
8752 if (OpNo == 1) {
8753 DataOp = GetWidenedVector(Op: DataOp);
8754 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8755
8756 // Widen index.
8757 EVT IndexVT = Index.getValueType();
8758 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8759 VT: IndexVT.getVectorElementType(), EC: WideEC);
8760 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8761
8762 // The mask should be widened as well.
8763 EVT MaskVT = Mask.getValueType();
8764 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8765 VT: MaskVT.getVectorElementType(), EC: WideEC);
8766 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8767
8768 // Widen the MemoryType
8769 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8770 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8771 } else if (OpNo == 4) {
8772 // Just widen the index. It's allowed to have extra elements.
8773 Index = GetWidenedVector(Op: Index);
8774 } else
8775 llvm_unreachable("Can't widen this operand of mscatter");
8776
8777 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8778 Scale};
8779 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8780 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8781 IsTruncating: MSC->isTruncatingStore());
8782}
8783
8784SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8785 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8786 SDValue DataOp = VPSC->getValue();
8787 SDValue Mask = VPSC->getMask();
8788 SDValue Index = VPSC->getIndex();
8789 SDValue Scale = VPSC->getScale();
8790 EVT WideMemVT = VPSC->getMemoryVT();
8791
8792 if (OpNo == 1) {
8793 DataOp = GetWidenedVector(Op: DataOp);
8794 Index = GetWidenedVector(Op: Index);
8795 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8796 Mask = GetWidenedMask(Mask, EC: WideEC);
8797 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8798 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8799 } else if (OpNo == 3) {
8800 // Just widen the index. It's allowed to have extra elements.
8801 Index = GetWidenedVector(Op: Index);
8802 } else
8803 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8804
8805 SDValue Ops[] = {
8806 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8807 VPSC->getVectorLength()};
8808 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8809 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8810}
8811
8812SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8813 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8814 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8815 SDLoc dl(N);
8816 EVT VT = N->getValueType(ResNo: 0);
8817
8818 // WARNING: In this code we widen the compare instruction with garbage.
8819 // This garbage may contain denormal floats which may be slow. Is this a real
8820 // concern ? Should we zero the unused lanes if this is a float compare ?
8821
8822 // Get a new SETCC node to compare the newly widened operands.
8823 // Only some of the compared elements are legal.
8824 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8825 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8826 if (VT.getScalarType() == MVT::i1)
8827 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8828 EC: SVT.getVectorElementCount());
8829
8830 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8831 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8832
8833 // Extract the needed results from the result vector.
8834 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8835 VT: SVT.getVectorElementType(),
8836 EC: VT.getVectorElementCount());
8837 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8838
8839 EVT OpVT = N->getOperand(Num: 0).getValueType();
8840 ISD::NodeType ExtendCode =
8841 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8842 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8843}
8844
8845SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8846 SDValue Chain = N->getOperand(Num: 0);
8847 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8848 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8849 SDValue CC = N->getOperand(Num: 3);
8850 SDLoc dl(N);
8851
8852 EVT VT = N->getValueType(ResNo: 0);
8853 EVT EltVT = VT.getVectorElementType();
8854 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8855 unsigned NumElts = VT.getVectorNumElements();
8856
8857 // Unroll into a build vector.
8858 SmallVector<SDValue, 8> Scalars(NumElts);
8859 SmallVector<SDValue, 8> Chains(NumElts);
8860
8861 for (unsigned i = 0; i != NumElts; ++i) {
8862 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8863 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8864
8865 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8866 Ops: {Chain, LHSElem, RHSElem, CC});
8867 Chains[i] = Scalars[i].getValue(R: 1);
8868 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8869 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8870 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8871 }
8872
8873 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8874 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8875
8876 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8877}
8878
8879static unsigned getExtendForIntVecReduction(unsigned Opc) {
8880 switch (Opc) {
8881 default:
8882 llvm_unreachable("Expected integer vector reduction");
8883 case ISD::VECREDUCE_ADD:
8884 case ISD::VECREDUCE_MUL:
8885 case ISD::VECREDUCE_AND:
8886 case ISD::VECREDUCE_OR:
8887 case ISD::VECREDUCE_XOR:
8888 return ISD::ANY_EXTEND;
8889 case ISD::VECREDUCE_SMAX:
8890 case ISD::VECREDUCE_SMIN:
8891 return ISD::SIGN_EXTEND;
8892 case ISD::VECREDUCE_UMAX:
8893 case ISD::VECREDUCE_UMIN:
8894 return ISD::ZERO_EXTEND;
8895 }
8896}
8897
8898SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8899 SDLoc dl(N);
8900 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8901 EVT VT = N->getValueType(ResNo: 0);
8902 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8903 EVT WideVT = Op.getValueType();
8904 EVT ElemVT = OrigVT.getVectorElementType();
8905 SDNodeFlags Flags = N->getFlags();
8906
8907 unsigned Opc = N->getOpcode();
8908 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8909 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8910 assert(NeutralElem && "Neutral element must exist");
8911
8912 // Pad the vector with the neutral element.
8913 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8914 unsigned WideElts = WideVT.getVectorMinNumElements();
8915
8916 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8917 // needing to pad the source vector, because the inactive lanes can simply be
8918 // disabled and not contribute to the result.
8919 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8920 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8921 SDValue Start = NeutralElem;
8922 if (VT.isInteger())
8923 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8924 assert(Start.getValueType() == VT);
8925 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8926 EC: WideVT.getVectorElementCount());
8927 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8928 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8929 EC: OrigVT.getVectorElementCount());
8930 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8931 }
8932
8933 if (WideVT.isScalableVector()) {
8934 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8935 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8936 EC: ElementCount::getScalable(MinVal: GCD));
8937 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8938 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8939 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8940 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8941 }
8942
8943 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8944 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8945
8946 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8947}
8948
8949SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8950 SDLoc dl(N);
8951 SDValue AccOp = N->getOperand(Num: 0);
8952 SDValue VecOp = N->getOperand(Num: 1);
8953 SDValue Op = GetWidenedVector(Op: VecOp);
8954
8955 EVT VT = N->getValueType(ResNo: 0);
8956 EVT OrigVT = VecOp.getValueType();
8957 EVT WideVT = Op.getValueType();
8958 EVT ElemVT = OrigVT.getVectorElementType();
8959 SDNodeFlags Flags = N->getFlags();
8960
8961 unsigned Opc = N->getOpcode();
8962 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8963 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8964
8965 // Pad the vector with the neutral element.
8966 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8967 unsigned WideElts = WideVT.getVectorMinNumElements();
8968
8969 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8970 // needing to pad the source vector, because the inactive lanes can simply be
8971 // disabled and not contribute to the result.
8972 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8973 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8974 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8975 EC: WideVT.getVectorElementCount());
8976 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8977 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8978 EC: OrigVT.getVectorElementCount());
8979 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8980 }
8981
8982 if (WideVT.isScalableVector()) {
8983 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8984 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8985 EC: ElementCount::getScalable(MinVal: GCD));
8986 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8987 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8988 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8989 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8990 }
8991
8992 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8993 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8994
8995 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8996}
8997
8998SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8999 assert(N->isVPOpcode() && "Expected VP opcode");
9000
9001 SDLoc dl(N);
9002 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
9003 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
9004 EC: Op.getValueType().getVectorElementCount());
9005
9006 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
9007 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
9008 Flags: N->getFlags());
9009}
9010
9011SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
9012 // This only gets called in the case that the left and right inputs and
9013 // result are of a legal odd vector type, and the condition is illegal i1 of
9014 // the same odd width that needs widening.
9015 EVT VT = N->getValueType(ResNo: 0);
9016 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
9017
9018 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
9019 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
9020 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
9021 SDLoc DL(N);
9022
9023 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
9024 N2: LeftIn, N3: RightIn);
9025 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
9026}
9027
9028SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
9029 SDLoc DL(N);
9030 SDValue Source = N->getOperand(Num: 0);
9031 EVT SourceVT = Source.getValueType();
9032 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9033
9034 SDValue WideSource;
9035 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
9036 WideSource = GetWidenedVector(Op: Source);
9037 } else {
9038 // Pad the widened portion with all-ones so the extra lanes appear as
9039 // active (non-zero) elements and do not contribute trailing zeros.
9040 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
9041 if (WideVT.isFixedLengthVector() &&
9042 getTypeAction(VT: WideVT) == TargetLowering::TypeSplitVector) {
9043 WideSource = GetWidenedVector(Op: Source);
9044 unsigned WideElts = WideVT.getVectorNumElements();
9045 SmallVector<int> Mask(WideElts);
9046 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
9047 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
9048 Mask[I] += WideElts;
9049 WideSource = DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: WideSource, N2: AllOnes, Mask);
9050 } else {
9051 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
9052 }
9053 }
9054
9055 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
9056 Flags: N->getFlags());
9057}
9058
9059SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
9060 SDLoc DL(N);
9061 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
9062 EVT SrcVT = Source.getValueType();
9063 SDValue Mask =
9064 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
9065
9066 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
9067 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
9068}
9069
9070SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
9071 SDLoc DL(N);
9072 SDValue Mask = N->getOperand(Num: 0);
9073 EVT OrigMaskVT = Mask.getValueType();
9074 SDValue WideMask = GetWidenedVector(Op: Mask);
9075 EVT WideMaskVT = WideMask.getValueType();
9076
9077 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
9078 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
9079 unsigned WideElts = WideMaskVT.getVectorNumElements();
9080 if (OrigElts != WideElts) {
9081 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
9082 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
9083 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9084 }
9085
9086 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
9087 Operand: WideMask);
9088}
9089
9090SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
9091 if (OpNo == 0) {
9092 SDLoc DL(N);
9093 EVT ResVT = N->getValueType(ResNo: 0);
9094 EVT SourceVT = N->getOperand(Num: 0).getValueType();
9095 EVT WideSourceVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9096 EVT WidenVT =
9097 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
9098 EC: WideSourceVT.getVectorElementCount());
9099
9100 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
9101 SubVec: N->getOperand(Num: 0), Idx: 0);
9102 SDValue WideMask = DAG.getInsertSubvector(
9103 DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT), SubVec: N->getOperand(Num: 2), Idx: 0);
9104 SDValue WideMatch = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
9105 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
9106 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideMatch, Idx: 0);
9107 }
9108
9109 // Note: The Mask (OpNo == 2) should be widened with the result.
9110 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
9111
9112 SDLoc DL(N);
9113 SDValue Needle = N->getOperand(Num: 1);
9114 EVT NeedleVT = Needle.getValueType();
9115 if (NeedleVT.getVectorNumElements() == 1)
9116 return TLI.expandVectorMatch(N, DAG);
9117
9118 EVT WidenNeedleVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: NeedleVT);
9119
9120 SDValue Fill =
9121 DAG.getExtractVectorElt(DL, VT: NeedleVT.getVectorElementType(), Vec: Needle, Idx: 0);
9122 SDValue WideNeedle = DAG.getSplatVector(VT: WidenNeedleVT, DL, Op: Fill);
9123 WideNeedle = DAG.getInsertSubvector(DL, Vec: WideNeedle, SubVec: Needle, Idx: 0);
9124
9125 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0),
9126 N1: N->getOperand(Num: 0), N2: WideNeedle, N3: N->getOperand(Num: 2),
9127 Flags: N->getFlags());
9128}
9129
9130//===----------------------------------------------------------------------===//
9131// Vector Widening Utilities
9132//===----------------------------------------------------------------------===//
9133
9134// Utility function to find the type to chop up a widen vector for load/store
9135// TLI: Target lowering used to determine legal types.
9136// Width: Width left need to load/store.
9137// WidenVT: The widen vector type to load to/store from
9138// Align: If 0, don't allow use of a wider type
9139// WidenEx: If Align is not 0, the amount additional we can load/store from.
9140
9141static std::optional<EVT> findMemType(SelectionDAG &DAG,
9142 const TargetLowering &TLI, unsigned Width,
9143 EVT WidenVT, unsigned Align = 0,
9144 unsigned WidenEx = 0) {
9145 EVT WidenEltVT = WidenVT.getVectorElementType();
9146 const bool Scalable = WidenVT.isScalableVector();
9147 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
9148 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
9149 unsigned AlignInBits = Align*8;
9150
9151 EVT RetVT = WidenEltVT;
9152 // Don't bother looking for an integer type if the vector is scalable, skip
9153 // to vector types.
9154 if (!Scalable) {
9155 // If we have one element to load/store, return it.
9156 if (Width == WidenEltWidth)
9157 return RetVT;
9158
9159 // See if there is larger legal integer than the element type to load/store.
9160 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
9161 unsigned MemVTWidth = MemVT.getSizeInBits();
9162 if (MemVT.getSizeInBits() <= WidenEltWidth)
9163 break;
9164 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9165 if ((Action == TargetLowering::TypeLegal ||
9166 Action == TargetLowering::TypePromoteInteger) &&
9167 (WidenWidth % MemVTWidth) == 0 &&
9168 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9169 (MemVTWidth <= Width ||
9170 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9171 if (MemVTWidth == WidenWidth)
9172 return MemVT;
9173 RetVT = MemVT;
9174 break;
9175 }
9176 }
9177 }
9178
9179 // See if there is a larger vector type to load/store that has the same vector
9180 // element type and is evenly divisible with the WidenVT.
9181 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
9182 // Skip vector MVTs which don't match the scalable property of WidenVT.
9183 if (Scalable != MemVT.isScalableVector())
9184 continue;
9185 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
9186 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9187 if ((Action == TargetLowering::TypeLegal ||
9188 Action == TargetLowering::TypePromoteInteger) &&
9189 WidenEltVT == MemVT.getVectorElementType() &&
9190 (WidenWidth % MemVTWidth) == 0 &&
9191 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9192 (MemVTWidth <= Width ||
9193 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9194 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
9195 return MemVT;
9196 }
9197 }
9198
9199 // Using element-wise loads and stores for widening operations is not
9200 // supported for scalable vectors
9201 if (Scalable)
9202 return std::nullopt;
9203
9204 return RetVT;
9205}
9206
9207// Builds a vector type from scalar loads
9208// VecTy: Resulting Vector type
9209// LDOps: Load operators to build a vector type
9210// [Start,End) the list of loads to use.
9211static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
9212 SmallVectorImpl<SDValue> &LdOps,
9213 unsigned Start, unsigned End) {
9214 SDLoc dl(LdOps[Start]);
9215 EVT LdTy = LdOps[Start].getValueType();
9216 unsigned Width = VecTy.getSizeInBits();
9217 unsigned NumElts = Width / LdTy.getSizeInBits();
9218 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
9219
9220 unsigned Idx = 1;
9221 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
9222
9223 for (unsigned i = Start + 1; i != End; ++i) {
9224 EVT NewLdTy = LdOps[i].getValueType();
9225 if (NewLdTy != LdTy) {
9226 NumElts = Width / NewLdTy.getSizeInBits();
9227 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
9228 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
9229 // Readjust position and vector position based on new load type.
9230 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
9231 LdTy = NewLdTy;
9232 }
9233 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
9234 }
9235 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
9236}
9237
9238SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
9239 LoadSDNode *LD) {
9240 // The strategy assumes that we can efficiently load power-of-two widths.
9241 // The routine chops the vector into the largest vector loads with the same
9242 // element type or scalar loads and then recombines it to the widen vector
9243 // type.
9244 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9245 EVT LdVT = LD->getMemoryVT();
9246 SDLoc dl(LD);
9247 assert(LdVT.isVector() && WidenVT.isVector());
9248 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9249 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
9250
9251 // Load information
9252 SDValue Chain = LD->getChain();
9253 SDValue BasePtr = LD->getBasePtr();
9254 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9255 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9256
9257 TypeSize LdWidth = LdVT.getSizeInBits();
9258 TypeSize WidenWidth = WidenVT.getSizeInBits();
9259 TypeSize WidthDiff = WidenWidth - LdWidth;
9260 // Allow wider loads if they are sufficiently aligned to avoid memory faults
9261 // and if the original load is simple.
9262 unsigned LdAlign =
9263 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
9264
9265 // Find the vector type that can load from.
9266 std::optional<EVT> FirstVT =
9267 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
9268 WidenEx: WidthDiff.getKnownMinValue());
9269
9270 if (!FirstVT)
9271 return SDValue();
9272
9273 SmallVector<EVT, 8> MemVTs;
9274 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9275
9276 // Unless we're able to load in one instruction we must work out how to load
9277 // the remainder.
9278 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
9279 std::optional<EVT> NewVT = FirstVT;
9280 TypeSize RemainingWidth = LdWidth;
9281 TypeSize NewVTWidth = FirstVTWidth;
9282 do {
9283 RemainingWidth -= NewVTWidth;
9284 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
9285 // The current type we are using is too large. Find a better size.
9286 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
9287 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
9288 if (!NewVT)
9289 return SDValue();
9290 NewVTWidth = NewVT->getSizeInBits();
9291 }
9292 MemVTs.push_back(Elt: *NewVT);
9293 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
9294 }
9295
9296 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9297 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9298 LdChain.push_back(Elt: LdOp.getValue(R: 1));
9299
9300 // Check if we can load the element with one instruction.
9301 if (MemVTs.empty())
9302 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
9303 DAG);
9304
9305 // Load vector by using multiple loads from largest vector to scalar.
9306 SmallVector<SDValue, 16> LdOps;
9307 LdOps.push_back(Elt: LdOp);
9308
9309 uint64_t ScaledOffset = 0;
9310 MachinePointerInfo MPI = LD->getPointerInfo();
9311
9312 // First incremement past the first load.
9313 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
9314 ScaledOffset: &ScaledOffset);
9315
9316 for (EVT MemVT : MemVTs) {
9317 Align NewAlign = ScaledOffset == 0
9318 ? LD->getBaseAlign()
9319 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
9320 SDValue L = DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags,
9321 Metadata);
9322
9323 LdOps.push_back(Elt: L);
9324 LdChain.push_back(Elt: L.getValue(R: 1));
9325 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
9326 }
9327
9328 // Build the vector from the load operations.
9329 unsigned End = LdOps.size();
9330 if (!LdOps[0].getValueType().isVector())
9331 // All the loads are scalar loads.
9332 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
9333
9334 // If the load contains vectors, build the vector using concat vector.
9335 // All of the vectors used to load are power-of-2, and the scalar loads can be
9336 // combined to make a power-of-2 vector.
9337 SmallVector<SDValue, 16> ConcatOps(End);
9338 int i = End - 1;
9339 int Idx = End;
9340 EVT LdTy = LdOps[i].getValueType();
9341 // First, combine the scalar loads to a vector.
9342 if (!LdTy.isVector()) {
9343 for (--i; i >= 0; --i) {
9344 LdTy = LdOps[i].getValueType();
9345 if (LdTy.isVector())
9346 break;
9347 }
9348 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
9349 }
9350
9351 ConcatOps[--Idx] = LdOps[i];
9352 for (--i; i >= 0; --i) {
9353 EVT NewLdTy = LdOps[i].getValueType();
9354 if (NewLdTy != LdTy) {
9355 // Create a larger vector.
9356 TypeSize LdTySize = LdTy.getSizeInBits();
9357 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9358 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9359 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9360 unsigned NumOps =
9361 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9362 SmallVector<SDValue, 16> WidenOps(NumOps);
9363 unsigned j = 0;
9364 for (; j != End-Idx; ++j)
9365 WidenOps[j] = ConcatOps[Idx+j];
9366 for (; j != NumOps; ++j)
9367 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9368
9369 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9370 Ops: WidenOps);
9371 Idx = End - 1;
9372 LdTy = NewLdTy;
9373 }
9374 ConcatOps[--Idx] = LdOps[i];
9375 }
9376
9377 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9378 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9379 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9380
9381 // We need to fill the rest with undefs to build the vector.
9382 unsigned NumOps =
9383 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9384 SmallVector<SDValue, 16> WidenOps(NumOps);
9385 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9386 {
9387 unsigned i = 0;
9388 for (; i != End-Idx; ++i)
9389 WidenOps[i] = ConcatOps[Idx+i];
9390 for (; i != NumOps; ++i)
9391 WidenOps[i] = UndefVal;
9392 }
9393 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9394}
9395
9396SDValue
9397DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9398 LoadSDNode *LD,
9399 ISD::LoadExtType ExtType) {
9400 // For extension loads, it may not be more efficient to chop up the vector
9401 // and then extend it. Instead, we unroll the load and build a new vector.
9402 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9403 EVT LdVT = LD->getMemoryVT();
9404 SDLoc dl(LD);
9405 assert(LdVT.isVector() && WidenVT.isVector());
9406 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9407
9408 // Load information
9409 SDValue Chain = LD->getChain();
9410 SDValue BasePtr = LD->getBasePtr();
9411 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9412 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9413
9414 if (LdVT.isScalableVector())
9415 return SDValue();
9416
9417 EVT EltVT = WidenVT.getVectorElementType();
9418 EVT LdEltVT = LdVT.getVectorElementType();
9419 unsigned NumElts = LdVT.getVectorNumElements();
9420
9421 // Load each element and widen.
9422 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9423 SmallVector<SDValue, 16> Ops(WidenNumElts);
9424 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9425 Ops[0] =
9426 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9427 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9428 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9429 unsigned i = 0, Offset = Increment;
9430 for (i=1; i < NumElts; ++i, Offset += Increment) {
9431 SDValue NewBasePtr =
9432 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9433 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9434 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9435 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9436 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9437 }
9438
9439 // Fill the rest with undefs.
9440 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9441 for (; i != WidenNumElts; ++i)
9442 Ops[i] = UndefVal;
9443
9444 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9445}
9446
9447bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9448 StoreSDNode *ST) {
9449 // The strategy assumes that we can efficiently store power-of-two widths.
9450 // The routine chops the vector into the largest vector stores with the same
9451 // element type or scalar stores.
9452 SDValue Chain = ST->getChain();
9453 SDValue BasePtr = ST->getBasePtr();
9454 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9455 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
9456 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9457 SDLoc dl(ST);
9458
9459 EVT StVT = ST->getMemoryVT();
9460 TypeSize StWidth = StVT.getSizeInBits();
9461 EVT ValVT = ValOp.getValueType();
9462 TypeSize ValWidth = ValVT.getSizeInBits();
9463 EVT ValEltVT = ValVT.getVectorElementType();
9464 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9465 assert(StVT.getVectorElementType() == ValEltVT);
9466 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9467 "Mismatch between store and value types");
9468
9469 int Idx = 0; // current index to store
9470
9471 MachinePointerInfo MPI = ST->getPointerInfo();
9472 uint64_t ScaledOffset = 0;
9473
9474 // A breakdown of how to widen this vector store. Each element of the vector
9475 // is a memory VT combined with the number of times it is to be stored to,
9476 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9477 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9478
9479 while (StWidth.isNonZero()) {
9480 // Find the largest vector type we can store with.
9481 std::optional<EVT> NewVT =
9482 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9483 if (!NewVT)
9484 return false;
9485 MemVTs.push_back(Elt: {*NewVT, 0});
9486 TypeSize NewVTWidth = NewVT->getSizeInBits();
9487
9488 do {
9489 StWidth -= NewVTWidth;
9490 MemVTs.back().second++;
9491 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9492 }
9493
9494 for (const auto &Pair : MemVTs) {
9495 EVT NewVT = Pair.first;
9496 unsigned Count = Pair.second;
9497 TypeSize NewVTWidth = NewVT.getSizeInBits();
9498
9499 if (NewVT.isVector()) {
9500 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9501 do {
9502 Align NewAlign = ScaledOffset == 0
9503 ? ST->getBaseAlign()
9504 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9505 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9506 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9507 MMOFlags, Metadata);
9508 StChain.push_back(Elt: PartStore);
9509
9510 Idx += NumVTElts;
9511 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9512 ScaledOffset: &ScaledOffset);
9513 } while (--Count);
9514 } else {
9515 // Cast the vector to the scalar type we can store.
9516 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9517 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9518 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9519 // Readjust index position based on new vector type.
9520 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9521 do {
9522 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9523 SDValue PartStore =
9524 DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getBaseAlign(),
9525 MMOFlags, Metadata);
9526 StChain.push_back(Elt: PartStore);
9527
9528 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9529 } while (--Count);
9530 // Restore index back to be relative to the original widen element type.
9531 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9532 }
9533 }
9534
9535 return true;
9536}
9537
9538/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9539/// input vector must have the same element type as NVT.
9540/// FillWithZeroes specifies that the vector should be widened with zeroes.
9541SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9542 bool FillWithZeroes) {
9543 // Note that InOp might have been widened so it might already have
9544 // the right width or it might need be narrowed.
9545 EVT InVT = InOp.getValueType();
9546 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9547 "input and widen element type must match");
9548 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9549 "cannot modify scalable vectors in this way");
9550 SDLoc dl(InOp);
9551
9552 // Check if InOp already has the right width.
9553 if (InVT == NVT)
9554 return InOp;
9555
9556 ElementCount InEC = InVT.getVectorElementCount();
9557 ElementCount WidenEC = NVT.getVectorElementCount();
9558 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9559 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9560 SmallVector<SDValue, 16> Ops(NumConcat);
9561 SDValue FillVal =
9562 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9563 Ops[0] = InOp;
9564 for (unsigned i = 1; i != NumConcat; ++i)
9565 Ops[i] = FillVal;
9566
9567 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9568 }
9569
9570 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9571 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9572
9573 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9574 // Split the input into the largest equal-sized scalable subvectors.
9575 unsigned InNumElts = InVT.getVectorMinNumElements();
9576 unsigned NewNumElts = NVT.getVectorMinNumElements();
9577 unsigned CommonFactor = std::gcd(m: InNumElts, n: NewNumElts);
9578 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
9579 EC: ElementCount::getScalable(MinVal: CommonFactor));
9580
9581 SmallVector<SDValue, 16> Ops;
9582 unsigned NumCopiedParts = std::min(a: InNumElts, b: NewNumElts) / CommonFactor;
9583 for (unsigned I = 0; I != NumCopiedParts; ++I)
9584 Ops.push_back(
9585 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: I * CommonFactor));
9586
9587 unsigned NumResultParts = NewNumElts / CommonFactor;
9588 if (NumResultParts > NumCopiedParts) {
9589 SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: PartVT)
9590 : DAG.getPOISON(VT: PartVT);
9591 Ops.append(NumInputs: NumResultParts - NumCopiedParts, Elt: FillVal);
9592 }
9593
9594 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9595 }
9596
9597 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9598 "Scalable vectors should have been handled already.");
9599
9600 unsigned InNumElts = InEC.getFixedValue();
9601 unsigned WidenNumElts = WidenEC.getFixedValue();
9602
9603 // Fall back to extract and build (+ mask, if padding with zeros).
9604 SmallVector<SDValue, 16> Ops(WidenNumElts);
9605 EVT EltVT = NVT.getVectorElementType();
9606 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9607 unsigned Idx;
9608 for (Idx = 0; Idx < MinNumElts; ++Idx)
9609 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9610
9611 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9612 for (; Idx < WidenNumElts; ++Idx)
9613 Ops[Idx] = UndefVal;
9614
9615 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9616 if (!FillWithZeroes)
9617 return Widened;
9618
9619 assert(NVT.isInteger() &&
9620 "We expect to never want to FillWithZeroes for non-integral types.");
9621
9622 SmallVector<SDValue, 16> MaskOps;
9623 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9624 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9625
9626 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9627 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9628}
9629