1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
67 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
68 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
69 case ISD::CONVERT_FROM_ARBITRARY_FP:
70 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
71 break;
72 case ISD::CONVERT_TO_ARBITRARY_FP:
73 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
74 break;
75 case ISD::AssertZext:
76 case ISD::AssertSext:
77 case ISD::FPOWI:
78 case ISD::AssertNoFPClass:
79 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
80 break;
81 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
82 case ISD::ATOMIC_LOAD:
83 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
84 break;
85 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
86 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
87 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
88 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
89 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
90 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
91 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
92 case ISD::POISON:
93 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
94 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
95 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
96 case ISD::ANY_EXTEND_VECTOR_INREG:
97 case ISD::SIGN_EXTEND_VECTOR_INREG:
98 case ISD::ZERO_EXTEND_VECTOR_INREG:
99 R = ScalarizeVecRes_VecInregOp(N);
100 break;
101 case ISD::ABS:
102 case ISD::ABS_MIN_POISON:
103 case ISD::ANY_EXTEND:
104 case ISD::BITREVERSE:
105 case ISD::BSWAP:
106 case ISD::CTLZ:
107 case ISD::CTLZ_ZERO_POISON:
108 case ISD::CTPOP:
109 case ISD::CTTZ:
110 case ISD::CTTZ_ZERO_POISON:
111 case ISD::FABS:
112 case ISD::FACOS:
113 case ISD::FASIN:
114 case ISD::FATAN:
115 case ISD::FCEIL:
116 case ISD::FCOS:
117 case ISD::FCOSH:
118 case ISD::FEXP:
119 case ISD::FEXP2:
120 case ISD::FEXP10:
121 case ISD::FFLOOR:
122 case ISD::FLOG:
123 case ISD::FLOG10:
124 case ISD::FLOG2:
125 case ISD::FNEARBYINT:
126 case ISD::FNEG:
127 case ISD::FREEZE:
128 case ISD::ARITH_FENCE:
129 case ISD::FP_EXTEND:
130 case ISD::FP_TO_SINT:
131 case ISD::FP_TO_UINT:
132 case ISD::FRINT:
133 case ISD::LRINT:
134 case ISD::LLRINT:
135 case ISD::FROUND:
136 case ISD::FROUNDEVEN:
137 case ISD::LROUND:
138 case ISD::LLROUND:
139 case ISD::FSIN:
140 case ISD::FSINH:
141 case ISD::FSQRT:
142 case ISD::FTAN:
143 case ISD::FTANH:
144 case ISD::FTRUNC:
145 case ISD::SIGN_EXTEND:
146 case ISD::SINT_TO_FP:
147 case ISD::TRUNCATE:
148 case ISD::UINT_TO_FP:
149 case ISD::ZERO_EXTEND:
150 case ISD::FCANONICALIZE:
151 R = ScalarizeVecRes_UnaryOp(N);
152 break;
153 case ISD::ADDRSPACECAST:
154 R = ScalarizeVecRes_ADDRSPACECAST(N);
155 break;
156 case ISD::FMODF:
157 case ISD::FFREXP:
158 case ISD::FSINCOS:
159 case ISD::FSINCOSPI:
160 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
161 break;
162 case ISD::ADD:
163 case ISD::AND:
164 case ISD::AVGCEILS:
165 case ISD::AVGCEILU:
166 case ISD::AVGFLOORS:
167 case ISD::AVGFLOORU:
168 case ISD::FADD:
169 case ISD::FCOPYSIGN:
170 case ISD::FDIV:
171 case ISD::FMUL:
172 case ISD::FMINNUM:
173 case ISD::FMAXNUM:
174 case ISD::FMINNUM_IEEE:
175 case ISD::FMAXNUM_IEEE:
176 case ISD::FMINIMUM:
177 case ISD::FMAXIMUM:
178 case ISD::FMINIMUMNUM:
179 case ISD::FMAXIMUMNUM:
180 case ISD::FLDEXP:
181 case ISD::ABDS:
182 case ISD::ABDU:
183 case ISD::SMIN:
184 case ISD::SMAX:
185 case ISD::UMIN:
186 case ISD::UMAX:
187
188 case ISD::SADDSAT:
189 case ISD::UADDSAT:
190 case ISD::SSUBSAT:
191 case ISD::USUBSAT:
192 case ISD::SSHLSAT:
193 case ISD::USHLSAT:
194
195 case ISD::FPOW:
196 case ISD::FATAN2:
197 case ISD::FREM:
198 case ISD::FSUB:
199 case ISD::MUL:
200 case ISD::MULHS:
201 case ISD::MULHU:
202 case ISD::OR:
203 case ISD::SDIV:
204 case ISD::SREM:
205 case ISD::SUB:
206 case ISD::UDIV:
207 case ISD::UREM:
208 case ISD::XOR:
209 case ISD::SHL:
210 case ISD::SRA:
211 case ISD::SRL:
212 case ISD::ROTL:
213 case ISD::ROTR:
214 case ISD::CLMUL:
215 case ISD::CLMULR:
216 case ISD::CLMULH:
217 case ISD::PEXT:
218 case ISD::PDEP:
219 R = ScalarizeVecRes_BinOp(N);
220 break;
221
222 case ISD::MASKED_UDIV:
223 case ISD::MASKED_SDIV:
224 case ISD::MASKED_UREM:
225 case ISD::MASKED_SREM:
226 R = ScalarizeVecRes_MaskedBinOp(N);
227 break;
228
229 case ISD::SCMP:
230 case ISD::UCMP:
231 R = ScalarizeVecRes_CMP(N);
232 break;
233
234 case ISD::FMA:
235 case ISD::FSHL:
236 case ISD::FSHR:
237 R = ScalarizeVecRes_TernaryOp(N);
238 break;
239
240#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
241 case ISD::STRICT_##DAGN:
242#include "llvm/IR/ConstrainedOps.def"
243 R = ScalarizeVecRes_StrictFPOp(N);
244 break;
245
246 case ISD::FP_TO_UINT_SAT:
247 case ISD::FP_TO_SINT_SAT:
248 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
249 break;
250
251 case ISD::UADDO:
252 case ISD::SADDO:
253 case ISD::USUBO:
254 case ISD::SSUBO:
255 case ISD::UMULO:
256 case ISD::SMULO:
257 R = ScalarizeVecRes_OverflowOp(N, ResNo);
258 break;
259 case ISD::SMULFIX:
260 case ISD::SMULFIXSAT:
261 case ISD::UMULFIX:
262 case ISD::UMULFIXSAT:
263 case ISD::SDIVFIX:
264 case ISD::SDIVFIXSAT:
265 case ISD::UDIVFIX:
266 case ISD::UDIVFIXSAT:
267 R = ScalarizeVecRes_FIX(N);
268 break;
269 }
270
271 // If R is null, the sub-method took care of registering the result.
272 if (R.getNode())
273 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
274}
275
276SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
277 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
278 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
279 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
280 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
281}
282
283SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
284 SDLoc DL(N);
285 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
286 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
287 SDValue Mask = N->getOperand(Num: 2);
288 EVT MaskVT = Mask.getValueType();
289 // The vselect result and input vectors need scalarizing, but it's
290 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
291 // See the similar logic in ScalarizeVecRes_SETCC.
292 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
293 Mask = GetScalarizedVector(Op: Mask);
294 else
295 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
296 // Vectors may have a different boolean contents to scalars, so truncate to i1
297 // and let type legalization promote appropriately.
298 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
299 // Masked binary ops don't have UB on disabled lanes but produce poison, so
300 // use 1 as the divisor to avoid division by zero and overflow.
301 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
302 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
303 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
304 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
305}
306
307SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
308 SDLoc DL(N);
309
310 SDValue LHS = N->getOperand(Num: 0);
311 SDValue RHS = N->getOperand(Num: 1);
312 if (getTypeAction(VT: LHS.getValueType()) ==
313 TargetLowering::TypeScalarizeVector) {
314 LHS = GetScalarizedVector(Op: LHS);
315 RHS = GetScalarizedVector(Op: RHS);
316 } else {
317 EVT VT = LHS.getValueType().getVectorElementType();
318 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
319 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
320 }
321
322 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
323 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
324}
325
326SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
327 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
328 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
329 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
330 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
331 N3: Op2, Flags: N->getFlags());
332}
333
334SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
335 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
336 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
337 SDValue Op2 = N->getOperand(Num: 2);
338 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
339 N3: Op2, Flags: N->getFlags());
340}
341
342SDValue
343DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
344 unsigned ResNo) {
345 assert(N->getValueType(0).getVectorNumElements() == 1 &&
346 "Unexpected vector type!");
347 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
348
349 EVT VT0 = N->getValueType(ResNo: 0);
350 EVT VT1 = N->getValueType(ResNo: 1);
351 SDLoc dl(N);
352
353 SDNode *ScalarNode =
354 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
355 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
356 .getNode();
357
358 // Replace the other vector result not being explicitly scalarized here.
359 unsigned OtherNo = 1 - ResNo;
360 EVT OtherVT = N->getValueType(ResNo: OtherNo);
361 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
362 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
363 } else {
364 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
365 Operand: SDValue(ScalarNode, OtherNo));
366 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
367 }
368
369 return SDValue(ScalarNode, ResNo);
370}
371
372SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
373 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
374 unsigned NumOpers = N->getNumOperands();
375 SDValue Chain = N->getOperand(Num: 0);
376 EVT ValueVTs[] = {VT, MVT::Other};
377 SDLoc dl(N);
378
379 SmallVector<SDValue, 4> Opers(NumOpers);
380
381 // The Chain is the first operand.
382 Opers[0] = Chain;
383
384 // Now process the remaining operands.
385 for (unsigned i = 1; i < NumOpers; ++i) {
386 SDValue Oper = N->getOperand(Num: i);
387 EVT OperVT = Oper.getValueType();
388
389 if (OperVT.isVector()) {
390 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
391 Oper = GetScalarizedVector(Op: Oper);
392 else
393 Oper =
394 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
395 }
396
397 Opers[i] = Oper;
398 }
399
400 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
401 Ops: Opers, Flags: N->getFlags());
402
403 // Legalize the chain result - switch anything that used the old chain to
404 // use the new one.
405 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
406 return Result;
407}
408
409SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
410 unsigned ResNo) {
411 SDLoc DL(N);
412 EVT ResVT = N->getValueType(ResNo: 0);
413 EVT OvVT = N->getValueType(ResNo: 1);
414
415 SDValue ScalarLHS, ScalarRHS;
416 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
417 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
418 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
419 } else {
420 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
421 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
422 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
423 ScalarLHS = ElemsLHS[0];
424 ScalarRHS = ElemsRHS[0];
425 }
426
427 SDVTList ScalarVTs = DAG.getVTList(
428 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
429 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
430 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
431 .getNode();
432
433 // Replace the other vector result not being explicitly scalarized here.
434 unsigned OtherNo = 1 - ResNo;
435 EVT OtherVT = N->getValueType(ResNo: OtherNo);
436 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
437 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
438 } else {
439 SDValue OtherVal = DAG.getNode(
440 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
441 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
442 }
443
444 return SDValue(ScalarNode, ResNo);
445}
446
447SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
448 unsigned ResNo) {
449 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
450 return GetScalarizedVector(Op);
451}
452
453SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
454 SDLoc DL(N);
455 // Reuse the expansion (which should scalarize).
456 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
457 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
458 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
459 N2: DAG.getVectorIdxConstant(Val: 0, DL));
460}
461
462SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
463 SDValue Op = N->getOperand(Num: 0);
464 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
465 Op = GetScalarizedVector(Op);
466 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
467 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
468 VT: NewVT, Operand: Op);
469}
470
471SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
472 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
473 SDValue InOp = N->getOperand(Num: 0);
474 // The BUILD_VECTOR operands may be of wider element types and
475 // we may need to truncate them back to the requested return type.
476 if (EltVT.isInteger())
477 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
478 return InOp;
479}
480
481SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
482 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
483 VT: N->getValueType(ResNo: 0).getVectorElementType(),
484 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
485}
486
487SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
488 SDLoc DL(N);
489 SDValue Op = N->getOperand(Num: 0);
490 EVT OpVT = Op.getValueType();
491 // The result needs scalarizing, but it's not a given that the source does.
492 // See similar logic in ScalarizeVecRes_UnaryOp.
493 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
494 Op = GetScalarizedVector(Op);
495 } else {
496 EVT VT = OpVT.getVectorElementType();
497 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
498 }
499 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
500 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
501 N2: N->getOperand(Num: 1));
502}
503
504SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
505 SDLoc DL(N);
506 SDValue Op = N->getOperand(Num: 0);
507 EVT OpVT = Op.getValueType();
508 // The result needs scalarizing, but it's not a given that the source does.
509 // See similar logic in ScalarizeVecRes_UnaryOp.
510 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
511 Op = GetScalarizedVector(Op);
512 } else {
513 EVT VT = OpVT.getVectorElementType();
514 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
515 }
516 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
517 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
518 N2: N->getOperand(Num: 1));
519}
520
521SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
522 SDLoc DL(N);
523 SDValue Op = N->getOperand(Num: 0);
524 EVT OpVT = Op.getValueType();
525 // The result needs scalarizing, but it's not a given that the source does.
526 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
527 Op = GetScalarizedVector(Op);
528 } else {
529 EVT VT = OpVT.getVectorElementType();
530 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
531 }
532 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
533 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
534 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
535}
536
537SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
538 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
539 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
540 N2: N->getOperand(Num: 1));
541}
542
543SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
544 // The value to insert may have a wider type than the vector element type,
545 // so be sure to truncate it to the element type if necessary.
546 SDValue Op = N->getOperand(Num: 1);
547 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
548 if (Op.getValueType() != EltVT)
549 // FIXME: Can this happen for floating point types?
550 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
551 return Op;
552}
553
554SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
555 SDValue Result = DAG.getAtomicLoad(
556 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
557 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
558 MMO: N->getMemOperand());
559
560 // Legalize the chain result - switch anything that used the old chain to
561 // use the new one.
562 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
563 return Result;
564}
565
566SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
567 assert(N->isUnindexed() && "Indexed vector load?");
568
569 SDValue Result = DAG.getLoad(
570 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
571 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
572 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
573 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
574 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
575
576 // Legalize the chain result - switch anything that used the old chain to
577 // use the new one.
578 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
579 return Result;
580}
581
582SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
583 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
584 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
585 SDValue Op = N->getOperand(Num: 0);
586 EVT OpVT = Op.getValueType();
587 SDLoc DL(N);
588 // The result needs scalarizing, but it's not a given that the source does.
589 // This is a workaround for targets where it's impossible to scalarize the
590 // result of a conversion, because the source type is legal.
591 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
592 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
593 // legal and was not scalarized.
594 // See the similar logic in ScalarizeVecRes_SETCC
595 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
596 Op = GetScalarizedVector(Op);
597 } else {
598 EVT VT = OpVT.getVectorElementType();
599 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
600 }
601 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
602}
603
604SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
605 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
606 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
607 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
608 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
609 N1: LHS, N2: DAG.getValueType(ExtVT));
610}
611
612SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
613 SDLoc DL(N);
614 SDValue Op = N->getOperand(Num: 0);
615
616 EVT OpVT = Op.getValueType();
617 EVT OpEltVT = OpVT.getVectorElementType();
618 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
619
620 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
621 Op = GetScalarizedVector(Op);
622 } else {
623 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
624 }
625
626 switch (N->getOpcode()) {
627 case ISD::ANY_EXTEND_VECTOR_INREG:
628 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
629 case ISD::SIGN_EXTEND_VECTOR_INREG:
630 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
631 case ISD::ZERO_EXTEND_VECTOR_INREG:
632 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
633 }
634
635 llvm_unreachable("Illegal extend_vector_inreg opcode");
636}
637
638SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
639 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
640 SDValue Op = N->getOperand(Num: 0);
641 EVT OpVT = Op.getValueType();
642 SDLoc DL(N);
643 // The result needs scalarizing, but it's not a given that the source does.
644 // This is a workaround for targets where it's impossible to scalarize the
645 // result of a conversion, because the source type is legal.
646 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
647 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
648 // legal and was not scalarized.
649 // See the similar logic in ScalarizeVecRes_SETCC
650 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
651 Op = GetScalarizedVector(Op);
652 } else {
653 EVT VT = OpVT.getVectorElementType();
654 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
655 }
656 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
657 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
658 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
659 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
660}
661
662SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
663 // If the operand is wider than the vector element type then it is implicitly
664 // truncated. Make that explicit here.
665 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
666 SDValue InOp = N->getOperand(Num: 0);
667 if (InOp.getValueType() != EltVT)
668 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
669 return InOp;
670}
671
672SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
673 SDValue Cond = N->getOperand(Num: 0);
674 EVT OpVT = Cond.getValueType();
675 SDLoc DL(N);
676 // The vselect result and true/value operands needs scalarizing, but it's
677 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
678 // See the similar logic in ScalarizeVecRes_SETCC
679 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
680 Cond = GetScalarizedVector(Op: Cond);
681 } else {
682 EVT VT = OpVT.getVectorElementType();
683 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
684 }
685
686 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
687 TargetLowering::BooleanContent ScalarBool =
688 TLI.getBooleanContents(isVec: false, isFloat: false);
689 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
690
691 // If integer and float booleans have different contents then we can't
692 // reliably optimize in all cases. There is a full explanation for this in
693 // DAGCombiner::visitSELECT() where the same issue affects folding
694 // (select C, 0, 1) to (xor C, 1).
695 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
696 TLI.getBooleanContents(isVec: false, isFloat: true)) {
697 // At least try the common case where the boolean is generated by a
698 // comparison.
699 if (Cond->getOpcode() == ISD::SETCC) {
700 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
701 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
702 VecBool = TLI.getBooleanContents(Type: OpVT);
703 } else
704 ScalarBool = TargetLowering::UndefinedBooleanContent;
705 }
706
707 EVT CondVT = Cond.getValueType();
708 if (ScalarBool != VecBool) {
709 switch (ScalarBool) {
710 case TargetLowering::UndefinedBooleanContent:
711 break;
712 case TargetLowering::ZeroOrOneBooleanContent:
713 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
714 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
715 // Vector read from all ones, scalar expects a single 1 so mask.
716 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
717 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
718 break;
719 case TargetLowering::ZeroOrNegativeOneBooleanContent:
720 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
721 VecBool == TargetLowering::ZeroOrOneBooleanContent);
722 // Vector reads from a one, scalar from all ones so sign extend.
723 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
724 N1: Cond, N2: DAG.getValueType(MVT::i1));
725 break;
726 }
727 }
728
729 // Truncate the condition if needed
730 auto BoolVT = getSetCCResultType(VT: CondVT);
731 if (BoolVT.bitsLT(VT: CondVT))
732 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
733
734 return DAG.getSelect(DL: SDLoc(N),
735 VT: LHS.getValueType(), Cond, LHS,
736 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
737}
738
739SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
740 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
741 return DAG.getSelect(DL: SDLoc(N),
742 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
743 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
744}
745
746SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
747 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
748 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
749 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
750 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
751 N5: N->getOperand(Num: 4));
752}
753
754SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
755 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
756}
757
758SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
759 // Figure out if the scalar is the LHS or RHS and return it.
760 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
761 if (Arg.isUndef())
762 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
763 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
764 return GetScalarizedVector(Op: N->getOperand(Num: Op));
765}
766
767SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
768 SDValue Src = N->getOperand(Num: 0);
769 EVT SrcVT = Src.getValueType();
770 SDLoc dl(N);
771
772 // Handle case where result is scalarized but operand is not
773 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
774 Src = GetScalarizedVector(Op: Src);
775 else
776 Src = DAG.getNode(
777 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
778 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
779
780 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
781 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
782}
783
784SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
785 assert(N->getValueType(0).isVector() &&
786 N->getOperand(0).getValueType().isVector() &&
787 "Operand types must be vectors");
788 SDValue LHS = N->getOperand(Num: 0);
789 SDValue RHS = N->getOperand(Num: 1);
790 EVT OpVT = LHS.getValueType();
791 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
792 SDLoc DL(N);
793
794 // The result needs scalarizing, but it's not a given that the source does.
795 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
796 LHS = GetScalarizedVector(Op: LHS);
797 RHS = GetScalarizedVector(Op: RHS);
798 } else {
799 EVT VT = OpVT.getVectorElementType();
800 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
801 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
802 }
803
804 // Turn it into a scalar SETCC.
805 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
806 N3: N->getOperand(Num: 2));
807 // Vectors may have a different boolean contents to scalars. Promote the
808 // value appropriately.
809 ISD::NodeType ExtendCode =
810 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
811 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
812}
813
814SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
815 SDLoc DL(N);
816 SDValue Arg = N->getOperand(Num: 0);
817 SDValue Test = N->getOperand(Num: 1);
818 EVT ArgVT = Arg.getValueType();
819 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
820
821 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
822 Arg = GetScalarizedVector(Op: Arg);
823 } else {
824 EVT VT = ArgVT.getVectorElementType();
825 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
826 }
827
828 SDValue Res =
829 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
830 // Vectors may have a different boolean contents to scalars. Promote the
831 // value appropriately.
832 ISD::NodeType ExtendCode =
833 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
834 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
835}
836
837//===----------------------------------------------------------------------===//
838// Operand Vector Scalarization <1 x ty> -> ty.
839//===----------------------------------------------------------------------===//
840
841bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
842 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
843 N->dump(&DAG));
844 SDValue Res = SDValue();
845
846 // See if the target wants to custom scalarize this node.
847 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
848 return false;
849
850 switch (N->getOpcode()) {
851 default:
852#ifndef NDEBUG
853 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
854 N->dump(&DAG);
855 dbgs() << "\n";
856#endif
857 report_fatal_error(reason: "Do not know how to scalarize this operator's "
858 "operand!\n");
859 case ISD::BITCAST:
860 Res = ScalarizeVecOp_BITCAST(N);
861 break;
862 case ISD::FAKE_USE:
863 Res = ScalarizeVecOp_FAKE_USE(N);
864 break;
865 case ISD::ANY_EXTEND:
866 case ISD::ZERO_EXTEND:
867 case ISD::SIGN_EXTEND:
868 case ISD::TRUNCATE:
869 case ISD::FP_TO_SINT:
870 case ISD::FP_TO_UINT:
871 case ISD::SINT_TO_FP:
872 case ISD::UINT_TO_FP:
873 case ISD::LROUND:
874 case ISD::LLROUND:
875 case ISD::LRINT:
876 case ISD::LLRINT:
877 Res = ScalarizeVecOp_UnaryOp(N);
878 break;
879 case ISD::FP_TO_SINT_SAT:
880 case ISD::FP_TO_UINT_SAT:
881 case ISD::CONVERT_FROM_ARBITRARY_FP:
882 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
883 break;
884 case ISD::CONVERT_TO_ARBITRARY_FP: {
885 assert(N->getValueType(0).getVectorNumElements() == 1 &&
886 "Unexpected vector type!");
887 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
888 SDValue Op = DAG.getNode(
889 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
890 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
891 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
892 break;
893 }
894 case ISD::STRICT_SINT_TO_FP:
895 case ISD::STRICT_UINT_TO_FP:
896 case ISD::STRICT_FP_TO_SINT:
897 case ISD::STRICT_FP_TO_UINT:
898 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
899 break;
900 case ISD::CONCAT_VECTORS:
901 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
902 break;
903 case ISD::INSERT_SUBVECTOR:
904 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
905 break;
906 case ISD::EXTRACT_VECTOR_ELT:
907 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
908 break;
909 case ISD::VSELECT:
910 Res = ScalarizeVecOp_VSELECT(N);
911 break;
912 case ISD::SETCC:
913 Res = ScalarizeVecOp_VSETCC(N);
914 break;
915 case ISD::STRICT_FSETCC:
916 case ISD::STRICT_FSETCCS:
917 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
918 break;
919 case ISD::STORE:
920 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
921 break;
922 case ISD::ATOMIC_STORE:
923 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
924 break;
925 case ISD::STRICT_FP_ROUND:
926 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
927 break;
928 case ISD::FP_ROUND:
929 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
930 break;
931 case ISD::STRICT_FP_EXTEND:
932 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
933 break;
934 case ISD::FP_EXTEND:
935 Res = ScalarizeVecOp_FP_EXTEND(N);
936 break;
937 case ISD::VECREDUCE_FADD:
938 case ISD::VECREDUCE_FMUL:
939 case ISD::VECREDUCE_ADD:
940 case ISD::VECREDUCE_MUL:
941 case ISD::VECREDUCE_AND:
942 case ISD::VECREDUCE_OR:
943 case ISD::VECREDUCE_XOR:
944 case ISD::VECREDUCE_SMAX:
945 case ISD::VECREDUCE_SMIN:
946 case ISD::VECREDUCE_UMAX:
947 case ISD::VECREDUCE_UMIN:
948 case ISD::VECREDUCE_FMAX:
949 case ISD::VECREDUCE_FMIN:
950 case ISD::VECREDUCE_FMAXIMUM:
951 case ISD::VECREDUCE_FMINIMUM:
952 Res = ScalarizeVecOp_VECREDUCE(N);
953 break;
954 case ISD::VECREDUCE_SEQ_FADD:
955 case ISD::VECREDUCE_SEQ_FMUL:
956 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
957 break;
958 case ISD::SCMP:
959 case ISD::UCMP:
960 Res = ScalarizeVecOp_CMP(N);
961 break;
962 case ISD::VECTOR_FIND_LAST_ACTIVE:
963 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
964 break;
965 case ISD::CTTZ_ELTS:
966 case ISD::CTTZ_ELTS_ZERO_POISON:
967 Res = ScalarizeVecOp_CTTZ_ELTS(N);
968 break;
969 case ISD::MASKED_UDIV:
970 case ISD::MASKED_SDIV:
971 case ISD::MASKED_UREM:
972 case ISD::MASKED_SREM:
973 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
974 break;
975 }
976
977 // If the result is null, the sub-method took care of registering results etc.
978 if (!Res.getNode()) return false;
979
980 // If the result is N, the sub-method updated N in place. Tell the legalizer
981 // core about this.
982 if (Res.getNode() == N)
983 return true;
984
985 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
986 "Invalid operand expansion");
987
988 ReplaceValueWith(From: SDValue(N, 0), To: Res);
989 return false;
990}
991
992/// If the value to convert is a vector that needs to be scalarized, it must be
993/// <1 x ty>. Convert the element instead.
994SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
995 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
996 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
997 VT: N->getValueType(ResNo: 0), Operand: Elt);
998}
999
1000// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1001SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1002 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1003 "Fake Use: Unexpected vector type!");
1004 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1005 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1006}
1007
1008/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1009/// Do the operation on the element instead.
1010SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1011 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1012 "Unexpected vector type!");
1013 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1014 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1015 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1016 // Revectorize the result so the types line up with what the uses of this
1017 // expression expect.
1018 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1019}
1020
1021/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1022/// typesize).
1023SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1024 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1025 "Unexpected vector type!");
1026 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1027 SDValue Op =
1028 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1029 N1: Elt, N2: N->getOperand(Num: 1));
1030 // Revectorize the result so the types line up with what the uses of this
1031 // expression expect.
1032 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1033}
1034
1035/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1036/// Do the strict FP operation on the element instead.
1037SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1038 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1039 "Unexpected vector type!");
1040 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1041 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1042 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1043 Ops: { N->getOperand(Num: 0), Elt });
1044 // Legalize the chain result - switch anything that used the old chain to
1045 // use the new one.
1046 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1047 // Revectorize the result so the types line up with what the uses of this
1048 // expression expect.
1049 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1050
1051 // Do our own replacement and return SDValue() to tell the caller that we
1052 // handled all replacements since caller can only handle a single result.
1053 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1054 return SDValue();
1055}
1056
1057/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1058SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1059 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1060 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1061 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1062 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1063}
1064
1065/// The inserted subvector is to be scalarized - use insert vector element
1066/// instead.
1067SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1068 unsigned OpNo) {
1069 // We should not be attempting to scalarize the containing vector
1070 assert(OpNo == 1);
1071 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1072 SDValue ContainingVec = N->getOperand(Num: 0);
1073 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1074 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1075 N3: N->getOperand(Num: 2));
1076}
1077
1078/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1079/// so just return the element, ignoring the index.
1080SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1081 EVT VT = N->getValueType(ResNo: 0);
1082 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1083 if (Res.getValueType() != VT)
1084 Res = VT.isFloatingPoint()
1085 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1086 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1087 return Res;
1088}
1089
1090/// If the input condition is a vector that needs to be scalarized, it must be
1091/// <1 x i1>, so just convert to a normal ISD::SELECT
1092/// (still with vector output type since that was acceptable if we got here).
1093SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1094 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1095 EVT VT = N->getValueType(ResNo: 0);
1096
1097 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1098 N3: N->getOperand(Num: 2));
1099}
1100
1101/// If the operand is a vector that needs to be scalarized then the
1102/// result must be v1i1, so just convert to a scalar SETCC and wrap
1103/// with a scalar_to_vector since the res type is legal if we got here
1104SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1105 assert(N->getValueType(0).isVector() &&
1106 N->getOperand(0).getValueType().isVector() &&
1107 "Operand types must be vectors");
1108 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1109
1110 EVT VT = N->getValueType(ResNo: 0);
1111 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1112 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1113
1114 EVT OpVT = N->getOperand(Num: 0).getValueType();
1115 EVT NVT = VT.getVectorElementType();
1116 SDLoc DL(N);
1117 // Turn it into a scalar SETCC.
1118 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1119 N3: N->getOperand(Num: 2));
1120
1121 // Vectors may have a different boolean contents to scalars. Promote the
1122 // value appropriately.
1123 ISD::NodeType ExtendCode =
1124 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1125
1126 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1127
1128 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1129}
1130
1131// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1132SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1133 unsigned OpNo) {
1134 assert(OpNo == 1 && "Wrong operand for scalarization!");
1135 assert(N->getValueType(0).isVector() &&
1136 N->getOperand(1).getValueType().isVector() &&
1137 "Operand types must be vectors");
1138 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1139
1140 EVT VT = N->getValueType(ResNo: 0);
1141 SDValue Ch = N->getOperand(Num: 0);
1142 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1143 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1144 SDValue CC = N->getOperand(Num: 3);
1145
1146 EVT OpVT = N->getOperand(Num: 1).getValueType();
1147 EVT NVT = VT.getVectorElementType();
1148 SDLoc DL(N);
1149 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1150 Ops: {Ch, LHS, RHS, CC});
1151
1152 // Legalize the chain result - switch anything that used the old chain to
1153 // use the new one.
1154 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1155
1156 ISD::NodeType ExtendCode =
1157 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1158
1159 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1160 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1161
1162 // Do our own replacement and return SDValue() to tell the caller that we
1163 // handled all replacements since caller can only handle a single result.
1164 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1165 return SDValue();
1166}
1167
1168/// If the value to store is a vector that needs to be scalarized, it must be
1169/// <1 x ty>. Just store the element.
1170SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1171 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1172 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1173 SDLoc dl(N);
1174
1175 if (N->isTruncatingStore())
1176 return DAG.getTruncStore(
1177 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1178 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1179 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1180 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1181
1182 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1183 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1184 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1185}
1186
1187/// If the value to store is a vector that needs to be scalarized, it must be
1188/// <1 x ty>. Just store the element.
1189SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1190 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1191 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1192 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1193 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1194}
1195
1196/// If the value to round is a vector that needs to be scalarized, it must be
1197/// <1 x ty>. Convert the element instead.
1198SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1199 assert(OpNo == 0 && "Wrong operand for scalarization!");
1200 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1201 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1202 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1203 N2: N->getOperand(Num: 1));
1204 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1205}
1206
1207SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1208 unsigned OpNo) {
1209 assert(OpNo == 1 && "Wrong operand for scalarization!");
1210 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1211 SDValue Res =
1212 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1213 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1214 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1215 // Legalize the chain result - switch anything that used the old chain to
1216 // use the new one.
1217 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1218
1219 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1220
1221 // Do our own replacement and return SDValue() to tell the caller that we
1222 // handled all replacements since caller can only handle a single result.
1223 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1224 return SDValue();
1225}
1226
1227/// If the value to extend is a vector that needs to be scalarized, it must be
1228/// <1 x ty>. Convert the element instead.
1229SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1230 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1231 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1232 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1233 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1234}
1235
1236/// If the value to extend is a vector that needs to be scalarized, it must be
1237/// <1 x ty>. Convert the element instead.
1238SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1239 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1240 SDValue Res =
1241 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1242 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1243 Ops: {N->getOperand(Num: 0), Elt});
1244 // Legalize the chain result - switch anything that used the old chain to
1245 // use the new one.
1246 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1247
1248 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1249
1250 // Do our own replacement and return SDValue() to tell the caller that we
1251 // handled all replacements since caller can only handle a single result.
1252 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1253 return SDValue();
1254}
1255
1256SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1257 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1258 // Result type may be wider than element type.
1259 if (Res.getValueType() != N->getValueType(ResNo: 0))
1260 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1261 return Res;
1262}
1263
1264SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1265 SDValue AccOp = N->getOperand(Num: 0);
1266 SDValue VecOp = N->getOperand(Num: 1);
1267
1268 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1269
1270 SDValue Op = GetScalarizedVector(Op: VecOp);
1271 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1272 N1: AccOp, N2: Op, Flags: N->getFlags());
1273}
1274
1275SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1276 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1277 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1278
1279 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1280 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1281 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1282}
1283
1284SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1285 // Since there is no "none-active" result, the only valid return for <1 x ty>
1286 // is 0. Note: Since we check the high mask during splitting this is safe.
1287 // As e.g., a <2 x ty> operation would split to:
1288 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1289 // : `last_active(%lo_mask)`
1290 // Which then scalarizes to:
1291 // %mask[1] ? 1 : 0
1292 EVT VT = N->getValueType(ResNo: 0);
1293 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1294}
1295
1296SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1297 // The number of trailing zero elements is 1 if the element is 0, and 0
1298 // otherwise.
1299 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1300 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1301 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1302 SDValue SetCC =
1303 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1304 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1305 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1306}
1307
1308SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1309 assert(OpNo == 2 && "Can only scalarize mask operand");
1310 SDLoc DL(N);
1311 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1312 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1313 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1314 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1315 // Vectors may have a different boolean contents to scalars, so truncate to i1
1316 // and let type legalization promote appropriately.
1317 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1318 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1319 // use 1 as the divisor to avoid division by zero and overflow.
1320 SDValue BinOp =
1321 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1322 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1323 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1324}
1325
1326//===----------------------------------------------------------------------===//
1327// Result Vector Splitting
1328//===----------------------------------------------------------------------===//
1329
1330/// This method is called when the specified result of the specified node is
1331/// found to need vector splitting. At this point, the node may also have
1332/// invalid operands or may have other results that need legalization, we just
1333/// know that (at least) one result needs vector splitting.
1334void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1335 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1336 SDValue Lo, Hi;
1337
1338 // See if the target wants to custom expand this node.
1339 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1340 return;
1341
1342 switch (N->getOpcode()) {
1343 default:
1344#ifndef NDEBUG
1345 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1346 N->dump(&DAG);
1347 dbgs() << "\n";
1348#endif
1349 report_fatal_error(reason: "Do not know how to split the result of this "
1350 "operator!\n");
1351
1352 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1353 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1354 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1355 break;
1356 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1357 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1358 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1359 case ISD::VSELECT:
1360 case ISD::SELECT:
1361 case ISD::VP_MERGE:
1362 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1363 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1364 case ISD::POISON:
1365 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1366 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1367 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1368 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1369 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1370 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1371 case ISD::FPOWI:
1372 case ISD::FLDEXP:
1373 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1374 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1375 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1376 case ISD::SPLAT_VECTOR:
1377 case ISD::SCALAR_TO_VECTOR:
1378 SplitVecRes_ScalarOp(N, Lo, Hi);
1379 break;
1380 case ISD::STEP_VECTOR:
1381 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1382 break;
1383 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1384 case ISD::ATOMIC_LOAD:
1385 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1386 break;
1387 case ISD::LOAD:
1388 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1389 break;
1390 case ISD::VP_LOAD:
1391 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1392 break;
1393 case ISD::VP_LOAD_FF:
1394 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1395 break;
1396 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1397 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1398 break;
1399 case ISD::MLOAD:
1400 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1401 break;
1402 case ISD::MGATHER:
1403 case ISD::VP_GATHER:
1404 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1405 break;
1406 case ISD::VECTOR_COMPRESS:
1407 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1408 break;
1409 case ISD::SETCC:
1410 case ISD::VP_SETCC:
1411 SplitVecRes_SETCC(N, Lo, Hi);
1412 break;
1413 case ISD::VECTOR_REVERSE:
1414 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1415 break;
1416 case ISD::VECTOR_SHUFFLE:
1417 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1418 break;
1419 case ISD::VECTOR_SPLICE_LEFT:
1420 case ISD::VECTOR_SPLICE_RIGHT:
1421 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1422 break;
1423 case ISD::VECTOR_DEINTERLEAVE:
1424 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1425 return;
1426 case ISD::VECTOR_INTERLEAVE:
1427 SplitVecRes_VECTOR_INTERLEAVE(N);
1428 return;
1429 case ISD::VAARG:
1430 SplitVecRes_VAARG(N, Lo, Hi);
1431 break;
1432
1433 case ISD::ANY_EXTEND_VECTOR_INREG:
1434 case ISD::SIGN_EXTEND_VECTOR_INREG:
1435 case ISD::ZERO_EXTEND_VECTOR_INREG:
1436 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1437 break;
1438
1439 case ISD::ABS:
1440 case ISD::ABS_MIN_POISON:
1441 case ISD::VP_ABS:
1442 case ISD::BITREVERSE:
1443 case ISD::VP_BITREVERSE:
1444 case ISD::BSWAP:
1445 case ISD::VP_BSWAP:
1446 case ISD::CTLZ:
1447 case ISD::VP_CTLZ:
1448 case ISD::CTTZ:
1449 case ISD::VP_CTTZ:
1450 case ISD::CTLZ_ZERO_POISON:
1451 case ISD::VP_CTLZ_ZERO_POISON:
1452 case ISD::CTTZ_ZERO_POISON:
1453 case ISD::VP_CTTZ_ZERO_POISON:
1454 case ISD::CTPOP:
1455 case ISD::VP_CTPOP:
1456 case ISD::FABS: case ISD::VP_FABS:
1457 case ISD::FACOS:
1458 case ISD::FASIN:
1459 case ISD::FATAN:
1460 case ISD::FCEIL:
1461 case ISD::VP_FCEIL:
1462 case ISD::FCOS:
1463 case ISD::FCOSH:
1464 case ISD::FEXP:
1465 case ISD::FEXP2:
1466 case ISD::FEXP10:
1467 case ISD::FFLOOR:
1468 case ISD::VP_FFLOOR:
1469 case ISD::FLOG:
1470 case ISD::FLOG10:
1471 case ISD::FLOG2:
1472 case ISD::FNEARBYINT:
1473 case ISD::VP_FNEARBYINT:
1474 case ISD::FNEG: case ISD::VP_FNEG:
1475 case ISD::FREEZE:
1476 case ISD::ARITH_FENCE:
1477 case ISD::FP_EXTEND:
1478 case ISD::VP_FP_EXTEND:
1479 case ISD::FP_ROUND:
1480 case ISD::VP_FP_ROUND:
1481 case ISD::FP_TO_SINT:
1482 case ISD::VP_FP_TO_SINT:
1483 case ISD::FP_TO_UINT:
1484 case ISD::VP_FP_TO_UINT:
1485 case ISD::FRINT:
1486 case ISD::VP_FRINT:
1487 case ISD::LRINT:
1488 case ISD::VP_LRINT:
1489 case ISD::LLRINT:
1490 case ISD::VP_LLRINT:
1491 case ISD::FROUND:
1492 case ISD::VP_FROUND:
1493 case ISD::FROUNDEVEN:
1494 case ISD::VP_FROUNDEVEN:
1495 case ISD::LROUND:
1496 case ISD::LLROUND:
1497 case ISD::FSIN:
1498 case ISD::FSINH:
1499 case ISD::FSQRT: case ISD::VP_SQRT:
1500 case ISD::FTAN:
1501 case ISD::FTANH:
1502 case ISD::FTRUNC:
1503 case ISD::VP_FROUNDTOZERO:
1504 case ISD::SINT_TO_FP:
1505 case ISD::VP_SINT_TO_FP:
1506 case ISD::TRUNCATE:
1507 case ISD::VP_TRUNCATE:
1508 case ISD::UINT_TO_FP:
1509 case ISD::VP_UINT_TO_FP:
1510 case ISD::FCANONICALIZE:
1511 case ISD::AssertNoFPClass:
1512 case ISD::CONVERT_FROM_ARBITRARY_FP:
1513 case ISD::CONVERT_TO_ARBITRARY_FP:
1514 SplitVecRes_UnaryOp(N, Lo, Hi);
1515 break;
1516 case ISD::ADDRSPACECAST:
1517 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1518 break;
1519 case ISD::FMODF:
1520 case ISD::FFREXP:
1521 case ISD::FSINCOS:
1522 case ISD::FSINCOSPI:
1523 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1524 break;
1525
1526 case ISD::ANY_EXTEND:
1527 case ISD::SIGN_EXTEND:
1528 case ISD::ZERO_EXTEND:
1529 case ISD::VP_SIGN_EXTEND:
1530 case ISD::VP_ZERO_EXTEND:
1531 SplitVecRes_ExtendOp(N, Lo, Hi);
1532 break;
1533
1534 case ISD::ADD: case ISD::VP_ADD:
1535 case ISD::SUB: case ISD::VP_SUB:
1536 case ISD::MUL: case ISD::VP_MUL:
1537 case ISD::CLMUL:
1538 case ISD::CLMULR:
1539 case ISD::CLMULH:
1540 case ISD::PEXT:
1541 case ISD::PDEP:
1542 case ISD::MULHS:
1543 case ISD::MULHU:
1544 case ISD::ABDS:
1545 case ISD::ABDU:
1546 case ISD::AVGCEILS:
1547 case ISD::AVGCEILU:
1548 case ISD::AVGFLOORS:
1549 case ISD::AVGFLOORU:
1550 case ISD::FADD: case ISD::VP_FADD:
1551 case ISD::FSUB: case ISD::VP_FSUB:
1552 case ISD::FMUL: case ISD::VP_FMUL:
1553 case ISD::FMINNUM:
1554 case ISD::FMINNUM_IEEE:
1555 case ISD::VP_FMINNUM:
1556 case ISD::FMAXNUM:
1557 case ISD::FMAXNUM_IEEE:
1558 case ISD::VP_FMAXNUM:
1559 case ISD::FMINIMUM:
1560 case ISD::VP_FMINIMUM:
1561 case ISD::FMAXIMUM:
1562 case ISD::VP_FMAXIMUM:
1563 case ISD::FMINIMUMNUM:
1564 case ISD::FMAXIMUMNUM:
1565 case ISD::SDIV: case ISD::VP_SDIV:
1566 case ISD::UDIV: case ISD::VP_UDIV:
1567 case ISD::FDIV: case ISD::VP_FDIV:
1568 case ISD::FPOW:
1569 case ISD::FATAN2:
1570 case ISD::AND: case ISD::VP_AND:
1571 case ISD::OR: case ISD::VP_OR:
1572 case ISD::XOR: case ISD::VP_XOR:
1573 case ISD::SHL: case ISD::VP_SHL:
1574 case ISD::SRA: case ISD::VP_SRA:
1575 case ISD::SRL: case ISD::VP_SRL:
1576 case ISD::UREM: case ISD::VP_UREM:
1577 case ISD::SREM: case ISD::VP_SREM:
1578 case ISD::FREM: case ISD::VP_FREM:
1579 case ISD::SMIN: case ISD::VP_SMIN:
1580 case ISD::SMAX: case ISD::VP_SMAX:
1581 case ISD::UMIN: case ISD::VP_UMIN:
1582 case ISD::UMAX: case ISD::VP_UMAX:
1583 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1584 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1585 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1586 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1587 case ISD::SSHLSAT:
1588 case ISD::USHLSAT:
1589 case ISD::ROTL:
1590 case ISD::ROTR:
1591 case ISD::VP_FCOPYSIGN:
1592 SplitVecRes_BinOp(N, Lo, Hi);
1593 break;
1594 case ISD::MASKED_UDIV:
1595 case ISD::MASKED_SDIV:
1596 case ISD::MASKED_UREM:
1597 case ISD::MASKED_SREM:
1598 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1599 break;
1600 case ISD::FMA: case ISD::VP_FMA:
1601 case ISD::FSHL:
1602 case ISD::VP_FSHL:
1603 case ISD::FSHR:
1604 case ISD::VP_FSHR:
1605 SplitVecRes_TernaryOp(N, Lo, Hi);
1606 break;
1607
1608 case ISD::SCMP: case ISD::UCMP:
1609 SplitVecRes_CMP(N, Lo, Hi);
1610 break;
1611
1612#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1613 case ISD::STRICT_##DAGN:
1614#include "llvm/IR/ConstrainedOps.def"
1615 SplitVecRes_StrictFPOp(N, Lo, Hi);
1616 break;
1617
1618 case ISD::FP_TO_UINT_SAT:
1619 case ISD::FP_TO_SINT_SAT:
1620 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1621 break;
1622
1623 case ISD::UADDO:
1624 case ISD::SADDO:
1625 case ISD::USUBO:
1626 case ISD::SSUBO:
1627 case ISD::UMULO:
1628 case ISD::SMULO:
1629 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1630 break;
1631 case ISD::SMULFIX:
1632 case ISD::SMULFIXSAT:
1633 case ISD::UMULFIX:
1634 case ISD::UMULFIXSAT:
1635 case ISD::SDIVFIX:
1636 case ISD::SDIVFIXSAT:
1637 case ISD::UDIVFIX:
1638 case ISD::UDIVFIXSAT:
1639 SplitVecRes_FIX(N, Lo, Hi);
1640 break;
1641 case ISD::EXPERIMENTAL_VP_SPLICE:
1642 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1643 break;
1644 case ISD::EXPERIMENTAL_VP_REVERSE:
1645 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1646 break;
1647 case ISD::PARTIAL_REDUCE_UMLA:
1648 case ISD::PARTIAL_REDUCE_SMLA:
1649 case ISD::PARTIAL_REDUCE_SUMLA:
1650 case ISD::PARTIAL_REDUCE_FMLA:
1651 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1652 break;
1653 case ISD::GET_ACTIVE_LANE_MASK:
1654 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1655 break;
1656 }
1657
1658 // If Lo/Hi is null, the sub-method took care of registering results etc.
1659 if (Lo.getNode())
1660 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1661}
1662
1663void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1664 MachinePointerInfo &MPI, SDValue &Ptr,
1665 uint64_t *ScaledOffset) {
1666 SDLoc DL(N);
1667 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1668
1669 if (MemVT.isScalableVector()) {
1670 SDValue BytesIncrement = DAG.getVScale(
1671 DL, VT: Ptr.getValueType(),
1672 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1673 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1674 if (ScaledOffset)
1675 *ScaledOffset += IncrementSize;
1676 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1677 Flags: SDNodeFlags::NoUnsignedWrap);
1678 } else {
1679 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1680 // Increment the pointer to the other half.
1681 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1682 }
1683}
1684
1685std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1686 return SplitMask(Mask, DL: SDLoc(Mask));
1687}
1688
1689std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1690 const SDLoc &DL) {
1691 SDValue MaskLo, MaskHi;
1692 EVT MaskVT = Mask.getValueType();
1693 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1694 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1695 else
1696 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1697 return std::make_pair(x&: MaskLo, y&: MaskHi);
1698}
1699
1700void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1701 SDValue LHSLo, LHSHi;
1702 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1703 SDValue RHSLo, RHSHi;
1704 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1705 SDLoc dl(N);
1706
1707 const SDNodeFlags Flags = N->getFlags();
1708 unsigned Opcode = N->getOpcode();
1709 if (N->getNumOperands() == 2) {
1710 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1711 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1712 return;
1713 }
1714
1715 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1716 assert(N->isVPOpcode() && "Expected VP opcode");
1717
1718 SDValue MaskLo, MaskHi;
1719 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1720
1721 SDValue EVLLo, EVLHi;
1722 std::tie(args&: EVLLo, args&: EVLHi) =
1723 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1724
1725 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1726 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1727 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1728 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1729}
1730
1731void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1732 SDValue &Hi) {
1733 SDValue LHSLo, LHSHi;
1734 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1735 SDValue RHSLo, RHSHi;
1736 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1737 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 2));
1738 SDLoc dl(N);
1739
1740 const SDNodeFlags Flags = N->getFlags();
1741 unsigned Opcode = N->getOpcode();
1742 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1743 Flags);
1744 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1745 Flags);
1746}
1747
1748void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1749 SDValue &Hi) {
1750 SDValue Op0Lo, Op0Hi;
1751 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1752 SDValue Op1Lo, Op1Hi;
1753 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1754 SDValue Op2Lo, Op2Hi;
1755 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1756 SDLoc dl(N);
1757
1758 const SDNodeFlags Flags = N->getFlags();
1759 unsigned Opcode = N->getOpcode();
1760 if (N->getNumOperands() == 3) {
1761 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1762 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1763 return;
1764 }
1765
1766 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1767 assert(N->isVPOpcode() && "Expected VP opcode");
1768
1769 SDValue MaskLo, MaskHi;
1770 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1771
1772 SDValue EVLLo, EVLHi;
1773 std::tie(args&: EVLLo, args&: EVLHi) =
1774 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1775
1776 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1777 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1778 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1779 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1780}
1781
1782void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1783 LLVMContext &Ctxt = *DAG.getContext();
1784 SDLoc dl(N);
1785
1786 SDValue LHS = N->getOperand(Num: 0);
1787 SDValue RHS = N->getOperand(Num: 1);
1788
1789 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1790 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1791 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1792 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1793 } else {
1794 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1795 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1796 }
1797
1798 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1799 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1800 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1801}
1802
1803void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1804 SDValue LHSLo, LHSHi;
1805 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1806 SDValue RHSLo, RHSHi;
1807 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1808 SDLoc dl(N);
1809 SDValue Op2 = N->getOperand(Num: 2);
1810
1811 unsigned Opcode = N->getOpcode();
1812 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1813 Flags: N->getFlags());
1814 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1815 Flags: N->getFlags());
1816}
1817
1818void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1819 SDValue &Hi) {
1820 // We know the result is a vector. The input may be either a vector or a
1821 // scalar value.
1822 EVT LoVT, HiVT;
1823 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1824 SDLoc dl(N);
1825
1826 SDValue InOp = N->getOperand(Num: 0);
1827 EVT InVT = InOp.getValueType();
1828
1829 // Handle some special cases efficiently.
1830 switch (getTypeAction(VT: InVT)) {
1831 case TargetLowering::TypeLegal:
1832 case TargetLowering::TypePromoteInteger:
1833 case TargetLowering::TypeSoftPromoteHalf:
1834 case TargetLowering::TypeSoftenFloat:
1835 case TargetLowering::TypeScalarizeVector:
1836 case TargetLowering::TypeWidenVector:
1837 break;
1838 case TargetLowering::TypeExpandInteger:
1839 case TargetLowering::TypeExpandFloat:
1840 // A scalar to vector conversion, where the scalar needs expansion.
1841 // If the vector is being split in two then we can just convert the
1842 // expanded pieces.
1843 if (LoVT == HiVT) {
1844 GetExpandedOp(Op: InOp, Lo, Hi);
1845 if (DAG.getDataLayout().isBigEndian())
1846 std::swap(a&: Lo, b&: Hi);
1847 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1848 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1849 return;
1850 }
1851 break;
1852 case TargetLowering::TypeSplitVector:
1853 // If the input is a vector that needs to be split, convert each split
1854 // piece of the input now.
1855 GetSplitVector(Op: InOp, Lo, Hi);
1856 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1857 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1858 return;
1859 case TargetLowering::TypeScalarizeScalableVector:
1860 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1861 }
1862
1863 if (LoVT.isScalableVector()) {
1864 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1865 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1866 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1867 return;
1868 }
1869
1870 // In the general case, convert the input to an integer and split it by hand.
1871 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1872 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1873 if (DAG.getDataLayout().isBigEndian())
1874 std::swap(a&: LoIntVT, b&: HiIntVT);
1875
1876 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1877
1878 if (DAG.getDataLayout().isBigEndian())
1879 std::swap(a&: Lo, b&: Hi);
1880 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1881 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1882}
1883
1884void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1885 SDValue &Hi) {
1886 SDLoc DL(N);
1887 EVT LoVT, HiVT;
1888 SDValue PtrA = N->getOperand(Num: 0);
1889 SDValue PtrB = N->getOperand(Num: 1);
1890 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1891
1892 // The lane offset for the "Lo" half of the mask is unchanged.
1893 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1894 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1895 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1896 // The lane offset for the "Hi" half of the mask is incremented by the number
1897 // of elements in the "Lo" half.
1898 unsigned LaneOffset =
1899 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1900 // Note: The lane offset is implicitly scalable for scalable masks.
1901 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1902 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1903 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1904}
1905
1906void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1907 SDValue &Hi) {
1908 EVT LoVT, HiVT;
1909 SDLoc dl(N);
1910 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1911 unsigned LoNumElts = LoVT.getVectorNumElements();
1912 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1913 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1914
1915 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1916 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1917}
1918
1919void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1920 SDValue &Hi) {
1921 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1922 SDLoc dl(N);
1923 unsigned NumSubvectors = N->getNumOperands() / 2;
1924 if (NumSubvectors == 1) {
1925 Lo = N->getOperand(Num: 0);
1926 Hi = N->getOperand(Num: 1);
1927 return;
1928 }
1929
1930 EVT LoVT, HiVT;
1931 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1932
1933 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1934 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1935
1936 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1937 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1938}
1939
1940void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1941 SDValue &Hi) {
1942 SDValue Vec = N->getOperand(Num: 0);
1943 SDValue Idx = N->getOperand(Num: 1);
1944 SDLoc dl(N);
1945
1946 EVT LoVT, HiVT;
1947 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1948
1949 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1950 uint64_t IdxVal = Idx->getAsZExtVal();
1951 Hi = DAG.getNode(
1952 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1953 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1954}
1955
1956void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1957 SDValue &Hi) {
1958 SDValue Vec = N->getOperand(Num: 0);
1959 SDValue SubVec = N->getOperand(Num: 1);
1960 SDValue Idx = N->getOperand(Num: 2);
1961 SDLoc dl(N);
1962 GetSplitVector(Op: Vec, Lo, Hi);
1963
1964 EVT VecVT = Vec.getValueType();
1965 EVT LoVT = Lo.getValueType();
1966 EVT SubVecVT = SubVec.getValueType();
1967 unsigned VecElems = VecVT.getVectorMinNumElements();
1968 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1969 unsigned LoElems = LoVT.getVectorMinNumElements();
1970
1971 // If we know the index is in the first half, and we know the subvector
1972 // doesn't cross the boundary between the halves, we can avoid spilling the
1973 // vector, and insert into the lower half of the split vector directly.
1974 unsigned IdxVal = Idx->getAsZExtVal();
1975 if (IdxVal + SubElems <= LoElems) {
1976 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1977 return;
1978 }
1979 // Similarly if the subvector is fully in the high half, but mind that we
1980 // can't tell whether a fixed-length subvector is fully within the high half
1981 // of a scalable vector.
1982 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1983 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1984 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1985 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1986 return;
1987 }
1988
1989 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1990 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1991 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1992 if (WideSubVec.getValueType() == VecVT) {
1993 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1994 return;
1995 }
1996 }
1997
1998 // Spill the vector to the stack.
1999 // In cases where the vector is illegal it will be broken down into parts
2000 // and stored in parts - we should use the alignment for the smallest part.
2001 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2002 SDValue StackPtr =
2003 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2004 auto &MF = DAG.getMachineFunction();
2005 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2006 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2007
2008 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2009 Alignment: SmallestAlign);
2010
2011 // Store the new subvector into the specified index.
2012 SDValue SubVecPtr =
2013 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2014 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2015 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2016
2017 // Load the Lo part from the stack slot.
2018 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2019 Alignment: SmallestAlign);
2020
2021 // Increment the pointer to the other part.
2022 auto *Load = cast<LoadSDNode>(Val&: Lo);
2023 MachinePointerInfo MPI = Load->getPointerInfo();
2024 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2025
2026 // Load the Hi part from the stack slot.
2027 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2028}
2029
2030// Handle splitting an FP where the second operand does not match the first
2031// type. The second operand may be a scalar, or a vector that has exactly as
2032// many elements as the first
2033void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2034 SDValue &Hi) {
2035 SDValue LHSLo, LHSHi;
2036 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2037 SDLoc DL(N);
2038
2039 SDValue RHSLo, RHSHi;
2040 SDValue RHS = N->getOperand(Num: 1);
2041 EVT RHSVT = RHS.getValueType();
2042 if (RHSVT.isVector()) {
2043 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2044 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2045 else
2046 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2047
2048 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2049 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2050 } else {
2051 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2052 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2053 }
2054}
2055
2056void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2057 SDValue &Hi) {
2058 SDLoc DL(N);
2059 SDValue ArgLo, ArgHi;
2060 SDValue Test = N->getOperand(Num: 1);
2061 SDValue FpValue = N->getOperand(Num: 0);
2062 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2063 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2064 else
2065 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2066 EVT LoVT, HiVT;
2067 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2068
2069 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2070 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2071}
2072
2073void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2074 SDValue &Hi) {
2075 SDValue LHSLo, LHSHi;
2076 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2077 SDLoc dl(N);
2078
2079 EVT LoVT, HiVT;
2080 std::tie(args&: LoVT, args&: HiVT) =
2081 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2082
2083 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2084 N2: DAG.getValueType(LoVT));
2085 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2086 N2: DAG.getValueType(HiVT));
2087}
2088
2089void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2090 SDValue &Hi) {
2091 unsigned Opcode = N->getOpcode();
2092 SDValue N0 = N->getOperand(Num: 0);
2093
2094 SDLoc dl(N);
2095 SDValue InLo, InHi;
2096
2097 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2098 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2099 else
2100 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2101
2102 EVT InLoVT = InLo.getValueType();
2103 unsigned InNumElements = InLoVT.getVectorNumElements();
2104
2105 EVT OutLoVT, OutHiVT;
2106 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2107 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2108 assert((2 * OutNumElements) <= InNumElements &&
2109 "Illegal extend vector in reg split");
2110
2111 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2112 // input vector (i.e. we only use InLo):
2113 // OutLo will extend the first OutNumElements from InLo.
2114 // OutHi will extend the next OutNumElements from InLo.
2115
2116 // Shuffle the elements from InLo for OutHi into the bottom elements to
2117 // create a 'fake' InHi.
2118 SmallVector<int, 8> SplitHi(InNumElements, -1);
2119 for (unsigned i = 0; i != OutNumElements; ++i)
2120 SplitHi[i] = i + OutNumElements;
2121 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2122
2123 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2124 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2125}
2126
2127void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2128 SDValue &Hi) {
2129 unsigned NumOps = N->getNumOperands();
2130 SDValue Chain = N->getOperand(Num: 0);
2131 EVT LoVT, HiVT;
2132 SDLoc dl(N);
2133 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2134
2135 SmallVector<SDValue, 4> OpsLo(NumOps);
2136 SmallVector<SDValue, 4> OpsHi(NumOps);
2137
2138 // The Chain is the first operand.
2139 OpsLo[0] = Chain;
2140 OpsHi[0] = Chain;
2141
2142 // Now process the remaining operands.
2143 for (unsigned i = 1; i < NumOps; ++i) {
2144 SDValue Op = N->getOperand(Num: i);
2145 SDValue OpLo = Op;
2146 SDValue OpHi = Op;
2147
2148 EVT InVT = Op.getValueType();
2149 if (InVT.isVector()) {
2150 // If the input also splits, handle it directly for a
2151 // compile time speedup. Otherwise split it by hand.
2152 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2153 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2154 else
2155 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2156 }
2157
2158 OpsLo[i] = OpLo;
2159 OpsHi[i] = OpHi;
2160 }
2161
2162 EVT LoValueVTs[] = {LoVT, MVT::Other};
2163 EVT HiValueVTs[] = {HiVT, MVT::Other};
2164 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2165 Flags: N->getFlags());
2166 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2167 Flags: N->getFlags());
2168
2169 // Build a factor node to remember that this Op is independent of the
2170 // other one.
2171 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2172 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2173
2174 // Legalize the chain result - switch anything that used the old chain to
2175 // use the new one.
2176 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2177}
2178
2179SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2180 SDValue Chain = N->getOperand(Num: 0);
2181 EVT VT = N->getValueType(ResNo: 0);
2182 unsigned NE = VT.getVectorNumElements();
2183 EVT EltVT = VT.getVectorElementType();
2184 SDLoc dl(N);
2185
2186 SmallVector<SDValue, 8> Scalars;
2187 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2188
2189 // If ResNE is 0, fully unroll the vector op.
2190 if (ResNE == 0)
2191 ResNE = NE;
2192 else if (NE > ResNE)
2193 NE = ResNE;
2194
2195 //The results of each unrolled operation, including the chain.
2196 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2197 SmallVector<SDValue, 8> Chains;
2198
2199 unsigned i;
2200 for (i = 0; i != NE; ++i) {
2201 Operands[0] = Chain;
2202 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2203 SDValue Operand = N->getOperand(Num: j);
2204 EVT OperandVT = Operand.getValueType();
2205 if (OperandVT.isVector()) {
2206 EVT OperandEltVT = OperandVT.getVectorElementType();
2207 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2208 } else {
2209 Operands[j] = Operand;
2210 }
2211 }
2212 SDValue Scalar =
2213 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2214
2215 //Add in the scalar as well as its chain value to the
2216 //result vectors.
2217 Scalars.push_back(Elt: Scalar);
2218 Chains.push_back(Elt: Scalar.getValue(R: 1));
2219 }
2220
2221 for (; i < ResNE; ++i)
2222 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2223
2224 // Build a new factor node to connect the chain back together.
2225 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2226 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2227
2228 // Create a new BUILD_VECTOR node
2229 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2230 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2231}
2232
2233void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2234 SDValue &Lo, SDValue &Hi) {
2235 SDLoc dl(N);
2236 EVT ResVT = N->getValueType(ResNo: 0);
2237 EVT OvVT = N->getValueType(ResNo: 1);
2238 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2239 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2240 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2241
2242 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2243 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2244 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2245 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2246 } else {
2247 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2248 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2249 }
2250
2251 unsigned Opcode = N->getOpcode();
2252 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2253 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2254 SDNode *LoNode =
2255 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2256 SDNode *HiNode =
2257 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2258
2259 Lo = SDValue(LoNode, ResNo);
2260 Hi = SDValue(HiNode, ResNo);
2261
2262 // Replace the other vector result not being explicitly split here.
2263 unsigned OtherNo = 1 - ResNo;
2264 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2265 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2266 SetSplitVector(Op: SDValue(N, OtherNo),
2267 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2268 } else {
2269 SDValue OtherVal = DAG.getNode(
2270 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2271 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2272 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2273 }
2274}
2275
2276void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2277 SDValue &Hi) {
2278 SDValue Vec = N->getOperand(Num: 0);
2279 SDValue Elt = N->getOperand(Num: 1);
2280 SDValue Idx = N->getOperand(Num: 2);
2281 SDLoc dl(N);
2282 GetSplitVector(Op: Vec, Lo, Hi);
2283
2284 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2285 unsigned IdxVal = CIdx->getZExtValue();
2286 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2287 if (IdxVal < LoNumElts) {
2288 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2289 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2290 return;
2291 } else if (!Vec.getValueType().isScalableVector()) {
2292 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2293 return;
2294 }
2295 }
2296
2297 // Make the vector elements byte-addressable if they aren't already.
2298 EVT VecVT = Vec.getValueType();
2299 EVT EltVT = VecVT.getVectorElementType();
2300 if (!EltVT.isByteSized()) {
2301 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2302 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2303 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2304 // Extend the element type to match if needed.
2305 if (EltVT.bitsGT(VT: Elt.getValueType()))
2306 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2307 }
2308
2309 // Spill the vector to the stack.
2310 // In cases where the vector is illegal it will be broken down into parts
2311 // and stored in parts - we should use the alignment for the smallest part.
2312 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2313 SDValue StackPtr =
2314 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2315 auto &MF = DAG.getMachineFunction();
2316 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2317 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2318
2319 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2320 Alignment: SmallestAlign);
2321
2322 // Store the new element. This may be larger than the vector element type,
2323 // so use a truncating store.
2324 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2325 Store = DAG.getTruncStore(
2326 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2327 Alignment: commonAlignment(A: SmallestAlign,
2328 Offset: EltVT.getFixedSizeInBits() / 8));
2329
2330 EVT LoVT, HiVT;
2331 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2332
2333 // Load the Lo part from the stack slot.
2334 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2335
2336 // Increment the pointer to the other part.
2337 auto Load = cast<LoadSDNode>(Val&: Lo);
2338 MachinePointerInfo MPI = Load->getPointerInfo();
2339 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2340
2341 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2342
2343 // If we adjusted the original type, we need to truncate the results.
2344 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2345 if (LoVT != Lo.getValueType())
2346 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2347 if (HiVT != Hi.getValueType())
2348 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2349}
2350
2351void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2352 SDValue &Hi) {
2353 EVT LoVT, HiVT;
2354 SDLoc dl(N);
2355 assert(N->getValueType(0).isScalableVector() &&
2356 "Only scalable vectors are supported for STEP_VECTOR");
2357 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2358 SDValue Step = N->getOperand(Num: 0);
2359
2360 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2361
2362 // Hi = Lo + (EltCnt * Step)
2363 EVT EltVT = Step.getValueType();
2364 APInt StepVal = Step->getAsAPIntVal();
2365 SDValue StartOfHi =
2366 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2367 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2368 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2369
2370 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2371 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2372}
2373
2374void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2375 SDValue &Hi) {
2376 EVT LoVT, HiVT;
2377 SDLoc dl(N);
2378 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2379 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2380 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2381 Hi = DAG.getPOISON(VT: HiVT);
2382 } else {
2383 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2384 Hi = Lo;
2385 }
2386}
2387
2388void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2389 SDValue &Hi) {
2390 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2391 "Extended load during type legalization!");
2392 SDLoc dl(LD);
2393 EVT VT = LD->getValueType(ResNo: 0);
2394 EVT LoVT, HiVT;
2395 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2396
2397 SDValue Ch = LD->getChain();
2398 SDValue Ptr = LD->getBasePtr();
2399
2400 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2401 EVT MemIntVT =
2402 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2403 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2404 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2405
2406 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2407 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2408 SDValue ExtractLo, ExtractHi;
2409 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2410
2411 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2412 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2413
2414 // Legalize the chain result - switch anything that used the old chain to
2415 // use the new one.
2416 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2417}
2418
2419void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2420 SDValue &Hi) {
2421 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2422 EVT LoVT, HiVT;
2423 SDLoc dl(LD);
2424 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2425
2426 ISD::LoadExtType ExtType = LD->getExtensionType();
2427 SDValue Ch = LD->getChain();
2428 SDValue Ptr = LD->getBasePtr();
2429 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2430 EVT MemoryVT = LD->getMemoryVT();
2431 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2432 AAMDNodes AAInfo = LD->getAAInfo();
2433
2434 EVT LoMemVT, HiMemVT;
2435 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2436
2437 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2438 SDValue Value, NewChain;
2439 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2440 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2441 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2442 return;
2443 }
2444
2445 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2446 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2447 AAInfo);
2448
2449 MachinePointerInfo MPI;
2450 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2451
2452 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2453 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2454
2455 // Build a factor node to remember that this load is independent of the
2456 // other one.
2457 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2458 N2: Hi.getValue(R: 1));
2459
2460 // Legalize the chain result - switch anything that used the old chain to
2461 // use the new one.
2462 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2463}
2464
2465void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2466 SDValue &Hi) {
2467 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2468 EVT LoVT, HiVT;
2469 SDLoc dl(LD);
2470 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2471
2472 ISD::LoadExtType ExtType = LD->getExtensionType();
2473 SDValue Ch = LD->getChain();
2474 SDValue Ptr = LD->getBasePtr();
2475 SDValue Offset = LD->getOffset();
2476 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2477 Align Alignment = LD->getBaseAlign();
2478 SDValue Mask = LD->getMask();
2479 SDValue EVL = LD->getVectorLength();
2480 EVT MemoryVT = LD->getMemoryVT();
2481
2482 EVT LoMemVT, HiMemVT;
2483 bool HiIsEmpty = false;
2484 std::tie(args&: LoMemVT, args&: HiMemVT) =
2485 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2486
2487 // Split Mask operand
2488 SDValue MaskLo, MaskHi;
2489 if (Mask.getOpcode() == ISD::SETCC) {
2490 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2491 } else {
2492 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2493 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2494 else
2495 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2496 }
2497
2498 // Split EVL operand
2499 SDValue EVLLo, EVLHi;
2500 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2501
2502 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2503 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2504 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2505 Ranges: LD->getRanges());
2506
2507 Lo =
2508 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2509 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2510
2511 if (HiIsEmpty) {
2512 // The hi vp_load has zero storage size. We therefore simply set it to
2513 // the low vp_load and rely on subsequent removal from the chain.
2514 Hi = Lo;
2515 } else {
2516 // Generate hi vp_load.
2517 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2518 IsCompressedMemory: LD->isExpandingLoad());
2519
2520 MachinePointerInfo MPI;
2521 if (LoMemVT.isScalableVector())
2522 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2523 else
2524 MPI = LD->getPointerInfo().getWithOffset(
2525 O: LoMemVT.getStoreSize().getFixedValue());
2526
2527 MMO = DAG.getMachineFunction().getMachineMemOperand(
2528 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2529 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2530
2531 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2532 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2533 IsExpanding: LD->isExpandingLoad());
2534 }
2535
2536 // Build a factor node to remember that this load is independent of the
2537 // other one.
2538 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2539 N2: Hi.getValue(R: 1));
2540
2541 // Legalize the chain result - switch anything that used the old chain to
2542 // use the new one.
2543 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2544}
2545
2546void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2547 SDValue &Hi) {
2548 SDLoc dl(LD);
2549 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2550
2551 SDValue Ch = LD->getChain();
2552 SDValue Ptr = LD->getBasePtr();
2553 Align Alignment = LD->getBaseAlign();
2554 SDValue Mask = LD->getMask();
2555 SDValue EVL = LD->getVectorLength();
2556
2557 // Split Mask operand
2558 SDValue MaskLo, MaskHi;
2559 if (Mask.getOpcode() == ISD::SETCC) {
2560 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2561 } else {
2562 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2563 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2564 else
2565 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2566 }
2567
2568 // Split EVL operand
2569 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2570
2571 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2572 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2573 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2574 Ranges: LD->getRanges());
2575
2576 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2577
2578 // Fill the upper half with poison.
2579 Hi = DAG.getPOISON(VT: HiVT);
2580
2581 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2582 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2583}
2584
2585void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2586 SDValue &Lo, SDValue &Hi) {
2587 assert(SLD->isUnindexed() &&
2588 "Indexed VP strided load during type legalization!");
2589 assert(SLD->getOffset().isUndef() &&
2590 "Unexpected indexed variable-length load offset");
2591
2592 SDLoc DL(SLD);
2593
2594 EVT LoVT, HiVT;
2595 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2596
2597 EVT LoMemVT, HiMemVT;
2598 bool HiIsEmpty = false;
2599 std::tie(args&: LoMemVT, args&: HiMemVT) =
2600 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2601
2602 SDValue Mask = SLD->getMask();
2603 SDValue LoMask, HiMask;
2604 if (Mask.getOpcode() == ISD::SETCC) {
2605 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2606 } else {
2607 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2608 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2609 else
2610 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2611 }
2612
2613 SDValue LoEVL, HiEVL;
2614 std::tie(args&: LoEVL, args&: HiEVL) =
2615 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2616
2617 // Generate the low vp_strided_load
2618 Lo = DAG.getStridedLoadVP(
2619 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2620 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2621 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2622
2623 if (HiIsEmpty) {
2624 // The high vp_strided_load has zero storage size. We therefore simply set
2625 // it to the low vp_strided_load and rely on subsequent removal from the
2626 // chain.
2627 Hi = Lo;
2628 } else {
2629 // Generate the high vp_strided_load.
2630 // To calculate the high base address, we need to sum to the low base
2631 // address stride number of bytes for each element already loaded by low,
2632 // that is: Ptr = Ptr + (LoEVL * Stride)
2633 EVT PtrVT = SLD->getBasePtr().getValueType();
2634 SDValue Increment =
2635 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2636 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2637 SDValue Ptr =
2638 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2639
2640 Align Alignment = SLD->getBaseAlign();
2641 if (LoMemVT.isScalableVector())
2642 Alignment = commonAlignment(
2643 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2644
2645 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2646 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2647 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2648 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2649
2650 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2651 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2652 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2653 IsExpanding: SLD->isExpandingLoad());
2654 }
2655
2656 // Build a factor node to remember that this load is independent of the
2657 // other one.
2658 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2659 N2: Hi.getValue(R: 1));
2660
2661 // Legalize the chain result - switch anything that used the old chain to
2662 // use the new one.
2663 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2664}
2665
2666void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2667 SDValue &Lo, SDValue &Hi) {
2668 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2669 EVT LoVT, HiVT;
2670 SDLoc dl(MLD);
2671 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2672
2673 SDValue Ch = MLD->getChain();
2674 SDValue Ptr = MLD->getBasePtr();
2675 SDValue Offset = MLD->getOffset();
2676 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2677 SDValue Mask = MLD->getMask();
2678 SDValue PassThru = MLD->getPassThru();
2679 Align Alignment = MLD->getBaseAlign();
2680 ISD::LoadExtType ExtType = MLD->getExtensionType();
2681 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2682
2683 // Split Mask operand
2684 SDValue MaskLo, MaskHi;
2685 if (Mask.getOpcode() == ISD::SETCC) {
2686 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2687 } else {
2688 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2689 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2690 else
2691 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2692 }
2693
2694 EVT MemoryVT = MLD->getMemoryVT();
2695 EVT LoMemVT, HiMemVT;
2696 bool HiIsEmpty = false;
2697 std::tie(args&: LoMemVT, args&: HiMemVT) =
2698 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2699
2700 SDValue PassThruLo, PassThruHi;
2701 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2702 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2703 else
2704 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2705
2706 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2707 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2708 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2709
2710 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2711 MMO, AM: MLD->getAddressingMode(), ExtType,
2712 IsExpanding: MLD->isExpandingLoad());
2713
2714 if (HiIsEmpty) {
2715 // The hi masked load has zero storage size. We therefore simply set it to
2716 // the low masked load and rely on subsequent removal from the chain.
2717 Hi = Lo;
2718 } else {
2719 // Generate hi masked load.
2720 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2721 IsCompressedMemory: MLD->isExpandingLoad());
2722
2723 MachinePointerInfo MPI;
2724 if (LoMemVT.isScalableVector())
2725 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2726 else
2727 MPI = MLD->getPointerInfo().getWithOffset(
2728 O: LoMemVT.getStoreSize().getFixedValue());
2729
2730 MMO = DAG.getMachineFunction().getMachineMemOperand(
2731 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2732 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2733
2734 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2735 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2736 IsExpanding: MLD->isExpandingLoad());
2737 }
2738
2739 // Build a factor node to remember that this load is independent of the
2740 // other one.
2741 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2742 N2: Hi.getValue(R: 1));
2743
2744 // Legalize the chain result - switch anything that used the old chain to
2745 // use the new one.
2746 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2747
2748}
2749
2750void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2751 SDValue &Hi, bool SplitSETCC) {
2752 EVT LoVT, HiVT;
2753 SDLoc dl(N);
2754 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2755
2756 SDValue Ch = N->getChain();
2757 SDValue Ptr = N->getBasePtr();
2758 struct Operands {
2759 SDValue Mask;
2760 SDValue Index;
2761 SDValue Scale;
2762 } Ops = [&]() -> Operands {
2763 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2764 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2765 }
2766 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2767 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2768 }();
2769
2770 EVT MemoryVT = N->getMemoryVT();
2771 Align Alignment = N->getBaseAlign();
2772
2773 // Split Mask operand
2774 SDValue MaskLo, MaskHi;
2775 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2776 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2777 } else {
2778 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2779 }
2780
2781 EVT LoMemVT, HiMemVT;
2782 // Split MemoryVT
2783 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2784
2785 SDValue IndexHi, IndexLo;
2786 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2787 TargetLowering::TypeSplitVector)
2788 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2789 else
2790 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2791
2792 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2793 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2794 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2795 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2796
2797 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2798 SDValue PassThru = MGT->getPassThru();
2799 SDValue PassThruLo, PassThruHi;
2800 if (getTypeAction(VT: PassThru.getValueType()) ==
2801 TargetLowering::TypeSplitVector)
2802 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2803 else
2804 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2805
2806 ISD::LoadExtType ExtType = MGT->getExtensionType();
2807 ISD::MemIndexType IndexTy = MGT->getIndexType();
2808
2809 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2810 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2811 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2812
2813 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2814 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2815 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2816 } else {
2817 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2818 SDValue EVLLo, EVLHi;
2819 std::tie(args&: EVLLo, args&: EVLHi) =
2820 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2821
2822 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2823 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2824 MMO, IndexType: VPGT->getIndexType());
2825
2826 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2827 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2828 MMO, IndexType: VPGT->getIndexType());
2829 }
2830
2831 // Build a factor node to remember that this load is independent of the
2832 // other one.
2833 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2834 N2: Hi.getValue(R: 1));
2835
2836 // Legalize the chain result - switch anything that used the old chain to
2837 // use the new one.
2838 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2839}
2840
2841void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2842 SDValue &Hi) {
2843 // This is not "trivial", as there is a dependency between the two subvectors.
2844 // Depending on the number of 1s in the mask, the elements from the Hi vector
2845 // need to be moved to the Lo vector. Passthru values make this even harder.
2846 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2847 // smaller types and passthru is undef, as it is most likely faster than the
2848 // fully expand path. Otherwise, just do the full expansion as one "big"
2849 // operation and then extract the Lo and Hi vectors from that. This gets
2850 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2851 SDLoc DL(N);
2852 EVT VecVT = N->getValueType(ResNo: 0);
2853
2854 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2855 bool HasCustomLowering = false;
2856 EVT CheckVT = LoVT;
2857 while (CheckVT.getVectorMinNumElements() > 1) {
2858 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2859 // custom lowering for illegal types. So we do the checks separately.
2860 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2861 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2862 HasCustomLowering = true;
2863 break;
2864 }
2865 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2866 }
2867
2868 SDValue Passthru = N->getOperand(Num: 2);
2869 if (!HasCustomLowering) {
2870 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2871 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2872 return;
2873 }
2874
2875 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2876 SDValue Mask = N->getOperand(Num: 1);
2877 SDValue LoMask, HiMask;
2878 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2879 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2880
2881 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2882 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2883 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2884
2885 SDValue StackPtr = DAG.CreateStackTemporary(
2886 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2887 MachineFunction &MF = DAG.getMachineFunction();
2888 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2889 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2890
2891 EVT MaskVT = LoMask.getValueType();
2892 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2893
2894 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2895 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2896 EC: MaskVT.getVectorElementCount());
2897 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2898 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2899 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2900
2901 SDValue Chain = DAG.getEntryNode();
2902 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2903 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2904 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2905
2906 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2907 if (!Passthru.isUndef()) {
2908 Compressed =
2909 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2910 }
2911 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2912}
2913
2914void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2915 assert(N->getValueType(0).isVector() &&
2916 N->getOperand(0).getValueType().isVector() &&
2917 "Operand types must be vectors");
2918
2919 EVT LoVT, HiVT;
2920 SDLoc DL(N);
2921 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2922
2923 // If the input also splits, handle it directly. Otherwise split it by hand.
2924 SDValue LL, LH, RL, RH;
2925 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2926 TargetLowering::TypeSplitVector)
2927 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2928 else
2929 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2930
2931 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2932 TargetLowering::TypeSplitVector)
2933 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2934 else
2935 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2936
2937 if (N->getOpcode() == ISD::SETCC) {
2938 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2939 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2940 } else {
2941 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2942 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2943 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2944 std::tie(args&: EVLLo, args&: EVLHi) =
2945 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2946 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2947 N5: EVLLo);
2948 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2949 N5: EVLHi);
2950 }
2951}
2952
2953void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2954 SDValue &Hi) {
2955 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2956 EVT LoVT, HiVT;
2957 SDLoc dl(N);
2958 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2959
2960 // If the input also splits, handle it directly for a compile time speedup.
2961 // Otherwise split it by hand.
2962 EVT InVT = N->getOperand(Num: 0).getValueType();
2963 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2964 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2965 else
2966 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2967
2968 const SDNodeFlags Flags = N->getFlags();
2969 unsigned Opcode = N->getOpcode();
2970 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
2971 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2972 N4: N->getOperand(Num: 3), Flags);
2973 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2974 N4: N->getOperand(Num: 3), Flags);
2975 return;
2976 }
2977 if (N->getNumOperands() <= 2) {
2978 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2979 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2980 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2981 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2982 } else {
2983 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2984 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2985 }
2986 return;
2987 }
2988
2989 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2990 assert(N->isVPOpcode() && "Expected VP opcode");
2991
2992 SDValue MaskLo, MaskHi;
2993 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2994
2995 SDValue EVLLo, EVLHi;
2996 std::tie(args&: EVLLo, args&: EVLHi) =
2997 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2998
2999 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
3000 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
3001}
3002
3003void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3004 SDValue &Hi) {
3005 SDLoc dl(N);
3006 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3007
3008 // If the input also splits, handle it directly for a compile time speedup.
3009 // Otherwise split it by hand.
3010 EVT InVT = N->getOperand(Num: 0).getValueType();
3011 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3012 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3013 else
3014 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3015
3016 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3017 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3018 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3019 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
3020 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
3021}
3022
3023void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3024 unsigned ResNo,
3025 SDValue &Lo,
3026 SDValue &Hi) {
3027 SDLoc dl(N);
3028 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3029 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3030
3031 // If the input also splits, handle it directly for a compile time speedup.
3032 // Otherwise split it by hand.
3033 EVT InVT = N->getOperand(Num: 0).getValueType();
3034 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3035 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3036 else
3037 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3038
3039 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3040 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3041
3042 SDNode *HiNode = Hi.getNode();
3043 SDNode *LoNode = Lo.getNode();
3044
3045 // Replace the other vector result not being explicitly split here.
3046 unsigned OtherNo = 1 - ResNo;
3047 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3048 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3049 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3050 Hi: SDValue(HiNode, OtherNo));
3051 } else {
3052 SDValue OtherVal =
3053 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3054 N2: SDValue(HiNode, OtherNo));
3055 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3056 }
3057}
3058
3059void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3060 SDValue &Hi) {
3061 SDLoc dl(N);
3062 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3063 EVT DestVT = N->getValueType(ResNo: 0);
3064 EVT LoVT, HiVT;
3065 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3066
3067 // We can do better than a generic split operation if the extend is doing
3068 // more than just doubling the width of the elements and the following are
3069 // true:
3070 // - The number of vector elements is even,
3071 // - the source type is legal,
3072 // - the type of a split source is illegal,
3073 // - the type of an extended (by doubling element size) source is legal, and
3074 // - the type of that extended source when split is legal.
3075 //
3076 // This won't necessarily completely legalize the operation, but it will
3077 // more effectively move in the right direction and prevent falling down
3078 // to scalarization in many cases due to the input vector being split too
3079 // far.
3080 if (SrcVT.getVectorElementCount().isKnownEven() &&
3081 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3082 LLVMContext &Ctx = *DAG.getContext();
3083 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3084 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3085
3086 EVT SplitLoVT, SplitHiVT;
3087 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3088 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3089 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3090 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3091 N->dump(&DAG); dbgs() << "\n");
3092 if (!N->isVPOpcode()) {
3093 // Extend the source vector by one step.
3094 SDValue NewSrc =
3095 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3096 // Get the low and high halves of the new, extended one step, vector.
3097 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3098 // Extend those vector halves the rest of the way.
3099 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3100 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3101 return;
3102 }
3103
3104 // Extend the source vector by one step.
3105 SDValue NewSrc =
3106 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
3107 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
3108 // Get the low and high halves of the new, extended one step, vector.
3109 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3110
3111 SDValue MaskLo, MaskHi;
3112 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3113
3114 SDValue EVLLo, EVLHi;
3115 std::tie(args&: EVLLo, args&: EVLHi) =
3116 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3117 // Extend those vector halves the rest of the way.
3118 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
3119 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
3120 return;
3121 }
3122 }
3123 // Fall back to the generic unary operator splitting otherwise.
3124 SplitVecRes_UnaryOp(N, Lo, Hi);
3125}
3126
3127void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3128 SDValue &Lo, SDValue &Hi) {
3129 // The low and high parts of the original input give four input vectors.
3130 SDValue Inputs[4];
3131 SDLoc DL(N);
3132 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3133 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3134 EVT NewVT = Inputs[0].getValueType();
3135 unsigned NewElts = NewVT.getVectorNumElements();
3136
3137 auto &&IsConstant = [](const SDValue &N) {
3138 APInt SplatValue;
3139 return N.getResNo() == 0 &&
3140 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3141 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3142 };
3143 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3144 SDValue &Input2,
3145 ArrayRef<int> Mask) {
3146 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3147 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3148 "Expected build vector node.");
3149 EVT EltVT = NewVT.getVectorElementType();
3150 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3151 for (unsigned I = 0; I < NewElts; ++I) {
3152 if (Mask[I] == PoisonMaskElem)
3153 continue;
3154 unsigned Idx = Mask[I];
3155 if (Idx >= NewElts)
3156 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3157 else
3158 Ops[I] = Input1.getOperand(i: Idx);
3159 // Make the type of all elements the same as the element type.
3160 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3161 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3162 }
3163 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3164 };
3165
3166 // If Lo or Hi uses elements from at most two of the four input vectors, then
3167 // express it as a vector shuffle of those two inputs. Otherwise extract the
3168 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3169 SmallVector<int> OrigMask(N->getMask());
3170 // Try to pack incoming shuffles/inputs.
3171 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3172 &DL](SmallVectorImpl<int> &Mask) {
3173 // Check if all inputs are shuffles of the same operands or non-shuffles.
3174 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3175 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3176 SDValue Input = Inputs[Idx];
3177 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3178 if (!Shuffle ||
3179 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3180 continue;
3181 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3182 .push_back(Elt: Idx);
3183 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3184 .push_back(Elt: Idx);
3185 }
3186 for (auto &P : ShufflesIdxs) {
3187 if (P.second.size() < 2)
3188 continue;
3189 // Use shuffles operands instead of shuffles themselves.
3190 // 1. Adjust mask.
3191 for (int &Idx : Mask) {
3192 if (Idx == PoisonMaskElem)
3193 continue;
3194 unsigned SrcRegIdx = Idx / NewElts;
3195 if (Inputs[SrcRegIdx].isUndef()) {
3196 Idx = PoisonMaskElem;
3197 continue;
3198 }
3199 auto *Shuffle =
3200 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3201 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3202 continue;
3203 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3204 if (MaskElt == PoisonMaskElem) {
3205 Idx = PoisonMaskElem;
3206 continue;
3207 }
3208 Idx = MaskElt % NewElts +
3209 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3210 ? 0
3211 : 1] *
3212 NewElts;
3213 }
3214 // 2. Update inputs.
3215 Inputs[P.second[0]] = P.first.first;
3216 Inputs[P.second[1]] = P.first.second;
3217 // Clear the pair data.
3218 P.second.clear();
3219 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3220 }
3221 // Check if any concat_vectors can be simplified.
3222 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3223 for (int &Idx : Mask) {
3224 if (Idx == PoisonMaskElem)
3225 continue;
3226 unsigned SrcRegIdx = Idx / NewElts;
3227 if (Inputs[SrcRegIdx].isUndef()) {
3228 Idx = PoisonMaskElem;
3229 continue;
3230 }
3231 TargetLowering::LegalizeTypeAction TypeAction =
3232 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3233 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3234 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3235 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3236 (TypeAction == TargetLowering::TypeLegal ||
3237 TypeAction == TargetLowering::TypeWidenVector))
3238 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3239 }
3240 if (UsedSubVector.count() > 1) {
3241 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3242 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3243 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3244 continue;
3245 if (Pairs.empty() || Pairs.back().size() == 2)
3246 Pairs.emplace_back();
3247 if (UsedSubVector.test(Idx: 2 * I)) {
3248 Pairs.back().emplace_back(Args&: I, Args: 0);
3249 } else {
3250 assert(UsedSubVector.test(2 * I + 1) &&
3251 "Expected to be used one of the subvectors.");
3252 Pairs.back().emplace_back(Args&: I, Args: 1);
3253 }
3254 }
3255 if (!Pairs.empty() && Pairs.front().size() > 1) {
3256 // Adjust mask.
3257 for (int &Idx : Mask) {
3258 if (Idx == PoisonMaskElem)
3259 continue;
3260 unsigned SrcRegIdx = Idx / NewElts;
3261 auto *It = find_if(
3262 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3263 return Idxs.front().first == SrcRegIdx ||
3264 Idxs.back().first == SrcRegIdx;
3265 });
3266 if (It == Pairs.end())
3267 continue;
3268 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3269 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3270 }
3271 // Adjust inputs.
3272 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3273 Inputs[Idxs.front().first] = DAG.getNode(
3274 Opcode: ISD::CONCAT_VECTORS, DL,
3275 VT: Inputs[Idxs.front().first].getValueType(),
3276 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3277 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3278 }
3279 }
3280 }
3281 bool Changed;
3282 do {
3283 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3284 // reused operands.
3285 Changed = false;
3286 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3287 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3288 if (!Shuffle)
3289 continue;
3290 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3291 continue;
3292 int Op = -1;
3293 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3294 !Shuffle->isSplat()) {
3295 Op = 0;
3296 } else if (!Inputs[I].hasOneUse() &&
3297 !Shuffle->getOperand(Num: 1).isUndef()) {
3298 // Find the only used operand, if possible.
3299 for (int &Idx : Mask) {
3300 if (Idx == PoisonMaskElem)
3301 continue;
3302 unsigned SrcRegIdx = Idx / NewElts;
3303 if (SrcRegIdx != I)
3304 continue;
3305 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3306 if (MaskElt == PoisonMaskElem) {
3307 Idx = PoisonMaskElem;
3308 continue;
3309 }
3310 int OpIdx = MaskElt / NewElts;
3311 if (Op == -1) {
3312 Op = OpIdx;
3313 continue;
3314 }
3315 if (Op != OpIdx) {
3316 Op = -1;
3317 break;
3318 }
3319 }
3320 }
3321 if (Op < 0) {
3322 // Try to check if one of the shuffle operands is used already.
3323 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3324 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3325 continue;
3326 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3327 if (It == std::end(arr&: Inputs))
3328 continue;
3329 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3330 // Found that operand is used already.
3331 // 1. Fix the mask for the reused operand.
3332 for (int &Idx : Mask) {
3333 if (Idx == PoisonMaskElem)
3334 continue;
3335 unsigned SrcRegIdx = Idx / NewElts;
3336 if (SrcRegIdx != I)
3337 continue;
3338 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3339 if (MaskElt == PoisonMaskElem) {
3340 Idx = PoisonMaskElem;
3341 continue;
3342 }
3343 int MaskIdx = MaskElt / NewElts;
3344 if (OpIdx == MaskIdx)
3345 Idx = MaskElt % NewElts + FoundOp * NewElts;
3346 }
3347 // 2. Set Op to the unused OpIdx.
3348 Op = (OpIdx + 1) % 2;
3349 break;
3350 }
3351 }
3352 if (Op >= 0) {
3353 Changed = true;
3354 Inputs[I] = Shuffle->getOperand(Num: Op);
3355 // Adjust mask.
3356 for (int &Idx : Mask) {
3357 if (Idx == PoisonMaskElem)
3358 continue;
3359 unsigned SrcRegIdx = Idx / NewElts;
3360 if (SrcRegIdx != I)
3361 continue;
3362 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3363 int OpIdx = MaskElt / NewElts;
3364 if (OpIdx != Op)
3365 continue;
3366 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3367 }
3368 }
3369 }
3370 } while (Changed);
3371 };
3372 TryPeekThroughShufflesInputs(OrigMask);
3373 // Proces unique inputs.
3374 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3375 NewElts](SmallVectorImpl<int> &Mask) {
3376 SetVector<SDValue> UniqueInputs;
3377 SetVector<SDValue> UniqueConstantInputs;
3378 for (const auto &I : Inputs) {
3379 if (IsConstant(I))
3380 UniqueConstantInputs.insert(X: I);
3381 else if (!I.isUndef())
3382 UniqueInputs.insert(X: I);
3383 }
3384 // Adjust mask in case of reused inputs. Also, need to insert constant
3385 // inputs at first, otherwise it affects the final outcome.
3386 if (UniqueInputs.size() != std::size(Inputs)) {
3387 auto &&UniqueVec = UniqueInputs.takeVector();
3388 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3389 unsigned ConstNum = UniqueConstantVec.size();
3390 for (int &Idx : Mask) {
3391 if (Idx == PoisonMaskElem)
3392 continue;
3393 unsigned SrcRegIdx = Idx / NewElts;
3394 if (Inputs[SrcRegIdx].isUndef()) {
3395 Idx = PoisonMaskElem;
3396 continue;
3397 }
3398 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3399 if (It != UniqueConstantVec.end()) {
3400 Idx = (Idx % NewElts) +
3401 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3402 assert(Idx >= 0 && "Expected defined mask idx.");
3403 continue;
3404 }
3405 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3406 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3407 Idx = (Idx % NewElts) +
3408 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3409 assert(Idx >= 0 && "Expected defined mask idx.");
3410 }
3411 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3412 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3413 }
3414 };
3415 MakeUniqueInputs(OrigMask);
3416 SDValue OrigInputs[4];
3417 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3418 for (unsigned High = 0; High < 2; ++High) {
3419 SDValue &Output = High ? Hi : Lo;
3420
3421 // Build a shuffle mask for the output, discovering on the fly which
3422 // input vectors to use as shuffle operands.
3423 unsigned FirstMaskIdx = High * NewElts;
3424 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3425 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3426 assert(!Output && "Expected default initialized initial value.");
3427 TryPeekThroughShufflesInputs(Mask);
3428 MakeUniqueInputs(Mask);
3429 SDValue TmpInputs[4];
3430 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3431 // Track changes in the output registers.
3432 int UsedIdx = -1;
3433 bool SecondIteration = false;
3434 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3435 if (UsedIdx < 0) {
3436 UsedIdx = Idx;
3437 return false;
3438 }
3439 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3440 SecondIteration = true;
3441 return SecondIteration;
3442 };
3443 processShuffleMasks(
3444 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3445 /*NumOfUsedRegs=*/1,
3446 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3447 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3448 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3449 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3450 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3451 else
3452 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3453 N2: DAG.getPOISON(VT: NewVT), Mask);
3454 Inputs[Idx] = Output;
3455 },
3456 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3457 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3458 unsigned Idx2, bool /*Unused*/) {
3459 if (AccumulateResults(Idx1)) {
3460 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3461 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3462 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3463 else
3464 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3465 N2: Inputs[Idx2], Mask);
3466 } else {
3467 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3468 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3469 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3470 else
3471 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3472 N2: TmpInputs[Idx2], Mask);
3473 }
3474 Inputs[Idx1] = Output;
3475 });
3476 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3477 }
3478}
3479
3480void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3481 EVT OVT = N->getValueType(ResNo: 0);
3482 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3483 SDValue Chain = N->getOperand(Num: 0);
3484 SDValue Ptr = N->getOperand(Num: 1);
3485 SDValue SV = N->getOperand(Num: 2);
3486 SDLoc dl(N);
3487
3488 const Align Alignment =
3489 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3490
3491 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3492 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3493 Chain = Hi.getValue(R: 1);
3494
3495 // Modified the chain - switch anything that used the old chain to use
3496 // the new one.
3497 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3498}
3499
3500void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3501 SDValue &Hi) {
3502 EVT DstVTLo, DstVTHi;
3503 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3504 SDLoc dl(N);
3505
3506 SDValue SrcLo, SrcHi;
3507 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3508 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3509 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3510 else
3511 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3512
3513 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3514 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3515}
3516
3517void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3518 SDValue &Hi) {
3519 SDValue InLo, InHi;
3520 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3521 SDLoc DL(N);
3522
3523 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3524 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3525}
3526
3527void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3528 SDValue &Hi) {
3529 SDLoc DL(N);
3530
3531 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3532 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3533}
3534
3535void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3536 SDValue &Hi) {
3537 EVT VT = N->getValueType(ResNo: 0);
3538 SDValue Val = N->getOperand(Num: 0);
3539 SDValue Mask = N->getOperand(Num: 1);
3540 SDValue EVL = N->getOperand(Num: 2);
3541 SDLoc DL(N);
3542
3543 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3544 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3545 // then truncate back.
3546 EVT OrigVT = VT;
3547 if (!VT.getVectorElementType().isByteSized()) {
3548 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3549 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3550 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3551 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3552 }
3553
3554 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3555 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3556
3557 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3558 EC: VT.getVectorElementCount());
3559 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3560 EVT PtrVT = StackPtr.getValueType();
3561 auto &MF = DAG.getMachineFunction();
3562 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3563 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3564
3565 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3566 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3567 BaseAlignment: Alignment);
3568 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3569 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3570 BaseAlignment: Alignment);
3571
3572 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3573 SDValue NumElemMinus1 =
3574 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3575 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3576 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3577 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3578 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3579 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3580
3581 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3582 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3583 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3584 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3585
3586 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3587
3588 // Truncate back if we widened above.
3589 if (OrigVT != VT)
3590 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3591
3592 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3593}
3594
3595void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3596 SDValue &Hi) {
3597 EVT VT = N->getValueType(ResNo: 0);
3598 SDValue V1 = N->getOperand(Num: 0);
3599 SDValue V2 = N->getOperand(Num: 1);
3600 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3601 SDValue Mask = N->getOperand(Num: 3);
3602 SDValue EVL1 = N->getOperand(Num: 4);
3603 SDValue EVL2 = N->getOperand(Num: 5);
3604 SDLoc DL(N);
3605
3606 // Since EVL2 is considered the real VL it gets promoted during
3607 // SelectionDAGBuilder. Promote EVL1 here if needed.
3608 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3609 EVL1 = ZExtPromotedInteger(Op: EVL1);
3610
3611 // The stack splice addresses elements by byte offset/stride, which breaks for
3612 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3613 // stride is 0. Widen to a byte integer, splice, then truncate back.
3614 EVT OrigVT = VT;
3615 if (!VT.getVectorElementType().isByteSized()) {
3616 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3617 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3618 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3619 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3620 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3621 }
3622
3623 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3624
3625 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3626 EC: VT.getVectorElementCount() * 2);
3627 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3628 EVT PtrVT = StackPtr.getValueType();
3629 auto &MF = DAG.getMachineFunction();
3630 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3631 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3632
3633 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3634 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3635 BaseAlignment: Alignment);
3636 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3637 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3638 BaseAlignment: Alignment);
3639
3640 SDValue EltByteSize =
3641 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3642 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3643 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3644 // Clip EVL1Bytes to make sure we stay within the stack object.
3645 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3646 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3647 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3648 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3649
3650 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3651 SDValue StoreV1 =
3652 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3653 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3654
3655 SDValue StoreV2 =
3656 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3657 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3658
3659 SDValue Load;
3660 if (Imm >= 0) {
3661 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3662 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3663 } else {
3664 uint64_t TrailingElts = -Imm;
3665 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3666 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3667
3668 // Make sure TrailingBytes doesn't exceed the size of vec1.
3669 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3670 TrailingBytes =
3671 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3672
3673 // Calculate the start address of the spliced result.
3674 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3675 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3676 }
3677
3678 // Truncate back if we widened above.
3679 if (OrigVT != VT)
3680 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3681
3682 EVT LoVT, HiVT;
3683 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3684 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3685 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3686 Hi =
3687 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3688 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3689}
3690
3691void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3692 SDValue &Hi) {
3693 SDLoc DL(N);
3694 SDValue Acc = N->getOperand(Num: 0);
3695 SDValue Input1 = N->getOperand(Num: 1);
3696 SDValue Input2 = N->getOperand(Num: 2);
3697
3698 SDValue AccLo, AccHi;
3699 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3700 unsigned Opcode = N->getOpcode();
3701
3702 // If the input types don't need splitting, just accumulate into the
3703 // low part of the accumulator.
3704 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3705 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3706 Hi = AccHi;
3707 return;
3708 }
3709
3710 SDValue Input1Lo, Input1Hi;
3711 SDValue Input2Lo, Input2Hi;
3712 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3713 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3714 EVT ResultVT = AccLo.getValueType();
3715
3716 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3717 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3718}
3719
3720void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3721 SDValue &Hi) {
3722 SDLoc DL(N);
3723 SDValue Op0 = N->getOperand(Num: 0);
3724 SDValue Op1 = N->getOperand(Num: 1);
3725 EVT OpVT = Op0.getValueType();
3726
3727 EVT LoVT, HiVT;
3728 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3729
3730 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3731 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3732 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3733 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3734}
3735
3736void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3737 unsigned Factor = N->getNumOperands();
3738
3739 SmallVector<SDValue, 8> Ops(Factor * 2);
3740 for (unsigned i = 0; i != Factor; ++i) {
3741 SDValue OpLo, OpHi;
3742 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3743 Ops[i * 2] = OpLo;
3744 Ops[i * 2 + 1] = OpHi;
3745 }
3746
3747 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3748
3749 SDLoc DL(N);
3750 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3751 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3752 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3753 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3754
3755 for (unsigned i = 0; i != Factor; ++i)
3756 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3757}
3758
3759void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3760 unsigned Factor = N->getNumOperands();
3761
3762 SmallVector<SDValue, 8> Ops(Factor * 2);
3763 for (unsigned i = 0; i != Factor; ++i) {
3764 SDValue OpLo, OpHi;
3765 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3766 Ops[i] = OpLo;
3767 Ops[i + Factor] = OpHi;
3768 }
3769
3770 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3771
3772 SDLoc DL(N);
3773 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3774 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3775 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3776 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3777
3778 for (unsigned i = 0; i != Factor; ++i) {
3779 unsigned IdxLo = 2 * i;
3780 unsigned IdxHi = 2 * i + 1;
3781 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3782 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3783 }
3784}
3785
3786//===----------------------------------------------------------------------===//
3787// Operand Vector Splitting
3788//===----------------------------------------------------------------------===//
3789
3790/// This method is called when the specified operand of the specified node is
3791/// found to need vector splitting. At this point, all of the result types of
3792/// the node are known to be legal, but other operands of the node may need
3793/// legalization as well as the specified one.
3794bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3795 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3796 SDValue Res = SDValue();
3797
3798 // See if the target wants to custom split this node.
3799 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3800 return false;
3801
3802 switch (N->getOpcode()) {
3803 default:
3804#ifndef NDEBUG
3805 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3806 N->dump(&DAG);
3807 dbgs() << "\n";
3808#endif
3809 report_fatal_error(reason: "Do not know how to split this operator's "
3810 "operand!\n");
3811
3812 case ISD::VP_SETCC:
3813 case ISD::STRICT_FSETCC:
3814 case ISD::STRICT_FSETCCS:
3815 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3816 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3817 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3818 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3819 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3820 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3821 case ISD::VECTOR_FIND_LAST_ACTIVE:
3822 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3823 break;
3824 case ISD::VP_TRUNCATE:
3825 case ISD::TRUNCATE:
3826 Res = SplitVecOp_TruncateHelper(N);
3827 break;
3828 case ISD::STRICT_FP_ROUND:
3829 case ISD::VP_FP_ROUND:
3830 case ISD::FP_ROUND:
3831 case ISD::CONVERT_FROM_ARBITRARY_FP:
3832 case ISD::CONVERT_TO_ARBITRARY_FP:
3833 Res = SplitVecOp_FP_ROUND(N);
3834 break;
3835 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3836 case ISD::STORE:
3837 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3838 break;
3839 case ISD::ATOMIC_STORE:
3840 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3841 break;
3842 case ISD::VP_STORE:
3843 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3844 break;
3845 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3846 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3847 break;
3848 case ISD::MSTORE:
3849 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3850 break;
3851 case ISD::MSCATTER:
3852 case ISD::VP_SCATTER:
3853 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3854 break;
3855 case ISD::MGATHER:
3856 case ISD::VP_GATHER:
3857 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3858 break;
3859 case ISD::VSELECT:
3860 Res = SplitVecOp_VSELECT(N, OpNo);
3861 break;
3862 case ISD::VECTOR_COMPRESS:
3863 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3864 break;
3865 case ISD::STRICT_SINT_TO_FP:
3866 case ISD::STRICT_UINT_TO_FP:
3867 case ISD::SINT_TO_FP:
3868 case ISD::UINT_TO_FP:
3869 case ISD::VP_SINT_TO_FP:
3870 case ISD::VP_UINT_TO_FP:
3871 if (N->getValueType(ResNo: 0).bitsLT(
3872 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3873 Res = SplitVecOp_TruncateHelper(N);
3874 else
3875 Res = SplitVecOp_UnaryOp(N);
3876 break;
3877 case ISD::FP_TO_SINT_SAT:
3878 case ISD::FP_TO_UINT_SAT:
3879 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3880 break;
3881 case ISD::FP_TO_SINT:
3882 case ISD::FP_TO_UINT:
3883 case ISD::VP_FP_TO_SINT:
3884 case ISD::VP_FP_TO_UINT:
3885 case ISD::STRICT_FP_TO_SINT:
3886 case ISD::STRICT_FP_TO_UINT:
3887 case ISD::STRICT_FP_EXTEND:
3888 case ISD::FP_EXTEND:
3889 case ISD::SIGN_EXTEND:
3890 case ISD::ZERO_EXTEND:
3891 case ISD::ANY_EXTEND:
3892 case ISD::FTRUNC:
3893 case ISD::LROUND:
3894 case ISD::LLROUND:
3895 case ISD::LRINT:
3896 case ISD::LLRINT:
3897 Res = SplitVecOp_UnaryOp(N);
3898 break;
3899 case ISD::FLDEXP:
3900 Res = SplitVecOp_FPOpDifferentTypes(N);
3901 break;
3902
3903 case ISD::SCMP:
3904 case ISD::UCMP:
3905 Res = SplitVecOp_CMP(N);
3906 break;
3907
3908 case ISD::FAKE_USE:
3909 Res = SplitVecOp_FAKE_USE(N);
3910 break;
3911 case ISD::ANY_EXTEND_VECTOR_INREG:
3912 case ISD::SIGN_EXTEND_VECTOR_INREG:
3913 case ISD::ZERO_EXTEND_VECTOR_INREG:
3914 Res = SplitVecOp_ExtVecInRegOp(N);
3915 break;
3916
3917 case ISD::VECREDUCE_FADD:
3918 case ISD::VECREDUCE_FMUL:
3919 case ISD::VECREDUCE_ADD:
3920 case ISD::VECREDUCE_MUL:
3921 case ISD::VECREDUCE_AND:
3922 case ISD::VECREDUCE_OR:
3923 case ISD::VECREDUCE_XOR:
3924 case ISD::VECREDUCE_SMAX:
3925 case ISD::VECREDUCE_SMIN:
3926 case ISD::VECREDUCE_UMAX:
3927 case ISD::VECREDUCE_UMIN:
3928 case ISD::VECREDUCE_FMAX:
3929 case ISD::VECREDUCE_FMIN:
3930 case ISD::VECREDUCE_FMAXIMUM:
3931 case ISD::VECREDUCE_FMINIMUM:
3932 Res = SplitVecOp_VECREDUCE(N, OpNo);
3933 break;
3934 case ISD::VECREDUCE_SEQ_FADD:
3935 case ISD::VECREDUCE_SEQ_FMUL:
3936 Res = SplitVecOp_VECREDUCE_SEQ(N);
3937 break;
3938 case ISD::VP_REDUCE_FADD:
3939 case ISD::VP_REDUCE_SEQ_FADD:
3940 case ISD::VP_REDUCE_FMUL:
3941 case ISD::VP_REDUCE_SEQ_FMUL:
3942 case ISD::VP_REDUCE_ADD:
3943 case ISD::VP_REDUCE_MUL:
3944 case ISD::VP_REDUCE_AND:
3945 case ISD::VP_REDUCE_OR:
3946 case ISD::VP_REDUCE_XOR:
3947 case ISD::VP_REDUCE_SMAX:
3948 case ISD::VP_REDUCE_SMIN:
3949 case ISD::VP_REDUCE_UMAX:
3950 case ISD::VP_REDUCE_UMIN:
3951 case ISD::VP_REDUCE_FMAX:
3952 case ISD::VP_REDUCE_FMIN:
3953 case ISD::VP_REDUCE_FMAXIMUM:
3954 case ISD::VP_REDUCE_FMINIMUM:
3955 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3956 break;
3957 case ISD::CTTZ_ELTS:
3958 case ISD::CTTZ_ELTS_ZERO_POISON:
3959 Res = SplitVecOp_CttzElts(N);
3960 break;
3961 case ISD::VP_CTTZ_ELTS:
3962 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
3963 Res = SplitVecOp_VP_CttzElements(N);
3964 break;
3965 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3966 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3967 break;
3968 case ISD::PARTIAL_REDUCE_UMLA:
3969 case ISD::PARTIAL_REDUCE_SMLA:
3970 case ISD::PARTIAL_REDUCE_SUMLA:
3971 case ISD::PARTIAL_REDUCE_FMLA:
3972 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3973 break;
3974 }
3975
3976 // If the result is null, the sub-method took care of registering results etc.
3977 if (!Res.getNode()) return false;
3978
3979 // If the result is N, the sub-method updated N in place. Tell the legalizer
3980 // core about this.
3981 if (Res.getNode() == N)
3982 return true;
3983
3984 if (N->isStrictFPOpcode())
3985 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3986 "Invalid operand expansion");
3987 else
3988 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3989 "Invalid operand expansion");
3990
3991 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3992 return false;
3993}
3994
3995SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3996 SDLoc DL(N);
3997
3998 SDValue LoMask, HiMask;
3999 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
4000
4001 EVT VT = N->getValueType(ResNo: 0);
4002 EVT SplitVT = LoMask.getValueType();
4003 ElementCount SplitEC = SplitVT.getVectorElementCount();
4004
4005 // Find the last active in both the low and the high masks.
4006 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
4007 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
4008
4009 // Check if any lane is active in the high mask.
4010 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4011 // sentinel value for "none active".
4012 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4013 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4014 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4015
4016 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4017 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4018 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4019 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4020 N3: LoFind);
4021}
4022
4023SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4024 // The only possibility for an illegal operand is the mask, since result type
4025 // legalization would have handled this node already otherwise.
4026 assert(OpNo == 0 && "Illegal operand must be mask");
4027
4028 SDValue Mask = N->getOperand(Num: 0);
4029 SDValue Src0 = N->getOperand(Num: 1);
4030 SDValue Src1 = N->getOperand(Num: 2);
4031 EVT Src0VT = Src0.getValueType();
4032 SDLoc DL(N);
4033 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4034
4035 SDValue Lo, Hi;
4036 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4037 assert(Lo.getValueType() == Hi.getValueType() &&
4038 "Lo and Hi have differing types");
4039
4040 EVT LoOpVT, HiOpVT;
4041 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4042 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4043
4044 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4045 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4046 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4047 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4048
4049 SDValue LoSelect =
4050 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4051 SDValue HiSelect =
4052 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4053
4054 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4055}
4056
4057SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4058 // The only possibility for an illegal operand is the mask, since result type
4059 // legalization would have handled this node already otherwise.
4060 assert(OpNo == 1 && "Illegal operand must be mask");
4061
4062 // To split the mask, we need to split the result type too, so we can just
4063 // reuse that logic here.
4064 SDValue Lo, Hi;
4065 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4066
4067 EVT VecVT = N->getValueType(ResNo: 0);
4068 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4069}
4070
4071SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4072 EVT ResVT = N->getValueType(ResNo: 0);
4073 SDValue Lo, Hi;
4074 SDLoc dl(N);
4075
4076 SDValue VecOp = N->getOperand(Num: OpNo);
4077 EVT VecVT = VecOp.getValueType();
4078 assert(VecVT.isVector() && "Can only split reduce vector operand");
4079 GetSplitVector(Op: VecOp, Lo, Hi);
4080 EVT LoOpVT, HiOpVT;
4081 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4082
4083 // Use the appropriate scalar instruction on the split subvectors before
4084 // reducing the now partially reduced smaller vector.
4085 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4086 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4087 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4088}
4089
4090SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4091 EVT ResVT = N->getValueType(ResNo: 0);
4092 SDValue Lo, Hi;
4093 SDLoc dl(N);
4094
4095 SDValue AccOp = N->getOperand(Num: 0);
4096 SDValue VecOp = N->getOperand(Num: 1);
4097 SDNodeFlags Flags = N->getFlags();
4098
4099 EVT VecVT = VecOp.getValueType();
4100 assert(VecVT.isVector() && "Can only split reduce vector operand");
4101 GetSplitVector(Op: VecOp, Lo, Hi);
4102 EVT LoOpVT, HiOpVT;
4103 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4104
4105 // Reduce low half.
4106 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4107
4108 // Reduce high half, using low half result as initial value.
4109 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4110}
4111
4112SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4113 assert(N->isVPOpcode() && "Expected VP opcode");
4114 assert(OpNo == 1 && "Can only split reduce vector operand");
4115
4116 unsigned Opc = N->getOpcode();
4117 EVT ResVT = N->getValueType(ResNo: 0);
4118 SDValue Lo, Hi;
4119 SDLoc dl(N);
4120
4121 SDValue VecOp = N->getOperand(Num: OpNo);
4122 EVT VecVT = VecOp.getValueType();
4123 assert(VecVT.isVector() && "Can only split reduce vector operand");
4124 GetSplitVector(Op: VecOp, Lo, Hi);
4125
4126 SDValue MaskLo, MaskHi;
4127 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4128
4129 SDValue EVLLo, EVLHi;
4130 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4131
4132 const SDNodeFlags Flags = N->getFlags();
4133
4134 SDValue ResLo =
4135 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4136 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4137}
4138
4139SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4140 // The result has a legal vector type, but the input needs splitting.
4141 EVT ResVT = N->getValueType(ResNo: 0);
4142 SDValue Lo, Hi;
4143 SDLoc dl(N);
4144 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4145 EVT InVT = Lo.getValueType();
4146
4147 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4148 EC: InVT.getVectorElementCount());
4149
4150 if (N->isStrictFPOpcode()) {
4151 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4152 Ops: {N->getOperand(Num: 0), Lo});
4153 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4154 Ops: {N->getOperand(Num: 0), Hi});
4155
4156 // Build a factor node to remember that this operation is independent
4157 // of the other one.
4158 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4159 N2: Hi.getValue(R: 1));
4160
4161 // Legalize the chain result - switch anything that used the old chain to
4162 // use the new one.
4163 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4164 } else if (N->getNumOperands() == 3) {
4165 assert(N->isVPOpcode() && "Expected VP opcode");
4166 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4167 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4168 std::tie(args&: EVLLo, args&: EVLHi) =
4169 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
4170 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4171 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4172 } else {
4173 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4174 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4175 }
4176
4177 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4178}
4179
4180// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4181SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4182 SDValue Lo, Hi;
4183 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4184 SDValue Chain =
4185 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4186 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4187}
4188
4189SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4190 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4191 // end up being split all the way down to individual components. Convert the
4192 // split pieces into integers and reassemble.
4193 EVT ResVT = N->getValueType(ResNo: 0);
4194 SDValue Lo, Hi;
4195 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4196 SDLoc dl(N);
4197
4198 if (ResVT.isScalableVector()) {
4199 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4200 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4201 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4202 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4203 }
4204
4205 Lo = BitConvertToInteger(Op: Lo);
4206 Hi = BitConvertToInteger(Op: Hi);
4207
4208 if (DAG.getDataLayout().isBigEndian())
4209 std::swap(a&: Lo, b&: Hi);
4210
4211 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4212}
4213
4214SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4215 unsigned OpNo) {
4216 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4217 // We know that the result type is legal.
4218 EVT ResVT = N->getValueType(ResNo: 0);
4219
4220 SDValue Vec = N->getOperand(Num: 0);
4221 SDValue SubVec = N->getOperand(Num: 1);
4222 SDValue Idx = N->getOperand(Num: 2);
4223 SDLoc dl(N);
4224
4225 SDValue Lo, Hi;
4226 GetSplitVector(Op: SubVec, Lo, Hi);
4227
4228 uint64_t IdxVal = Idx->getAsZExtVal();
4229 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4230
4231 SDValue FirstInsertion =
4232 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4233 SDValue SecondInsertion =
4234 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4235 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4236
4237 return SecondInsertion;
4238}
4239
4240SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4241 // We know that the extracted result type is legal.
4242 EVT SubVT = N->getValueType(ResNo: 0);
4243 SDValue Idx = N->getOperand(Num: 1);
4244 SDLoc dl(N);
4245 SDValue Lo, Hi;
4246
4247 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4248
4249 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4250 // Note: For scalable vectors, the index is scaled by vscale.
4251 ElementCount IdxVal =
4252 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4253 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4254
4255 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4256 ElementCount NumResultElts = SubVT.getVectorElementCount();
4257
4258 // If the extracted elements are all in the low half, do a simple extract.
4259 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4260 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4261
4262 unsigned LoEltsMin = LoElts.getKnownMinValue();
4263 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4264 SrcVT.isFixedLengthVector()) {
4265 // Extracted subvector crosses vector split, so we need to blend the two
4266 // halves.
4267 // TODO: May be able to emit partial extract_subvector.
4268 SmallVector<SDValue, 8> Elts;
4269 Elts.reserve(N: NumResultElts.getFixedValue());
4270
4271 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4272 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4273 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4274 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4275 /*Count=*/LoEltsMin - IdxValMin);
4276 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4277 /*Count=*/SubVT.getVectorNumElements() -
4278 Elts.size());
4279 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4280 }
4281
4282 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4283 ElementCount ExtractIdx = IdxVal - LoElts;
4284 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4285 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4286 Idx: ExtractIdx.getKnownMinValue());
4287
4288 EVT HiVT = Hi.getValueType();
4289 assert(HiVT.isFixedLengthVector() &&
4290 "Only fixed-vector extracts are supported in this case");
4291
4292 // We cannot create an extract_subvector that isn't a multiple of the
4293 // result size, which may go out of bounds for the last elements. Shuffle
4294 // the desired elements down to 0 and do a simple 0 extract.
4295 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4296 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4297 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4298
4299 SDValue Shuffle =
4300 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4301 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4302 }
4303
4304 // After this point the DAG node only permits extracting fixed-width
4305 // subvectors from scalable vectors.
4306 assert(SubVT.isFixedLengthVector() &&
4307 "Extracting scalable subvector from fixed-width unsupported");
4308
4309 // If the element type is i1 and we're not promoting the result, then we may
4310 // end up loading the wrong data since the bits are packed tightly into
4311 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4312 // type at index 4, then we will load a byte starting at index 0.
4313 if (SubVT.getScalarType() == MVT::i1)
4314 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4315 "subvector from a scalable predicate vector");
4316
4317 // Spill the vector to the stack. We should use the alignment for
4318 // the smallest part.
4319 SDValue Vec = N->getOperand(Num: 0);
4320 EVT VecVT = Vec.getValueType();
4321 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4322 SDValue StackPtr =
4323 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4324 auto &MF = DAG.getMachineFunction();
4325 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4326 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4327
4328 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4329 Alignment: SmallestAlign);
4330
4331 // Extract the subvector by loading the correct part.
4332 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4333
4334 return DAG.getLoad(
4335 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4336 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4337}
4338
4339SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4340 SDValue Vec = N->getOperand(Num: 0);
4341 SDValue Idx = N->getOperand(Num: 1);
4342 EVT VecVT = Vec.getValueType();
4343
4344 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4345 uint64_t IdxVal = Index->getZExtValue();
4346
4347 SDValue Lo, Hi;
4348 GetSplitVector(Op: Vec, Lo, Hi);
4349
4350 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4351
4352 if (IdxVal < LoElts)
4353 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4354 else if (!Vec.getValueType().isScalableVector())
4355 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4356 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4357 VT: Idx.getValueType())), 0);
4358 }
4359
4360 // See if the target wants to custom expand this node.
4361 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4362 return SDValue();
4363
4364 // Make the vector elements byte-addressable if they aren't already.
4365 SDLoc dl(N);
4366 EVT EltVT = VecVT.getVectorElementType();
4367 if (!EltVT.isByteSized()) {
4368 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4369 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4370 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4371 SDValue NewExtract =
4372 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4373 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4374 }
4375
4376 // Store the vector to the stack.
4377 // In cases where the vector is illegal it will be broken down into parts
4378 // and stored in parts - we should use the alignment for the smallest part.
4379 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4380 SDValue StackPtr =
4381 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4382 auto &MF = DAG.getMachineFunction();
4383 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4384 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4385 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4386 Alignment: SmallestAlign);
4387
4388 // Load back the required element.
4389 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4390
4391 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4392 // type, leaving the high bits undefined. But it can't truncate.
4393 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4394
4395 return DAG.getExtLoad(
4396 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4397 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4398 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4399}
4400
4401SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4402 SDValue Lo, Hi;
4403
4404 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4405 // splitting the result has the same effect as splitting the input operand.
4406 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4407
4408 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4409}
4410
4411SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4412 (void)OpNo;
4413 SDValue Lo, Hi;
4414 SplitVecRes_Gather(N, Lo, Hi);
4415
4416 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4417 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4418 return SDValue();
4419}
4420
4421SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4422 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4423 SDValue Ch = N->getChain();
4424 SDValue Ptr = N->getBasePtr();
4425 SDValue Offset = N->getOffset();
4426 assert(Offset.isUndef() && "Unexpected VP store offset");
4427 SDValue Mask = N->getMask();
4428 SDValue EVL = N->getVectorLength();
4429 SDValue Data = N->getValue();
4430 Align Alignment = N->getBaseAlign();
4431 SDLoc DL(N);
4432
4433 SDValue DataLo, DataHi;
4434 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4435 // Split Data operand
4436 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4437 else
4438 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4439
4440 // Split Mask operand
4441 SDValue MaskLo, MaskHi;
4442 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4443 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4444 } else {
4445 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4446 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4447 else
4448 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4449 }
4450
4451 EVT MemoryVT = N->getMemoryVT();
4452 EVT LoMemVT, HiMemVT;
4453 bool HiIsEmpty = false;
4454 std::tie(args&: LoMemVT, args&: HiMemVT) =
4455 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4456
4457 // Split EVL
4458 SDValue EVLLo, EVLHi;
4459 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4460
4461 SDValue Lo, Hi;
4462 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4463 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4464 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4465 Ranges: N->getRanges());
4466
4467 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4468 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4469 IsCompressing: N->isCompressingStore());
4470
4471 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4472 if (HiIsEmpty)
4473 return Lo;
4474
4475 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4476 IsCompressedMemory: N->isCompressingStore());
4477
4478 MachinePointerInfo MPI;
4479 if (LoMemVT.isScalableVector()) {
4480 Alignment = commonAlignment(A: Alignment,
4481 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4482 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4483 } else
4484 MPI = N->getPointerInfo().getWithOffset(
4485 O: LoMemVT.getStoreSize().getFixedValue());
4486
4487 MMO = DAG.getMachineFunction().getMachineMemOperand(
4488 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4489 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4490
4491 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4492 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4493 IsCompressing: N->isCompressingStore());
4494
4495 // Build a factor node to remember that this store is independent of the
4496 // other one.
4497 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4498}
4499
4500SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4501 unsigned OpNo) {
4502 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4503 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4504
4505 SDLoc DL(N);
4506
4507 SDValue Data = N->getValue();
4508 SDValue LoData, HiData;
4509 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4510 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4511 else
4512 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4513
4514 EVT LoMemVT, HiMemVT;
4515 bool HiIsEmpty = false;
4516 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4517 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4518
4519 SDValue Mask = N->getMask();
4520 SDValue LoMask, HiMask;
4521 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4522 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4523 else if (getTypeAction(VT: Mask.getValueType()) ==
4524 TargetLowering::TypeSplitVector)
4525 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4526 else
4527 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4528
4529 SDValue LoEVL, HiEVL;
4530 std::tie(args&: LoEVL, args&: HiEVL) =
4531 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4532
4533 // Generate the low vp_strided_store
4534 SDValue Lo = DAG.getStridedStoreVP(
4535 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4536 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4537 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4538
4539 // If the high vp_strided_store has zero storage size, only the low
4540 // vp_strided_store is needed.
4541 if (HiIsEmpty)
4542 return Lo;
4543
4544 // Generate the high vp_strided_store.
4545 // To calculate the high base address, we need to sum to the low base
4546 // address stride number of bytes for each element already stored by low,
4547 // that is: Ptr = Ptr + (LoEVL * Stride)
4548 EVT PtrVT = N->getBasePtr().getValueType();
4549 SDValue Increment =
4550 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4551 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4552 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4553
4554 Align Alignment = N->getBaseAlign();
4555 if (LoMemVT.isScalableVector())
4556 Alignment = commonAlignment(A: Alignment,
4557 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4558
4559 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4560 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4561 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4562 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4563
4564 SDValue Hi = DAG.getStridedStoreVP(
4565 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4566 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4567 IsCompressing: N->isCompressingStore());
4568
4569 // Build a factor node to remember that this store is independent of the
4570 // other one.
4571 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4572}
4573
4574SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4575 unsigned OpNo) {
4576 assert(N->isUnindexed() && "Indexed masked store of vector?");
4577 SDValue Ch = N->getChain();
4578 SDValue Ptr = N->getBasePtr();
4579 SDValue Offset = N->getOffset();
4580 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4581 SDValue Mask = N->getMask();
4582 SDValue Data = N->getValue();
4583 Align Alignment = N->getBaseAlign();
4584 SDLoc DL(N);
4585
4586 SDValue DataLo, DataHi;
4587 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4588 // Split Data operand
4589 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4590 else
4591 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4592
4593 // Split Mask operand
4594 SDValue MaskLo, MaskHi;
4595 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4596 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4597 } else {
4598 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4599 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4600 else
4601 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4602 }
4603
4604 EVT MemoryVT = N->getMemoryVT();
4605 EVT LoMemVT, HiMemVT;
4606 bool HiIsEmpty = false;
4607 std::tie(args&: LoMemVT, args&: HiMemVT) =
4608 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4609
4610 SDValue Lo, Hi, Res;
4611 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4612 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4613 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4614 Ranges: N->getRanges());
4615
4616 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4617 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4618 IsCompressing: N->isCompressingStore());
4619
4620 if (HiIsEmpty) {
4621 // The hi masked store has zero storage size.
4622 // Only the lo masked store is needed.
4623 Res = Lo;
4624 } else {
4625
4626 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4627 IsCompressedMemory: N->isCompressingStore());
4628
4629 MachinePointerInfo MPI;
4630 if (LoMemVT.isScalableVector()) {
4631 Alignment = commonAlignment(
4632 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4633 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4634 } else
4635 MPI = N->getPointerInfo().getWithOffset(
4636 O: LoMemVT.getStoreSize().getFixedValue());
4637
4638 MMO = DAG.getMachineFunction().getMachineMemOperand(
4639 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4640 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4641
4642 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4643 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4644 IsCompressing: N->isCompressingStore());
4645
4646 // Build a factor node to remember that this store is independent of the
4647 // other one.
4648 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4649 }
4650
4651 return Res;
4652}
4653
4654SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4655 SDValue Ch = N->getChain();
4656 SDValue Ptr = N->getBasePtr();
4657 EVT MemoryVT = N->getMemoryVT();
4658 Align Alignment = N->getBaseAlign();
4659 SDLoc DL(N);
4660 struct Operands {
4661 SDValue Mask;
4662 SDValue Index;
4663 SDValue Scale;
4664 SDValue Data;
4665 } Ops = [&]() -> Operands {
4666 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4667 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4668 .Data: MSC->getValue()};
4669 }
4670 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4671 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4672 .Data: VPSC->getValue()};
4673 }();
4674 // Split all operands
4675
4676 EVT LoMemVT, HiMemVT;
4677 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4678
4679 SDValue DataLo, DataHi;
4680 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4681 // Split Data operand
4682 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4683 else
4684 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4685
4686 // Split Mask operand
4687 SDValue MaskLo, MaskHi;
4688 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4689 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4690 } else {
4691 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4692 }
4693
4694 SDValue IndexHi, IndexLo;
4695 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4696 TargetLowering::TypeSplitVector)
4697 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4698 else
4699 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4700
4701 SDValue Lo;
4702 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4703 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4704 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4705 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4706
4707 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4708 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4709 Lo =
4710 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4711 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4712
4713 // The order of the Scatter operation after split is well defined. The "Hi"
4714 // part comes after the "Lo". So these two operations should be chained one
4715 // after another.
4716 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4717 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4718 MMO, IndexType: MSC->getIndexType(),
4719 IsTruncating: MSC->isTruncatingStore());
4720 }
4721 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4722 SDValue EVLLo, EVLHi;
4723 std::tie(args&: EVLLo, args&: EVLHi) =
4724 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4725
4726 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4727 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4728 IndexType: VPSC->getIndexType());
4729
4730 // The order of the Scatter operation after split is well defined. The "Hi"
4731 // part comes after the "Lo". So these two operations should be chained one
4732 // after another.
4733 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4734 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4735 IndexType: VPSC->getIndexType());
4736}
4737
4738SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4739 assert(N->isUnindexed() && "Indexed store of vector?");
4740 assert(OpNo == 1 && "Can only split the stored value");
4741 SDLoc DL(N);
4742
4743 bool isTruncating = N->isTruncatingStore();
4744 SDValue Ch = N->getChain();
4745 SDValue Ptr = N->getBasePtr();
4746 EVT MemoryVT = N->getMemoryVT();
4747 Align Alignment = N->getBaseAlign();
4748 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4749 AAMDNodes AAInfo = N->getAAInfo();
4750 SDValue Lo, Hi;
4751 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4752
4753 EVT LoMemVT, HiMemVT;
4754 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4755
4756 // Scalarize if the split halves are not byte-sized.
4757 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4758 return TLI.scalarizeVectorStore(ST: N, DAG);
4759
4760 if (isTruncating)
4761 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4762 Alignment, MMOFlags, AAInfo);
4763 else
4764 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4765 AAInfo);
4766
4767 MachinePointerInfo MPI;
4768 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4769
4770 if (isTruncating)
4771 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4772 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4773 else
4774 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4775
4776 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4777}
4778
4779SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4780 SDLoc DL(N);
4781 LLVMContext &Ctx = *DAG.getContext();
4782 SDValue StVal = N->getVal();
4783 EVT VT = StVal.getValueType();
4784 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4785
4786 // The store needs a single value spanning the full memory width. If the
4787 // value can be held in a legal vector register, keep it there and extract
4788 // the low integer element of the memory width. This lets the store be issued
4789 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4790 // bitcasting the split vector straight to a scalar integer, which would
4791 // reassemble the value element by element in GPRs.
4792 //
4793 // Reinterpret the value as a same-shaped integer vector first: an FP element
4794 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4795 // integer-of-element-size form does. Ask the target which legal vector type
4796 // it widens to.
4797 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4798 EVT IntEltVT = IntVecVT.getVectorElementType();
4799 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4800 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4801 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4802 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4803 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4804 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4805 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4806 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4807 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4808 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4809 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4810 Val: N->getBasePtr(), MMO: N->getMemOperand());
4811 }
4812
4813 // Otherwise issue a single atomic store of an integer that spans the full
4814 // memory width. Bitcasting the (illegal) vector value to that integer lets
4815 // the type legalizer further legalize the BITCAST input as needed, while the
4816 // ATOMIC_STORE itself uses only the legal integer type.
4817 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4818 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4819 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4820 Val: N->getBasePtr(), MMO: N->getMemOperand());
4821}
4822
4823SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4824 SDLoc DL(N);
4825
4826 // The input operands all must have the same type, and we know the result
4827 // type is valid. Convert this to a buildvector which extracts all the
4828 // input elements.
4829 // TODO: If the input elements are power-two vectors, we could convert this to
4830 // a new CONCAT_VECTORS node with elements that are half-wide.
4831 SmallVector<SDValue, 32> Elts;
4832 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4833 for (const SDValue &Op : N->op_values()) {
4834 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4835 i != e; ++i) {
4836 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4837 }
4838 }
4839
4840 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4841}
4842
4843SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4844 // The result type is legal, but the input type is illegal. If splitting
4845 // ends up with the result type of each half still being legal, just
4846 // do that. If, however, that would result in an illegal result type,
4847 // we can try to get more clever with power-two vectors. Specifically,
4848 // split the input type, but also widen the result element size, then
4849 // concatenate the halves and truncate again. For example, consider a target
4850 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4851 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4852 // %inlo = v4i32 extract_subvector %in, 0
4853 // %inhi = v4i32 extract_subvector %in, 4
4854 // %lo16 = v4i16 trunc v4i32 %inlo
4855 // %hi16 = v4i16 trunc v4i32 %inhi
4856 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4857 // %res = v8i8 trunc v8i16 %in16
4858 //
4859 // Without this transform, the original truncate would end up being
4860 // scalarized, which is pretty much always a last resort.
4861 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4862 SDValue InVec = N->getOperand(Num: OpNo);
4863 EVT InVT = InVec->getValueType(ResNo: 0);
4864 EVT OutVT = N->getValueType(ResNo: 0);
4865 ElementCount NumElements = OutVT.getVectorElementCount();
4866 bool IsFloat = OutVT.isFloatingPoint();
4867
4868 unsigned InElementSize = InVT.getScalarSizeInBits();
4869 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4870
4871 // Determine the split output VT. If its legal we can just split dirctly.
4872 EVT LoOutVT, HiOutVT;
4873 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4874 assert(LoOutVT == HiOutVT && "Unequal split?");
4875
4876 // If the input elements are only 1/2 the width of the result elements,
4877 // just use the normal splitting. Our trick only work if there's room
4878 // to split more than once.
4879 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4880 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4881 return SplitVecOp_UnaryOp(N);
4882 SDLoc DL(N);
4883
4884 // Don't touch if this will be scalarized.
4885 EVT FinalVT = InVT;
4886 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4887 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4888
4889 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4890 return SplitVecOp_UnaryOp(N);
4891
4892 // Get the split input vector.
4893 SDValue InLoVec, InHiVec;
4894 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4895
4896 // Truncate them to 1/2 the element size.
4897 //
4898 // This assumes the number of elements is a power of two; any vector that
4899 // isn't should be widened, not split.
4900 EVT HalfElementVT = IsFloat ?
4901 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4902 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4903 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4904 EC: NumElements.divideCoefficientBy(RHS: 2));
4905
4906 SDValue HalfLo;
4907 SDValue HalfHi;
4908 SDValue Chain;
4909 if (N->isStrictFPOpcode()) {
4910 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4911 Ops: {N->getOperand(Num: 0), InLoVec});
4912 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4913 Ops: {N->getOperand(Num: 0), InHiVec});
4914 // Legalize the chain result - switch anything that used the old chain to
4915 // use the new one.
4916 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4917 N2: HalfHi.getValue(R: 1));
4918 } else {
4919 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4920 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4921 }
4922
4923 // Concatenate them to get the full intermediate truncation result.
4924 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4925 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4926 N2: HalfHi);
4927 // Now finish up by truncating all the way down to the original result
4928 // type. This should normally be something that ends up being legal directly,
4929 // but in theory if a target has very wide vectors and an annoyingly
4930 // restricted set of legal types, this split can chain to build things up.
4931
4932 if (N->isStrictFPOpcode()) {
4933 SDValue Res = DAG.getNode(
4934 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4935 Ops: {Chain, InterVec,
4936 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4937 // Relink the chain
4938 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4939 return Res;
4940 }
4941
4942 return IsFloat
4943 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4944 N2: DAG.getTargetConstant(
4945 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4946 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4947}
4948
4949SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4950 unsigned Opc = N->getOpcode();
4951 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4952 assert(N->getValueType(0).isVector() &&
4953 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4954 "Operand types must be vectors");
4955 // The result has a legal vector type, but the input needs splitting.
4956 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4957 SDLoc DL(N);
4958 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4959 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4960
4961 EVT VT = N->getValueType(ResNo: 0);
4962 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
4963
4964 if (Opc == ISD::SETCC) {
4965 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4966 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4967 } else if (isStrict) {
4968 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4969 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4970 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4971 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4972 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4973 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4974 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4975 } else {
4976 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4977 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4978 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4979 std::tie(args&: EVLLo, args&: EVLHi) =
4980 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4981 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4982 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4983 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4984 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4985 }
4986
4987 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4988 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
4989 if (VT == ConcatVT)
4990 return Con;
4991
4992 EVT OpVT = N->getOperand(Num: 0).getValueType();
4993 ISD::NodeType ExtendCode =
4994 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
4995 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
4996}
4997
4998
4999SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5000 // The result has a legal vector type, but the input needs splitting.
5001 EVT ResVT = N->getValueType(ResNo: 0);
5002 SDValue Lo, Hi;
5003 SDLoc DL(N);
5004 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5005 EVT InVT = Lo.getValueType();
5006
5007 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5008 EC: InVT.getVectorElementCount());
5009
5010 if (N->isStrictFPOpcode()) {
5011 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5012 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5013 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5014 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5015 // Legalize the chain result - switch anything that used the old chain to
5016 // use the new one.
5017 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5018 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5019 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5020 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
5021 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
5022 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
5023 std::tie(args&: EVLLo, args&: EVLHi) =
5024 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
5025 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5026 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5027 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5028 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5029 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5030 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5031 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5032 } else {
5033 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5034 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5035 }
5036
5037 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5038}
5039
5040// Split a vector type in an FP binary operation where the second operand has a
5041// different type from the first.
5042//
5043// The result (and the first input) has a legal vector type, but the second
5044// input needs splitting.
5045SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5046 SDLoc DL(N);
5047
5048 EVT LHSLoVT, LHSHiVT;
5049 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5050
5051 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5052 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5053
5054 SDValue LHSLo, LHSHi;
5055 std::tie(args&: LHSLo, args&: LHSHi) =
5056 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5057
5058 SDValue RHSLo, RHSHi;
5059 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5060
5061 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5062 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5063
5064 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5065}
5066
5067SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5068 LLVMContext &Ctxt = *DAG.getContext();
5069 SDLoc dl(N);
5070
5071 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5072 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5073 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5074
5075 EVT ResVT = N->getValueType(ResNo: 0);
5076 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5077 EVT NewResVT =
5078 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5079
5080 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5081 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5082
5083 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5084}
5085
5086SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5087 EVT ResVT = N->getValueType(ResNo: 0);
5088 SDValue Lo, Hi;
5089 SDLoc dl(N);
5090 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5091 EVT InVT = Lo.getValueType();
5092
5093 EVT NewResVT =
5094 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5095 EC: InVT.getVectorElementCount());
5096
5097 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5098 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5099
5100 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5101}
5102
5103SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5104 SDLoc DL(N);
5105 EVT ResVT = N->getValueType(ResNo: 0);
5106
5107 SDValue Lo, Hi;
5108 SDValue VecOp = N->getOperand(Num: 0);
5109 GetSplitVector(Op: VecOp, Lo, Hi);
5110
5111 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5112 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5113 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5114 SDValue VL =
5115 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5116 SDValue ResLoNotVL =
5117 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5118 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5119 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5120 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5121}
5122
5123SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5124 SDLoc DL(N);
5125 EVT ResVT = N->getValueType(ResNo: 0);
5126
5127 SDValue Lo, Hi;
5128 SDValue VecOp = N->getOperand(Num: 0);
5129 GetSplitVector(Op: VecOp, Lo, Hi);
5130
5131 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5132 auto [EVLLo, EVLHi] =
5133 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5134 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5135
5136 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5137 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5138 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5139 SDValue ResLoNotEVL =
5140 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5141 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5142 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5143 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5144}
5145
5146SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5147 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5148 SDLoc DL(HG);
5149 SDValue Inc = HG->getInc();
5150 SDValue Ptr = HG->getBasePtr();
5151 SDValue Scale = HG->getScale();
5152 SDValue IntID = HG->getIntID();
5153 EVT MemVT = HG->getMemoryVT();
5154 MachineMemOperand *MMO = HG->getMemOperand();
5155 ISD::MemIndexType IndexType = HG->getIndexType();
5156
5157 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5158 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5159 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5160 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5161 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5162 Ops: OpsLo, MMO, IndexType);
5163 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5164 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5165 MMO, IndexType);
5166}
5167
5168SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5169 SDValue Acc = N->getOperand(Num: 0);
5170 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5171 "Accumulator should already be a legal type, and shouldn't need "
5172 "further splitting");
5173
5174 SDLoc DL(N);
5175 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5176 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5177 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5178 unsigned Opcode = N->getOpcode();
5179 EVT ResultVT = Acc.getValueType();
5180
5181 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5182 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5183}
5184
5185//===----------------------------------------------------------------------===//
5186// Result Vector Widening
5187//===----------------------------------------------------------------------===//
5188
5189void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5190 unsigned WidenResNo) {
5191 unsigned NumResults = N->getNumValues();
5192 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5193 if (ResNo == WidenResNo)
5194 continue;
5195 EVT ResVT = N->getValueType(ResNo);
5196 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5197 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5198 } else {
5199 SDLoc DL(N);
5200 SDValue ResVal =
5201 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5202 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5203 }
5204 }
5205}
5206
5207void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5208 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5209
5210 // See if the target wants to custom widen this node.
5211 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5212 return;
5213
5214 SDValue Res = SDValue();
5215
5216 auto unrollExpandedOp = [&]() {
5217 // We're going to widen this vector op to a legal type by padding with undef
5218 // elements. If the wide vector op is eventually going to be expanded to
5219 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5220 // libcalls on the undef elements.
5221 EVT VT = N->getValueType(ResNo: 0);
5222 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5223 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5224 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
5225 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5226 if (N->getNumValues() > 1)
5227 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
5228 return true;
5229 }
5230 return false;
5231 };
5232
5233 switch (N->getOpcode()) {
5234 default:
5235#ifndef NDEBUG
5236 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5237 N->dump(&DAG);
5238 dbgs() << "\n";
5239#endif
5240 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
5241
5242 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5243 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5244 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5245 break;
5246 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5247 case ISD::ADDRSPACECAST:
5248 Res = WidenVecRes_ADDRSPACECAST(N);
5249 break;
5250 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5251 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5252 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5253 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5254 case ISD::INSERT_SUBVECTOR:
5255 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5256 break;
5257 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5258 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5259 case ISD::ATOMIC_LOAD:
5260 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5261 break;
5262 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5263 case ISD::STEP_VECTOR:
5264 case ISD::SPLAT_VECTOR:
5265 case ISD::SCALAR_TO_VECTOR:
5266 Res = WidenVecRes_ScalarOp(N);
5267 break;
5268 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5269 case ISD::VSELECT:
5270 case ISD::SELECT:
5271 case ISD::VP_SELECT:
5272 case ISD::VP_MERGE:
5273 Res = WidenVecRes_Select(N);
5274 break;
5275 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5276 case ISD::VP_SETCC:
5277 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5278 case ISD::POISON:
5279 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5280 case ISD::VECTOR_SHUFFLE:
5281 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5282 break;
5283 case ISD::VP_LOAD:
5284 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5285 break;
5286 case ISD::VP_LOAD_FF:
5287 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5288 break;
5289 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5290 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5291 break;
5292 case ISD::VECTOR_COMPRESS:
5293 Res = WidenVecRes_VECTOR_COMPRESS(N);
5294 break;
5295 case ISD::MLOAD:
5296 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5297 break;
5298 case ISD::MGATHER:
5299 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5300 break;
5301 case ISD::VP_GATHER:
5302 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5303 break;
5304 case ISD::VECTOR_REVERSE:
5305 Res = WidenVecRes_VECTOR_REVERSE(N);
5306 break;
5307 case ISD::GET_ACTIVE_LANE_MASK:
5308 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5309 break;
5310 case ISD::VECTOR_DEINTERLEAVE:
5311 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5312 break;
5313
5314 case ISD::ADD: case ISD::VP_ADD:
5315 case ISD::AND: case ISD::VP_AND:
5316 case ISD::MUL: case ISD::VP_MUL:
5317 case ISD::MULHS:
5318 case ISD::MULHU:
5319 case ISD::ABDS:
5320 case ISD::ABDU:
5321 case ISD::OR: case ISD::VP_OR:
5322 case ISD::SUB: case ISD::VP_SUB:
5323 case ISD::XOR: case ISD::VP_XOR:
5324 case ISD::SHL: case ISD::VP_SHL:
5325 case ISD::SRA: case ISD::VP_SRA:
5326 case ISD::SRL: case ISD::VP_SRL:
5327 case ISD::CLMUL:
5328 case ISD::CLMULR:
5329 case ISD::CLMULH:
5330 case ISD::PEXT:
5331 case ISD::PDEP:
5332 case ISD::FMINNUM:
5333 case ISD::FMINNUM_IEEE:
5334 case ISD::VP_FMINNUM:
5335 case ISD::FMAXNUM:
5336 case ISD::FMAXNUM_IEEE:
5337 case ISD::VP_FMAXNUM:
5338 case ISD::FMINIMUM:
5339 case ISD::VP_FMINIMUM:
5340 case ISD::FMAXIMUM:
5341 case ISD::VP_FMAXIMUM:
5342 case ISD::FMINIMUMNUM:
5343 case ISD::FMAXIMUMNUM:
5344 case ISD::SMIN: case ISD::VP_SMIN:
5345 case ISD::SMAX: case ISD::VP_SMAX:
5346 case ISD::UMIN: case ISD::VP_UMIN:
5347 case ISD::UMAX: case ISD::VP_UMAX:
5348 case ISD::UADDSAT: case ISD::VP_UADDSAT:
5349 case ISD::SADDSAT: case ISD::VP_SADDSAT:
5350 case ISD::USUBSAT: case ISD::VP_USUBSAT:
5351 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
5352 case ISD::SSHLSAT:
5353 case ISD::USHLSAT:
5354 case ISD::ROTL:
5355 case ISD::ROTR:
5356 case ISD::AVGFLOORS:
5357 case ISD::AVGFLOORU:
5358 case ISD::AVGCEILS:
5359 case ISD::AVGCEILU:
5360 // Vector-predicated binary op widening. Note that -- unlike the
5361 // unpredicated versions -- we don't have to worry about trapping on
5362 // operations like UDIV, FADD, etc., as we pass on the original vector
5363 // length parameter. This means the widened elements containing garbage
5364 // aren't active.
5365 case ISD::VP_SDIV:
5366 case ISD::VP_UDIV:
5367 case ISD::VP_SREM:
5368 case ISD::VP_UREM:
5369 case ISD::VP_FADD:
5370 case ISD::VP_FSUB:
5371 case ISD::VP_FMUL:
5372 case ISD::VP_FDIV:
5373 case ISD::VP_FREM:
5374 case ISD::VP_FCOPYSIGN:
5375 Res = WidenVecRes_Binary(N);
5376 break;
5377
5378 case ISD::MASKED_UDIV:
5379 case ISD::MASKED_SDIV:
5380 case ISD::MASKED_UREM:
5381 case ISD::MASKED_SREM:
5382 Res = WidenVecRes_MaskedBinary(N);
5383 break;
5384
5385 case ISD::SCMP:
5386 case ISD::UCMP:
5387 Res = WidenVecRes_CMP(N);
5388 break;
5389
5390 case ISD::FPOW:
5391 case ISD::FATAN2:
5392 case ISD::FREM:
5393 if (unrollExpandedOp())
5394 break;
5395 // If the target has custom/legal support for the scalar FP intrinsic ops
5396 // (they are probably not destined to become libcalls), then widen those
5397 // like any other binary ops.
5398 [[fallthrough]];
5399
5400 case ISD::FADD:
5401 case ISD::FMUL:
5402 case ISD::FSUB:
5403 case ISD::FDIV:
5404 case ISD::SDIV:
5405 case ISD::UDIV:
5406 case ISD::SREM:
5407 case ISD::UREM:
5408 Res = WidenVecRes_BinaryCanTrap(N);
5409 break;
5410
5411 case ISD::SMULFIX:
5412 case ISD::SMULFIXSAT:
5413 case ISD::UMULFIX:
5414 case ISD::UMULFIXSAT:
5415 // These are binary operations, but with an extra operand that shouldn't
5416 // be widened (the scale).
5417 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5418 break;
5419
5420#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5421 case ISD::STRICT_##DAGN:
5422#include "llvm/IR/ConstrainedOps.def"
5423 Res = WidenVecRes_StrictFP(N);
5424 break;
5425
5426 case ISD::UADDO:
5427 case ISD::SADDO:
5428 case ISD::USUBO:
5429 case ISD::SSUBO:
5430 case ISD::UMULO:
5431 case ISD::SMULO:
5432 Res = WidenVecRes_OverflowOp(N, ResNo);
5433 break;
5434
5435 case ISD::FCOPYSIGN:
5436 Res = WidenVecRes_FCOPYSIGN(N);
5437 break;
5438
5439 case ISD::IS_FPCLASS:
5440 case ISD::FPTRUNC_ROUND:
5441 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5442 break;
5443
5444 case ISD::FLDEXP:
5445 case ISD::FPOWI:
5446 if (!unrollExpandedOp())
5447 Res = WidenVecRes_ExpOp(N);
5448 break;
5449
5450 case ISD::ANY_EXTEND_VECTOR_INREG:
5451 case ISD::SIGN_EXTEND_VECTOR_INREG:
5452 case ISD::ZERO_EXTEND_VECTOR_INREG:
5453 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5454 break;
5455
5456 case ISD::ANY_EXTEND:
5457 case ISD::FP_EXTEND:
5458 case ISD::VP_FP_EXTEND:
5459 case ISD::FP_ROUND:
5460 case ISD::VP_FP_ROUND:
5461 case ISD::FP_TO_SINT:
5462 case ISD::VP_FP_TO_SINT:
5463 case ISD::FP_TO_UINT:
5464 case ISD::VP_FP_TO_UINT:
5465 case ISD::SIGN_EXTEND:
5466 case ISD::VP_SIGN_EXTEND:
5467 case ISD::SINT_TO_FP:
5468 case ISD::VP_SINT_TO_FP:
5469 case ISD::VP_TRUNCATE:
5470 case ISD::TRUNCATE:
5471 case ISD::UINT_TO_FP:
5472 case ISD::VP_UINT_TO_FP:
5473 case ISD::ZERO_EXTEND:
5474 case ISD::VP_ZERO_EXTEND:
5475 case ISD::CONVERT_FROM_ARBITRARY_FP:
5476 case ISD::CONVERT_TO_ARBITRARY_FP:
5477 Res = WidenVecRes_Convert(N);
5478 break;
5479
5480 case ISD::FP_TO_SINT_SAT:
5481 case ISD::FP_TO_UINT_SAT:
5482 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5483 break;
5484
5485 case ISD::LRINT:
5486 case ISD::LLRINT:
5487 case ISD::VP_LRINT:
5488 case ISD::VP_LLRINT:
5489 case ISD::LROUND:
5490 case ISD::LLROUND:
5491 Res = WidenVecRes_XROUND(N);
5492 break;
5493
5494 case ISD::FACOS:
5495 case ISD::FASIN:
5496 case ISD::FATAN:
5497 case ISD::FCEIL:
5498 case ISD::FCOS:
5499 case ISD::FCOSH:
5500 case ISD::FEXP:
5501 case ISD::FEXP2:
5502 case ISD::FEXP10:
5503 case ISD::FFLOOR:
5504 case ISD::FLOG:
5505 case ISD::FLOG10:
5506 case ISD::FLOG2:
5507 case ISD::FNEARBYINT:
5508 case ISD::FRINT:
5509 case ISD::FROUND:
5510 case ISD::FROUNDEVEN:
5511 case ISD::FSIN:
5512 case ISD::FSINH:
5513 case ISD::FSQRT:
5514 case ISD::FTAN:
5515 case ISD::FTANH:
5516 case ISD::FTRUNC:
5517 if (unrollExpandedOp())
5518 break;
5519 // If the target has custom/legal support for the scalar FP intrinsic ops
5520 // (they are probably not destined to become libcalls), then widen those
5521 // like any other unary ops.
5522 [[fallthrough]];
5523
5524 case ISD::ABS:
5525 case ISD::ABS_MIN_POISON:
5526 case ISD::VP_ABS:
5527 case ISD::BITREVERSE:
5528 case ISD::VP_BITREVERSE:
5529 case ISD::BSWAP:
5530 case ISD::VP_BSWAP:
5531 case ISD::CTLZ:
5532 case ISD::VP_CTLZ:
5533 case ISD::CTLZ_ZERO_POISON:
5534 case ISD::VP_CTLZ_ZERO_POISON:
5535 case ISD::CTPOP:
5536 case ISD::VP_CTPOP:
5537 case ISD::CTTZ:
5538 case ISD::VP_CTTZ:
5539 case ISD::CTTZ_ZERO_POISON:
5540 case ISD::VP_CTTZ_ZERO_POISON:
5541 case ISD::FNEG: case ISD::VP_FNEG:
5542 case ISD::FABS: case ISD::VP_FABS:
5543 case ISD::VP_SQRT:
5544 case ISD::VP_FCEIL:
5545 case ISD::VP_FFLOOR:
5546 case ISD::VP_FRINT:
5547 case ISD::VP_FNEARBYINT:
5548 case ISD::VP_FROUND:
5549 case ISD::VP_FROUNDEVEN:
5550 case ISD::VP_FROUNDTOZERO:
5551 case ISD::FREEZE:
5552 case ISD::ARITH_FENCE:
5553 case ISD::FCANONICALIZE:
5554 case ISD::AssertNoFPClass:
5555 Res = WidenVecRes_Unary(N);
5556 break;
5557 case ISD::FMA: case ISD::VP_FMA:
5558 case ISD::FSHL:
5559 case ISD::VP_FSHL:
5560 case ISD::FSHR:
5561 case ISD::VP_FSHR:
5562 Res = WidenVecRes_Ternary(N);
5563 break;
5564 case ISD::FMODF:
5565 case ISD::FFREXP:
5566 case ISD::FSINCOS:
5567 case ISD::FSINCOSPI: {
5568 if (!unrollExpandedOp())
5569 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5570 break;
5571 }
5572 }
5573
5574 // If Res is null, the sub-method took care of registering the result.
5575 if (Res.getNode())
5576 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5577}
5578
5579SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5580 // Ternary op widening.
5581 SDLoc dl(N);
5582 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5583 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5584 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5585 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5586 if (N->getNumOperands() == 3)
5587 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5588
5589 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5590 assert(N->isVPOpcode() && "Expected VP opcode");
5591
5592 SDValue Mask =
5593 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5594 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5595 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5596}
5597
5598SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5599 // Binary op widening.
5600 SDLoc dl(N);
5601 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5602 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5603 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5604 if (N->getNumOperands() == 2)
5605 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5606 Flags: N->getFlags());
5607
5608 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5609 assert(N->isVPOpcode() && "Expected VP opcode");
5610
5611 SDValue Mask =
5612 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5613 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5614 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5615}
5616
5617SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5618 SDLoc dl(N);
5619 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5620 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5621 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5622 SDValue Mask = N->getOperand(Num: 2);
5623 EVT WideMaskVT = WidenVT.changeVectorElementType(
5624 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5625 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5626 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5627 Flags: N->getFlags());
5628}
5629
5630SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5631 LLVMContext &Ctxt = *DAG.getContext();
5632 SDLoc dl(N);
5633
5634 SDValue LHS = N->getOperand(Num: 0);
5635 SDValue RHS = N->getOperand(Num: 1);
5636 EVT OpVT = LHS.getValueType();
5637 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5638 LHS = GetWidenedVector(Op: LHS);
5639 RHS = GetWidenedVector(Op: RHS);
5640 OpVT = LHS.getValueType();
5641 }
5642
5643 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5644 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5645 if (WidenResEC == OpVT.getVectorElementCount()) {
5646 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5647 }
5648
5649 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5650}
5651
5652SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5653 // Binary op widening, but with an extra operand that shouldn't be widened.
5654 SDLoc dl(N);
5655 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5656 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5657 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5658 SDValue InOp3 = N->getOperand(Num: 2);
5659 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5660 Flags: N->getFlags());
5661}
5662
5663// Given a vector of operations that have been broken up to widen, see
5664// if we can collect them together into the next widest legal VT. This
5665// implementation is trap-safe.
5666static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5667 SmallVectorImpl<SDValue> &ConcatOps,
5668 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5669 EVT WidenVT) {
5670 // Check to see if we have a single operation with the widen type.
5671 if (ConcatEnd == 1) {
5672 VT = ConcatOps[0].getValueType();
5673 if (VT == WidenVT)
5674 return ConcatOps[0];
5675 }
5676
5677 SDLoc dl(ConcatOps[0]);
5678 EVT WidenEltVT = WidenVT.getVectorElementType();
5679
5680 // while (Some element of ConcatOps is not of type MaxVT) {
5681 // From the end of ConcatOps, collect elements of the same type and put
5682 // them into an op of the next larger supported type
5683 // }
5684 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5685 int Idx = ConcatEnd - 1;
5686 VT = ConcatOps[Idx--].getValueType();
5687 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5688 Idx--;
5689
5690 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5691 EVT NextVT;
5692 do {
5693 NextSize *= 2;
5694 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5695 } while (!TLI.isTypeLegal(VT: NextVT));
5696
5697 if (!VT.isVector()) {
5698 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5699 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5700 unsigned NumToInsert = ConcatEnd - Idx - 1;
5701 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5702 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5703 ConcatOps[Idx+1] = VecOp;
5704 ConcatEnd = Idx + 2;
5705 } else {
5706 // Vector type, create a CONCAT_VECTORS of type NextVT
5707 SDValue undefVec = DAG.getPOISON(VT);
5708 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5709 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5710 unsigned RealVals = ConcatEnd - Idx - 1;
5711 unsigned SubConcatEnd = 0;
5712 unsigned SubConcatIdx = Idx + 1;
5713 while (SubConcatEnd < RealVals)
5714 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5715 while (SubConcatEnd < OpsToConcat)
5716 SubConcatOps[SubConcatEnd++] = undefVec;
5717 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5718 VT: NextVT, Ops: SubConcatOps);
5719 ConcatEnd = SubConcatIdx + 1;
5720 }
5721 }
5722
5723 // Check to see if we have a single operation with the widen type.
5724 if (ConcatEnd == 1) {
5725 VT = ConcatOps[0].getValueType();
5726 if (VT == WidenVT)
5727 return ConcatOps[0];
5728 }
5729
5730 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5731 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5732 if (NumOps != ConcatEnd ) {
5733 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5734 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5735 ConcatOps[j] = UndefVal;
5736 }
5737 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5738 Ops: ArrayRef(ConcatOps.data(), NumOps));
5739}
5740
5741SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5742 // Binary op widening for operations that can trap.
5743 unsigned Opcode = N->getOpcode();
5744 SDLoc dl(N);
5745 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5746 EVT WidenEltVT = WidenVT.getVectorElementType();
5747 EVT VT = WidenVT;
5748 unsigned NumElts = VT.getVectorMinNumElements();
5749 const SDNodeFlags Flags = N->getFlags();
5750 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5751 NumElts = NumElts / 2;
5752 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5753 }
5754
5755 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5756 // Operation doesn't trap so just widen as normal.
5757 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5758 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5759 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5760 }
5761
5762 // Generate a vp.op if it is custom/legal for the target. This avoids need
5763 // to split and tile the subvectors (below), because the inactive lanes can
5764 // simply be disabled. To avoid possible recursion, only do this if the
5765 // widened mask type is legal.
5766 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5767 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5768 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5769 EC: WidenVT.getVectorElementCount());
5770 TLI.isTypeLegal(VT: WideMaskVT)) {
5771 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5772 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5773 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5774 SDValue EVL =
5775 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5776 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5777 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5778 Flags);
5779 }
5780 }
5781
5782 // FIXME: Improve support for scalable vectors.
5783 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5784
5785 // No legal vector version so unroll the vector operation and then widen.
5786 if (NumElts == 1)
5787 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5788
5789 // Since the operation can trap, apply operation on the original vector.
5790 EVT MaxVT = VT;
5791 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5792 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5793 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5794
5795 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5796 unsigned ConcatEnd = 0; // Current ConcatOps index.
5797 int Idx = 0; // Current Idx into input vectors.
5798
5799 // NumElts := greatest legal vector size (at most WidenVT)
5800 // while (orig. vector has unhandled elements) {
5801 // take munches of size NumElts from the beginning and add to ConcatOps
5802 // NumElts := next smaller supported vector size or 1
5803 // }
5804 while (CurNumElts != 0) {
5805 while (CurNumElts >= NumElts) {
5806 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5807 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5808 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5809 Idx += NumElts;
5810 CurNumElts -= NumElts;
5811 }
5812 do {
5813 NumElts = NumElts / 2;
5814 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5815 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5816
5817 if (NumElts == 1) {
5818 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5819 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5820 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5821 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5822 N1: EOp1, N2: EOp2, Flags);
5823 }
5824 CurNumElts = 0;
5825 }
5826 }
5827
5828 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5829}
5830
5831SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5832 switch (N->getOpcode()) {
5833 case ISD::STRICT_FSETCC:
5834 case ISD::STRICT_FSETCCS:
5835 return WidenVecRes_STRICT_FSETCC(N);
5836 case ISD::STRICT_FP_EXTEND:
5837 case ISD::STRICT_FP_ROUND:
5838 case ISD::STRICT_FP_TO_SINT:
5839 case ISD::STRICT_FP_TO_UINT:
5840 case ISD::STRICT_SINT_TO_FP:
5841 case ISD::STRICT_UINT_TO_FP:
5842 return WidenVecRes_Convert_StrictFP(N);
5843 default:
5844 break;
5845 }
5846
5847 // StrictFP op widening for operations that can trap.
5848 unsigned NumOpers = N->getNumOperands();
5849 unsigned Opcode = N->getOpcode();
5850 SDLoc dl(N);
5851 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5852 EVT WidenEltVT = WidenVT.getVectorElementType();
5853 EVT VT = WidenVT;
5854 unsigned NumElts = VT.getVectorNumElements();
5855 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5856 NumElts = NumElts / 2;
5857 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5858 }
5859
5860 // No legal vector version so unroll the vector operation and then widen.
5861 if (NumElts == 1)
5862 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5863
5864 // Since the operation can trap, apply operation on the original vector.
5865 EVT MaxVT = VT;
5866 SmallVector<SDValue, 4> InOps;
5867 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5868
5869 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5870 SmallVector<SDValue, 16> Chains;
5871 unsigned ConcatEnd = 0; // Current ConcatOps index.
5872 int Idx = 0; // Current Idx into input vectors.
5873
5874 // The Chain is the first operand.
5875 InOps.push_back(Elt: N->getOperand(Num: 0));
5876
5877 // Now process the remaining operands.
5878 for (unsigned i = 1; i < NumOpers; ++i) {
5879 SDValue Oper = N->getOperand(Num: i);
5880
5881 EVT OpVT = Oper.getValueType();
5882 if (OpVT.isVector()) {
5883 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5884 Oper = GetWidenedVector(Op: Oper);
5885 else {
5886 EVT WideOpVT =
5887 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5888 EC: WidenVT.getVectorElementCount());
5889 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5890 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5891 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5892 }
5893 }
5894
5895 InOps.push_back(Elt: Oper);
5896 }
5897
5898 // NumElts := greatest legal vector size (at most WidenVT)
5899 // while (orig. vector has unhandled elements) {
5900 // take munches of size NumElts from the beginning and add to ConcatOps
5901 // NumElts := next smaller supported vector size or 1
5902 // }
5903 while (CurNumElts != 0) {
5904 while (CurNumElts >= NumElts) {
5905 SmallVector<SDValue, 4> EOps;
5906
5907 for (unsigned i = 0; i < NumOpers; ++i) {
5908 SDValue Op = InOps[i];
5909
5910 EVT OpVT = Op.getValueType();
5911 if (OpVT.isVector()) {
5912 EVT OpExtractVT =
5913 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5914 EC: VT.getVectorElementCount());
5915 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5916 }
5917
5918 EOps.push_back(Elt: Op);
5919 }
5920
5921 EVT OperVT[] = {VT, MVT::Other};
5922 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5923 ConcatOps[ConcatEnd++] = Oper;
5924 Chains.push_back(Elt: Oper.getValue(R: 1));
5925 Idx += NumElts;
5926 CurNumElts -= NumElts;
5927 }
5928 do {
5929 NumElts = NumElts / 2;
5930 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5931 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5932
5933 if (NumElts == 1) {
5934 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5935 SmallVector<SDValue, 4> EOps;
5936
5937 for (unsigned i = 0; i < NumOpers; ++i) {
5938 SDValue Op = InOps[i];
5939
5940 EVT OpVT = Op.getValueType();
5941 if (OpVT.isVector())
5942 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5943 Idx);
5944
5945 EOps.push_back(Elt: Op);
5946 }
5947
5948 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5949 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5950 ConcatOps[ConcatEnd++] = Oper;
5951 Chains.push_back(Elt: Oper.getValue(R: 1));
5952 }
5953 CurNumElts = 0;
5954 }
5955 }
5956
5957 // Build a factor node to remember all the Ops that have been created.
5958 SDValue NewChain;
5959 if (Chains.size() == 1)
5960 NewChain = Chains[0];
5961 else
5962 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5963 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5964
5965 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5966}
5967
5968SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5969 SDLoc DL(N);
5970 EVT ResVT = N->getValueType(ResNo: 0);
5971 EVT OvVT = N->getValueType(ResNo: 1);
5972 EVT WideResVT, WideOvVT;
5973 SDValue WideLHS, WideRHS;
5974
5975 // TODO: This might result in a widen/split loop.
5976 if (ResNo == 0) {
5977 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5978 WideOvVT = EVT::getVectorVT(
5979 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5980 NumElements: WideResVT.getVectorNumElements());
5981
5982 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5983 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5984 } else {
5985 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5986 WideResVT = EVT::getVectorVT(
5987 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5988 NumElements: WideOvVT.getVectorNumElements());
5989
5990 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5991 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5992
5993 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5994 N2: N->getOperand(Num: 0), N3: Zero);
5995 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5996 N2: N->getOperand(Num: 1), N3: Zero);
5997 }
5998
5999 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
6000 SDNode *WideNode = DAG.getNode(
6001 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
6002
6003 // Replace the other vector result not being explicitly widened here.
6004 unsigned OtherNo = 1 - ResNo;
6005 EVT OtherVT = N->getValueType(ResNo: OtherNo);
6006 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
6007 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
6008 } else {
6009 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6010 SDValue OtherVal = DAG.getNode(
6011 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
6012 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
6013 }
6014
6015 return SDValue(WideNode, ResNo);
6016}
6017
6018SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6019 LLVMContext &Ctx = *DAG.getContext();
6020 SDValue InOp = N->getOperand(Num: 0);
6021 SDLoc DL(N);
6022
6023 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6024 ElementCount WidenEC = WidenVT.getVectorElementCount();
6025
6026 EVT InVT = InOp.getValueType();
6027
6028 unsigned Opcode = N->getOpcode();
6029 const SDNodeFlags Flags = N->getFlags();
6030
6031 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6032 // not equal that of WidenVT.
6033 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6034 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6035 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6036 WidenVT.getScalarSizeInBits()) {
6037 InOp = ZExtPromotedInteger(Op: InOp);
6038 InVT = InOp.getValueType();
6039 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6040 Opcode = ISD::TRUNCATE;
6041 }
6042
6043 EVT InEltVT = InVT.getVectorElementType();
6044 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6045 ElementCount InVTEC = InVT.getVectorElementCount();
6046
6047 // Helper to build node with all scalar trailing operands.
6048 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6049 if (N->getNumOperands() == 1)
6050 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6051 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6052 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6053 N4: N->getOperand(Num: 3), Flags);
6054 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6055 };
6056
6057 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6058 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6059 InVT = InOp.getValueType();
6060 InVTEC = InVT.getVectorElementCount();
6061 if (InVTEC == WidenEC) {
6062 if (N->getNumOperands() == 3 && N->isVPOpcode()) {
6063 SDValue Mask =
6064 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6065 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
6066 }
6067 return MakeConvertNode(WidenVT, InOp);
6068 }
6069 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6070 // If both input and result vector types are of same width, extend
6071 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6072 // accepts fewer elements in the result than in the input.
6073 if (Opcode == ISD::ANY_EXTEND)
6074 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6075 if (Opcode == ISD::SIGN_EXTEND)
6076 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6077 if (Opcode == ISD::ZERO_EXTEND)
6078 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6079 }
6080
6081 // For TRUNCATE, try to widen using the legal EC of the input type instead
6082 // if the legalisation action for that intermediate type is not widening.
6083 // E.g. for trunc nxv1i64 -> nxv1i8 where
6084 // - nxv1i64 input gets widened to nxv2i64
6085 // - nxv1i8 output gets widened to nxv16i8
6086 // Then one can try widening the result to nxv2i8 (instead of going all the
6087 // way to nxv16i8) if this later allows type promotion.
6088 EVT MidResVT =
6089 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6090 if (N->getOpcode() == ISD::TRUNCATE &&
6091 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6092 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6093 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6094 }
6095 }
6096
6097 if (TLI.isTypeLegal(VT: InWidenVT)) {
6098 // Because the result and the input are different vector types, widening
6099 // the result could create a legal type but widening the input might make
6100 // it an illegal type that might lead to repeatedly splitting the input
6101 // and then widening it. To avoid this, we widen the input only if
6102 // it results in a legal type.
6103 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6104 // Widen the input and call convert on the widened input vector.
6105 unsigned NumConcat =
6106 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6107 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6108 Ops[0] = InOp;
6109 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6110 return MakeConvertNode(WidenVT, InVec);
6111 }
6112
6113 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6114 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6115 // Extract the input and convert the shorten input vector.
6116 return MakeConvertNode(WidenVT, InVal);
6117 }
6118 }
6119
6120 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6121 EVT EltVT = WidenVT.getVectorElementType();
6122 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6123 // Use the original element count so we don't do more scalar opts than
6124 // necessary.
6125 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6126 for (unsigned i=0; i < MinElts; ++i) {
6127 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6128 Ops[i] = MakeConvertNode(EltVT, Val);
6129 }
6130
6131 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6132}
6133
6134SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6135 SDLoc dl(N);
6136 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6137 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6138
6139 SDValue Src = N->getOperand(Num: 0);
6140 EVT SrcVT = Src.getValueType();
6141
6142 // Also widen the input.
6143 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6144 Src = GetWidenedVector(Op: Src);
6145 SrcVT = Src.getValueType();
6146 }
6147
6148 // Input and output not widened to the same size, give up.
6149 if (WidenNumElts != SrcVT.getVectorElementCount())
6150 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6151
6152 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6153}
6154
6155SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6156 SDLoc dl(N);
6157 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6158 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6159
6160 SDValue Src = N->getOperand(Num: 0);
6161 EVT SrcVT = Src.getValueType();
6162
6163 // Also widen the input.
6164 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6165 Src = GetWidenedVector(Op: Src);
6166 SrcVT = Src.getValueType();
6167 }
6168
6169 // Input and output not widened to the same size, give up.
6170 if (WidenNumElts != SrcVT.getVectorElementCount())
6171 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6172
6173 if (N->getNumOperands() == 1)
6174 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6175
6176 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6177 assert(N->isVPOpcode() && "Expected VP opcode");
6178
6179 SDValue Mask =
6180 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6181 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
6182}
6183
6184SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6185 SDValue InOp = N->getOperand(Num: 1);
6186 SDLoc DL(N);
6187 SmallVector<SDValue, 4> NewOps(N->ops());
6188
6189 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6190 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6191
6192 EVT InVT = InOp.getValueType();
6193 EVT InEltVT = InVT.getVectorElementType();
6194
6195 unsigned Opcode = N->getOpcode();
6196
6197 // FIXME: Optimizations need to be implemented here.
6198
6199 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6200 EVT EltVT = WidenVT.getVectorElementType();
6201 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6202 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6203 SmallVector<SDValue, 32> OpChains;
6204 // Use the original element count so we don't do more scalar opts than
6205 // necessary.
6206 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6207 for (unsigned i=0; i < MinElts; ++i) {
6208 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6209 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6210 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6211 }
6212 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6213 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6214
6215 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6216}
6217
6218SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6219 unsigned Opcode = N->getOpcode();
6220 SDValue InOp = N->getOperand(Num: 0);
6221 SDLoc DL(N);
6222
6223 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6224 EVT WidenSVT = WidenVT.getVectorElementType();
6225 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6226
6227 EVT InVT = InOp.getValueType();
6228 EVT InSVT = InVT.getVectorElementType();
6229 unsigned InVTNumElts = InVT.getVectorNumElements();
6230
6231 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6232 InOp = GetWidenedVector(Op: InOp);
6233 InVT = InOp.getValueType();
6234 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6235 switch (Opcode) {
6236 case ISD::ANY_EXTEND_VECTOR_INREG:
6237 case ISD::SIGN_EXTEND_VECTOR_INREG:
6238 case ISD::ZERO_EXTEND_VECTOR_INREG:
6239 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6240 }
6241 }
6242 }
6243
6244 // Unroll, extend the scalars and rebuild the vector.
6245 SmallVector<SDValue, 16> Ops;
6246 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6247 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6248 switch (Opcode) {
6249 case ISD::ANY_EXTEND_VECTOR_INREG:
6250 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6251 break;
6252 case ISD::SIGN_EXTEND_VECTOR_INREG:
6253 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6254 break;
6255 case ISD::ZERO_EXTEND_VECTOR_INREG:
6256 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6257 break;
6258 default:
6259 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6260 }
6261 Ops.push_back(Elt: Val);
6262 }
6263
6264 while (Ops.size() != WidenNumElts)
6265 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6266
6267 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6268}
6269
6270SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6271 // If this is an FCOPYSIGN with same input types, we can treat it as a
6272 // normal (can trap) binary op.
6273 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6274 return WidenVecRes_BinaryCanTrap(N);
6275
6276 // If the types are different, fall back to unrolling.
6277 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6278 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6279}
6280
6281/// Result and first source operand are different scalar types, but must have
6282/// the same number of elements. There is an additional control argument which
6283/// should be passed through unchanged.
6284SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6285 SDValue FpValue = N->getOperand(Num: 0);
6286 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6287 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6288 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6289 SDValue Arg = GetWidenedVector(Op: FpValue);
6290 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6291 Flags: N->getFlags());
6292}
6293
6294SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6295 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6296 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6297 SDValue RHS = N->getOperand(Num: 1);
6298 EVT ExpVT = RHS.getValueType();
6299 SDValue ExpOp = RHS;
6300 if (ExpVT.isVector()) {
6301 EVT WideExpVT = WidenVT.changeVectorElementType(
6302 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6303 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6304 }
6305
6306 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6307}
6308
6309SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6310 // Unary op widening.
6311 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6312 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6313 if (N->getNumOperands() == 1)
6314 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6315 if (N->getOpcode() == ISD::AssertNoFPClass)
6316 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
6317 N2: N->getOperand(Num: 1), Flags: N->getFlags());
6318
6319 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6320 assert(N->isVPOpcode() && "Expected VP opcode");
6321
6322 SDValue Mask =
6323 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6324 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
6325 Ops: {InOp, Mask, N->getOperand(Num: 2)});
6326}
6327
6328SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6329 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6330 EVT ExtVT = EVT::getVectorVT(
6331 Context&: *DAG.getContext(),
6332 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6333 EC: WidenVT.getVectorElementCount());
6334 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6335 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6336 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6337}
6338
6339SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6340 unsigned ResNo) {
6341 EVT VT0 = N->getValueType(ResNo: 0);
6342 EVT VT1 = N->getValueType(ResNo: 1);
6343
6344 assert(VT0.isVector() && VT1.isVector() &&
6345 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6346 "expected both results to be vectors of matching element count");
6347
6348 LLVMContext &Ctx = *DAG.getContext();
6349 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6350
6351 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6352 ElementCount WidenEC = WidenVT.getVectorElementCount();
6353
6354 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6355 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6356
6357 SDNode *WidenNode =
6358 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6359 .getNode();
6360
6361 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6362 return SDValue(WidenNode, ResNo);
6363}
6364
6365SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6366 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6367 return GetWidenedVector(Op: WidenVec);
6368}
6369
6370SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6371 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6372 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6373 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6374
6375 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
6376 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6377 DestAS: AddrSpaceCastN->getDestAddressSpace());
6378}
6379
6380SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6381 SDValue InOp = N->getOperand(Num: 0);
6382 EVT InVT = InOp.getValueType();
6383 EVT VT = N->getValueType(ResNo: 0);
6384 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6385 SDLoc dl(N);
6386
6387 switch (getTypeAction(VT: InVT)) {
6388 case TargetLowering::TypeLegal:
6389 break;
6390 case TargetLowering::TypeScalarizeScalableVector:
6391 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6392 case TargetLowering::TypePromoteInteger: {
6393 // If the incoming type is a vector that is being promoted, then
6394 // we know that the elements are arranged differently and that we
6395 // must perform the conversion using a stack slot.
6396 if (InVT.isVector())
6397 break;
6398
6399 // If the InOp is promoted to the same size, convert it. Otherwise,
6400 // fall out of the switch and widen the promoted input.
6401 SDValue NInOp = GetPromotedInteger(Op: InOp);
6402 EVT NInVT = NInOp.getValueType();
6403 if (WidenVT.bitsEq(VT: NInVT)) {
6404 // For big endian targets we need to shift the input integer or the
6405 // interesting bits will end up at the wrong place.
6406 if (DAG.getDataLayout().isBigEndian()) {
6407 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6408 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6409 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6410 }
6411 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6412 }
6413 InOp = NInOp;
6414 InVT = NInVT;
6415 break;
6416 }
6417 case TargetLowering::TypeSoftenFloat:
6418 case TargetLowering::TypeSoftPromoteHalf:
6419 case TargetLowering::TypeExpandInteger:
6420 case TargetLowering::TypeExpandFloat:
6421 case TargetLowering::TypeScalarizeVector:
6422 case TargetLowering::TypeSplitVector:
6423 break;
6424 case TargetLowering::TypeWidenVector:
6425 // If the InOp is widened to the same size, convert it. Otherwise, fall
6426 // out of the switch and widen the widened input.
6427 InOp = GetWidenedVector(Op: InOp);
6428 InVT = InOp.getValueType();
6429 if (WidenVT.bitsEq(VT: InVT))
6430 // The input widens to the same size. Convert to the widen value.
6431 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6432 break;
6433 }
6434
6435 unsigned WidenSize = WidenVT.getSizeInBits();
6436 unsigned InSize = InVT.getSizeInBits();
6437 unsigned InScalarSize = InVT.getScalarSizeInBits();
6438 // x86mmx is not an acceptable vector element type, so don't try.
6439 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6440 // Determine new input vector type. The new input vector type will use
6441 // the same element type (if its a vector) or use the input type as a
6442 // vector. It is the same size as the type to widen to.
6443 EVT NewInVT;
6444 unsigned NewNumParts = WidenSize / InSize;
6445 if (InVT.isVector()) {
6446 EVT InEltVT = InVT.getVectorElementType();
6447 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6448 NumElements: WidenSize / InEltVT.getSizeInBits());
6449 } else {
6450 // For big endian systems, using the promoted input scalar type
6451 // to produce the scalar_to_vector would put the desired bits into
6452 // the least significant byte(s) of the wider element zero. This
6453 // will mean that the users of the result vector are using incorrect
6454 // bits. Use the original input type instead. Although either input
6455 // type can be used on little endian systems, for consistency we
6456 // use the original type there as well.
6457 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6458 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6459 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6460 }
6461
6462 if (TLI.isTypeLegal(VT: NewInVT)) {
6463 SDValue NewVec;
6464 if (InVT.isVector()) {
6465 // Because the result and the input are different vector types, widening
6466 // the result could create a legal type but widening the input might
6467 // make it an illegal type that might lead to repeatedly splitting the
6468 // input and then widening it. To avoid this, we widen the input only if
6469 // it results in a legal type.
6470 if (WidenSize % InSize == 0) {
6471 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6472 Ops[0] = InOp;
6473
6474 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6475 } else {
6476 SmallVector<SDValue, 16> Ops;
6477 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6478 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6479 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6480
6481 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6482 }
6483 } else {
6484 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6485 }
6486 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6487 }
6488 }
6489
6490 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6491}
6492
6493SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6494 return DAG.getNode(
6495 Opcode: N->getOpcode(), DL: SDLoc(N),
6496 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6497 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6498}
6499
6500SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6501 SDLoc dl(N);
6502 // Build a vector with poison for the new nodes.
6503 EVT VT = N->getValueType(ResNo: 0);
6504
6505 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6506 // type. The POISONs need to have the same type as the existing operands.
6507 EVT EltVT = N->getOperand(Num: 0).getValueType();
6508 unsigned NumElts = VT.getVectorNumElements();
6509
6510 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6511 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6512
6513 SmallVector<SDValue, 16> NewOps(N->ops());
6514 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6515 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6516
6517 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6518}
6519
6520SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6521 EVT InVT = N->getOperand(Num: 0).getValueType();
6522 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6523 SDLoc dl(N);
6524 unsigned NumOperands = N->getNumOperands();
6525
6526 bool InputWidened = false; // Indicates we need to widen the input.
6527 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6528 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6529 unsigned NumInElts = InVT.getVectorMinNumElements();
6530 if (WidenNumElts % NumInElts == 0) {
6531 // Add undef vectors to widen to correct length.
6532 unsigned NumConcat = WidenNumElts / NumInElts;
6533 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6534 SmallVector<SDValue, 16> Ops(NumConcat);
6535 for (unsigned i=0; i < NumOperands; ++i)
6536 Ops[i] = N->getOperand(Num: i);
6537 for (unsigned i = NumOperands; i != NumConcat; ++i)
6538 Ops[i] = UndefVal;
6539 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6540 }
6541 } else {
6542 InputWidened = true;
6543 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6544 // The inputs and the result are widen to the same value.
6545 unsigned i;
6546 for (i=1; i < NumOperands; ++i)
6547 if (!N->getOperand(Num: i).isUndef())
6548 break;
6549
6550 if (i == NumOperands)
6551 // Everything but the first operand is an UNDEF so just return the
6552 // widened first operand.
6553 return GetWidenedVector(Op: N->getOperand(Num: 0));
6554
6555 if (NumOperands == 2) {
6556 assert(!WidenVT.isScalableVector() &&
6557 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6558 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6559 unsigned NumInElts = InVT.getVectorNumElements();
6560
6561 // Replace concat of two operands with a shuffle.
6562 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6563 for (unsigned i = 0; i < NumInElts; ++i) {
6564 MaskOps[i] = i;
6565 MaskOps[i + NumInElts] = i + WidenNumElts;
6566 }
6567 return DAG.getVectorShuffle(VT: WidenVT, dl,
6568 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6569 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6570 Mask: MaskOps);
6571 }
6572 }
6573 }
6574
6575 assert(!WidenVT.isScalableVector() &&
6576 "Cannot use build vectors to widen CONCAT_VECTOR result");
6577 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6578 unsigned NumInElts = InVT.getVectorNumElements();
6579
6580 // Fall back to use extracts and build vector.
6581 EVT EltVT = WidenVT.getVectorElementType();
6582 SmallVector<SDValue, 16> Ops(WidenNumElts);
6583 unsigned Idx = 0;
6584 for (unsigned i=0; i < NumOperands; ++i) {
6585 SDValue InOp = N->getOperand(Num: i);
6586 if (InputWidened)
6587 InOp = GetWidenedVector(Op: InOp);
6588 for (unsigned j = 0; j < NumInElts; ++j)
6589 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6590 }
6591 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6592 for (; Idx < WidenNumElts; ++Idx)
6593 Ops[Idx] = UndefVal;
6594 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6595}
6596
6597SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6598 EVT VT = N->getValueType(ResNo: 0);
6599 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6600 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6601 SDValue InOp2 = N->getOperand(Num: 1);
6602 SDValue Idx = N->getOperand(Num: 2);
6603 SDLoc dl(N);
6604 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6605}
6606
6607SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6608 EVT VT = N->getValueType(ResNo: 0);
6609 EVT EltVT = VT.getVectorElementType();
6610 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6611 SDValue InOp = N->getOperand(Num: 0);
6612 SDValue Idx = N->getOperand(Num: 1);
6613 SDLoc dl(N);
6614
6615 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6616 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6617 InOp = GetWidenedVector(Op: InOp);
6618
6619 EVT InVT = InOp.getValueType();
6620
6621 // Check if we can just return the input vector after widening.
6622 uint64_t IdxVal = Idx->getAsZExtVal();
6623 if (IdxVal == 0 && InVT == WidenVT)
6624 return InOp;
6625
6626 // Check if we can extract from the vector.
6627 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6628 unsigned InNumElts = InVT.getVectorMinNumElements();
6629 unsigned VTNumElts = VT.getVectorMinNumElements();
6630 assert(IdxVal % VTNumElts == 0 &&
6631 "Expected Idx to be a multiple of subvector minimum vector length");
6632 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6633 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6634
6635 if (VT.isScalableVector()) {
6636 // Try to split the operation up into smaller extracts and concat the
6637 // results together, e.g.
6638 // nxv6i64 extract_subvector(nxv12i64, 6)
6639 // <->
6640 // nxv8i64 concat(
6641 // nxv2i64 extract_subvector(nxv16i64, 6)
6642 // nxv2i64 extract_subvector(nxv16i64, 8)
6643 // nxv2i64 extract_subvector(nxv16i64, 10)
6644 // undef)
6645 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6646 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6647 "down type's element count");
6648 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6649 EC: ElementCount::getScalable(MinVal: GCD));
6650 // Avoid recursion around e.g. nxv1i8.
6651 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6652 SmallVector<SDValue> Parts;
6653 unsigned I = 0;
6654 for (; I < VTNumElts / GCD; ++I)
6655 Parts.push_back(
6656 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6657 for (; I < WidenNumElts / GCD; ++I)
6658 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6659
6660 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6661 }
6662
6663 // Fallback to extracting through memory.
6664
6665 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6666 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6667 MachineFunction &MF = DAG.getMachineFunction();
6668 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6669 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6670
6671 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6672 PtrInfo, F: MachineMemOperand::MOStore,
6673 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6674 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6675 PtrInfo, F: MachineMemOperand::MOLoad,
6676 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6677
6678 // Write out the input vector.
6679 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6680
6681 // Build a mask to match the length of the non-widened result.
6682 SDValue Mask =
6683 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6684
6685 // Read back the sub-vector setting the remaining lanes to poison.
6686 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6687 return DAG.getMaskedLoad(
6688 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6689 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6690 }
6691
6692 // We could try widening the input to the right length but for now, extract
6693 // the original elements, fill the rest with undefs and build a vector.
6694 SmallVector<SDValue, 16> Ops(WidenNumElts);
6695 unsigned i;
6696 for (i = 0; i < VTNumElts; ++i)
6697 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6698
6699 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6700 for (; i < WidenNumElts; ++i)
6701 Ops[i] = UndefVal;
6702 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6703}
6704
6705SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6706 SDValue InOp = ModifyToType(
6707 InOp: N->getOperand(Num: 0),
6708 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6709 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6710 N2: N->getOperand(Num: 1));
6711}
6712
6713SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6714 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6715 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6716 VT: InOp.getValueType(), N1: InOp,
6717 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6718}
6719
6720/// Either return the same load or provide appropriate casts
6721/// from the load and return that.
6722static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6723 TypeSize LdWidth, TypeSize FirstVTWidth,
6724 SDLoc dl, SelectionDAG &DAG) {
6725 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6726 "Load width must be less than or equal to first value type width");
6727 TypeSize WidenWidth = WidenVT.getSizeInBits();
6728 if (!FirstVT.isVector()) {
6729 unsigned NumElts =
6730 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6731 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6732 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6733 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6734 }
6735 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6736 return LdOp;
6737}
6738
6739/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6740/// widened value so it can be issued in a single atomic store.
6741static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6742 TypeSize FirstVTWidth, const SDLoc &dl,
6743 SelectionDAG &DAG) {
6744 TypeSize WidenWidth = WidenVT.getSizeInBits();
6745 if (!FirstVT.isVector()) {
6746 unsigned NumElts =
6747 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6748 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6749 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6750 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6751 }
6752 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6753 return StVal;
6754}
6755
6756static std::optional<EVT> findMemType(SelectionDAG &DAG,
6757 const TargetLowering &TLI, unsigned Width,
6758 EVT WidenVT, unsigned Align,
6759 unsigned WidenEx);
6760
6761SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6762 EVT WidenVT =
6763 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6764 EVT LdVT = LD->getMemoryVT();
6765 SDLoc dl(LD);
6766
6767 // Load information
6768 SDValue Chain = LD->getChain();
6769 SDValue BasePtr = LD->getBasePtr();
6770
6771 TypeSize LdWidth = LdVT.getSizeInBits();
6772 TypeSize WidenWidth = WidenVT.getSizeInBits();
6773 TypeSize WidthDiff = WidenWidth - LdWidth;
6774
6775 // Find the vector type that can load from.
6776 std::optional<EVT> FirstVT =
6777 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6778 WidenEx: WidthDiff.getKnownMinValue());
6779
6780 if (!FirstVT)
6781 return SDValue();
6782
6783 SmallVector<EVT, 8> MemVTs;
6784 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6785
6786 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6787 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6788
6789 // Load the element with one instruction.
6790 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6791 FirstVTWidth, dl, DAG);
6792
6793 // Modified the chain - switch anything that used the old chain to use
6794 // the new one.
6795 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6796 return Result;
6797}
6798
6799SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6800 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6801 ISD::LoadExtType ExtType = LD->getExtensionType();
6802
6803 // A vector must always be stored in memory as-is, i.e. without any padding
6804 // between the elements, since various code depend on it, e.g. in the
6805 // handling of a bitcast of a vector type to int, which may be done with a
6806 // vector store followed by an integer load. A vector that does not have
6807 // elements that are byte-sized must therefore be stored as an integer
6808 // built out of the extracted vector elements.
6809 if (!LD->getMemoryVT().isByteSized()) {
6810 SDValue Value, NewChain;
6811 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6812 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6813 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6814 return SDValue();
6815 }
6816
6817 // Generate a vector-predicated load if it is custom/legal on the target. To
6818 // avoid possible recursion, only do this if the widened mask type is legal.
6819 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6820 // removed from the IR by the ExpandVectorPredication pass but we're
6821 // reintroducing them here.
6822 EVT VT = LD->getValueType(ResNo: 0);
6823 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6824 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6825
6826 if (ExtType == ISD::NON_EXTLOAD &&
6827 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6828 TLI.isTypeLegal(VT: WideMaskVT)) {
6829 SDLoc DL(N);
6830 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6831 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6832 EC: VT.getVectorElementCount());
6833 SDValue NewLoad =
6834 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6835 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6836 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6837
6838 // Modified the chain - switch anything that used the old chain to use
6839 // the new one.
6840 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6841
6842 return NewLoad;
6843 }
6844
6845 SDValue Result;
6846 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6847 if (ExtType != ISD::NON_EXTLOAD)
6848 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6849 else
6850 Result = GenWidenVectorLoads(LdChain, LD);
6851
6852 if (Result) {
6853 // If we generate a single load, we can use that for the chain. Otherwise,
6854 // build a factor node to remember the multiple loads are independent and
6855 // chain to that.
6856 SDValue NewChain;
6857 if (LdChain.size() == 1)
6858 NewChain = LdChain[0];
6859 else
6860 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6861
6862 // Modified the chain - switch anything that used the old chain to use
6863 // the new one.
6864 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6865
6866 return Result;
6867 }
6868
6869 if (VT.isVector()) {
6870 // If all else fails replace the load with a wide masked load.
6871 SDLoc DL(N);
6872 SDValue Mask =
6873 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6874
6875 SDValue NewLoad = DAG.getMaskedLoad(
6876 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6877 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6878 AM: LD->getAddressingMode(), LD->getExtensionType());
6879
6880 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6881 return NewLoad;
6882 }
6883
6884 report_fatal_error(reason: "Unable to widen vector load");
6885}
6886
6887SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6888 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6889 SDValue Mask = N->getMask();
6890 SDValue EVL = N->getVectorLength();
6891 ISD::LoadExtType ExtType = N->getExtensionType();
6892 SDLoc dl(N);
6893
6894 // The mask should be widened as well
6895 assert(getTypeAction(Mask.getValueType()) ==
6896 TargetLowering::TypeWidenVector &&
6897 "Unable to widen binary VP op");
6898 Mask = GetWidenedVector(Op: Mask);
6899 assert(Mask.getValueType().getVectorElementCount() ==
6900 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6901 .getVectorElementCount() &&
6902 "Unable to widen vector load");
6903
6904 SDValue Res =
6905 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6906 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6907 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6908 // Legalize the chain result - switch anything that used the old chain to
6909 // use the new one.
6910 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6911 return Res;
6912}
6913
6914SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6915 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6916 SDValue Mask = N->getMask();
6917 SDValue EVL = N->getVectorLength();
6918 SDLoc dl(N);
6919
6920 // The mask should be widened as well
6921 assert(getTypeAction(Mask.getValueType()) ==
6922 TargetLowering::TypeWidenVector &&
6923 "Unable to widen binary VP op");
6924 Mask = GetWidenedVector(Op: Mask);
6925 assert(Mask.getValueType().getVectorElementCount() ==
6926 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6927 .getVectorElementCount() &&
6928 "Unable to widen vector load");
6929
6930 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6931 Mask, EVL, MMO: N->getMemOperand());
6932 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6933 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6934 return Res;
6935}
6936
6937SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6938 SDLoc DL(N);
6939
6940 // The mask should be widened as well
6941 SDValue Mask = N->getMask();
6942 assert(getTypeAction(Mask.getValueType()) ==
6943 TargetLowering::TypeWidenVector &&
6944 "Unable to widen VP strided load");
6945 Mask = GetWidenedVector(Op: Mask);
6946
6947 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6948 assert(Mask.getValueType().getVectorElementCount() ==
6949 WidenVT.getVectorElementCount() &&
6950 "Data and mask vectors should have the same number of elements");
6951
6952 SDValue Res = DAG.getStridedLoadVP(
6953 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6954 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6955 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6956 IsExpanding: N->isExpandingLoad());
6957
6958 // Legalize the chain result - switch anything that used the old chain to
6959 // use the new one.
6960 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6961 return Res;
6962}
6963
6964SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6965 SDValue Vec = N->getOperand(Num: 0);
6966 SDValue Mask = N->getOperand(Num: 1);
6967 SDValue Passthru = N->getOperand(Num: 2);
6968 EVT WideVecVT =
6969 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6970 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6971 VT: Mask.getValueType().getVectorElementType(),
6972 EC: WideVecVT.getVectorElementCount());
6973
6974 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6975 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6976 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6977 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6978 N2: WideMask, N3: WidePassthru);
6979}
6980
6981SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6982 EVT VT = N->getValueType(ResNo: 0);
6983 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6984 SDValue Mask = N->getMask();
6985 EVT MaskVT = Mask.getValueType();
6986 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6987 ISD::LoadExtType ExtType = N->getExtensionType();
6988 SDLoc dl(N);
6989
6990 EVT WideMaskVT =
6991 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6992 EC: WidenVT.getVectorElementCount());
6993
6994 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
6995 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6996 TLI.isTypeLegal(VT: WideMaskVT) &&
6997 // If there is a passthru, we shouldn't use vp.load. However,
6998 // type legalizer will struggle on masked.load with
6999 // scalable vectors, so for scalable vectors, we still use vp.load
7000 // but manually merge the load result with the passthru using vp.select.
7001 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7002 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7003 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7004 EC: VT.getVectorElementCount());
7005 SDValue NewLoad =
7006 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
7007 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7008 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
7009 SDValue NewVal = NewLoad;
7010
7011 // Manually merge with vselect
7012 if (!N->getPassThru()->isUndef()) {
7013 assert(WidenVT.isScalableVector());
7014 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
7015 // The lanes past EVL are poison.
7016 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
7017 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
7018 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
7019 }
7020
7021 // Modified the chain - switch anything that used the old chain to use
7022 // the new one.
7023 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7024
7025 return NewVal;
7026 }
7027
7028 // The mask should be widened as well
7029 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7030
7031 SDValue Res = DAG.getMaskedLoad(
7032 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7033 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7034 ExtType, IsExpanding: N->isExpandingLoad());
7035 // Legalize the chain result - switch anything that used the old chain to
7036 // use the new one.
7037 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7038 return Res;
7039}
7040
7041SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7042
7043 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7044 SDValue Mask = N->getMask();
7045 EVT MaskVT = Mask.getValueType();
7046 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7047 SDValue Scale = N->getScale();
7048 ElementCount WideEC = WideVT.getVectorElementCount();
7049 SDLoc dl(N);
7050
7051 // The mask should be widened as well
7052 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7053 VT: MaskVT.getVectorElementType(), EC: WideEC);
7054 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7055
7056 // Widen the Index operand
7057 SDValue Index = N->getIndex();
7058 EVT WideIndexVT = EVT::getVectorVT(
7059 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7060 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7061 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7062 Scale };
7063
7064 // Widen the MemoryType
7065 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7066 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7067 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7068 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7069 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7070
7071 // Legalize the chain result - switch anything that used the old chain to
7072 // use the new one.
7073 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7074 return Res;
7075}
7076
7077SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7078 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7079 SDValue Mask = N->getMask();
7080 SDValue Scale = N->getScale();
7081 ElementCount WideEC = WideVT.getVectorElementCount();
7082 SDLoc dl(N);
7083
7084 SDValue Index = GetWidenedVector(Op: N->getIndex());
7085 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7086 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7087 Mask = GetWidenedMask(Mask, EC: WideEC);
7088
7089 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7090 Mask, N->getVectorLength()};
7091 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7092 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7093
7094 // Legalize the chain result - switch anything that used the old chain to
7095 // use the new one.
7096 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7097 return Res;
7098}
7099
7100SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7101 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7102 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7103}
7104
7105// Return true is this is a SETCC node or a strict version of it.
7106static inline bool isSETCCOp(unsigned Opcode) {
7107 switch (Opcode) {
7108 case ISD::SETCC:
7109 case ISD::STRICT_FSETCC:
7110 case ISD::STRICT_FSETCCS:
7111 return true;
7112 }
7113 return false;
7114}
7115
7116// Return true if this is a node that could have two SETCCs as operands.
7117static inline bool isLogicalMaskOp(unsigned Opcode) {
7118 switch (Opcode) {
7119 case ISD::AND:
7120 case ISD::OR:
7121 case ISD::XOR:
7122 return true;
7123 }
7124 return false;
7125}
7126
7127// If N is a SETCC or a strict variant of it, return the type
7128// of the compare operands.
7129static inline EVT getSETCCOperandType(SDValue N) {
7130 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7131 return N->getOperand(Num: OpNo).getValueType();
7132}
7133
7134// This is used just for the assert in convertMask(). Check that this either
7135// a SETCC or a previously handled SETCC by convertMask().
7136#ifndef NDEBUG
7137static inline bool isSETCCorConvertedSETCC(SDValue N) {
7138 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7139 N = N.getOperand(0);
7140 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7141 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7142 if (!N->getOperand(i)->isUndef())
7143 return false;
7144 N = N.getOperand(0);
7145 }
7146
7147 if (N.getOpcode() == ISD::TRUNCATE)
7148 N = N.getOperand(0);
7149 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7150 N = N.getOperand(0);
7151
7152 if (isLogicalMaskOp(N.getOpcode()))
7153 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7154 isSETCCorConvertedSETCC(N.getOperand(1));
7155
7156 return (isSETCCOp(N.getOpcode()) ||
7157 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7158}
7159#endif
7160
7161// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7162// to ToMaskVT if needed with vector extension or truncation.
7163SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7164 EVT ToMaskVT) {
7165 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
7166 // FIXME: This code seems to be too restrictive, we might consider
7167 // generalizing it or dropping it.
7168 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7169
7170 // Make a new Mask node, with a legal result VT.
7171 SDValue Mask;
7172 SmallVector<SDValue, 4> Ops;
7173 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7174 Ops.push_back(Elt: InMask->getOperand(Num: i));
7175 if (InMask->isStrictFPOpcode()) {
7176 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7177 ResultTys: { MaskVT, MVT::Other }, Ops);
7178 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7179 }
7180 else
7181 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7182 Flags: InMask->getFlags());
7183
7184 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
7185 // extend or truncate is needed.
7186 LLVMContext &Ctx = *DAG.getContext();
7187 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7188 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7189 if (MaskScalarBits < ToMaskScalBits) {
7190 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7191 NumElements: MaskVT.getVectorNumElements());
7192 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7193 } else if (MaskScalarBits > ToMaskScalBits) {
7194 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7195 NumElements: MaskVT.getVectorNumElements());
7196 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7197 }
7198
7199 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7200 ToMaskVT.getScalarSizeInBits() &&
7201 "Mask should have the right element size by now.");
7202
7203 // Adjust Mask to the right number of elements.
7204 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7205 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7206 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7207 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7208 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7209 EVT SubVT = Mask->getValueType(ResNo: 0);
7210 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7211 SubOps[0] = Mask;
7212 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7213 }
7214
7215 assert((Mask->getValueType(0) == ToMaskVT) &&
7216 "A mask of ToMaskVT should have been produced by now.");
7217
7218 return Mask;
7219}
7220
7221// This method tries to handle some special cases for the vselect mask
7222// and if needed adjusting the mask vector type to match that of the VSELECT.
7223// Without it, many cases end up with scalarization of the SETCC, with many
7224// unnecessary instructions.
7225SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7226 LLVMContext &Ctx = *DAG.getContext();
7227 SDValue Cond = N->getOperand(Num: 0);
7228
7229 if (N->getOpcode() != ISD::VSELECT)
7230 return SDValue();
7231
7232 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
7233 return SDValue();
7234
7235 // If this is a splitted VSELECT that was previously already handled, do
7236 // nothing.
7237 EVT CondVT = Cond->getValueType(ResNo: 0);
7238 if (CondVT.getScalarSizeInBits() != 1)
7239 return SDValue();
7240
7241 EVT VSelVT = N->getValueType(ResNo: 0);
7242
7243 // This method can't handle scalable vector types.
7244 // FIXME: This support could be added in the future.
7245 if (VSelVT.isScalableVector())
7246 return SDValue();
7247
7248 // Only handle vector types which are a power of 2.
7249 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7250 return SDValue();
7251
7252 // Don't touch if this will be scalarized.
7253 EVT FinalVT = VSelVT;
7254 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7255 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7256
7257 if (FinalVT.getVectorNumElements() == 1)
7258 return SDValue();
7259
7260 // If there is support for an i1 vector mask, don't touch.
7261 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7262 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7263 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7264 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7265 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7266 if (SetCCResVT.getScalarSizeInBits() == 1)
7267 return SDValue();
7268 } else if (CondVT.getScalarType() == MVT::i1) {
7269 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7270 // don't touch.
7271 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7272 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7273
7274 if (CondVT.getScalarType() == MVT::i1)
7275 return SDValue();
7276 }
7277
7278 // Widen the vselect result type if needed.
7279 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7280 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7281
7282 // The mask of the VSELECT should have integer elements.
7283 EVT ToMaskVT = VSelVT;
7284 if (!ToMaskVT.getScalarType().isInteger())
7285 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7286
7287 SDValue Mask;
7288 if (isSETCCOp(Opcode: Cond->getOpcode())) {
7289 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
7290 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7291 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
7292 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
7293 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
7294 // Cond is (AND/OR/XOR (SETCC, SETCC))
7295 SDValue SETCC0 = Cond->getOperand(Num: 0);
7296 SDValue SETCC1 = Cond->getOperand(Num: 1);
7297 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
7298 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
7299 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
7300 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
7301 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
7302 EVT MaskVT;
7303 // If the two SETCCs have different VTs, either extend/truncate one of
7304 // them to the other "towards" ToMaskVT, or truncate one and extend the
7305 // other to ToMaskVT.
7306 if (ScalarBits0 != ScalarBits1) {
7307 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
7308 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
7309 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
7310 MaskVT = WideVT;
7311 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
7312 MaskVT = NarrowVT;
7313 else
7314 MaskVT = ToMaskVT;
7315 } else
7316 // If the two SETCCs have the same VT, don't change it.
7317 MaskVT = VT0;
7318
7319 // Make new SETCCs and logical nodes.
7320 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
7321 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
7322 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
7323
7324 // Convert the logical op for VSELECT if needed.
7325 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7326 } else
7327 return SDValue();
7328
7329 return Mask;
7330}
7331
7332SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7333 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7334 ElementCount WidenEC = WidenVT.getVectorElementCount();
7335
7336 SDValue Cond1 = N->getOperand(Num: 0);
7337 EVT CondVT = Cond1.getValueType();
7338 unsigned Opcode = N->getOpcode();
7339 if (CondVT.isVector()) {
7340 if (SDValue WideCond = WidenVSELECTMask(N)) {
7341 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7342 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7343 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7344 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7345 }
7346
7347 EVT CondEltVT = CondVT.getVectorElementType();
7348 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7349 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7350 Cond1 = GetWidenedVector(Op: Cond1);
7351
7352 // If we have to split the condition there is no point in widening the
7353 // select. This would result in an cycle of widening the select ->
7354 // widening the condition operand -> splitting the condition operand ->
7355 // splitting the select -> widening the select. Instead split this select
7356 // further and widen the resulting type.
7357 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7358 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7359 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7360 return Res;
7361 }
7362
7363 if (Cond1.getValueType() != CondWidenVT)
7364 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7365 }
7366
7367 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7368 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7369 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7370 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
7371 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7372 N4: N->getOperand(Num: 3));
7373 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7374}
7375
7376SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7377 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7378 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7379 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7380 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7381 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7382}
7383
7384SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7385 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7386 return DAG.getUNDEF(VT: WidenVT);
7387}
7388
7389SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7390 EVT VT = N->getValueType(ResNo: 0);
7391 SDLoc dl(N);
7392
7393 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7394 unsigned NumElts = VT.getVectorNumElements();
7395 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7396
7397 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7398 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7399
7400 // Adjust mask based on new input vector length.
7401 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7402 for (unsigned i = 0; i != NumElts; ++i) {
7403 int Idx = N->getMaskElt(Idx: i);
7404 if (Idx < (int)NumElts)
7405 NewMask[i] = Idx;
7406 else
7407 NewMask[i] = Idx - NumElts + WidenNumElts;
7408 }
7409 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7410}
7411
7412SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7413 EVT VT = N->getValueType(ResNo: 0);
7414 EVT EltVT = VT.getVectorElementType();
7415 SDLoc dl(N);
7416
7417 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7418 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7419 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7420
7421 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7422 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7423 unsigned VTNumElts = VT.getVectorMinNumElements();
7424 unsigned IdxVal = WidenNumElts - VTNumElts;
7425
7426 if (VT.isScalableVector()) {
7427 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7428 // results together, e.g.(nxv6i64 -> nxv8i64)
7429 // nxv8i64 vector_reverse
7430 // <->
7431 // nxv8i64 concat(
7432 // nxv2i64 extract_subvector(nxv8i64, 2)
7433 // nxv2i64 extract_subvector(nxv8i64, 4)
7434 // nxv2i64 extract_subvector(nxv8i64, 6)
7435 // nxv2i64 undef)
7436
7437 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7438 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7439 EC: ElementCount::getScalable(MinVal: GCD));
7440 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7441 "down type's element count");
7442 SmallVector<SDValue> Parts;
7443 unsigned i = 0;
7444 for (; i < VTNumElts / GCD; ++i)
7445 Parts.push_back(
7446 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7447 for (; i < WidenNumElts / GCD; ++i)
7448 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7449
7450 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7451 }
7452
7453 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7454 // fixed-vectors.
7455 SmallVector<int, 16> Mask(WidenNumElts, -1);
7456 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7457
7458 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7459 Mask);
7460}
7461
7462SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7463 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7464 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7465}
7466
7467void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7468 EVT VT = N->getValueType(ResNo: 0);
7469 EVT EltVT = VT.getVectorElementType();
7470 ElementCount OrigEC = VT.getVectorElementCount();
7471 unsigned Factor = N->getNumOperands();
7472 SDLoc DL(N);
7473
7474 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7475 ElementCount WidenEC = WidenVT.getVectorElementCount();
7476 // We cannot just use the widened operands directly: since they might be
7477 // individually widened, using them directly will result in de-interleaving
7478 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7479 // not concat the widened operands but the original ones to effectively
7480 // generate a "packed" concated and widened vector, before extracting new
7481 // operand vectors with the widened type.
7482 EVT PackedWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7483 EC: WidenEC.multiplyCoefficientBy(RHS: Factor));
7484 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7485 EC: OrigEC.multiplyCoefficientBy(RHS: Factor));
7486 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7487 SDValue PackedWidenVec = DAG.getInsertSubvector(
7488 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7489
7490 // Extract the new widened operand vectors.
7491 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7492 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7493 NewOps[Idx] = DAG.getExtractSubvector(
7494 DL, VT: WidenVT, Vec: PackedWidenVec,
7495 Idx: WidenEC.multiplyCoefficientBy(RHS: Idx).getKnownMinValue());
7496 }
7497
7498 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7499 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7500 // Set the widened results manually.
7501 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7502 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7503}
7504
7505SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7506 assert(N->getValueType(0).isVector() &&
7507 N->getOperand(0).getValueType().isVector() &&
7508 "Operands must be vectors");
7509 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7510 ElementCount WidenEC = WidenVT.getVectorElementCount();
7511
7512 SDValue InOp1 = N->getOperand(Num: 0);
7513 EVT InVT = InOp1.getValueType();
7514 assert(InVT.isVector() && "can not widen non-vector type");
7515 EVT WidenInVT =
7516 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7517
7518 // The input and output types often differ here, and it could be that while
7519 // we'd prefer to widen the result type, the input operands have been split.
7520 // In this case, we also need to split the result of this node as well.
7521 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7522 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7523 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7524 return Res;
7525 }
7526
7527 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7528 SDValue InOp2 = N->getOperand(Num: 1);
7529 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7530 InOp1 = GetWidenedVector(Op: InOp1);
7531 InOp2 = GetWidenedVector(Op: InOp2);
7532 } else {
7533 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7534 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7535 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7536 N2: InOp1, N3: ZeroIdx);
7537 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7538 N2: InOp2, N3: ZeroIdx);
7539 }
7540
7541 // Assume that the input and output will be widen appropriately. If not,
7542 // we will have to unroll it at some point.
7543 assert(InOp1.getValueType() == WidenInVT &&
7544 InOp2.getValueType() == WidenInVT &&
7545 "Input not widened to expected type!");
7546 (void)WidenInVT;
7547 if (N->getOpcode() == ISD::VP_SETCC) {
7548 SDValue Mask =
7549 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7550 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7551 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7552 }
7553 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7554 N3: N->getOperand(Num: 2));
7555}
7556
7557SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7558 assert(N->getValueType(0).isVector() &&
7559 N->getOperand(1).getValueType().isVector() &&
7560 "Operands must be vectors");
7561 EVT VT = N->getValueType(ResNo: 0);
7562 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7563 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7564 unsigned NumElts = VT.getVectorNumElements();
7565 EVT EltVT = VT.getVectorElementType();
7566
7567 SDLoc dl(N);
7568 SDValue Chain = N->getOperand(Num: 0);
7569 SDValue LHS = N->getOperand(Num: 1);
7570 SDValue RHS = N->getOperand(Num: 2);
7571 SDValue CC = N->getOperand(Num: 3);
7572 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7573
7574 // Fully unroll and reassemble.
7575 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7576 SmallVector<SDValue, 8> Chains(NumElts);
7577 for (unsigned i = 0; i != NumElts; ++i) {
7578 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7579 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7580
7581 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7582 Ops: {Chain, LHSElem, RHSElem, CC});
7583 Chains[i] = Scalars[i].getValue(R: 1);
7584 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7585 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7586 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7587 }
7588
7589 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7590 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7591
7592 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7593}
7594
7595//===----------------------------------------------------------------------===//
7596// Widen Vector Operand
7597//===----------------------------------------------------------------------===//
7598bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7599 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7600 SDValue Res = SDValue();
7601
7602 // See if the target wants to custom widen this node.
7603 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7604 return false;
7605
7606 switch (N->getOpcode()) {
7607 default:
7608#ifndef NDEBUG
7609 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7610 N->dump(&DAG);
7611 dbgs() << "\n";
7612#endif
7613 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7614
7615 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7616 case ISD::FAKE_USE:
7617 Res = WidenVecOp_FAKE_USE(N);
7618 break;
7619 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7620 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7621 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7622 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7623 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7624 case ISD::ATOMIC_STORE:
7625 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7626 break;
7627 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7628 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7629 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7630 break;
7631 case ISD::ANY_EXTEND_VECTOR_INREG:
7632 case ISD::SIGN_EXTEND_VECTOR_INREG:
7633 case ISD::ZERO_EXTEND_VECTOR_INREG:
7634 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7635 break;
7636 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7637 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7638 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7639 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7640 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7641 case ISD::STRICT_FSETCC:
7642 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7643 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7644 case ISD::FLDEXP:
7645 case ISD::FCOPYSIGN:
7646 case ISD::LROUND:
7647 case ISD::LLROUND:
7648 case ISD::LRINT:
7649 case ISD::LLRINT:
7650 Res = WidenVecOp_UnrollVectorOp(N);
7651 break;
7652 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7653
7654 case ISD::ANY_EXTEND:
7655 case ISD::SIGN_EXTEND:
7656 case ISD::ZERO_EXTEND:
7657 Res = WidenVecOp_EXTEND(N);
7658 break;
7659
7660 case ISD::SCMP:
7661 case ISD::UCMP:
7662 Res = WidenVecOp_CMP(N);
7663 break;
7664
7665 case ISD::FP_EXTEND:
7666 case ISD::STRICT_FP_EXTEND:
7667 case ISD::FP_ROUND:
7668 case ISD::STRICT_FP_ROUND:
7669 case ISD::FP_TO_SINT:
7670 case ISD::STRICT_FP_TO_SINT:
7671 case ISD::FP_TO_UINT:
7672 case ISD::STRICT_FP_TO_UINT:
7673 case ISD::SINT_TO_FP:
7674 case ISD::STRICT_SINT_TO_FP:
7675 case ISD::UINT_TO_FP:
7676 case ISD::STRICT_UINT_TO_FP:
7677 case ISD::TRUNCATE:
7678 case ISD::CONVERT_FROM_ARBITRARY_FP:
7679 case ISD::CONVERT_TO_ARBITRARY_FP:
7680 Res = WidenVecOp_Convert(N);
7681 break;
7682
7683 case ISD::FP_TO_SINT_SAT:
7684 case ISD::FP_TO_UINT_SAT:
7685 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7686 break;
7687
7688 case ISD::VECREDUCE_FADD:
7689 case ISD::VECREDUCE_FMUL:
7690 case ISD::VECREDUCE_ADD:
7691 case ISD::VECREDUCE_MUL:
7692 case ISD::VECREDUCE_AND:
7693 case ISD::VECREDUCE_OR:
7694 case ISD::VECREDUCE_XOR:
7695 case ISD::VECREDUCE_SMAX:
7696 case ISD::VECREDUCE_SMIN:
7697 case ISD::VECREDUCE_UMAX:
7698 case ISD::VECREDUCE_UMIN:
7699 case ISD::VECREDUCE_FMAX:
7700 case ISD::VECREDUCE_FMIN:
7701 case ISD::VECREDUCE_FMAXIMUM:
7702 case ISD::VECREDUCE_FMINIMUM:
7703 Res = WidenVecOp_VECREDUCE(N);
7704 break;
7705 case ISD::VECREDUCE_SEQ_FADD:
7706 case ISD::VECREDUCE_SEQ_FMUL:
7707 Res = WidenVecOp_VECREDUCE_SEQ(N);
7708 break;
7709 case ISD::VP_REDUCE_FADD:
7710 case ISD::VP_REDUCE_SEQ_FADD:
7711 case ISD::VP_REDUCE_FMUL:
7712 case ISD::VP_REDUCE_SEQ_FMUL:
7713 case ISD::VP_REDUCE_ADD:
7714 case ISD::VP_REDUCE_MUL:
7715 case ISD::VP_REDUCE_AND:
7716 case ISD::VP_REDUCE_OR:
7717 case ISD::VP_REDUCE_XOR:
7718 case ISD::VP_REDUCE_SMAX:
7719 case ISD::VP_REDUCE_SMIN:
7720 case ISD::VP_REDUCE_UMAX:
7721 case ISD::VP_REDUCE_UMIN:
7722 case ISD::VP_REDUCE_FMAX:
7723 case ISD::VP_REDUCE_FMIN:
7724 case ISD::VP_REDUCE_FMAXIMUM:
7725 case ISD::VP_REDUCE_FMINIMUM:
7726 Res = WidenVecOp_VP_REDUCE(N);
7727 break;
7728 case ISD::CTTZ_ELTS:
7729 case ISD::CTTZ_ELTS_ZERO_POISON:
7730 Res = WidenVecOp_CttzElements(N);
7731 break;
7732 case ISD::VP_CTTZ_ELTS:
7733 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
7734 Res = WidenVecOp_VP_CttzElements(N);
7735 break;
7736 case ISD::VECTOR_FIND_LAST_ACTIVE:
7737 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7738 break;
7739 }
7740
7741 // If Res is null, the sub-method took care of registering the result.
7742 if (!Res.getNode()) return false;
7743
7744 // If the result is N, the sub-method updated N in place. Tell the legalizer
7745 // core about this.
7746 if (Res.getNode() == N)
7747 return true;
7748
7749
7750 if (N->isStrictFPOpcode())
7751 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7752 "Invalid operand expansion");
7753 else
7754 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7755 "Invalid operand expansion");
7756
7757 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7758 return false;
7759}
7760
7761SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7762 SDLoc DL(N);
7763 EVT VT = N->getValueType(ResNo: 0);
7764
7765 SDValue InOp = N->getOperand(Num: 0);
7766 assert(getTypeAction(InOp.getValueType()) ==
7767 TargetLowering::TypeWidenVector &&
7768 "Unexpected type action");
7769 InOp = GetWidenedVector(Op: InOp);
7770 assert(VT.getVectorNumElements() <
7771 InOp.getValueType().getVectorNumElements() &&
7772 "Input wasn't widened!");
7773
7774 // We may need to further widen the operand until it has the same total
7775 // vector size as the result.
7776 EVT InVT = InOp.getValueType();
7777 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7778 EVT InEltVT = InVT.getVectorElementType();
7779 for (EVT FixedVT : MVT::vector_valuetypes()) {
7780 EVT FixedEltVT = FixedVT.getVectorElementType();
7781 if (TLI.isTypeLegal(VT: FixedVT) &&
7782 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7783 FixedEltVT == InEltVT) {
7784 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7785 "Not enough elements in the fixed type for the operand!");
7786 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7787 "We can't have the same type as we started with!");
7788 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7789 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7790 else
7791 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7792 break;
7793 }
7794 }
7795 InVT = InOp.getValueType();
7796 if (InVT.getSizeInBits() != VT.getSizeInBits())
7797 // We couldn't find a legal vector type that was a widening of the input
7798 // and could be extended in-register to the result type, so we have to
7799 // scalarize.
7800 return WidenVecOp_Convert(N);
7801 }
7802
7803 // Use special DAG nodes to represent the operation of extending the
7804 // low lanes.
7805 switch (N->getOpcode()) {
7806 default:
7807 llvm_unreachable("Extend legalization on extend operation!");
7808 case ISD::ANY_EXTEND:
7809 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7810 case ISD::SIGN_EXTEND:
7811 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7812 case ISD::ZERO_EXTEND:
7813 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7814 }
7815}
7816
7817SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7818 SDLoc dl(N);
7819
7820 EVT OpVT = N->getOperand(Num: 0).getValueType();
7821 EVT ResVT = N->getValueType(ResNo: 0);
7822 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7823 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7824
7825 // 1. EXTRACT_SUBVECTOR
7826 // 2. SIGN_EXTEND/ZERO_EXTEND
7827 // 3. CMP
7828 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7829 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7830
7831 // At this point the result type is guaranteed to be valid, so we can use it
7832 // as the operand type by extending it appropriately
7833 ISD::NodeType ExtendOpcode =
7834 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7835 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7836 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7837
7838 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7839}
7840
7841SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7842 // The result (and first input) is legal, but the second input is illegal.
7843 // We can't do much to fix that, so just unroll and let the extracts off of
7844 // the second input be widened as needed later.
7845 return DAG.UnrollVectorOp(N);
7846}
7847
7848SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7849 SDLoc DL(N);
7850 EVT ResultVT = N->getValueType(ResNo: 0);
7851 SDValue Test = N->getOperand(Num: 1);
7852 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7853
7854 // Process this node similarly to SETCC.
7855 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7856 if (ResultVT.getScalarType() == MVT::i1)
7857 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7858 NumElements: WideResultVT.getVectorNumElements());
7859
7860 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7861 Ops: {WideArg, Test}, Flags: N->getFlags());
7862
7863 // Extract the needed results from the result vector.
7864 EVT ResVT =
7865 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7866 NumElements: ResultVT.getVectorNumElements());
7867 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7868
7869 EVT OpVT = N->getOperand(Num: 0).getValueType();
7870 ISD::NodeType ExtendCode =
7871 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7872 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7873}
7874
7875SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7876 // Since the result is legal and the input is illegal.
7877 EVT VT = N->getValueType(ResNo: 0);
7878 EVT EltVT = VT.getVectorElementType();
7879 SDLoc dl(N);
7880 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7881 assert(getTypeAction(InOp.getValueType()) ==
7882 TargetLowering::TypeWidenVector &&
7883 "Unexpected type action");
7884 InOp = GetWidenedVector(Op: InOp);
7885 EVT InVT = InOp.getValueType();
7886 unsigned Opcode = N->getOpcode();
7887
7888 // Helper to build a convert node with all scalar trailing operands.
7889 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
7890 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
7891 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
7892 N4: N->getOperand(Num: 3));
7893 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7894 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
7895 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
7896 };
7897
7898 // See if a widened result type would be legal, if so widen the node.
7899 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7900 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7901 EC: InVT.getVectorElementCount());
7902 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7903 SDValue Res;
7904 if (N->isStrictFPOpcode()) {
7905 if (Opcode == ISD::STRICT_FP_ROUND)
7906 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7907 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7908 else
7909 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7910 Ops: { N->getOperand(Num: 0), InOp });
7911 // Legalize the chain result - switch anything that used the old chain to
7912 // use the new one.
7913 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7914 } else {
7915 Res = MakeConvertNode(WideVT, InOp);
7916 }
7917 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7918 }
7919
7920 EVT InEltVT = InVT.getVectorElementType();
7921
7922 // Unroll the convert into some scalar code and create a nasty build vector.
7923 unsigned NumElts = VT.getVectorNumElements();
7924 SmallVector<SDValue, 16> Ops(NumElts);
7925 if (N->isStrictFPOpcode()) {
7926 SmallVector<SDValue, 4> NewOps(N->ops());
7927 SmallVector<SDValue, 32> OpChains;
7928 for (unsigned i=0; i < NumElts; ++i) {
7929 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7930 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7931 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7932 }
7933 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7934 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7935 } else {
7936 for (unsigned i = 0; i < NumElts; ++i) {
7937 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7938 Ops[i] = MakeConvertNode(EltVT, Elt);
7939 }
7940 }
7941
7942 return DAG.getBuildVector(VT, DL: dl, Ops);
7943}
7944
7945SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7946 EVT DstVT = N->getValueType(ResNo: 0);
7947 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7948 EVT SrcVT = Src.getValueType();
7949 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7950 SDLoc dl(N);
7951
7952 // See if a widened result type would be legal, if so widen the node.
7953 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7954 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7955 if (TLI.isTypeLegal(VT: WideDstVT)) {
7956 SDValue Res =
7957 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7958 return DAG.getNode(
7959 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7960 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7961 }
7962
7963 // Give up and unroll.
7964 return DAG.UnrollVectorOp(N);
7965}
7966
7967SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7968 EVT VT = N->getValueType(ResNo: 0);
7969 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7970 EVT InWidenVT = InOp.getValueType();
7971 SDLoc dl(N);
7972
7973 // Check if we can convert between two legal vector types and extract.
7974 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7975 TypeSize Size = VT.getSizeInBits();
7976 // x86mmx is not an acceptable vector element type, so don't try.
7977 if (!VT.isVector() && VT != MVT::x86mmx &&
7978 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7979 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7980 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7981 if (TLI.isTypeLegal(VT: NewVT)) {
7982 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7983 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7984 }
7985 }
7986
7987 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7988 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7989 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7990 // having to copy via memory.
7991 if (VT.isVector()) {
7992 EVT EltVT = VT.getVectorElementType();
7993 unsigned EltSize = EltVT.getFixedSizeInBits();
7994 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7995 ElementCount NewNumElts =
7996 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
7997 .divideCoefficientBy(RHS: EltSize);
7998 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
7999 if (TLI.isTypeLegal(VT: NewVT)) {
8000 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8001 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8002 }
8003 }
8004 }
8005
8006 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8007}
8008
8009// Vectors with sizes that are not powers of 2 need to be widened to the
8010// next largest power of 2. For example, we may get a vector of 3 32-bit
8011// integers or of 6 16-bit integers, both of which have to be widened to a
8012// 128-bit vector.
8013SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8014 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8015 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8016 N2: WidenedOp);
8017}
8018
8019SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8020 EVT VT = N->getValueType(ResNo: 0);
8021 EVT EltVT = VT.getVectorElementType();
8022 EVT InVT = N->getOperand(Num: 0).getValueType();
8023 SDLoc dl(N);
8024
8025 // If the widen width for this operand is the same as the width of the concat
8026 // and all but the first operand is undef, just use the widened operand.
8027 unsigned NumOperands = N->getNumOperands();
8028 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8029 unsigned i;
8030 for (i = 1; i < NumOperands; ++i)
8031 if (!N->getOperand(Num: i).isUndef())
8032 break;
8033
8034 if (i == NumOperands)
8035 return GetWidenedVector(Op: N->getOperand(Num: 0));
8036 }
8037
8038 // Otherwise, fall back to a nasty build vector.
8039 unsigned NumElts = VT.getVectorNumElements();
8040 SmallVector<SDValue, 16> Ops(NumElts);
8041
8042 unsigned NumInElts = InVT.getVectorNumElements();
8043
8044 unsigned Idx = 0;
8045 for (unsigned i=0; i < NumOperands; ++i) {
8046 SDValue InOp = N->getOperand(Num: i);
8047 assert(getTypeAction(InOp.getValueType()) ==
8048 TargetLowering::TypeWidenVector &&
8049 "Unexpected type action");
8050 InOp = GetWidenedVector(Op: InOp);
8051 for (unsigned j = 0; j < NumInElts; ++j)
8052 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8053 }
8054 return DAG.getBuildVector(VT, DL: dl, Ops);
8055}
8056
8057SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8058 EVT VT = N->getValueType(ResNo: 0);
8059 SDValue SubVec = N->getOperand(Num: 1);
8060 SDValue InVec = N->getOperand(Num: 0);
8061
8062 EVT OrigVT = SubVec.getValueType();
8063 SubVec = GetWidenedVector(Op: SubVec);
8064 EVT SubVT = SubVec.getValueType();
8065
8066 // Whether or not all the elements of the widened SubVec will be inserted into
8067 // valid indices of VT.
8068 bool IndicesValid = false;
8069 // If we statically know that VT can fit SubVT, the indices are valid.
8070 if (VT.knownBitsGE(VT: SubVT))
8071 IndicesValid = true;
8072 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8073 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8074 // we know the minimum vscale we can work out if it's valid ourselves.
8075 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8076 Kind: Attribute::VScaleRange);
8077 if (Attr.isValid()) {
8078 unsigned VScaleMin = Attr.getVScaleRangeMin();
8079 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8080 SubVT.getFixedSizeInBits())
8081 IndicesValid = true;
8082 }
8083 }
8084
8085 if (!IndicesValid)
8086 report_fatal_error(
8087 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8088
8089 SDLoc DL(N);
8090
8091 // We need to make sure that the indices are still valid, otherwise we might
8092 // widen what was previously well-defined to something undefined.
8093 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8094 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8095 N3: N->getOperand(Num: 2));
8096
8097 if (OrigVT.isScalableVector()) {
8098 // When the widened types match, overwriting the start of a vector is
8099 // effectively a merge operation that can be implement as a vselect.
8100 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8101 SDValue Mask =
8102 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8103 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8104 }
8105
8106 // Fallback to inserting through memory.
8107 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8108 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8109 MachineFunction &MF = DAG.getMachineFunction();
8110 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8111 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8112
8113 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8114 PtrInfo, F: MachineMemOperand::MOStore,
8115 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8116 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8117 PtrInfo, F: MachineMemOperand::MOLoad,
8118 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8119
8120 // Write out the vector being inserting into.
8121 SDValue Ch =
8122 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8123
8124 // Build a mask to match the length of the sub-vector.
8125 SDValue Mask =
8126 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8127
8128 // Overwrite the sub-vector at the required offset.
8129 SDValue SubVecPtr =
8130 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8131 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8132 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8133 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8134
8135 // Read back the result.
8136 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8137 }
8138
8139 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8140 // with a series of INSERT_VECTOR_ELT
8141 unsigned Idx = N->getConstantOperandVal(Num: 2);
8142
8143 SDValue InsertElt = InVec;
8144 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8145 SDValue ExtractElt =
8146 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8147 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8148 }
8149
8150 return InsertElt;
8151}
8152
8153SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8154 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8155 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8156 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8157}
8158
8159SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8160 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8161 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8162 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8163}
8164
8165SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8166 SDLoc DL(N);
8167 EVT ResVT = N->getValueType(ResNo: 0);
8168
8169 // Widen the input as requested by the legalizer.
8170 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8171 EVT WideInVT = WideInOp.getValueType();
8172
8173 // Simple case: if widened input is still smaller than or equal to result,
8174 // just use it directly.
8175 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8176 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8177
8178 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8179 // If widening makes the input larger than the original result, widen the
8180 // result to match, then extract back down.
8181 EVT ResEltVT = ResVT.getVectorElementType();
8182 unsigned EltBits = ResEltVT.getSizeInBits();
8183 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8184 "Widened input size must be a multiple of result element size");
8185
8186 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8187 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8188
8189 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8190 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8191}
8192
8193SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8194 // We have to widen the value, but we want only to store the original
8195 // vector type.
8196 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8197
8198 if (!ST->getMemoryVT().getScalarType().isByteSized())
8199 return TLI.scalarizeVectorStore(ST, DAG);
8200
8201 if (ST->isTruncatingStore())
8202 return TLI.scalarizeVectorStore(ST, DAG);
8203
8204 // Generate a vector-predicated store if it is custom/legal on the target.
8205 // To avoid possible recursion, only do this if the widened mask type is
8206 // legal.
8207 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8208 // removed from the IR by the ExpandVectorPredication pass but we're
8209 // reintroducing them here.
8210 SDValue StVal = ST->getValue();
8211 EVT StVT = StVal.getValueType();
8212 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8213 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8214
8215 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8216 TLI.isTypeLegal(VT: WideMaskVT)) {
8217 // Widen the value.
8218 SDLoc DL(N);
8219 StVal = GetWidenedVector(Op: StVal);
8220 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8221 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8222 EC: StVT.getVectorElementCount());
8223 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8224 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8225 AM: ST->getAddressingMode());
8226 }
8227
8228 SmallVector<SDValue, 16> StChain;
8229 if (GenWidenVectorStores(StChain, ST)) {
8230 if (StChain.size() == 1)
8231 return StChain[0];
8232
8233 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8234 }
8235
8236 if (StVT.isVector()) {
8237 // If all else fails replace the store with a wide masked store.
8238 SDLoc DL(N);
8239 SDValue WideStVal = GetWidenedVector(Op: StVal);
8240 SDValue Mask =
8241 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8242
8243 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8244 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8245 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8246 IsTruncating: ST->isTruncatingStore());
8247 }
8248
8249 report_fatal_error(reason: "Unable to widen vector store");
8250}
8251
8252SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8253 EVT StVT = ST->getMemoryVT();
8254 SDLoc dl(ST);
8255
8256 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8257 EVT WidenVT = StVal.getValueType();
8258
8259 TypeSize StWidth = StVT.getSizeInBits();
8260 TypeSize WidenWidth = WidenVT.getSizeInBits();
8261 TypeSize WidthDiff = WidenWidth - StWidth;
8262
8263 // Find the vector type that can store the original memory width in one
8264 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8265 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8266 // implies a 4-byte movl, writing undef bytes past its object).
8267 std::optional<EVT> FirstVT =
8268 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8269 WidenEx: WidthDiff.getKnownMinValue());
8270 if (!FirstVT)
8271 return SDValue();
8272
8273 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8274
8275 SDValue StOp =
8276 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8277
8278 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8279 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8280}
8281
8282SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8283 assert((OpNo == 1 || OpNo == 3) &&
8284 "Can widen only data or mask operand of vp_store");
8285 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8286 SDValue Mask = ST->getMask();
8287 SDValue StVal = ST->getValue();
8288 SDLoc dl(N);
8289
8290 if (OpNo == 1) {
8291 // Widen the value.
8292 StVal = GetWidenedVector(Op: StVal);
8293
8294 // We only handle the case where the mask needs widening to an
8295 // identically-sized type as the vector inputs.
8296 assert(getTypeAction(Mask.getValueType()) ==
8297 TargetLowering::TypeWidenVector &&
8298 "Unable to widen VP store");
8299 Mask = GetWidenedVector(Op: Mask);
8300 } else {
8301 Mask = GetWidenedVector(Op: Mask);
8302
8303 // We only handle the case where the stored value needs widening to an
8304 // identically-sized type as the mask.
8305 assert(getTypeAction(StVal.getValueType()) ==
8306 TargetLowering::TypeWidenVector &&
8307 "Unable to widen VP store");
8308 StVal = GetWidenedVector(Op: StVal);
8309 }
8310
8311 assert(Mask.getValueType().getVectorElementCount() ==
8312 StVal.getValueType().getVectorElementCount() &&
8313 "Mask and data vectors should have the same number of elements");
8314 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8315 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8316 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8317 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8318 IsCompressing: ST->isCompressingStore());
8319}
8320
8321SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8322 unsigned OpNo) {
8323 assert((OpNo == 1 || OpNo == 4) &&
8324 "Can widen only data or mask operand of vp_strided_store");
8325 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8326 SDValue Mask = SST->getMask();
8327 SDValue StVal = SST->getValue();
8328 SDLoc DL(N);
8329
8330 if (OpNo == 1)
8331 assert(getTypeAction(Mask.getValueType()) ==
8332 TargetLowering::TypeWidenVector &&
8333 "Unable to widen VP strided store");
8334 else
8335 assert(getTypeAction(StVal.getValueType()) ==
8336 TargetLowering::TypeWidenVector &&
8337 "Unable to widen VP strided store");
8338
8339 StVal = GetWidenedVector(Op: StVal);
8340 Mask = GetWidenedVector(Op: Mask);
8341
8342 assert(StVal.getValueType().getVectorElementCount() ==
8343 Mask.getValueType().getVectorElementCount() &&
8344 "Data and mask vectors should have the same number of elements");
8345
8346 return DAG.getStridedStoreVP(
8347 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8348 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8349 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8350 IsCompressing: SST->isCompressingStore());
8351}
8352
8353SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8354 assert((OpNo == 1 || OpNo == 4) &&
8355 "Can widen only data or mask operand of mstore");
8356 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8357 SDValue Mask = MST->getMask();
8358 EVT MaskVT = Mask.getValueType();
8359 SDValue StVal = MST->getValue();
8360 EVT VT = StVal.getValueType();
8361 SDLoc dl(N);
8362
8363 EVT WideVT, WideMaskVT;
8364 if (OpNo == 1) {
8365 // Widen the value.
8366 StVal = GetWidenedVector(Op: StVal);
8367
8368 WideVT = StVal.getValueType();
8369 WideMaskVT =
8370 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8371 EC: WideVT.getVectorElementCount());
8372 } else {
8373 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8374
8375 EVT ValueVT = StVal.getValueType();
8376 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8377 EC: WideMaskVT.getVectorElementCount());
8378 }
8379
8380 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8381 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8382 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8383 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8384 EC: VT.getVectorElementCount());
8385 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8386 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8387 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8388 }
8389
8390 if (OpNo == 1) {
8391 // The mask should be widened as well.
8392 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8393 } else {
8394 // Widen the mask.
8395 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8396
8397 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8398 }
8399
8400 assert(Mask.getValueType().getVectorElementCount() ==
8401 StVal.getValueType().getVectorElementCount() &&
8402 "Mask and data vectors should have the same number of elements");
8403 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8404 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8405 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8406 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8407}
8408
8409SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8410 assert(OpNo == 4 && "Can widen only the index of mgather");
8411 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8412 SDValue DataOp = MG->getPassThru();
8413 SDValue Mask = MG->getMask();
8414 SDValue Scale = MG->getScale();
8415
8416 // Just widen the index. It's allowed to have extra elements.
8417 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8418
8419 SDLoc dl(N);
8420 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8421 Scale};
8422 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8423 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8424 ExtTy: MG->getExtensionType());
8425 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8426 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8427 return SDValue();
8428}
8429
8430SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8431 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8432 SDValue DataOp = MSC->getValue();
8433 SDValue Mask = MSC->getMask();
8434 SDValue Index = MSC->getIndex();
8435 SDValue Scale = MSC->getScale();
8436 EVT WideMemVT = MSC->getMemoryVT();
8437
8438 if (OpNo == 1) {
8439 DataOp = GetWidenedVector(Op: DataOp);
8440 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8441
8442 // Widen index.
8443 EVT IndexVT = Index.getValueType();
8444 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8445 VT: IndexVT.getVectorElementType(), EC: WideEC);
8446 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8447
8448 // The mask should be widened as well.
8449 EVT MaskVT = Mask.getValueType();
8450 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8451 VT: MaskVT.getVectorElementType(), EC: WideEC);
8452 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8453
8454 // Widen the MemoryType
8455 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8456 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8457 } else if (OpNo == 4) {
8458 // Just widen the index. It's allowed to have extra elements.
8459 Index = GetWidenedVector(Op: Index);
8460 } else
8461 llvm_unreachable("Can't widen this operand of mscatter");
8462
8463 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8464 Scale};
8465 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8466 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8467 IsTruncating: MSC->isTruncatingStore());
8468}
8469
8470SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8471 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8472 SDValue DataOp = VPSC->getValue();
8473 SDValue Mask = VPSC->getMask();
8474 SDValue Index = VPSC->getIndex();
8475 SDValue Scale = VPSC->getScale();
8476 EVT WideMemVT = VPSC->getMemoryVT();
8477
8478 if (OpNo == 1) {
8479 DataOp = GetWidenedVector(Op: DataOp);
8480 Index = GetWidenedVector(Op: Index);
8481 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8482 Mask = GetWidenedMask(Mask, EC: WideEC);
8483 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8484 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8485 } else if (OpNo == 3) {
8486 // Just widen the index. It's allowed to have extra elements.
8487 Index = GetWidenedVector(Op: Index);
8488 } else
8489 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8490
8491 SDValue Ops[] = {
8492 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8493 VPSC->getVectorLength()};
8494 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8495 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8496}
8497
8498SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8499 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8500 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8501 SDLoc dl(N);
8502 EVT VT = N->getValueType(ResNo: 0);
8503
8504 // WARNING: In this code we widen the compare instruction with garbage.
8505 // This garbage may contain denormal floats which may be slow. Is this a real
8506 // concern ? Should we zero the unused lanes if this is a float compare ?
8507
8508 // Get a new SETCC node to compare the newly widened operands.
8509 // Only some of the compared elements are legal.
8510 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8511 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8512 if (VT.getScalarType() == MVT::i1)
8513 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8514 EC: SVT.getVectorElementCount());
8515
8516 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8517 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8518
8519 // Extract the needed results from the result vector.
8520 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8521 VT: SVT.getVectorElementType(),
8522 EC: VT.getVectorElementCount());
8523 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8524
8525 EVT OpVT = N->getOperand(Num: 0).getValueType();
8526 ISD::NodeType ExtendCode =
8527 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8528 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8529}
8530
8531SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8532 SDValue Chain = N->getOperand(Num: 0);
8533 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8534 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8535 SDValue CC = N->getOperand(Num: 3);
8536 SDLoc dl(N);
8537
8538 EVT VT = N->getValueType(ResNo: 0);
8539 EVT EltVT = VT.getVectorElementType();
8540 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8541 unsigned NumElts = VT.getVectorNumElements();
8542
8543 // Unroll into a build vector.
8544 SmallVector<SDValue, 8> Scalars(NumElts);
8545 SmallVector<SDValue, 8> Chains(NumElts);
8546
8547 for (unsigned i = 0; i != NumElts; ++i) {
8548 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8549 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8550
8551 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8552 Ops: {Chain, LHSElem, RHSElem, CC});
8553 Chains[i] = Scalars[i].getValue(R: 1);
8554 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8555 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8556 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8557 }
8558
8559 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8560 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8561
8562 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8563}
8564
8565static unsigned getExtendForIntVecReduction(unsigned Opc) {
8566 switch (Opc) {
8567 default:
8568 llvm_unreachable("Expected integer vector reduction");
8569 case ISD::VECREDUCE_ADD:
8570 case ISD::VECREDUCE_MUL:
8571 case ISD::VECREDUCE_AND:
8572 case ISD::VECREDUCE_OR:
8573 case ISD::VECREDUCE_XOR:
8574 return ISD::ANY_EXTEND;
8575 case ISD::VECREDUCE_SMAX:
8576 case ISD::VECREDUCE_SMIN:
8577 return ISD::SIGN_EXTEND;
8578 case ISD::VECREDUCE_UMAX:
8579 case ISD::VECREDUCE_UMIN:
8580 return ISD::ZERO_EXTEND;
8581 }
8582}
8583
8584SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8585 SDLoc dl(N);
8586 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8587 EVT VT = N->getValueType(ResNo: 0);
8588 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8589 EVT WideVT = Op.getValueType();
8590 EVT ElemVT = OrigVT.getVectorElementType();
8591 SDNodeFlags Flags = N->getFlags();
8592
8593 unsigned Opc = N->getOpcode();
8594 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8595 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8596 assert(NeutralElem && "Neutral element must exist");
8597
8598 // Pad the vector with the neutral element.
8599 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8600 unsigned WideElts = WideVT.getVectorMinNumElements();
8601
8602 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8603 // needing to pad the source vector, because the inactive lanes can simply be
8604 // disabled and not contribute to the result.
8605 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8606 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8607 SDValue Start = NeutralElem;
8608 if (VT.isInteger())
8609 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8610 assert(Start.getValueType() == VT);
8611 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8612 EC: WideVT.getVectorElementCount());
8613 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8614 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8615 EC: OrigVT.getVectorElementCount());
8616 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8617 }
8618
8619 if (WideVT.isScalableVector()) {
8620 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8621 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8622 EC: ElementCount::getScalable(MinVal: GCD));
8623 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8624 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8625 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8626 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8627 }
8628
8629 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8630 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8631
8632 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8633}
8634
8635SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8636 SDLoc dl(N);
8637 SDValue AccOp = N->getOperand(Num: 0);
8638 SDValue VecOp = N->getOperand(Num: 1);
8639 SDValue Op = GetWidenedVector(Op: VecOp);
8640
8641 EVT VT = N->getValueType(ResNo: 0);
8642 EVT OrigVT = VecOp.getValueType();
8643 EVT WideVT = Op.getValueType();
8644 EVT ElemVT = OrigVT.getVectorElementType();
8645 SDNodeFlags Flags = N->getFlags();
8646
8647 unsigned Opc = N->getOpcode();
8648 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8649 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8650
8651 // Pad the vector with the neutral element.
8652 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8653 unsigned WideElts = WideVT.getVectorMinNumElements();
8654
8655 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8656 // needing to pad the source vector, because the inactive lanes can simply be
8657 // disabled and not contribute to the result.
8658 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8659 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8660 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8661 EC: WideVT.getVectorElementCount());
8662 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8663 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8664 EC: OrigVT.getVectorElementCount());
8665 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8666 }
8667
8668 if (WideVT.isScalableVector()) {
8669 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8670 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8671 EC: ElementCount::getScalable(MinVal: GCD));
8672 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8673 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8674 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8675 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8676 }
8677
8678 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8679 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8680
8681 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8682}
8683
8684SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8685 assert(N->isVPOpcode() && "Expected VP opcode");
8686
8687 SDLoc dl(N);
8688 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8689 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8690 EC: Op.getValueType().getVectorElementCount());
8691
8692 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8693 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8694 Flags: N->getFlags());
8695}
8696
8697SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8698 // This only gets called in the case that the left and right inputs and
8699 // result are of a legal odd vector type, and the condition is illegal i1 of
8700 // the same odd width that needs widening.
8701 EVT VT = N->getValueType(ResNo: 0);
8702 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8703
8704 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8705 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8706 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8707 SDLoc DL(N);
8708
8709 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8710 N2: LeftIn, N3: RightIn);
8711 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8712}
8713
8714SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
8715 SDLoc DL(N);
8716 SDValue Source = N->getOperand(Num: 0);
8717 EVT WideVT =
8718 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Source.getValueType());
8719
8720 SDValue WideSource;
8721 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
8722 WideSource = GetWidenedVector(Op: Source);
8723 } else {
8724 // Pad the widened portion with all-ones so the extra lanes appear as
8725 // active (non-zero) elements and do not contribute trailing zeros.
8726 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
8727 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
8728 }
8729
8730 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
8731 Flags: N->getFlags());
8732}
8733
8734SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8735 SDLoc DL(N);
8736 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8737 EVT SrcVT = Source.getValueType();
8738 SDValue Mask =
8739 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8740
8741 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8742 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8743}
8744
8745SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8746 SDLoc DL(N);
8747 SDValue Mask = N->getOperand(Num: 0);
8748 EVT OrigMaskVT = Mask.getValueType();
8749 SDValue WideMask = GetWidenedVector(Op: Mask);
8750 EVT WideMaskVT = WideMask.getValueType();
8751
8752 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8753 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8754 unsigned WideElts = WideMaskVT.getVectorNumElements();
8755 if (OrigElts != WideElts) {
8756 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8757 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8758 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8759 }
8760
8761 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8762 Operand: WideMask);
8763}
8764
8765//===----------------------------------------------------------------------===//
8766// Vector Widening Utilities
8767//===----------------------------------------------------------------------===//
8768
8769// Utility function to find the type to chop up a widen vector for load/store
8770// TLI: Target lowering used to determine legal types.
8771// Width: Width left need to load/store.
8772// WidenVT: The widen vector type to load to/store from
8773// Align: If 0, don't allow use of a wider type
8774// WidenEx: If Align is not 0, the amount additional we can load/store from.
8775
8776static std::optional<EVT> findMemType(SelectionDAG &DAG,
8777 const TargetLowering &TLI, unsigned Width,
8778 EVT WidenVT, unsigned Align = 0,
8779 unsigned WidenEx = 0) {
8780 EVT WidenEltVT = WidenVT.getVectorElementType();
8781 const bool Scalable = WidenVT.isScalableVector();
8782 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8783 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8784 unsigned AlignInBits = Align*8;
8785
8786 EVT RetVT = WidenEltVT;
8787 // Don't bother looking for an integer type if the vector is scalable, skip
8788 // to vector types.
8789 if (!Scalable) {
8790 // If we have one element to load/store, return it.
8791 if (Width == WidenEltWidth)
8792 return RetVT;
8793
8794 // See if there is larger legal integer than the element type to load/store.
8795 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8796 unsigned MemVTWidth = MemVT.getSizeInBits();
8797 if (MemVT.getSizeInBits() <= WidenEltWidth)
8798 break;
8799 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8800 if ((Action == TargetLowering::TypeLegal ||
8801 Action == TargetLowering::TypePromoteInteger) &&
8802 (WidenWidth % MemVTWidth) == 0 &&
8803 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8804 (MemVTWidth <= Width ||
8805 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8806 if (MemVTWidth == WidenWidth)
8807 return MemVT;
8808 RetVT = MemVT;
8809 break;
8810 }
8811 }
8812 }
8813
8814 // See if there is a larger vector type to load/store that has the same vector
8815 // element type and is evenly divisible with the WidenVT.
8816 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8817 // Skip vector MVTs which don't match the scalable property of WidenVT.
8818 if (Scalable != MemVT.isScalableVector())
8819 continue;
8820 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8821 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8822 if ((Action == TargetLowering::TypeLegal ||
8823 Action == TargetLowering::TypePromoteInteger) &&
8824 WidenEltVT == MemVT.getVectorElementType() &&
8825 (WidenWidth % MemVTWidth) == 0 &&
8826 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8827 (MemVTWidth <= Width ||
8828 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8829 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8830 return MemVT;
8831 }
8832 }
8833
8834 // Using element-wise loads and stores for widening operations is not
8835 // supported for scalable vectors
8836 if (Scalable)
8837 return std::nullopt;
8838
8839 return RetVT;
8840}
8841
8842// Builds a vector type from scalar loads
8843// VecTy: Resulting Vector type
8844// LDOps: Load operators to build a vector type
8845// [Start,End) the list of loads to use.
8846static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8847 SmallVectorImpl<SDValue> &LdOps,
8848 unsigned Start, unsigned End) {
8849 SDLoc dl(LdOps[Start]);
8850 EVT LdTy = LdOps[Start].getValueType();
8851 unsigned Width = VecTy.getSizeInBits();
8852 unsigned NumElts = Width / LdTy.getSizeInBits();
8853 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8854
8855 unsigned Idx = 1;
8856 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8857
8858 for (unsigned i = Start + 1; i != End; ++i) {
8859 EVT NewLdTy = LdOps[i].getValueType();
8860 if (NewLdTy != LdTy) {
8861 NumElts = Width / NewLdTy.getSizeInBits();
8862 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8863 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8864 // Readjust position and vector position based on new load type.
8865 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8866 LdTy = NewLdTy;
8867 }
8868 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8869 }
8870 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8871}
8872
8873SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8874 LoadSDNode *LD) {
8875 // The strategy assumes that we can efficiently load power-of-two widths.
8876 // The routine chops the vector into the largest vector loads with the same
8877 // element type or scalar loads and then recombines it to the widen vector
8878 // type.
8879 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8880 EVT LdVT = LD->getMemoryVT();
8881 SDLoc dl(LD);
8882 assert(LdVT.isVector() && WidenVT.isVector());
8883 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8884 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8885
8886 // Load information
8887 SDValue Chain = LD->getChain();
8888 SDValue BasePtr = LD->getBasePtr();
8889 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8890 AAMDNodes AAInfo = LD->getAAInfo();
8891
8892 TypeSize LdWidth = LdVT.getSizeInBits();
8893 TypeSize WidenWidth = WidenVT.getSizeInBits();
8894 TypeSize WidthDiff = WidenWidth - LdWidth;
8895 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8896 // and if the original load is simple.
8897 unsigned LdAlign =
8898 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8899
8900 // Find the vector type that can load from.
8901 std::optional<EVT> FirstVT =
8902 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8903 WidenEx: WidthDiff.getKnownMinValue());
8904
8905 if (!FirstVT)
8906 return SDValue();
8907
8908 SmallVector<EVT, 8> MemVTs;
8909 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8910
8911 // Unless we're able to load in one instruction we must work out how to load
8912 // the remainder.
8913 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8914 std::optional<EVT> NewVT = FirstVT;
8915 TypeSize RemainingWidth = LdWidth;
8916 TypeSize NewVTWidth = FirstVTWidth;
8917 do {
8918 RemainingWidth -= NewVTWidth;
8919 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8920 // The current type we are using is too large. Find a better size.
8921 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8922 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8923 if (!NewVT)
8924 return SDValue();
8925 NewVTWidth = NewVT->getSizeInBits();
8926 }
8927 MemVTs.push_back(Elt: *NewVT);
8928 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8929 }
8930
8931 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8932 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8933 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8934
8935 // Check if we can load the element with one instruction.
8936 if (MemVTs.empty())
8937 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8938 DAG);
8939
8940 // Load vector by using multiple loads from largest vector to scalar.
8941 SmallVector<SDValue, 16> LdOps;
8942 LdOps.push_back(Elt: LdOp);
8943
8944 uint64_t ScaledOffset = 0;
8945 MachinePointerInfo MPI = LD->getPointerInfo();
8946
8947 // First incremement past the first load.
8948 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8949 ScaledOffset: &ScaledOffset);
8950
8951 for (EVT MemVT : MemVTs) {
8952 Align NewAlign = ScaledOffset == 0
8953 ? LD->getBaseAlign()
8954 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8955 SDValue L =
8956 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8957
8958 LdOps.push_back(Elt: L);
8959 LdChain.push_back(Elt: L.getValue(R: 1));
8960 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8961 }
8962
8963 // Build the vector from the load operations.
8964 unsigned End = LdOps.size();
8965 if (!LdOps[0].getValueType().isVector())
8966 // All the loads are scalar loads.
8967 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8968
8969 // If the load contains vectors, build the vector using concat vector.
8970 // All of the vectors used to load are power-of-2, and the scalar loads can be
8971 // combined to make a power-of-2 vector.
8972 SmallVector<SDValue, 16> ConcatOps(End);
8973 int i = End - 1;
8974 int Idx = End;
8975 EVT LdTy = LdOps[i].getValueType();
8976 // First, combine the scalar loads to a vector.
8977 if (!LdTy.isVector()) {
8978 for (--i; i >= 0; --i) {
8979 LdTy = LdOps[i].getValueType();
8980 if (LdTy.isVector())
8981 break;
8982 }
8983 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8984 }
8985
8986 ConcatOps[--Idx] = LdOps[i];
8987 for (--i; i >= 0; --i) {
8988 EVT NewLdTy = LdOps[i].getValueType();
8989 if (NewLdTy != LdTy) {
8990 // Create a larger vector.
8991 TypeSize LdTySize = LdTy.getSizeInBits();
8992 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8993 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8994 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8995 unsigned NumOps =
8996 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8997 SmallVector<SDValue, 16> WidenOps(NumOps);
8998 unsigned j = 0;
8999 for (; j != End-Idx; ++j)
9000 WidenOps[j] = ConcatOps[Idx+j];
9001 for (; j != NumOps; ++j)
9002 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9003
9004 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9005 Ops: WidenOps);
9006 Idx = End - 1;
9007 LdTy = NewLdTy;
9008 }
9009 ConcatOps[--Idx] = LdOps[i];
9010 }
9011
9012 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9013 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9014 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9015
9016 // We need to fill the rest with undefs to build the vector.
9017 unsigned NumOps =
9018 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9019 SmallVector<SDValue, 16> WidenOps(NumOps);
9020 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9021 {
9022 unsigned i = 0;
9023 for (; i != End-Idx; ++i)
9024 WidenOps[i] = ConcatOps[Idx+i];
9025 for (; i != NumOps; ++i)
9026 WidenOps[i] = UndefVal;
9027 }
9028 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9029}
9030
9031SDValue
9032DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9033 LoadSDNode *LD,
9034 ISD::LoadExtType ExtType) {
9035 // For extension loads, it may not be more efficient to chop up the vector
9036 // and then extend it. Instead, we unroll the load and build a new vector.
9037 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9038 EVT LdVT = LD->getMemoryVT();
9039 SDLoc dl(LD);
9040 assert(LdVT.isVector() && WidenVT.isVector());
9041 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9042
9043 // Load information
9044 SDValue Chain = LD->getChain();
9045 SDValue BasePtr = LD->getBasePtr();
9046 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9047 AAMDNodes AAInfo = LD->getAAInfo();
9048
9049 if (LdVT.isScalableVector())
9050 return SDValue();
9051
9052 EVT EltVT = WidenVT.getVectorElementType();
9053 EVT LdEltVT = LdVT.getVectorElementType();
9054 unsigned NumElts = LdVT.getVectorNumElements();
9055
9056 // Load each element and widen.
9057 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9058 SmallVector<SDValue, 16> Ops(WidenNumElts);
9059 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9060 Ops[0] =
9061 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9062 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
9063 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9064 unsigned i = 0, Offset = Increment;
9065 for (i=1; i < NumElts; ++i, Offset += Increment) {
9066 SDValue NewBasePtr =
9067 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9068 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9069 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9070 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
9071 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9072 }
9073
9074 // Fill the rest with undefs.
9075 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9076 for (; i != WidenNumElts; ++i)
9077 Ops[i] = UndefVal;
9078
9079 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9080}
9081
9082bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9083 StoreSDNode *ST) {
9084 // The strategy assumes that we can efficiently store power-of-two widths.
9085 // The routine chops the vector into the largest vector stores with the same
9086 // element type or scalar stores.
9087 SDValue Chain = ST->getChain();
9088 SDValue BasePtr = ST->getBasePtr();
9089 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9090 AAMDNodes AAInfo = ST->getAAInfo();
9091 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9092 SDLoc dl(ST);
9093
9094 EVT StVT = ST->getMemoryVT();
9095 TypeSize StWidth = StVT.getSizeInBits();
9096 EVT ValVT = ValOp.getValueType();
9097 TypeSize ValWidth = ValVT.getSizeInBits();
9098 EVT ValEltVT = ValVT.getVectorElementType();
9099 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9100 assert(StVT.getVectorElementType() == ValEltVT);
9101 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9102 "Mismatch between store and value types");
9103
9104 int Idx = 0; // current index to store
9105
9106 MachinePointerInfo MPI = ST->getPointerInfo();
9107 uint64_t ScaledOffset = 0;
9108
9109 // A breakdown of how to widen this vector store. Each element of the vector
9110 // is a memory VT combined with the number of times it is to be stored to,
9111 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9112 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9113
9114 while (StWidth.isNonZero()) {
9115 // Find the largest vector type we can store with.
9116 std::optional<EVT> NewVT =
9117 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9118 if (!NewVT)
9119 return false;
9120 MemVTs.push_back(Elt: {*NewVT, 0});
9121 TypeSize NewVTWidth = NewVT->getSizeInBits();
9122
9123 do {
9124 StWidth -= NewVTWidth;
9125 MemVTs.back().second++;
9126 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9127 }
9128
9129 for (const auto &Pair : MemVTs) {
9130 EVT NewVT = Pair.first;
9131 unsigned Count = Pair.second;
9132 TypeSize NewVTWidth = NewVT.getSizeInBits();
9133
9134 if (NewVT.isVector()) {
9135 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9136 do {
9137 Align NewAlign = ScaledOffset == 0
9138 ? ST->getBaseAlign()
9139 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9140 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9141 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9142 MMOFlags, AAInfo);
9143 StChain.push_back(Elt: PartStore);
9144
9145 Idx += NumVTElts;
9146 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9147 ScaledOffset: &ScaledOffset);
9148 } while (--Count);
9149 } else {
9150 // Cast the vector to the scalar type we can store.
9151 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9152 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9153 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9154 // Readjust index position based on new vector type.
9155 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9156 do {
9157 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9158 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
9159 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
9160 StChain.push_back(Elt: PartStore);
9161
9162 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9163 } while (--Count);
9164 // Restore index back to be relative to the original widen element type.
9165 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9166 }
9167 }
9168
9169 return true;
9170}
9171
9172/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9173/// input vector must have the same element type as NVT.
9174/// FillWithZeroes specifies that the vector should be widened with zeroes.
9175SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9176 bool FillWithZeroes) {
9177 // Note that InOp might have been widened so it might already have
9178 // the right width or it might need be narrowed.
9179 EVT InVT = InOp.getValueType();
9180 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9181 "input and widen element type must match");
9182 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9183 "cannot modify scalable vectors in this way");
9184 SDLoc dl(InOp);
9185
9186 // Check if InOp already has the right width.
9187 if (InVT == NVT)
9188 return InOp;
9189
9190 ElementCount InEC = InVT.getVectorElementCount();
9191 ElementCount WidenEC = NVT.getVectorElementCount();
9192 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9193 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9194 SmallVector<SDValue, 16> Ops(NumConcat);
9195 SDValue FillVal =
9196 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9197 Ops[0] = InOp;
9198 for (unsigned i = 1; i != NumConcat; ++i)
9199 Ops[i] = FillVal;
9200
9201 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9202 }
9203
9204 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9205 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9206
9207 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9208 "Scalable vectors should have been handled already.");
9209
9210 unsigned InNumElts = InEC.getFixedValue();
9211 unsigned WidenNumElts = WidenEC.getFixedValue();
9212
9213 // Fall back to extract and build (+ mask, if padding with zeros).
9214 SmallVector<SDValue, 16> Ops(WidenNumElts);
9215 EVT EltVT = NVT.getVectorElementType();
9216 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9217 unsigned Idx;
9218 for (Idx = 0; Idx < MinNumElts; ++Idx)
9219 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9220
9221 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9222 for (; Idx < WidenNumElts; ++Idx)
9223 Ops[Idx] = UndefVal;
9224
9225 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9226 if (!FillWithZeroes)
9227 return Widened;
9228
9229 assert(NVT.isInteger() &&
9230 "We expect to never want to FillWithZeroes for non-integral types.");
9231
9232 SmallVector<SDValue, 16> MaskOps;
9233 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9234 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9235
9236 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9237 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9238}
9239