1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
67 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
68 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
69 case ISD::CONVERT_FROM_ARBITRARY_FP:
70 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
71 break;
72 case ISD::CONVERT_TO_ARBITRARY_FP:
73 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
74 break;
75 case ISD::AssertZext:
76 case ISD::AssertSext:
77 case ISD::FPOWI:
78 case ISD::AssertNoFPClass:
79 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
80 break;
81 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
82 case ISD::ATOMIC_LOAD:
83 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
84 break;
85 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
86 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
87 case ISD::VECTOR_DEINTERLEAVE:
88 case ISD::VECTOR_INTERLEAVE:
89 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
90 break;
91 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
92 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
93 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
94 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
95 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
96 case ISD::VECTOR_MATCH:
97 R = ScalarizeVecRes_VECTOR_MATCH(N);
98 break;
99 case ISD::POISON:
100 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
101 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
102 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
103 case ISD::ANY_EXTEND_VECTOR_INREG:
104 case ISD::SIGN_EXTEND_VECTOR_INREG:
105 case ISD::ZERO_EXTEND_VECTOR_INREG:
106 R = ScalarizeVecRes_VecInregOp(N);
107 break;
108 case ISD::ABS:
109 case ISD::ABS_MIN_POISON:
110 case ISD::ANY_EXTEND:
111 case ISD::BITREVERSE:
112 case ISD::BSWAP:
113 case ISD::CTLZ:
114 case ISD::CTLZ_ZERO_POISON:
115 case ISD::CTPOP:
116 case ISD::CTTZ:
117 case ISD::CTTZ_ZERO_POISON:
118 case ISD::FABS:
119 case ISD::FACOS:
120 case ISD::FASIN:
121 case ISD::FATAN:
122 case ISD::FCEIL:
123 case ISD::FCOS:
124 case ISD::FCOSH:
125 case ISD::FEXP:
126 case ISD::FEXP2:
127 case ISD::FEXP10:
128 case ISD::FFLOOR:
129 case ISD::FLOG:
130 case ISD::FLOG10:
131 case ISD::FLOG2:
132 case ISD::FNEARBYINT:
133 case ISD::FNEG:
134 case ISD::FREEZE:
135 case ISD::ARITH_FENCE:
136 case ISD::FP_EXTEND:
137 case ISD::FP_TO_SINT:
138 case ISD::FP_TO_UINT:
139 case ISD::FRINT:
140 case ISD::LRINT:
141 case ISD::LLRINT:
142 case ISD::FROUND:
143 case ISD::FROUNDEVEN:
144 case ISD::LROUND:
145 case ISD::LLROUND:
146 case ISD::FSIN:
147 case ISD::FSINH:
148 case ISD::FSQRT:
149 case ISD::FTAN:
150 case ISD::FTANH:
151 case ISD::FTRUNC:
152 case ISD::SIGN_EXTEND:
153 case ISD::SINT_TO_FP:
154 case ISD::TRUNCATE:
155 case ISD::UINT_TO_FP:
156 case ISD::ZERO_EXTEND:
157 case ISD::FCANONICALIZE:
158 R = ScalarizeVecRes_UnaryOp(N);
159 break;
160 case ISD::ADDRSPACECAST:
161 R = ScalarizeVecRes_ADDRSPACECAST(N);
162 break;
163 case ISD::FMODF:
164 case ISD::FFREXP:
165 case ISD::FSINCOS:
166 case ISD::FSINCOSPI:
167 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
168 break;
169 case ISD::ADD:
170 case ISD::AND:
171 case ISD::AVGCEILS:
172 case ISD::AVGCEILU:
173 case ISD::AVGFLOORS:
174 case ISD::AVGFLOORU:
175 case ISD::FADD:
176 case ISD::FCOPYSIGN:
177 case ISD::FDIV:
178 case ISD::FMUL:
179 case ISD::FMINNUM:
180 case ISD::FMAXNUM:
181 case ISD::FMINNUM_IEEE:
182 case ISD::FMAXNUM_IEEE:
183 case ISD::FMINIMUM:
184 case ISD::FMAXIMUM:
185 case ISD::FMINIMUMNUM:
186 case ISD::FMAXIMUMNUM:
187 case ISD::FLDEXP:
188 case ISD::ABDS:
189 case ISD::ABDU:
190 case ISD::SMIN:
191 case ISD::SMAX:
192 case ISD::UMIN:
193 case ISD::UMAX:
194
195 case ISD::SADDSAT:
196 case ISD::UADDSAT:
197 case ISD::SSUBSAT:
198 case ISD::USUBSAT:
199 case ISD::SSHLSAT:
200 case ISD::USHLSAT:
201
202 case ISD::FPOW:
203 case ISD::FATAN2:
204 case ISD::FREM:
205 case ISD::FSUB:
206 case ISD::MUL:
207 case ISD::MULHS:
208 case ISD::MULHU:
209 case ISD::OR:
210 case ISD::SDIV:
211 case ISD::SREM:
212 case ISD::SUB:
213 case ISD::UDIV:
214 case ISD::UREM:
215 case ISD::XOR:
216 case ISD::SHL:
217 case ISD::SRA:
218 case ISD::SRL:
219 case ISD::ROTL:
220 case ISD::ROTR:
221 case ISD::CLMUL:
222 case ISD::CLMULR:
223 case ISD::CLMULH:
224 case ISD::PEXT:
225 case ISD::PDEP:
226 R = ScalarizeVecRes_BinOp(N);
227 break;
228
229 case ISD::MASKED_UDIV:
230 case ISD::MASKED_SDIV:
231 case ISD::MASKED_UREM:
232 case ISD::MASKED_SREM:
233 R = ScalarizeVecRes_MaskedBinOp(N);
234 break;
235
236 case ISD::SCMP:
237 case ISD::UCMP:
238 R = ScalarizeVecRes_CMP(N);
239 break;
240
241 case ISD::FMA:
242 case ISD::FSHL:
243 case ISD::FSHR:
244 R = ScalarizeVecRes_TernaryOp(N);
245 break;
246
247#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
248 case ISD::STRICT_##DAGN:
249#include "llvm/IR/ConstrainedOps.def"
250 R = ScalarizeVecRes_StrictFPOp(N);
251 break;
252
253 case ISD::FP_TO_UINT_SAT:
254 case ISD::FP_TO_SINT_SAT:
255 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
256 break;
257
258 case ISD::UADDO:
259 case ISD::SADDO:
260 case ISD::USUBO:
261 case ISD::SSUBO:
262 case ISD::UMULO:
263 case ISD::SMULO:
264 R = ScalarizeVecRes_OverflowOp(N, ResNo);
265 break;
266 case ISD::SMULFIX:
267 case ISD::SMULFIXSAT:
268 case ISD::UMULFIX:
269 case ISD::UMULFIXSAT:
270 case ISD::SDIVFIX:
271 case ISD::SDIVFIXSAT:
272 case ISD::UDIVFIX:
273 case ISD::UDIVFIXSAT:
274 R = ScalarizeVecRes_FIX(N);
275 break;
276 }
277
278 // If R is null, the sub-method took care of registering the result.
279 if (R.getNode())
280 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
281}
282
283SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
284 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
285 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
286 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
287 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
288}
289
290SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
291 SDLoc DL(N);
292 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
293 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
294 SDValue Mask = N->getOperand(Num: 2);
295 EVT MaskVT = Mask.getValueType();
296 // The vselect result and input vectors need scalarizing, but it's
297 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
298 // See the similar logic in ScalarizeVecRes_SETCC.
299 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
300 Mask = GetScalarizedVector(Op: Mask);
301 else
302 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
303 // Vectors may have a different boolean contents to scalars, so truncate to i1
304 // and let type legalization promote appropriately.
305 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
306 // Masked binary ops don't have UB on disabled lanes but produce poison, so
307 // use 1 as the divisor to avoid division by zero and overflow.
308 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
309 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
310 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
311 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
312}
313
314SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
315 SDLoc DL(N);
316
317 SDValue LHS = N->getOperand(Num: 0);
318 SDValue RHS = N->getOperand(Num: 1);
319 if (getTypeAction(VT: LHS.getValueType()) ==
320 TargetLowering::TypeScalarizeVector) {
321 LHS = GetScalarizedVector(Op: LHS);
322 RHS = GetScalarizedVector(Op: RHS);
323 } else {
324 EVT VT = LHS.getValueType().getVectorElementType();
325 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
326 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
327 }
328
329 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
330 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
331}
332
333SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
334 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
335 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
336 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
337 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
338 N3: Op2, Flags: N->getFlags());
339}
340
341SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
342 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
343 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
344 SDValue Op2 = N->getOperand(Num: 2);
345 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
346 N3: Op2, Flags: N->getFlags());
347}
348
349SDValue
350DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
351 unsigned ResNo) {
352 assert(N->getValueType(0).getVectorNumElements() == 1 &&
353 "Unexpected vector type!");
354 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
355
356 EVT VT0 = N->getValueType(ResNo: 0);
357 EVT VT1 = N->getValueType(ResNo: 1);
358 SDLoc dl(N);
359
360 SDNode *ScalarNode =
361 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
362 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
363 .getNode();
364
365 // Replace the other vector result not being explicitly scalarized here.
366 unsigned OtherNo = 1 - ResNo;
367 EVT OtherVT = N->getValueType(ResNo: OtherNo);
368 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
369 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
370 } else {
371 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
372 Operand: SDValue(ScalarNode, OtherNo));
373 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
374 }
375
376 return SDValue(ScalarNode, ResNo);
377}
378
379SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
380 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
381 unsigned NumOpers = N->getNumOperands();
382 SDValue Chain = N->getOperand(Num: 0);
383 EVT ValueVTs[] = {VT, MVT::Other};
384 SDLoc dl(N);
385
386 SmallVector<SDValue, 4> Opers(NumOpers);
387
388 // The Chain is the first operand.
389 Opers[0] = Chain;
390
391 // Now process the remaining operands.
392 for (unsigned i = 1; i < NumOpers; ++i) {
393 SDValue Oper = N->getOperand(Num: i);
394 EVT OperVT = Oper.getValueType();
395
396 if (OperVT.isVector()) {
397 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
398 Oper = GetScalarizedVector(Op: Oper);
399 else
400 Oper =
401 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
402 }
403
404 Opers[i] = Oper;
405 }
406
407 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
408 Ops: Opers, Flags: N->getFlags());
409
410 // Legalize the chain result - switch anything that used the old chain to
411 // use the new one.
412 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
413 return Result;
414}
415
416SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
417 unsigned ResNo) {
418 SDLoc DL(N);
419 EVT ResVT = N->getValueType(ResNo: 0);
420 EVT OvVT = N->getValueType(ResNo: 1);
421
422 SDValue ScalarLHS, ScalarRHS;
423 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
424 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
425 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
426 } else {
427 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
428 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
429 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
430 ScalarLHS = ElemsLHS[0];
431 ScalarRHS = ElemsRHS[0];
432 }
433
434 SDVTList ScalarVTs = DAG.getVTList(
435 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
436 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
437 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
438 .getNode();
439
440 // Replace the other vector result not being explicitly scalarized here.
441 unsigned OtherNo = 1 - ResNo;
442 EVT OtherVT = N->getValueType(ResNo: OtherNo);
443 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
444 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
445 } else {
446 SDValue OtherVal = DAG.getNode(
447 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
448 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
449 }
450
451 return SDValue(ScalarNode, ResNo);
452}
453
454SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
455 unsigned ResNo) {
456 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
457 return GetScalarizedVector(Op);
458}
459
460SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
461 SDLoc DL(N);
462 // Reuse the expansion (which should scalarize).
463 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
464 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
465 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
466 N2: DAG.getVectorIdxConstant(Val: 0, DL));
467}
468
469SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
470 SDValue Op = N->getOperand(Num: 0);
471 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
472 Op = GetScalarizedVector(Op);
473 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
474 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
475 VT: NewVT, Operand: Op);
476}
477
478SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
479 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
480 SDValue InOp = N->getOperand(Num: 0);
481 // The BUILD_VECTOR operands may be of wider element types and
482 // we may need to truncate them back to the requested return type.
483 if (EltVT.isInteger())
484 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
485 return InOp;
486}
487
488SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
489 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
490 VT: N->getValueType(ResNo: 0).getVectorElementType(),
491 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
492}
493
494SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
495 SDLoc DL(N);
496 SDValue Op = N->getOperand(Num: 0);
497 EVT OpVT = Op.getValueType();
498 // The result needs scalarizing, but it's not a given that the source does.
499 // See similar logic in ScalarizeVecRes_UnaryOp.
500 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
501 Op = GetScalarizedVector(Op);
502 } else {
503 EVT VT = OpVT.getVectorElementType();
504 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
505 }
506 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
507 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
508 N2: N->getOperand(Num: 1));
509}
510
511SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
512 SDLoc DL(N);
513 SDValue Op = N->getOperand(Num: 0);
514 EVT OpVT = Op.getValueType();
515 // The result needs scalarizing, but it's not a given that the source does.
516 // See similar logic in ScalarizeVecRes_UnaryOp.
517 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
518 Op = GetScalarizedVector(Op);
519 } else {
520 EVT VT = OpVT.getVectorElementType();
521 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
522 }
523 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
524 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
525 N2: N->getOperand(Num: 1));
526}
527
528SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
529 SDLoc DL(N);
530 SDValue Op = N->getOperand(Num: 0);
531 EVT OpVT = Op.getValueType();
532 // The result needs scalarizing, but it's not a given that the source does.
533 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
534 Op = GetScalarizedVector(Op);
535 } else {
536 EVT VT = OpVT.getVectorElementType();
537 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
538 }
539 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
540 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
541 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
542}
543
544SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
545 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
546 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
547 N2: N->getOperand(Num: 1));
548}
549
550SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
551 // The value to insert may have a wider type than the vector element type,
552 // so be sure to truncate it to the element type if necessary.
553 SDValue Op = N->getOperand(Num: 1);
554 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
555 if (Op.getValueType() != EltVT)
556 // FIXME: Can this happen for floating point types?
557 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
558 return Op;
559}
560
561SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
562 SDValue Result = DAG.getAtomicLoad(
563 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
564 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
565 MMO: N->getMemOperand());
566
567 // Legalize the chain result - switch anything that used the old chain to
568 // use the new one.
569 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
570 return Result;
571}
572
573SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
574 assert(N->isUnindexed() && "Indexed vector load?");
575
576 SDValue Result = DAG.getLoad(
577 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
578 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
579 Ptr: N->getBasePtr(), Offset: DAG.getPOISON(VT: N->getBasePtr().getValueType()),
580 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
581 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
582
583 // Legalize the chain result - switch anything that used the old chain to
584 // use the new one.
585 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
586 return Result;
587}
588
589SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
590 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
591 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
592 SDValue Op = N->getOperand(Num: 0);
593 EVT OpVT = Op.getValueType();
594 SDLoc DL(N);
595 // The result needs scalarizing, but it's not a given that the source does.
596 // This is a workaround for targets where it's impossible to scalarize the
597 // result of a conversion, because the source type is legal.
598 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
599 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
600 // legal and was not scalarized.
601 // See the similar logic in ScalarizeVecRes_SETCC
602 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
603 Op = GetScalarizedVector(Op);
604 } else {
605 EVT VT = OpVT.getVectorElementType();
606 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
607 }
608 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
609}
610
611SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
612 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
613 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
614 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
615 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
616 N1: LHS, N2: DAG.getValueType(ExtVT));
617}
618
619SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
620 SDLoc DL(N);
621 SDValue Op = N->getOperand(Num: 0);
622
623 EVT OpVT = Op.getValueType();
624 EVT OpEltVT = OpVT.getVectorElementType();
625 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
626
627 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
628 Op = GetScalarizedVector(Op);
629 } else {
630 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
631 }
632
633 switch (N->getOpcode()) {
634 case ISD::ANY_EXTEND_VECTOR_INREG:
635 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
636 case ISD::SIGN_EXTEND_VECTOR_INREG:
637 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
638 case ISD::ZERO_EXTEND_VECTOR_INREG:
639 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
640 }
641
642 llvm_unreachable("Illegal extend_vector_inreg opcode");
643}
644
645SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
646 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
647 SDValue Op = N->getOperand(Num: 0);
648 EVT OpVT = Op.getValueType();
649 SDLoc DL(N);
650 // The result needs scalarizing, but it's not a given that the source does.
651 // This is a workaround for targets where it's impossible to scalarize the
652 // result of a conversion, because the source type is legal.
653 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
654 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
655 // legal and was not scalarized.
656 // See the similar logic in ScalarizeVecRes_SETCC
657 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
658 Op = GetScalarizedVector(Op);
659 } else {
660 EVT VT = OpVT.getVectorElementType();
661 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
662 }
663 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
664 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
665 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
666 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS,
667 Flags: AddrSpaceCastN->getFlags());
668}
669
670SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
671 // If the operand is wider than the vector element type then it is implicitly
672 // truncated. Make that explicit here.
673 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
674 SDValue InOp = N->getOperand(Num: 0);
675 if (InOp.getValueType() != EltVT)
676 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
677 return InOp;
678}
679
680SDValue
681DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
682 assert(N->getNumValues() == N->getNumOperands() &&
683 "Expected one result per operand");
684
685 // Interleaving or deinterleaving one-element vectors leaves each result
686 // equal to the corresponding operand.
687 for (unsigned I = 0; I != N->getNumValues(); ++I)
688 SetScalarizedVector(Op: SDValue(N, I), Result: GetScalarizedVector(Op: N->getOperand(Num: I)));
689 return SDValue();
690}
691
692SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
693 SDValue Cond = N->getOperand(Num: 0);
694 EVT OpVT = Cond.getValueType();
695 SDLoc DL(N);
696 // The vselect result and true/value operands needs scalarizing, but it's
697 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
698 // See the similar logic in ScalarizeVecRes_SETCC
699 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
700 Cond = GetScalarizedVector(Op: Cond);
701 } else {
702 EVT VT = OpVT.getVectorElementType();
703 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
704 }
705
706 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
707 TargetLowering::BooleanContent ScalarBool =
708 TLI.getBooleanContents(isVec: false, isFloat: false);
709 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
710
711 // If integer and float booleans have different contents then we can't
712 // reliably optimize in all cases. There is a full explanation for this in
713 // DAGCombiner::visitSELECT() where the same issue affects folding
714 // (select C, 0, 1) to (xor C, 1).
715 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
716 TLI.getBooleanContents(isVec: false, isFloat: true)) {
717 // At least try the common case where the boolean is generated by a
718 // comparison.
719 if (Cond->getOpcode() == ISD::SETCC) {
720 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
721 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
722 VecBool = TLI.getBooleanContents(Type: OpVT);
723 } else
724 ScalarBool = TargetLowering::UndefinedBooleanContent;
725 }
726
727 EVT CondVT = Cond.getValueType();
728 if (ScalarBool != VecBool) {
729 switch (ScalarBool) {
730 case TargetLowering::UndefinedBooleanContent:
731 break;
732 case TargetLowering::ZeroOrOneBooleanContent:
733 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
734 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
735 // Vector read from all ones, scalar expects a single 1 so mask.
736 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
737 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
738 break;
739 case TargetLowering::ZeroOrNegativeOneBooleanContent:
740 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
741 VecBool == TargetLowering::ZeroOrOneBooleanContent);
742 // Vector reads from a one, scalar from all ones so sign extend.
743 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
744 N1: Cond, N2: DAG.getValueType(MVT::i1));
745 break;
746 }
747 }
748
749 // Truncate the condition if needed
750 auto BoolVT = getSetCCResultType(VT: CondVT);
751 if (BoolVT.bitsLT(VT: CondVT))
752 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
753
754 return DAG.getSelect(DL: SDLoc(N), VT: LHS.getValueType(), Cond, LHS,
755 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)), Flags: N->getFlags());
756}
757
758SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
759 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
760 return DAG.getSelect(DL: SDLoc(N),
761 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
762 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
763}
764
765SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
766 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
767 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
768 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
769 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
770 N5: N->getOperand(Num: 4));
771}
772
773SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
774 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
775}
776
777SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
778 // Figure out if the scalar is the LHS or RHS and return it.
779 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
780 if (Arg.isUndef())
781 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
782 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
783 return GetScalarizedVector(Op: N->getOperand(Num: Op));
784}
785
786SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
787 SDValue Src = N->getOperand(Num: 0);
788 EVT SrcVT = Src.getValueType();
789 SDLoc dl(N);
790
791 // Handle case where result is scalarized but operand is not
792 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
793 Src = GetScalarizedVector(Op: Src);
794 else
795 Src = DAG.getNode(
796 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
797 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
798
799 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
800 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
801}
802
803SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
804 assert(N->getValueType(0).isVector() &&
805 N->getOperand(0).getValueType().isVector() &&
806 "Operand types must be vectors");
807 SDValue LHS = N->getOperand(Num: 0);
808 SDValue RHS = N->getOperand(Num: 1);
809 EVT OpVT = LHS.getValueType();
810 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
811 SDLoc DL(N);
812
813 // The result needs scalarizing, but it's not a given that the source does.
814 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
815 LHS = GetScalarizedVector(Op: LHS);
816 RHS = GetScalarizedVector(Op: RHS);
817 } else {
818 EVT VT = OpVT.getVectorElementType();
819 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
820 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
821 }
822
823 // Turn it into a scalar SETCC.
824 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
825 N3: N->getOperand(Num: 2));
826 // Vectors may have a different boolean contents to scalars. Promote the
827 // value appropriately.
828 ISD::NodeType ExtendCode =
829 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
830 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
831}
832
833SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
834 SDLoc DL(N);
835 SDValue Arg = N->getOperand(Num: 0);
836 SDValue Test = N->getOperand(Num: 1);
837 EVT ArgVT = Arg.getValueType();
838 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
839
840 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
841 Arg = GetScalarizedVector(Op: Arg);
842 } else {
843 EVT VT = ArgVT.getVectorElementType();
844 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
845 }
846
847 SDValue Res =
848 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
849 // Vectors may have a different boolean contents to scalars. Promote the
850 // value appropriately.
851 ISD::NodeType ExtendCode =
852 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
853 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
854}
855
856//===----------------------------------------------------------------------===//
857// Operand Vector Scalarization <1 x ty> -> ty.
858//===----------------------------------------------------------------------===//
859
860bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
861 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
862 N->dump(&DAG));
863 SDValue Res = SDValue();
864
865 // See if the target wants to custom scalarize this node.
866 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
867 return false;
868
869 switch (N->getOpcode()) {
870 default:
871#ifndef NDEBUG
872 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
873 N->dump(&DAG);
874 dbgs() << "\n";
875#endif
876 report_fatal_error(reason: "Do not know how to scalarize this operator's "
877 "operand!\n");
878 case ISD::BITCAST:
879 Res = ScalarizeVecOp_BITCAST(N);
880 break;
881 case ISD::FAKE_USE:
882 Res = ScalarizeVecOp_FAKE_USE(N);
883 break;
884 case ISD::ANY_EXTEND:
885 case ISD::ZERO_EXTEND:
886 case ISD::SIGN_EXTEND:
887 case ISD::TRUNCATE:
888 case ISD::FP_TO_SINT:
889 case ISD::FP_TO_UINT:
890 case ISD::SINT_TO_FP:
891 case ISD::UINT_TO_FP:
892 case ISD::LROUND:
893 case ISD::LLROUND:
894 case ISD::LRINT:
895 case ISD::LLRINT:
896 Res = ScalarizeVecOp_UnaryOp(N);
897 break;
898 case ISD::FP_TO_SINT_SAT:
899 case ISD::FP_TO_UINT_SAT:
900 case ISD::CONVERT_FROM_ARBITRARY_FP:
901 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
902 break;
903 case ISD::CONVERT_TO_ARBITRARY_FP: {
904 assert(N->getValueType(0).getVectorNumElements() == 1 &&
905 "Unexpected vector type!");
906 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
907 SDValue Op = DAG.getNode(
908 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
909 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
910 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
911 break;
912 }
913 case ISD::STRICT_SINT_TO_FP:
914 case ISD::STRICT_UINT_TO_FP:
915 case ISD::STRICT_FP_TO_SINT:
916 case ISD::STRICT_FP_TO_UINT:
917 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
918 break;
919 case ISD::CONCAT_VECTORS:
920 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
921 break;
922 case ISD::INSERT_SUBVECTOR:
923 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
924 break;
925 case ISD::EXTRACT_VECTOR_ELT:
926 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
927 break;
928 case ISD::VSELECT:
929 Res = ScalarizeVecOp_VSELECT(N);
930 break;
931 case ISD::SETCC:
932 Res = ScalarizeVecOp_VSETCC(N);
933 break;
934 case ISD::STRICT_FSETCC:
935 case ISD::STRICT_FSETCCS:
936 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
937 break;
938 case ISD::STORE:
939 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
940 break;
941 case ISD::ATOMIC_STORE:
942 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
943 break;
944 case ISD::STRICT_FP_ROUND:
945 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
946 break;
947 case ISD::FP_ROUND:
948 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
949 break;
950 case ISD::STRICT_FP_EXTEND:
951 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
952 break;
953 case ISD::FP_EXTEND:
954 Res = ScalarizeVecOp_FP_EXTEND(N);
955 break;
956 case ISD::VECREDUCE_FADD:
957 case ISD::VECREDUCE_FMUL:
958 case ISD::VECREDUCE_ADD:
959 case ISD::VECREDUCE_MUL:
960 case ISD::VECREDUCE_AND:
961 case ISD::VECREDUCE_OR:
962 case ISD::VECREDUCE_XOR:
963 case ISD::VECREDUCE_SMAX:
964 case ISD::VECREDUCE_SMIN:
965 case ISD::VECREDUCE_UMAX:
966 case ISD::VECREDUCE_UMIN:
967 case ISD::VECREDUCE_FMAX:
968 case ISD::VECREDUCE_FMIN:
969 case ISD::VECREDUCE_FMAXIMUM:
970 case ISD::VECREDUCE_FMINIMUM:
971 case ISD::VECREDUCE_FMAXIMUMNUM:
972 case ISD::VECREDUCE_FMINIMUMNUM:
973 Res = ScalarizeVecOp_VECREDUCE(N);
974 break;
975 case ISD::VECREDUCE_SEQ_FADD:
976 case ISD::VECREDUCE_SEQ_FMUL:
977 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
978 break;
979 case ISD::SCMP:
980 case ISD::UCMP:
981 Res = ScalarizeVecOp_CMP(N);
982 break;
983 case ISD::VECTOR_FIND_LAST_ACTIVE:
984 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
985 break;
986 case ISD::CTTZ_ELTS:
987 case ISD::CTTZ_ELTS_ZERO_POISON:
988 Res = ScalarizeVecOp_CTTZ_ELTS(N);
989 break;
990 case ISD::VECTOR_MATCH:
991 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
992 break;
993 case ISD::MASKED_UDIV:
994 case ISD::MASKED_SDIV:
995 case ISD::MASKED_UREM:
996 case ISD::MASKED_SREM:
997 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
998 break;
999 }
1000
1001 // If the result is null, the sub-method took care of registering results etc.
1002 if (!Res.getNode()) return false;
1003
1004 // If the result is N, the sub-method updated N in place. Tell the legalizer
1005 // core about this.
1006 if (Res.getNode() == N)
1007 return true;
1008
1009 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1010 "Invalid operand expansion");
1011
1012 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1013 return false;
1014}
1015
1016/// If the value to convert is a vector that needs to be scalarized, it must be
1017/// <1 x ty>. Convert the element instead.
1018SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1019 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1020 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1021 VT: N->getValueType(ResNo: 0), Operand: Elt);
1022}
1023
1024// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1025SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1026 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1027 "Fake Use: Unexpected vector type!");
1028 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1029 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1030}
1031
1032/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1033/// Do the operation on the element instead.
1034SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1035 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1036 "Unexpected vector type!");
1037 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1038 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1039 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1040 // Revectorize the result so the types line up with what the uses of this
1041 // expression expect.
1042 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1043}
1044
1045/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1046/// typesize).
1047SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1048 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1049 "Unexpected vector type!");
1050 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1051 SDValue Op =
1052 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1053 N1: Elt, N2: N->getOperand(Num: 1));
1054 // Revectorize the result so the types line up with what the uses of this
1055 // expression expect.
1056 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1057}
1058
1059/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1060/// Do the strict FP operation on the element instead.
1061SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1062 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1063 "Unexpected vector type!");
1064 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1065 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1066 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1067 Ops: { N->getOperand(Num: 0), Elt });
1068 // Legalize the chain result - switch anything that used the old chain to
1069 // use the new one.
1070 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1071 // Revectorize the result so the types line up with what the uses of this
1072 // expression expect.
1073 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1074
1075 // Do our own replacement and return SDValue() to tell the caller that we
1076 // handled all replacements since caller can only handle a single result.
1077 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1078 return SDValue();
1079}
1080
1081/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1082SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1083 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1084 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1085 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1086 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1087}
1088
1089/// The inserted subvector is to be scalarized - use insert vector element
1090/// instead.
1091SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1092 unsigned OpNo) {
1093 // We should not be attempting to scalarize the containing vector
1094 assert(OpNo == 1);
1095 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1096 SDValue ContainingVec = N->getOperand(Num: 0);
1097 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1098 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1099 N3: N->getOperand(Num: 2));
1100}
1101
1102/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1103/// so just return the element, ignoring the index.
1104SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1105 EVT VT = N->getValueType(ResNo: 0);
1106 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1107 if (Res.getValueType() != VT)
1108 Res = VT.isFloatingPoint()
1109 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1110 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1111 return Res;
1112}
1113
1114/// If the input condition is a vector that needs to be scalarized, it must be
1115/// <1 x i1>, so just convert to a normal ISD::SELECT
1116/// (still with vector output type since that was acceptable if we got here).
1117SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1118 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1119 EVT VT = N->getValueType(ResNo: 0);
1120
1121 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1122 N3: N->getOperand(Num: 2));
1123}
1124
1125/// If the operand is a vector that needs to be scalarized then the
1126/// result must be a single-element vector, so just convert to a scalar
1127/// SETCC and wrap with a scalar_to_vector since the res type is legal
1128/// if we got here
1129SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1130 assert(N->getValueType(0).isVector() &&
1131 N->getOperand(0).getValueType().isVector() &&
1132 "Operand types must be vectors");
1133 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1134 "Expected single-element vector type");
1135
1136 EVT VT = N->getValueType(ResNo: 0);
1137 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1138 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1139
1140 EVT OpVT = N->getOperand(Num: 0).getValueType();
1141 EVT NVT = VT.getVectorElementType();
1142 SDLoc DL(N);
1143 // Turn it into a scalar SETCC.
1144 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1145 N3: N->getOperand(Num: 2));
1146
1147 // Vectors may have a different boolean contents to scalars. Promote the
1148 // value appropriately.
1149 ISD::NodeType ExtendCode =
1150 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1151
1152 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1153
1154 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1155}
1156
1157// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1158SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1159 unsigned OpNo) {
1160 assert(OpNo == 1 && "Wrong operand for scalarization!");
1161 assert(N->getValueType(0).isVector() &&
1162 N->getOperand(1).getValueType().isVector() &&
1163 "Operand types must be vectors");
1164 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1165 "Expected single-element vector type");
1166
1167 EVT VT = N->getValueType(ResNo: 0);
1168 SDValue Ch = N->getOperand(Num: 0);
1169 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1170 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1171 SDValue CC = N->getOperand(Num: 3);
1172
1173 EVT OpVT = N->getOperand(Num: 1).getValueType();
1174 EVT NVT = VT.getVectorElementType();
1175 SDLoc DL(N);
1176 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1177 Ops: {Ch, LHS, RHS, CC});
1178
1179 // Legalize the chain result - switch anything that used the old chain to
1180 // use the new one.
1181 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1182
1183 ISD::NodeType ExtendCode =
1184 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1185
1186 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1187 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1188
1189 // Do our own replacement and return SDValue() to tell the caller that we
1190 // handled all replacements since caller can only handle a single result.
1191 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1192 return SDValue();
1193}
1194
1195/// If the value to store is a vector that needs to be scalarized, it must be
1196/// <1 x ty>. Just store the element.
1197SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1198 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1199 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1200 SDLoc dl(N);
1201
1202 if (N->isTruncatingStore())
1203 return DAG.getTruncStore(
1204 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1205 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1206 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1207 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
1208
1209 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1210 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1211 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
1212}
1213
1214/// If the value to store is a vector that needs to be scalarized, it must be
1215/// <1 x ty>. Just store the element.
1216SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1217 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1218 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1219 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1220 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1221}
1222
1223/// If the value to round is a vector that needs to be scalarized, it must be
1224/// <1 x ty>. Convert the element instead.
1225SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1226 assert(OpNo == 0 && "Wrong operand for scalarization!");
1227 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1228 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1229 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1230 N2: N->getOperand(Num: 1));
1231 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1232}
1233
1234SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1235 unsigned OpNo) {
1236 assert(OpNo == 1 && "Wrong operand for scalarization!");
1237 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1238 SDValue Res =
1239 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1240 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1241 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1242 // Legalize the chain result - switch anything that used the old chain to
1243 // use the new one.
1244 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1245
1246 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1247
1248 // Do our own replacement and return SDValue() to tell the caller that we
1249 // handled all replacements since caller can only handle a single result.
1250 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1251 return SDValue();
1252}
1253
1254/// If the value to extend is a vector that needs to be scalarized, it must be
1255/// <1 x ty>. Convert the element instead.
1256SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1257 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1258 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1259 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1260 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1261}
1262
1263/// If the value to extend is a vector that needs to be scalarized, it must be
1264/// <1 x ty>. Convert the element instead.
1265SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1266 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1267 SDValue Res =
1268 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1269 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1270 Ops: {N->getOperand(Num: 0), Elt});
1271 // Legalize the chain result - switch anything that used the old chain to
1272 // use the new one.
1273 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1274
1275 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1276
1277 // Do our own replacement and return SDValue() to tell the caller that we
1278 // handled all replacements since caller can only handle a single result.
1279 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1280 return SDValue();
1281}
1282
1283SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1284 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1285 // Result type may be wider than element type.
1286 if (Res.getValueType() != N->getValueType(ResNo: 0))
1287 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1288 return Res;
1289}
1290
1291SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1292 SDValue AccOp = N->getOperand(Num: 0);
1293 SDValue VecOp = N->getOperand(Num: 1);
1294
1295 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1296
1297 SDValue Op = GetScalarizedVector(Op: VecOp);
1298 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1299 N1: AccOp, N2: Op, Flags: N->getFlags());
1300}
1301
1302SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1303 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1304 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1305
1306 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1307 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1308 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1309}
1310
1311SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1312 // Since there is no "none-active" result, the only valid return for <1 x ty>
1313 // is 0. Note: Since we check the high mask during splitting this is safe.
1314 // As e.g., a <2 x ty> operation would split to:
1315 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1316 // : `last_active(%lo_mask)`
1317 // Which then scalarizes to:
1318 // %mask[1] ? 1 : 0
1319 EVT VT = N->getValueType(ResNo: 0);
1320 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1321}
1322
1323SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1324 // The number of trailing zero elements is 1 if the element is 0, and 0
1325 // otherwise.
1326 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1327 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1328 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1329 SDValue SetCC =
1330 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1331 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1332 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1333}
1334
1335SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1336 SDLoc DL(N);
1337 // Reuse the expansion (which should scalarize).
1338 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1339 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
1340 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
1341 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1342}
1343
1344SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1345 unsigned OpNo) {
1346 return TLI.expandVectorMatch(N, DAG);
1347}
1348
1349SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1350 assert(OpNo == 2 && "Can only scalarize mask operand");
1351 SDLoc DL(N);
1352 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1353 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1354 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1355 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1356 // Vectors may have a different boolean contents to scalars, so truncate to i1
1357 // and let type legalization promote appropriately.
1358 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1359 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1360 // use 1 as the divisor to avoid division by zero and overflow.
1361 SDValue BinOp =
1362 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1363 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1364 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1365}
1366
1367//===----------------------------------------------------------------------===//
1368// Result Vector Splitting
1369//===----------------------------------------------------------------------===//
1370
1371/// This method is called when the specified result of the specified node is
1372/// found to need vector splitting. At this point, the node may also have
1373/// invalid operands or may have other results that need legalization, we just
1374/// know that (at least) one result needs vector splitting.
1375void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1376 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1377 SDValue Lo, Hi;
1378
1379 // See if the target wants to custom expand this node.
1380 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1381 return;
1382
1383 switch (N->getOpcode()) {
1384 default:
1385#ifndef NDEBUG
1386 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1387 N->dump(&DAG);
1388 dbgs() << "\n";
1389#endif
1390 report_fatal_error(reason: "Do not know how to split the result of this "
1391 "operator!\n");
1392
1393 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1394 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1395 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1396 break;
1397 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1398 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1399 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1400 case ISD::VSELECT:
1401 case ISD::SELECT:
1402 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1403 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1404 case ISD::POISON:
1405 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1406 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1407 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1408 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1409 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1410 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1411 case ISD::FPOWI:
1412 case ISD::FLDEXP:
1413 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1414 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1415 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1416 case ISD::SPLAT_VECTOR:
1417 case ISD::SCALAR_TO_VECTOR:
1418 SplitVecRes_ScalarOp(N, Lo, Hi);
1419 break;
1420 case ISD::STEP_VECTOR:
1421 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1422 break;
1423 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1424 case ISD::ATOMIC_LOAD:
1425 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1426 break;
1427 case ISD::LOAD:
1428 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1429 break;
1430 case ISD::VP_LOAD:
1431 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1432 break;
1433 case ISD::VP_LOAD_FF:
1434 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1435 break;
1436 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1437 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1438 break;
1439 case ISD::MLOAD:
1440 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1441 break;
1442 case ISD::MGATHER:
1443 case ISD::VP_GATHER:
1444 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1445 break;
1446 case ISD::VECTOR_COMPRESS:
1447 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1448 break;
1449 case ISD::SETCC:
1450 SplitVecRes_SETCC(N, Lo, Hi);
1451 break;
1452 case ISD::VECTOR_REVERSE:
1453 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1454 break;
1455 case ISD::VECTOR_SHUFFLE:
1456 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1457 break;
1458 case ISD::VECTOR_SPLICE_LEFT:
1459 case ISD::VECTOR_SPLICE_RIGHT:
1460 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1461 break;
1462 case ISD::VECTOR_DEINTERLEAVE:
1463 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1464 return;
1465 case ISD::VECTOR_INTERLEAVE:
1466 SplitVecRes_VECTOR_INTERLEAVE(N);
1467 return;
1468 case ISD::VAARG:
1469 SplitVecRes_VAARG(N, Lo, Hi);
1470 break;
1471
1472 case ISD::ANY_EXTEND_VECTOR_INREG:
1473 case ISD::SIGN_EXTEND_VECTOR_INREG:
1474 case ISD::ZERO_EXTEND_VECTOR_INREG:
1475 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1476 break;
1477
1478 case ISD::ABS:
1479 case ISD::ABS_MIN_POISON:
1480 case ISD::BITREVERSE:
1481 case ISD::BSWAP:
1482 case ISD::CTLZ:
1483 case ISD::CTTZ:
1484 case ISD::CTLZ_ZERO_POISON:
1485 case ISD::CTTZ_ZERO_POISON:
1486 case ISD::CTPOP:
1487 case ISD::FABS:
1488 case ISD::FACOS:
1489 case ISD::FASIN:
1490 case ISD::FATAN:
1491 case ISD::FCEIL:
1492 case ISD::FCOS:
1493 case ISD::FCOSH:
1494 case ISD::FEXP:
1495 case ISD::FEXP2:
1496 case ISD::FEXP10:
1497 case ISD::FFLOOR:
1498 case ISD::FLOG:
1499 case ISD::FLOG10:
1500 case ISD::FLOG2:
1501 case ISD::FNEARBYINT:
1502 case ISD::FNEG:
1503 case ISD::FREEZE:
1504 case ISD::ARITH_FENCE:
1505 case ISD::FP_EXTEND:
1506 case ISD::FP_ROUND:
1507 case ISD::FP_TO_SINT:
1508 case ISD::FP_TO_UINT:
1509 case ISD::FRINT:
1510 case ISD::LRINT:
1511 case ISD::LLRINT:
1512 case ISD::FROUND:
1513 case ISD::FROUNDEVEN:
1514 case ISD::LROUND:
1515 case ISD::LLROUND:
1516 case ISD::FSIN:
1517 case ISD::FSINH:
1518 case ISD::FSQRT:
1519 case ISD::FTAN:
1520 case ISD::FTANH:
1521 case ISD::FTRUNC:
1522 case ISD::SINT_TO_FP:
1523 case ISD::TRUNCATE:
1524 case ISD::UINT_TO_FP:
1525 case ISD::FCANONICALIZE:
1526 case ISD::AssertNoFPClass:
1527 case ISD::CONVERT_FROM_ARBITRARY_FP:
1528 case ISD::CONVERT_TO_ARBITRARY_FP:
1529 SplitVecRes_UnaryOp(N, Lo, Hi);
1530 break;
1531 case ISD::ADDRSPACECAST:
1532 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1533 break;
1534 case ISD::FMODF:
1535 case ISD::FFREXP:
1536 case ISD::FSINCOS:
1537 case ISD::FSINCOSPI:
1538 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1539 break;
1540
1541 case ISD::ANY_EXTEND:
1542 case ISD::SIGN_EXTEND:
1543 case ISD::ZERO_EXTEND:
1544 SplitVecRes_ExtendOp(N, Lo, Hi);
1545 break;
1546
1547 case ISD::ADD:
1548 case ISD::SUB:
1549 case ISD::MUL:
1550 case ISD::CLMUL:
1551 case ISD::CLMULR:
1552 case ISD::CLMULH:
1553 case ISD::PEXT:
1554 case ISD::PDEP:
1555 case ISD::MULHS:
1556 case ISD::MULHU:
1557 case ISD::ABDS:
1558 case ISD::ABDU:
1559 case ISD::AVGCEILS:
1560 case ISD::AVGCEILU:
1561 case ISD::AVGFLOORS:
1562 case ISD::AVGFLOORU:
1563 case ISD::FADD:
1564 case ISD::FSUB:
1565 case ISD::FMUL:
1566 case ISD::FMINNUM:
1567 case ISD::FMINNUM_IEEE:
1568 case ISD::FMAXNUM:
1569 case ISD::FMAXNUM_IEEE:
1570 case ISD::FMINIMUM:
1571 case ISD::FMAXIMUM:
1572 case ISD::FMINIMUMNUM:
1573 case ISD::FMAXIMUMNUM:
1574 case ISD::SDIV: case ISD::VP_SDIV:
1575 case ISD::UDIV: case ISD::VP_UDIV:
1576 case ISD::FDIV:
1577 case ISD::FPOW:
1578 case ISD::FATAN2:
1579 case ISD::AND:
1580 case ISD::OR:
1581 case ISD::XOR:
1582 case ISD::SHL:
1583 case ISD::SRA:
1584 case ISD::SRL:
1585 case ISD::UREM: case ISD::VP_UREM:
1586 case ISD::SREM: case ISD::VP_SREM:
1587 case ISD::FREM:
1588 case ISD::SMIN:
1589 case ISD::SMAX:
1590 case ISD::UMIN:
1591 case ISD::UMAX:
1592 case ISD::SADDSAT:
1593 case ISD::UADDSAT:
1594 case ISD::SSUBSAT:
1595 case ISD::USUBSAT:
1596 case ISD::SSHLSAT:
1597 case ISD::USHLSAT:
1598 case ISD::ROTL:
1599 case ISD::ROTR:
1600 SplitVecRes_BinOp(N, Lo, Hi);
1601 break;
1602 case ISD::MASKED_UDIV:
1603 case ISD::MASKED_SDIV:
1604 case ISD::MASKED_UREM:
1605 case ISD::MASKED_SREM:
1606 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1607 break;
1608 case ISD::FMA:
1609 case ISD::FSHL:
1610 case ISD::FSHR:
1611 SplitVecRes_TernaryOp(N, Lo, Hi);
1612 break;
1613
1614 case ISD::SCMP: case ISD::UCMP:
1615 SplitVecRes_CMP(N, Lo, Hi);
1616 break;
1617
1618#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1619 case ISD::STRICT_##DAGN:
1620#include "llvm/IR/ConstrainedOps.def"
1621 SplitVecRes_StrictFPOp(N, Lo, Hi);
1622 break;
1623
1624 case ISD::FP_TO_UINT_SAT:
1625 case ISD::FP_TO_SINT_SAT:
1626 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1627 break;
1628
1629 case ISD::UADDO:
1630 case ISD::SADDO:
1631 case ISD::USUBO:
1632 case ISD::SSUBO:
1633 case ISD::UMULO:
1634 case ISD::SMULO:
1635 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1636 break;
1637 case ISD::SMULFIX:
1638 case ISD::SMULFIXSAT:
1639 case ISD::UMULFIX:
1640 case ISD::UMULFIXSAT:
1641 case ISD::SDIVFIX:
1642 case ISD::SDIVFIXSAT:
1643 case ISD::UDIVFIX:
1644 case ISD::UDIVFIXSAT:
1645 SplitVecRes_FIX(N, Lo, Hi);
1646 break;
1647 case ISD::EXPERIMENTAL_VP_SPLICE:
1648 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1649 break;
1650 case ISD::EXPERIMENTAL_VP_REVERSE:
1651 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1652 break;
1653 case ISD::PARTIAL_REDUCE_UMLA:
1654 case ISD::PARTIAL_REDUCE_SMLA:
1655 case ISD::PARTIAL_REDUCE_SUMLA:
1656 case ISD::PARTIAL_REDUCE_FMLA:
1657 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1658 break;
1659 case ISD::GET_ACTIVE_LANE_MASK:
1660 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1661 break;
1662 case ISD::VECTOR_MATCH:
1663 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1664 break;
1665 }
1666
1667 // If Lo/Hi is null, the sub-method took care of registering results etc.
1668 if (Lo.getNode())
1669 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1670}
1671
1672void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1673 MachinePointerInfo &MPI, SDValue &Ptr,
1674 uint64_t *ScaledOffset) {
1675 SDLoc DL(N);
1676 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1677
1678 if (MemVT.isScalableVector()) {
1679 SDValue BytesIncrement = DAG.getVScale(
1680 DL, VT: Ptr.getValueType(),
1681 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1682 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1683 if (ScaledOffset)
1684 *ScaledOffset += IncrementSize;
1685 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1686 Flags: SDNodeFlags::NoUnsignedWrap);
1687 } else {
1688 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1689 // Increment the pointer to the other half.
1690 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1691 }
1692}
1693
1694std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1695 return SplitMask(Mask, DL: SDLoc(Mask));
1696}
1697
1698std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1699 const SDLoc &DL) {
1700 SDValue MaskLo, MaskHi;
1701 EVT MaskVT = Mask.getValueType();
1702 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1703 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1704 else
1705 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1706 return std::make_pair(x&: MaskLo, y&: MaskHi);
1707}
1708
1709void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1710 SDValue LHSLo, LHSHi;
1711 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1712 SDValue RHSLo, RHSHi;
1713 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1714 SDLoc dl(N);
1715
1716 const SDNodeFlags Flags = N->getFlags();
1717 unsigned Opcode = N->getOpcode();
1718 if (N->getNumOperands() == 2) {
1719 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1720 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1721 return;
1722 }
1723
1724 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1725 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1726 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1727 "Expected VP opcode");
1728
1729 SDValue MaskLo, MaskHi;
1730 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1731
1732 SDValue EVLLo, EVLHi;
1733 std::tie(args&: EVLLo, args&: EVLHi) =
1734 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1735
1736 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1737 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1738 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1739 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1740}
1741
1742void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1743 SDValue &Hi) {
1744 SDValue LHSLo, LHSHi;
1745 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1746 SDValue RHSLo, RHSHi;
1747 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1748
1749 SDValue MaskLo, MaskHi, Mask = N->getOperand(Num: 2);
1750 if (Mask.getOpcode() == ISD::SETCC)
1751 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1752 else
1753 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask);
1754
1755 SDLoc dl(N);
1756
1757 const SDNodeFlags Flags = N->getFlags();
1758 unsigned Opcode = N->getOpcode();
1759 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1760 Flags);
1761 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1762 Flags);
1763}
1764
1765void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1766 SDValue &Hi) {
1767 SDValue Op0Lo, Op0Hi;
1768 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1769 SDValue Op1Lo, Op1Hi;
1770 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1771 SDValue Op2Lo, Op2Hi;
1772 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1773 SDLoc dl(N);
1774
1775 const SDNodeFlags Flags = N->getFlags();
1776 unsigned Opcode = N->getOpcode();
1777 Lo =
1778 DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1779 Hi =
1780 DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1781}
1782
1783void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1784 LLVMContext &Ctxt = *DAG.getContext();
1785 SDLoc dl(N);
1786
1787 SDValue LHS = N->getOperand(Num: 0);
1788 SDValue RHS = N->getOperand(Num: 1);
1789
1790 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1791 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1792 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1793 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1794 } else {
1795 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1796 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1797 }
1798
1799 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1800 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1801 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1802}
1803
1804void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1805 SDValue LHSLo, LHSHi;
1806 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1807 SDValue RHSLo, RHSHi;
1808 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1809 SDLoc dl(N);
1810 SDValue Op2 = N->getOperand(Num: 2);
1811
1812 unsigned Opcode = N->getOpcode();
1813 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1814 Flags: N->getFlags());
1815 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1816 Flags: N->getFlags());
1817}
1818
1819void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1820 SDValue &Hi) {
1821 // We know the result is a vector. The input may be either a vector or a
1822 // scalar value.
1823 EVT LoVT, HiVT;
1824 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1825 SDLoc dl(N);
1826
1827 SDValue InOp = N->getOperand(Num: 0);
1828 EVT InVT = InOp.getValueType();
1829
1830 // Handle some special cases efficiently.
1831 switch (getTypeAction(VT: InVT)) {
1832 case TargetLowering::TypeLegal:
1833 case TargetLowering::TypePromoteInteger:
1834 case TargetLowering::TypeSoftPromoteHalf:
1835 case TargetLowering::TypeSoftenFloat:
1836 case TargetLowering::TypeScalarizeVector:
1837 case TargetLowering::TypeWidenVector:
1838 break;
1839 case TargetLowering::TypeExpandInteger:
1840 case TargetLowering::TypeExpandFloat:
1841 // A scalar to vector conversion, where the scalar needs expansion.
1842 // If the vector is being split in two then we can just convert the
1843 // expanded pieces.
1844 if (LoVT == HiVT) {
1845 GetExpandedOp(Op: InOp, Lo, Hi);
1846 if (DAG.getDataLayout().isBigEndian())
1847 std::swap(a&: Lo, b&: Hi);
1848 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1849 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1850 return;
1851 }
1852 break;
1853 case TargetLowering::TypeSplitVector:
1854 // If the input is a vector that needs to be split, convert each split
1855 // piece of the input now.
1856 GetSplitVector(Op: InOp, Lo, Hi);
1857 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1858 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1859 return;
1860 case TargetLowering::TypeScalarizeScalableVector:
1861 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1862 }
1863
1864 if (LoVT.isScalableVector()) {
1865 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1866 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1867 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1868 return;
1869 }
1870
1871 // In the general case, convert the input to an integer and split it by hand.
1872 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1873 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1874 if (DAG.getDataLayout().isBigEndian())
1875 std::swap(a&: LoIntVT, b&: HiIntVT);
1876
1877 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1878
1879 if (DAG.getDataLayout().isBigEndian())
1880 std::swap(a&: Lo, b&: Hi);
1881 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1882 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1883}
1884
1885void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1886 SDValue &Hi) {
1887 SDLoc DL(N);
1888 EVT LoVT, HiVT;
1889 SDValue PtrA = N->getOperand(Num: 0);
1890 SDValue PtrB = N->getOperand(Num: 1);
1891 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1892
1893 // The lane offset for the "Lo" half of the mask is unchanged.
1894 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1895 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1896 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1897 // The lane offset for the "Hi" half of the mask is incremented by the number
1898 // of elements in the "Lo" half.
1899 unsigned LaneOffset =
1900 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1901 // Note: The lane offset is implicitly scalable for scalable masks.
1902 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1903 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1904 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1905}
1906
1907void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1908 SDValue &Hi) {
1909 EVT LoVT, HiVT;
1910 SDLoc dl(N);
1911 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1912 unsigned LoNumElts = LoVT.getVectorNumElements();
1913 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1914 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1915
1916 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1917 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1918}
1919
1920void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1921 SDValue &Hi) {
1922 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1923 SDLoc dl(N);
1924 unsigned NumSubvectors = N->getNumOperands() / 2;
1925 if (NumSubvectors == 1) {
1926 Lo = N->getOperand(Num: 0);
1927 Hi = N->getOperand(Num: 1);
1928 return;
1929 }
1930
1931 EVT LoVT, HiVT;
1932 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1933
1934 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1935 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1936
1937 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1938 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1939}
1940
1941void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1942 SDValue &Hi) {
1943 SDValue Vec = N->getOperand(Num: 0);
1944 SDValue Idx = N->getOperand(Num: 1);
1945 SDLoc dl(N);
1946
1947 EVT LoVT, HiVT;
1948 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1949
1950 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1951 uint64_t IdxVal = Idx->getAsZExtVal();
1952 Hi = DAG.getNode(
1953 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1954 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1955}
1956
1957void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1958 SDValue &Hi) {
1959 SDValue Vec = N->getOperand(Num: 0);
1960 SDValue SubVec = N->getOperand(Num: 1);
1961 SDValue Idx = N->getOperand(Num: 2);
1962 SDLoc dl(N);
1963 GetSplitVector(Op: Vec, Lo, Hi);
1964
1965 EVT VecVT = Vec.getValueType();
1966 EVT LoVT = Lo.getValueType();
1967 EVT SubVecVT = SubVec.getValueType();
1968 unsigned VecElems = VecVT.getVectorMinNumElements();
1969 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1970 unsigned LoElems = LoVT.getVectorMinNumElements();
1971
1972 // If we know the index is in the first half, and we know the subvector
1973 // doesn't cross the boundary between the halves, we can avoid spilling the
1974 // vector, and insert into the lower half of the split vector directly.
1975 unsigned IdxVal = Idx->getAsZExtVal();
1976 if (IdxVal + SubElems <= LoElems) {
1977 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1978 return;
1979 }
1980 // Similarly if the subvector is fully in the high half, but mind that we
1981 // can't tell whether a fixed-length subvector is fully within the high half
1982 // of a scalable vector.
1983 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1984 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1985 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1986 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1987 return;
1988 }
1989
1990 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1991 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1992 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1993 if (WideSubVec.getValueType() == VecVT) {
1994 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1995 return;
1996 }
1997 }
1998
1999 // Spill the vector to the stack.
2000 // In cases where the vector is illegal it will be broken down into parts
2001 // and stored in parts - we should use the alignment for the smallest part.
2002 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2003 SDValue StackPtr =
2004 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2005 auto &MF = DAG.getMachineFunction();
2006 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2007 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2008
2009 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2010 Alignment: SmallestAlign);
2011
2012 // Store the new subvector into the specified index.
2013 SDValue SubVecPtr =
2014 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2015 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2016 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2017
2018 // Load the Lo part from the stack slot.
2019 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2020 Alignment: SmallestAlign);
2021
2022 // Increment the pointer to the other part.
2023 auto *Load = cast<LoadSDNode>(Val&: Lo);
2024 MachinePointerInfo MPI = Load->getPointerInfo();
2025 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2026
2027 // Load the Hi part from the stack slot.
2028 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2029}
2030
2031// Handle splitting an FP where the second operand does not match the first
2032// type. The second operand may be a scalar, or a vector that has exactly as
2033// many elements as the first
2034void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2035 SDValue &Hi) {
2036 SDValue LHSLo, LHSHi;
2037 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2038 SDLoc DL(N);
2039
2040 SDValue RHSLo, RHSHi;
2041 SDValue RHS = N->getOperand(Num: 1);
2042 EVT RHSVT = RHS.getValueType();
2043 if (RHSVT.isVector()) {
2044 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2045 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2046 else
2047 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2048
2049 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2050 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2051 } else {
2052 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2053 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2054 }
2055}
2056
2057void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2058 SDValue &Hi) {
2059 SDLoc DL(N);
2060 SDValue ArgLo, ArgHi;
2061 SDValue Test = N->getOperand(Num: 1);
2062 SDValue FpValue = N->getOperand(Num: 0);
2063 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2064 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2065 else
2066 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2067 EVT LoVT, HiVT;
2068 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2069
2070 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2071 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2072}
2073
2074void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2075 SDValue &Hi) {
2076 SDValue LHSLo, LHSHi;
2077 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2078 SDLoc dl(N);
2079
2080 EVT LoVT, HiVT;
2081 std::tie(args&: LoVT, args&: HiVT) =
2082 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2083
2084 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2085 N2: DAG.getValueType(LoVT));
2086 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2087 N2: DAG.getValueType(HiVT));
2088}
2089
2090void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2091 SDValue &Hi) {
2092 unsigned Opcode = N->getOpcode();
2093 SDValue N0 = N->getOperand(Num: 0);
2094
2095 SDLoc dl(N);
2096 SDValue InLo, InHi;
2097
2098 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2099 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2100 else
2101 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2102
2103 EVT InLoVT = InLo.getValueType();
2104 unsigned InNumElements = InLoVT.getVectorNumElements();
2105
2106 EVT OutLoVT, OutHiVT;
2107 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2108 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2109 assert((2 * OutNumElements) <= InNumElements &&
2110 "Illegal extend vector in reg split");
2111
2112 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2113 // input vector (i.e. we only use InLo):
2114 // OutLo will extend the first OutNumElements from InLo.
2115 // OutHi will extend the next OutNumElements from InLo.
2116
2117 // Shuffle the elements from InLo for OutHi into the bottom elements to
2118 // create a 'fake' InHi.
2119 SmallVector<int, 8> SplitHi(InNumElements, -1);
2120 for (unsigned i = 0; i != OutNumElements; ++i)
2121 SplitHi[i] = i + OutNumElements;
2122 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2123
2124 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2125 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2126}
2127
2128void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2129 SDValue &Hi) {
2130 unsigned NumOps = N->getNumOperands();
2131 SDValue Chain = N->getOperand(Num: 0);
2132 EVT LoVT, HiVT;
2133 SDLoc dl(N);
2134 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2135
2136 SmallVector<SDValue, 4> OpsLo(NumOps);
2137 SmallVector<SDValue, 4> OpsHi(NumOps);
2138
2139 // The Chain is the first operand.
2140 OpsLo[0] = Chain;
2141 OpsHi[0] = Chain;
2142
2143 // Now process the remaining operands.
2144 for (unsigned i = 1; i < NumOps; ++i) {
2145 SDValue Op = N->getOperand(Num: i);
2146 SDValue OpLo = Op;
2147 SDValue OpHi = Op;
2148
2149 EVT InVT = Op.getValueType();
2150 if (InVT.isVector()) {
2151 // If the input also splits, handle it directly for a
2152 // compile time speedup. Otherwise split it by hand.
2153 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2154 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2155 else
2156 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2157 }
2158
2159 OpsLo[i] = OpLo;
2160 OpsHi[i] = OpHi;
2161 }
2162
2163 EVT LoValueVTs[] = {LoVT, MVT::Other};
2164 EVT HiValueVTs[] = {HiVT, MVT::Other};
2165 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2166 Flags: N->getFlags());
2167 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2168 Flags: N->getFlags());
2169
2170 // Build a factor node to remember that this Op is independent of the
2171 // other one.
2172 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2173 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2174
2175 // Legalize the chain result - switch anything that used the old chain to
2176 // use the new one.
2177 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2178}
2179
2180SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2181 SDValue Chain = N->getOperand(Num: 0);
2182 EVT VT = N->getValueType(ResNo: 0);
2183 unsigned NE = VT.getVectorNumElements();
2184 EVT EltVT = VT.getVectorElementType();
2185 SDLoc dl(N);
2186
2187 SmallVector<SDValue, 8> Scalars;
2188 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2189
2190 // If ResNE is 0, fully unroll the vector op.
2191 if (ResNE == 0)
2192 ResNE = NE;
2193 else if (NE > ResNE)
2194 NE = ResNE;
2195
2196 //The results of each unrolled operation, including the chain.
2197 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2198 SmallVector<SDValue, 8> Chains;
2199
2200 unsigned i;
2201 for (i = 0; i != NE; ++i) {
2202 Operands[0] = Chain;
2203 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2204 SDValue Operand = N->getOperand(Num: j);
2205 EVT OperandVT = Operand.getValueType();
2206 if (OperandVT.isVector()) {
2207 EVT OperandEltVT = OperandVT.getVectorElementType();
2208 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2209 } else {
2210 Operands[j] = Operand;
2211 }
2212 }
2213 SDValue Scalar =
2214 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2215
2216 //Add in the scalar as well as its chain value to the
2217 //result vectors.
2218 Scalars.push_back(Elt: Scalar);
2219 Chains.push_back(Elt: Scalar.getValue(R: 1));
2220 }
2221
2222 for (; i < ResNE; ++i)
2223 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2224
2225 // Build a new factor node to connect the chain back together.
2226 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2227 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2228
2229 // Create a new BUILD_VECTOR node
2230 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2231 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2232}
2233
2234void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2235 SDValue &Lo, SDValue &Hi) {
2236 SDLoc dl(N);
2237 EVT ResVT = N->getValueType(ResNo: 0);
2238 EVT OvVT = N->getValueType(ResNo: 1);
2239 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2240 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2241 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2242
2243 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2244 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2245 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2246 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2247 } else {
2248 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2249 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2250 }
2251
2252 unsigned Opcode = N->getOpcode();
2253 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2254 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2255 SDNode *LoNode =
2256 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2257 SDNode *HiNode =
2258 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2259
2260 Lo = SDValue(LoNode, ResNo);
2261 Hi = SDValue(HiNode, ResNo);
2262
2263 // Replace the other vector result not being explicitly split here.
2264 unsigned OtherNo = 1 - ResNo;
2265 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2266 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2267 SetSplitVector(Op: SDValue(N, OtherNo),
2268 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2269 } else {
2270 SDValue OtherVal = DAG.getNode(
2271 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2272 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2273 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2274 }
2275}
2276
2277void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2278 SDValue &Hi) {
2279 SDValue Vec = N->getOperand(Num: 0);
2280 SDValue Elt = N->getOperand(Num: 1);
2281 SDValue Idx = N->getOperand(Num: 2);
2282 SDLoc dl(N);
2283 GetSplitVector(Op: Vec, Lo, Hi);
2284
2285 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2286 unsigned IdxVal = CIdx->getZExtValue();
2287 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2288 if (IdxVal < LoNumElts) {
2289 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2290 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2291 return;
2292 } else if (!Vec.getValueType().isScalableVector()) {
2293 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2294 return;
2295 }
2296 }
2297
2298 // Make the vector elements byte-addressable if they aren't already.
2299 EVT VecVT = Vec.getValueType();
2300 EVT EltVT = VecVT.getVectorElementType();
2301 if (!EltVT.isByteSized()) {
2302 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2303 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2304 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2305 // Extend the element type to match if needed.
2306 if (EltVT.bitsGT(VT: Elt.getValueType()))
2307 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2308 }
2309
2310 // Spill the vector to the stack.
2311 // In cases where the vector is illegal it will be broken down into parts
2312 // and stored in parts - we should use the alignment for the smallest part.
2313 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2314 SDValue StackPtr =
2315 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2316 auto &MF = DAG.getMachineFunction();
2317 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2318 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2319
2320 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2321 Alignment: SmallestAlign);
2322
2323 // Store the new element. This may be larger than the vector element type,
2324 // so use a truncating store.
2325 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2326 Store = DAG.getTruncStore(
2327 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2328 Alignment: commonAlignment(A: SmallestAlign,
2329 Offset: EltVT.getFixedSizeInBits() / 8));
2330
2331 EVT LoVT, HiVT;
2332 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2333
2334 // Load the Lo part from the stack slot.
2335 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2336
2337 // Increment the pointer to the other part.
2338 auto Load = cast<LoadSDNode>(Val&: Lo);
2339 MachinePointerInfo MPI = Load->getPointerInfo();
2340 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2341
2342 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2343
2344 // If we adjusted the original type, we need to truncate the results.
2345 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2346 if (LoVT != Lo.getValueType())
2347 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2348 if (HiVT != Hi.getValueType())
2349 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2350}
2351
2352void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2353 SDValue &Hi) {
2354 EVT LoVT, HiVT;
2355 SDLoc dl(N);
2356 assert(N->getValueType(0).isScalableVector() &&
2357 "Only scalable vectors are supported for STEP_VECTOR");
2358 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2359 SDValue Step = N->getOperand(Num: 0);
2360
2361 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2362
2363 // Hi = Lo + (EltCnt * Step)
2364 EVT EltVT = Step.getValueType();
2365 APInt StepVal = Step->getAsAPIntVal();
2366 SDValue StartOfHi =
2367 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2368 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2369 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2370
2371 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2372 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2373}
2374
2375void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2376 SDValue &Hi) {
2377 EVT LoVT, HiVT;
2378 SDLoc dl(N);
2379 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2380 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2381 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2382 Hi = DAG.getPOISON(VT: HiVT);
2383 } else {
2384 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2385 Hi = Lo;
2386 }
2387}
2388
2389void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2390 SDValue &Hi) {
2391 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2392 "Extended load during type legalization!");
2393 SDLoc dl(LD);
2394 EVT VT = LD->getValueType(ResNo: 0);
2395 EVT LoVT, HiVT;
2396 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2397
2398 SDValue Ch = LD->getChain();
2399 SDValue Ptr = LD->getBasePtr();
2400
2401 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2402 EVT MemIntVT =
2403 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2404 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2405 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2406
2407 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2408 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2409 SDValue ExtractLo, ExtractHi;
2410 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2411
2412 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2413 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2414
2415 // Legalize the chain result - switch anything that used the old chain to
2416 // use the new one.
2417 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2418}
2419
2420void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2421 SDValue &Hi) {
2422 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2423 EVT LoVT, HiVT;
2424 SDLoc dl(LD);
2425 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2426
2427 ISD::LoadExtType ExtType = LD->getExtensionType();
2428 SDValue Ch = LD->getChain();
2429 SDValue Ptr = LD->getBasePtr();
2430 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
2431 EVT MemoryVT = LD->getMemoryVT();
2432 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2433 AAMDNodes AAInfo = LD->getAAInfo();
2434
2435 EVT LoMemVT, HiMemVT;
2436 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2437
2438 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2439 SDValue Value, NewChain;
2440 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2441 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2442 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2443 return;
2444 }
2445
2446 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2447 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2448 Metadata: AAInfo);
2449
2450 MachinePointerInfo MPI;
2451 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2452
2453 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2454 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
2455
2456 // Build a factor node to remember that this load is independent of the
2457 // other one.
2458 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2459 N2: Hi.getValue(R: 1));
2460
2461 // Legalize the chain result - switch anything that used the old chain to
2462 // use the new one.
2463 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2464}
2465
2466void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2467 SDValue &Hi) {
2468 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2469 EVT LoVT, HiVT;
2470 SDLoc dl(LD);
2471 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2472
2473 ISD::LoadExtType ExtType = LD->getExtensionType();
2474 SDValue Ch = LD->getChain();
2475 SDValue Ptr = LD->getBasePtr();
2476 SDValue Offset = LD->getOffset();
2477 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2478 Align Alignment = LD->getBaseAlign();
2479 SDValue Mask = LD->getMask();
2480 SDValue EVL = LD->getVectorLength();
2481 EVT MemoryVT = LD->getMemoryVT();
2482
2483 EVT LoMemVT, HiMemVT;
2484 bool HiIsEmpty = false;
2485 std::tie(args&: LoMemVT, args&: HiMemVT) =
2486 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2487
2488 // Split Mask operand
2489 SDValue MaskLo, MaskHi;
2490 if (Mask.getOpcode() == ISD::SETCC) {
2491 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2492 } else {
2493 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2494 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2495 else
2496 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2497 }
2498
2499 // Split EVL operand
2500 SDValue EVLLo, EVLHi;
2501 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2502
2503 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2504 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2505 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2506 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2507
2508 Lo =
2509 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2510 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2511
2512 if (HiIsEmpty) {
2513 // The hi vp_load has zero storage size. We therefore simply set it to
2514 // the low vp_load and rely on subsequent removal from the chain.
2515 Hi = Lo;
2516 } else {
2517 // Generate hi vp_load.
2518 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2519 IsCompressedMemory: LD->isExpandingLoad());
2520
2521 MachinePointerInfo MPI;
2522 if (LoMemVT.isScalableVector())
2523 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2524 else
2525 MPI = LD->getPointerInfo().getWithOffset(
2526 O: LoMemVT.getStoreSize().getFixedValue());
2527
2528 MMO = DAG.getMachineFunction().getMachineMemOperand(
2529 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2530 BaseAlignment: Alignment, Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2531
2532 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2533 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2534 IsExpanding: LD->isExpandingLoad());
2535 }
2536
2537 // Build a factor node to remember that this load is independent of the
2538 // other one.
2539 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2540 N2: Hi.getValue(R: 1));
2541
2542 // Legalize the chain result - switch anything that used the old chain to
2543 // use the new one.
2544 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2545}
2546
2547void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2548 SDValue &Hi) {
2549 SDLoc dl(LD);
2550 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2551
2552 SDValue Ch = LD->getChain();
2553 SDValue Ptr = LD->getBasePtr();
2554 Align Alignment = LD->getBaseAlign();
2555 SDValue Mask = LD->getMask();
2556 SDValue EVL = LD->getVectorLength();
2557
2558 // Split Mask operand
2559 SDValue MaskLo, MaskHi;
2560 if (Mask.getOpcode() == ISD::SETCC) {
2561 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2562 } else {
2563 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2564 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2565 else
2566 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2567 }
2568
2569 // Split EVL operand
2570 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2571
2572 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2573 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2574 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2575 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2576
2577 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2578
2579 // Fill the upper half with poison.
2580 Hi = DAG.getPOISON(VT: HiVT);
2581
2582 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2583 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2584}
2585
2586void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2587 SDValue &Lo, SDValue &Hi) {
2588 assert(SLD->isUnindexed() &&
2589 "Indexed VP strided load during type legalization!");
2590 assert(SLD->getOffset().isUndef() &&
2591 "Unexpected indexed variable-length load offset");
2592
2593 SDLoc DL(SLD);
2594
2595 EVT LoVT, HiVT;
2596 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2597
2598 EVT LoMemVT, HiMemVT;
2599 bool HiIsEmpty = false;
2600 std::tie(args&: LoMemVT, args&: HiMemVT) =
2601 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2602
2603 SDValue Mask = SLD->getMask();
2604 SDValue LoMask, HiMask;
2605 if (Mask.getOpcode() == ISD::SETCC) {
2606 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2607 } else {
2608 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2609 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2610 else
2611 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2612 }
2613
2614 SDValue LoEVL, HiEVL;
2615 std::tie(args&: LoEVL, args&: HiEVL) =
2616 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2617
2618 // Generate the low vp_strided_load
2619 Lo = DAG.getStridedLoadVP(
2620 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2621 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2622 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2623
2624 if (HiIsEmpty) {
2625 // The high vp_strided_load has zero storage size. We therefore simply set
2626 // it to the low vp_strided_load and rely on subsequent removal from the
2627 // chain.
2628 Hi = Lo;
2629 } else {
2630 // Generate the high vp_strided_load.
2631 // To calculate the high base address, we need to sum to the low base
2632 // address stride number of bytes for each element already loaded by low,
2633 // that is: Ptr = Ptr + (LoEVL * Stride)
2634 EVT PtrVT = SLD->getBasePtr().getValueType();
2635 SDValue Increment =
2636 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2637 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2638 SDValue Ptr =
2639 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2640
2641 Align Alignment = SLD->getBaseAlign();
2642 if (LoMemVT.isScalableVector())
2643 Alignment = commonAlignment(
2644 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2645
2646 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2647 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2648 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2649 BaseAlignment: Alignment, Metadata: MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2650
2651 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2652 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2653 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2654 IsExpanding: SLD->isExpandingLoad());
2655 }
2656
2657 // Build a factor node to remember that this load is independent of the
2658 // other one.
2659 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2660 N2: Hi.getValue(R: 1));
2661
2662 // Legalize the chain result - switch anything that used the old chain to
2663 // use the new one.
2664 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2665}
2666
2667void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2668 SDValue &Lo, SDValue &Hi) {
2669 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2670 EVT LoVT, HiVT;
2671 SDLoc dl(MLD);
2672 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2673
2674 SDValue Ch = MLD->getChain();
2675 SDValue Ptr = MLD->getBasePtr();
2676 SDValue Offset = MLD->getOffset();
2677 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2678 SDValue Mask = MLD->getMask();
2679 SDValue PassThru = MLD->getPassThru();
2680 Align Alignment = MLD->getBaseAlign();
2681 ISD::LoadExtType ExtType = MLD->getExtensionType();
2682 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2683
2684 // Split Mask operand
2685 SDValue MaskLo, MaskHi;
2686 if (Mask.getOpcode() == ISD::SETCC) {
2687 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2688 } else {
2689 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2690 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2691 else
2692 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2693 }
2694
2695 EVT MemoryVT = MLD->getMemoryVT();
2696 EVT LoMemVT, HiMemVT;
2697 bool HiIsEmpty = false;
2698 std::tie(args&: LoMemVT, args&: HiMemVT) =
2699 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2700
2701 SDValue PassThruLo, PassThruHi;
2702 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2703 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2704 else
2705 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2706
2707 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2708 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2709 BaseAlignment: Alignment,
2710 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(), MLD->getMemCacheHint()));
2711
2712 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2713 MMO, AM: MLD->getAddressingMode(), ExtType,
2714 IsExpanding: MLD->isExpandingLoad());
2715
2716 if (HiIsEmpty) {
2717 // The hi masked load has zero storage size. We therefore simply set it to
2718 // the low masked load and rely on subsequent removal from the chain.
2719 Hi = Lo;
2720 } else {
2721 // Generate hi masked load.
2722 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2723 IsCompressedMemory: MLD->isExpandingLoad());
2724
2725 MachinePointerInfo MPI;
2726 if (LoMemVT.isScalableVector())
2727 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2728 else
2729 MPI = MLD->getPointerInfo().getWithOffset(
2730 O: LoMemVT.getStoreSize().getFixedValue());
2731
2732 MMO = DAG.getMachineFunction().getMachineMemOperand(
2733 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2734 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
2735 MLD->getMemCacheHint()));
2736
2737 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2738 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2739 IsExpanding: MLD->isExpandingLoad());
2740 }
2741
2742 // Build a factor node to remember that this load is independent of the
2743 // other one.
2744 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2745 N2: Hi.getValue(R: 1));
2746
2747 // Legalize the chain result - switch anything that used the old chain to
2748 // use the new one.
2749 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2750
2751}
2752
2753void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2754 SDValue &Hi, bool SplitSETCC) {
2755 EVT LoVT, HiVT;
2756 SDLoc dl(N);
2757 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2758
2759 SDValue Ch = N->getChain();
2760 SDValue Ptr = N->getBasePtr();
2761 struct Operands {
2762 SDValue Mask;
2763 SDValue Index;
2764 SDValue Scale;
2765 } Ops = [&]() -> Operands {
2766 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2767 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2768 }
2769 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2770 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2771 }();
2772
2773 EVT MemoryVT = N->getMemoryVT();
2774 Align Alignment = N->getBaseAlign();
2775
2776 // Split Mask operand
2777 SDValue MaskLo, MaskHi;
2778 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2779 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2780 } else {
2781 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2782 }
2783
2784 EVT LoMemVT, HiMemVT;
2785 // Split MemoryVT
2786 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2787
2788 SDValue IndexHi, IndexLo;
2789 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2790 TargetLowering::TypeSplitVector)
2791 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2792 else
2793 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2794
2795 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2796 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2797 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2798 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
2799
2800 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2801 SDValue PassThru = MGT->getPassThru();
2802 SDValue PassThruLo, PassThruHi;
2803 if (getTypeAction(VT: PassThru.getValueType()) ==
2804 TargetLowering::TypeSplitVector)
2805 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2806 else
2807 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2808
2809 ISD::LoadExtType ExtType = MGT->getExtensionType();
2810 ISD::MemIndexType IndexTy = MGT->getIndexType();
2811
2812 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2813 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2814 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2815
2816 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2817 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2818 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2819 } else {
2820 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2821 SDValue EVLLo, EVLHi;
2822 std::tie(args&: EVLLo, args&: EVLHi) =
2823 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2824
2825 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2826 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2827 MMO, IndexType: VPGT->getIndexType());
2828
2829 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2830 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2831 MMO, IndexType: VPGT->getIndexType());
2832 }
2833
2834 // Build a factor node to remember that this load is independent of the
2835 // other one.
2836 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2837 N2: Hi.getValue(R: 1));
2838
2839 // Legalize the chain result - switch anything that used the old chain to
2840 // use the new one.
2841 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2842}
2843
2844void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2845 SDValue &Hi) {
2846 // This is not "trivial", as there is a dependency between the two subvectors.
2847 // Depending on the number of 1s in the mask, the elements from the Hi vector
2848 // need to be moved to the Lo vector. Passthru values make this even harder.
2849 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2850 // smaller types and passthru is undef, as it is most likely faster than the
2851 // fully expand path. Otherwise, just do the full expansion as one "big"
2852 // operation and then extract the Lo and Hi vectors from that. This gets
2853 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2854 SDLoc DL(N);
2855 EVT VecVT = N->getValueType(ResNo: 0);
2856
2857 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2858 bool HasCustomLowering = false;
2859 EVT CheckVT = LoVT;
2860 while (CheckVT.getVectorMinNumElements() > 1) {
2861 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2862 // custom lowering for illegal types. So we do the checks separately.
2863 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2864 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2865 HasCustomLowering = true;
2866 break;
2867 }
2868 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2869 }
2870
2871 SDValue Passthru = N->getOperand(Num: 2);
2872 if (!HasCustomLowering) {
2873 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2874 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2875 return;
2876 }
2877
2878 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2879 SDValue Mask = N->getOperand(Num: 1);
2880 SDValue LoMask, HiMask;
2881 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2882 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2883
2884 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2885 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2886 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2887
2888 SDValue StackPtr = DAG.CreateStackTemporary(
2889 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2890 MachineFunction &MF = DAG.getMachineFunction();
2891 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2892 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2893
2894 EVT LoMaskVT = LoMask.getValueType();
2895 assert(LoMaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2896
2897 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2898 EVT WideLoMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2899 EC: LoMaskVT.getVectorElementCount());
2900 SDValue WideLoMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideLoMaskVT, Operand: LoMask);
2901 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideLoMask);
2902 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2903
2904 SDValue Chain = DAG.getEntryNode();
2905 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2906 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2907 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2908
2909 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2910 if (!Passthru.isUndef()) {
2911 // Compress the input mask so only inactive lanes of the result are replaced
2912 // by their passthrough value.
2913 EVT MaskVT = Mask.getValueType();
2914 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2915 EC: MaskVT.getVectorElementCount());
2916 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: Mask);
2917 SDValue NumActiveElts =
2918 DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2919
2920 SDValue StepVector = DAG.getStepVector(DL, ResVT: WideMaskVT);
2921 SDValue SplatNumActiveElts = DAG.getSplat(VT: WideMaskVT, DL, Op: NumActiveElts);
2922 SDValue CompressedMask =
2923 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVector, RHS: SplatNumActiveElts, Cond: ISD::SETULT);
2924
2925 Compressed = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: CompressedMask,
2926 N2: Compressed, N3: Passthru);
2927 }
2928 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2929}
2930
2931void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2932 assert(N->getValueType(0).isVector() &&
2933 N->getOperand(0).getValueType().isVector() &&
2934 "Operand types must be vectors");
2935
2936 EVT LoVT, HiVT;
2937 SDLoc DL(N);
2938 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2939
2940 // If the input also splits, handle it directly. Otherwise split it by hand.
2941 SDValue LL, LH, RL, RH;
2942 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2943 TargetLowering::TypeSplitVector)
2944 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2945 else
2946 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2947
2948 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2949 TargetLowering::TypeSplitVector)
2950 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2951 else
2952 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2953
2954 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2955 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2956}
2957
2958void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2959 SDValue &Hi) {
2960 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2961 EVT LoVT, HiVT;
2962 SDLoc dl(N);
2963 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2964
2965 // If the input also splits, handle it directly for a compile time speedup.
2966 // Otherwise split it by hand.
2967 EVT InVT = N->getOperand(Num: 0).getValueType();
2968 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2969 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2970 else
2971 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2972
2973 const SDNodeFlags Flags = N->getFlags();
2974 unsigned Opcode = N->getOpcode();
2975 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
2976 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2977 N4: N->getOperand(Num: 3), Flags);
2978 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2979 N4: N->getOperand(Num: 3), Flags);
2980 return;
2981 }
2982
2983 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2984 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2985 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2986 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2987 } else {
2988 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2989 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2990 }
2991}
2992
2993void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2994 SDValue &Hi) {
2995 SDLoc dl(N);
2996 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2997
2998 // If the input also splits, handle it directly for a compile time speedup.
2999 // Otherwise split it by hand.
3000 EVT InVT = N->getOperand(Num: 0).getValueType();
3001 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3002 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3003 else
3004 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3005
3006 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3007 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3008 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3009 SDNodeFlags Flags = AddrSpaceCastN->getFlags();
3010 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS, Flags);
3011 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS, Flags);
3012}
3013
3014void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3015 unsigned ResNo,
3016 SDValue &Lo,
3017 SDValue &Hi) {
3018 SDLoc dl(N);
3019 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3020 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3021
3022 // If the input also splits, handle it directly for a compile time speedup.
3023 // Otherwise split it by hand.
3024 EVT InVT = N->getOperand(Num: 0).getValueType();
3025 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3026 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3027 else
3028 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3029
3030 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3031 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3032
3033 SDNode *HiNode = Hi.getNode();
3034 SDNode *LoNode = Lo.getNode();
3035
3036 // Replace the other vector result not being explicitly split here.
3037 unsigned OtherNo = 1 - ResNo;
3038 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3039 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3040 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3041 Hi: SDValue(HiNode, OtherNo));
3042 } else {
3043 SDValue OtherVal =
3044 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3045 N2: SDValue(HiNode, OtherNo));
3046 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3047 }
3048}
3049
3050void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3051 SDValue &Hi) {
3052 SDLoc dl(N);
3053 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3054 EVT DestVT = N->getValueType(ResNo: 0);
3055 EVT LoVT, HiVT;
3056 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3057
3058 // We can do better than a generic split operation if the extend is doing
3059 // more than just doubling the width of the elements and the following are
3060 // true:
3061 // - The number of vector elements is even,
3062 // - the source type is legal,
3063 // - the type of a split source is illegal,
3064 // - the type of an extended (by doubling element size) source is legal, and
3065 // - the type of that extended source when split is legal.
3066 //
3067 // This won't necessarily completely legalize the operation, but it will
3068 // more effectively move in the right direction and prevent falling down
3069 // to scalarization in many cases due to the input vector being split too
3070 // far.
3071 if (SrcVT.getVectorElementCount().isKnownEven() &&
3072 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3073 LLVMContext &Ctx = *DAG.getContext();
3074 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3075 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3076
3077 EVT SplitLoVT, SplitHiVT;
3078 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3079 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3080 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3081 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3082 N->dump(&DAG); dbgs() << "\n");
3083 // Extend the source vector by one step.
3084 SDValue NewSrc =
3085 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3086 // Get the low and high halves of the new, extended one step, vector.
3087 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3088 // Extend those vector halves the rest of the way.
3089 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3090 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3091 return;
3092 }
3093 }
3094 // Fall back to the generic unary operator splitting otherwise.
3095 SplitVecRes_UnaryOp(N, Lo, Hi);
3096}
3097
3098void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3099 SDValue &Lo, SDValue &Hi) {
3100 // The low and high parts of the original input give four input vectors.
3101 SDValue Inputs[4];
3102 SDLoc DL(N);
3103 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3104 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3105 EVT NewVT = Inputs[0].getValueType();
3106 unsigned NewElts = NewVT.getVectorNumElements();
3107
3108 auto &&IsConstant = [](const SDValue &N) {
3109 APInt SplatValue;
3110 return N.getResNo() == 0 &&
3111 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3112 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3113 };
3114 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3115 SDValue &Input2,
3116 ArrayRef<int> Mask) {
3117 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3118 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3119 "Expected build vector node.");
3120 EVT EltVT = NewVT.getVectorElementType();
3121 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3122 for (unsigned I = 0; I < NewElts; ++I) {
3123 if (Mask[I] == PoisonMaskElem)
3124 continue;
3125 unsigned Idx = Mask[I];
3126 if (Idx >= NewElts)
3127 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3128 else
3129 Ops[I] = Input1.getOperand(i: Idx);
3130 // Make the type of all elements the same as the element type.
3131 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3132 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3133 }
3134 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3135 };
3136
3137 // If Lo or Hi uses elements from at most two of the four input vectors, then
3138 // express it as a vector shuffle of those two inputs. Otherwise extract the
3139 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3140 SmallVector<int> OrigMask(N->getMask());
3141 // Try to pack incoming shuffles/inputs.
3142 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3143 &DL](SmallVectorImpl<int> &Mask) {
3144 // Check if all inputs are shuffles of the same operands or non-shuffles.
3145 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3146 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3147 SDValue Input = Inputs[Idx];
3148 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3149 if (!Shuffle ||
3150 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3151 continue;
3152 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3153 .push_back(Elt: Idx);
3154 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3155 .push_back(Elt: Idx);
3156 }
3157 for (auto &P : ShufflesIdxs) {
3158 if (P.second.size() < 2)
3159 continue;
3160 // Use shuffles operands instead of shuffles themselves.
3161 // 1. Adjust mask.
3162 for (int &Idx : Mask) {
3163 if (Idx == PoisonMaskElem)
3164 continue;
3165 unsigned SrcRegIdx = Idx / NewElts;
3166 if (Inputs[SrcRegIdx].isUndef()) {
3167 Idx = PoisonMaskElem;
3168 continue;
3169 }
3170 auto *Shuffle =
3171 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3172 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3173 continue;
3174 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3175 if (MaskElt == PoisonMaskElem) {
3176 Idx = PoisonMaskElem;
3177 continue;
3178 }
3179 Idx = MaskElt % NewElts +
3180 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3181 ? 0
3182 : 1] *
3183 NewElts;
3184 }
3185 // 2. Update inputs.
3186 Inputs[P.second[0]] = P.first.first;
3187 Inputs[P.second[1]] = P.first.second;
3188 // Clear the pair data.
3189 P.second.clear();
3190 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3191 }
3192 // Check if any concat_vectors can be simplified.
3193 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3194 for (int &Idx : Mask) {
3195 if (Idx == PoisonMaskElem)
3196 continue;
3197 unsigned SrcRegIdx = Idx / NewElts;
3198 if (Inputs[SrcRegIdx].isUndef()) {
3199 Idx = PoisonMaskElem;
3200 continue;
3201 }
3202 TargetLowering::LegalizeTypeAction TypeAction =
3203 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3204 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3205 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3206 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3207 (TypeAction == TargetLowering::TypeLegal ||
3208 TypeAction == TargetLowering::TypeWidenVector))
3209 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3210 }
3211 if (UsedSubVector.count() > 1) {
3212 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3213 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3214 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3215 continue;
3216 if (Pairs.empty() || Pairs.back().size() == 2)
3217 Pairs.emplace_back();
3218 if (UsedSubVector.test(Idx: 2 * I)) {
3219 Pairs.back().emplace_back(Args&: I, Args: 0);
3220 } else {
3221 assert(UsedSubVector.test(2 * I + 1) &&
3222 "Expected to be used one of the subvectors.");
3223 Pairs.back().emplace_back(Args&: I, Args: 1);
3224 }
3225 }
3226 if (!Pairs.empty() && Pairs.front().size() > 1) {
3227 // Adjust mask.
3228 for (int &Idx : Mask) {
3229 if (Idx == PoisonMaskElem)
3230 continue;
3231 unsigned SrcRegIdx = Idx / NewElts;
3232 auto *It = find_if(
3233 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3234 return Idxs.front().first == SrcRegIdx ||
3235 Idxs.back().first == SrcRegIdx;
3236 });
3237 if (It == Pairs.end())
3238 continue;
3239 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3240 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3241 }
3242 // Adjust inputs.
3243 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3244 Inputs[Idxs.front().first] = DAG.getNode(
3245 Opcode: ISD::CONCAT_VECTORS, DL,
3246 VT: Inputs[Idxs.front().first].getValueType(),
3247 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3248 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3249 }
3250 }
3251 }
3252 bool Changed;
3253 do {
3254 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3255 // reused operands.
3256 Changed = false;
3257 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3258 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3259 if (!Shuffle)
3260 continue;
3261 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3262 continue;
3263 int Op = -1;
3264 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3265 !Shuffle->isSplat()) {
3266 Op = 0;
3267 } else if (!Inputs[I].hasOneUse() &&
3268 !Shuffle->getOperand(Num: 1).isUndef()) {
3269 // Find the only used operand, if possible.
3270 for (int &Idx : Mask) {
3271 if (Idx == PoisonMaskElem)
3272 continue;
3273 unsigned SrcRegIdx = Idx / NewElts;
3274 if (SrcRegIdx != I)
3275 continue;
3276 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3277 if (MaskElt == PoisonMaskElem) {
3278 Idx = PoisonMaskElem;
3279 continue;
3280 }
3281 int OpIdx = MaskElt / NewElts;
3282 if (Op == -1) {
3283 Op = OpIdx;
3284 continue;
3285 }
3286 if (Op != OpIdx) {
3287 Op = -1;
3288 break;
3289 }
3290 }
3291 }
3292 if (Op < 0) {
3293 // Try to check if one of the shuffle operands is used already.
3294 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3295 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3296 continue;
3297 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3298 if (It == std::end(arr&: Inputs))
3299 continue;
3300 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3301 // Found that operand is used already.
3302 // 1. Fix the mask for the reused operand.
3303 for (int &Idx : Mask) {
3304 if (Idx == PoisonMaskElem)
3305 continue;
3306 unsigned SrcRegIdx = Idx / NewElts;
3307 if (SrcRegIdx != I)
3308 continue;
3309 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3310 if (MaskElt == PoisonMaskElem) {
3311 Idx = PoisonMaskElem;
3312 continue;
3313 }
3314 int MaskIdx = MaskElt / NewElts;
3315 if (OpIdx == MaskIdx)
3316 Idx = MaskElt % NewElts + FoundOp * NewElts;
3317 }
3318 // 2. Set Op to the unused OpIdx.
3319 Op = (OpIdx + 1) % 2;
3320 break;
3321 }
3322 }
3323 if (Op >= 0) {
3324 Changed = true;
3325 Inputs[I] = Shuffle->getOperand(Num: Op);
3326 // Adjust mask.
3327 for (int &Idx : Mask) {
3328 if (Idx == PoisonMaskElem)
3329 continue;
3330 unsigned SrcRegIdx = Idx / NewElts;
3331 if (SrcRegIdx != I)
3332 continue;
3333 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3334 int OpIdx = MaskElt / NewElts;
3335 if (OpIdx != Op)
3336 continue;
3337 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3338 }
3339 }
3340 }
3341 } while (Changed);
3342 };
3343 TryPeekThroughShufflesInputs(OrigMask);
3344 // Proces unique inputs.
3345 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3346 NewElts](SmallVectorImpl<int> &Mask) {
3347 SetVector<SDValue> UniqueInputs;
3348 SetVector<SDValue> UniqueConstantInputs;
3349 for (const auto &I : Inputs) {
3350 if (IsConstant(I))
3351 UniqueConstantInputs.insert(X: I);
3352 else if (!I.isUndef())
3353 UniqueInputs.insert(X: I);
3354 }
3355 // Adjust mask in case of reused inputs. Also, need to insert constant
3356 // inputs at first, otherwise it affects the final outcome.
3357 if (UniqueInputs.size() != std::size(Inputs)) {
3358 auto &&UniqueVec = UniqueInputs.takeVector();
3359 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3360 unsigned ConstNum = UniqueConstantVec.size();
3361 for (int &Idx : Mask) {
3362 if (Idx == PoisonMaskElem)
3363 continue;
3364 unsigned SrcRegIdx = Idx / NewElts;
3365 if (Inputs[SrcRegIdx].isUndef()) {
3366 Idx = PoisonMaskElem;
3367 continue;
3368 }
3369 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3370 if (It != UniqueConstantVec.end()) {
3371 Idx = (Idx % NewElts) +
3372 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3373 assert(Idx >= 0 && "Expected defined mask idx.");
3374 continue;
3375 }
3376 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3377 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3378 Idx = (Idx % NewElts) +
3379 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3380 assert(Idx >= 0 && "Expected defined mask idx.");
3381 }
3382 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3383 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3384 }
3385 };
3386 MakeUniqueInputs(OrigMask);
3387 SDValue OrigInputs[4];
3388 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3389 for (unsigned High = 0; High < 2; ++High) {
3390 SDValue &Output = High ? Hi : Lo;
3391
3392 // Build a shuffle mask for the output, discovering on the fly which
3393 // input vectors to use as shuffle operands.
3394 unsigned FirstMaskIdx = High * NewElts;
3395 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3396 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3397 assert(!Output && "Expected default initialized initial value.");
3398 TryPeekThroughShufflesInputs(Mask);
3399 MakeUniqueInputs(Mask);
3400 SDValue TmpInputs[4];
3401 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3402 // Track changes in the output registers.
3403 int UsedIdx = -1;
3404 bool SecondIteration = false;
3405 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3406 if (UsedIdx < 0) {
3407 UsedIdx = Idx;
3408 return false;
3409 }
3410 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3411 SecondIteration = true;
3412 return SecondIteration;
3413 };
3414 processShuffleMasks(
3415 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3416 /*NumOfUsedRegs=*/1,
3417 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3418 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3419 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3420 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3421 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3422 else
3423 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3424 N2: DAG.getPOISON(VT: NewVT), Mask);
3425 Inputs[Idx] = Output;
3426 },
3427 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3428 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3429 unsigned Idx2, bool /*Unused*/) {
3430 if (AccumulateResults(Idx1)) {
3431 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3432 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3433 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3434 else
3435 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3436 N2: Inputs[Idx2], Mask);
3437 } else {
3438 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3439 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3440 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3441 else
3442 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3443 N2: TmpInputs[Idx2], Mask);
3444 }
3445 Inputs[Idx1] = Output;
3446 });
3447 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3448 }
3449}
3450
3451void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3452 EVT OVT = N->getValueType(ResNo: 0);
3453 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3454 SDValue Chain = N->getOperand(Num: 0);
3455 SDValue Ptr = N->getOperand(Num: 1);
3456 SDValue SV = N->getOperand(Num: 2);
3457 SDLoc dl(N);
3458
3459 const Align Alignment =
3460 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3461
3462 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3463 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3464 Chain = Hi.getValue(R: 1);
3465
3466 // Modified the chain - switch anything that used the old chain to use
3467 // the new one.
3468 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3469}
3470
3471void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3472 SDValue &Hi) {
3473 EVT DstVTLo, DstVTHi;
3474 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3475 SDLoc dl(N);
3476
3477 SDValue SrcLo, SrcHi;
3478 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3479 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3480 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3481 else
3482 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3483
3484 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3485 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3486}
3487
3488void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3489 SDValue &Hi) {
3490 SDValue InLo, InHi;
3491 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3492 SDLoc DL(N);
3493
3494 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3495 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3496}
3497
3498void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3499 SDValue &Hi) {
3500 SDLoc DL(N);
3501
3502 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3503 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3504}
3505
3506void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3507 SDValue &Hi) {
3508 EVT VT = N->getValueType(ResNo: 0);
3509 SDValue Val = N->getOperand(Num: 0);
3510 SDValue Mask = N->getOperand(Num: 1);
3511 SDValue EVL = N->getOperand(Num: 2);
3512 SDLoc DL(N);
3513
3514 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3515 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3516 // then truncate back.
3517 EVT OrigVT = VT;
3518 if (!VT.getVectorElementType().isByteSized()) {
3519 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3520 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3521 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3522 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3523 }
3524
3525 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3526 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3527
3528 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3529 EC: VT.getVectorElementCount());
3530 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3531 EVT PtrVT = StackPtr.getValueType();
3532 auto &MF = DAG.getMachineFunction();
3533 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3534 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3535
3536 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3537 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3538 BaseAlignment: Alignment);
3539 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3540 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3541 BaseAlignment: Alignment);
3542
3543 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3544 SDValue NumElemMinus1 =
3545 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3546 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3547 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3548 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3549 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3550 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3551
3552 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3553 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3554 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3555 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3556
3557 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3558
3559 // Truncate back if we widened above.
3560 if (OrigVT != VT)
3561 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3562
3563 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3564}
3565
3566void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3567 SDValue &Hi) {
3568 EVT VT = N->getValueType(ResNo: 0);
3569 SDValue V1 = N->getOperand(Num: 0);
3570 SDValue V2 = N->getOperand(Num: 1);
3571 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3572 SDValue Mask = N->getOperand(Num: 3);
3573 SDValue EVL1 = N->getOperand(Num: 4);
3574 SDValue EVL2 = N->getOperand(Num: 5);
3575 SDLoc DL(N);
3576
3577 // Since EVL2 is considered the real VL it gets promoted during
3578 // SelectionDAGBuilder. Promote EVL1 here if needed.
3579 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3580 EVL1 = ZExtPromotedInteger(Op: EVL1);
3581
3582 // The stack splice addresses elements by byte offset/stride, which breaks for
3583 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3584 // stride is 0. Widen to a byte integer, splice, then truncate back.
3585 EVT OrigVT = VT;
3586 if (!VT.getVectorElementType().isByteSized()) {
3587 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3588 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3589 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3590 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3591 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3592 }
3593
3594 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3595
3596 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3597 EC: VT.getVectorElementCount() * 2);
3598 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3599 EVT PtrVT = StackPtr.getValueType();
3600 auto &MF = DAG.getMachineFunction();
3601 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3602 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3603
3604 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3605 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3606 BaseAlignment: Alignment);
3607 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3608 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3609 BaseAlignment: Alignment);
3610
3611 SDValue EltByteSize =
3612 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3613 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3614 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3615 // Clip EVL1Bytes to make sure we stay within the stack object.
3616 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3617 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3618 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3619 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3620
3621 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3622 SDValue StoreV1 =
3623 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3624 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3625
3626 SDValue StoreV2 =
3627 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3628 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3629
3630 SDValue Load;
3631 if (Imm >= 0) {
3632 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3633 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3634 } else {
3635 uint64_t TrailingElts = -Imm;
3636 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3637 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3638
3639 // Make sure TrailingBytes doesn't exceed the size of vec1.
3640 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3641 TrailingBytes =
3642 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3643
3644 // Calculate the start address of the spliced result.
3645 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3646 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3647 }
3648
3649 // Truncate back if we widened above.
3650 if (OrigVT != VT)
3651 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3652
3653 EVT LoVT, HiVT;
3654 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3655 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3656 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3657 Hi =
3658 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3659 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3660}
3661
3662void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3663 SDValue &Hi) {
3664 SDLoc DL(N);
3665 SDValue Acc = N->getOperand(Num: 0);
3666 SDValue Input1 = N->getOperand(Num: 1);
3667 SDValue Input2 = N->getOperand(Num: 2);
3668
3669 SDValue AccLo, AccHi;
3670 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3671 unsigned Opcode = N->getOpcode();
3672
3673 // If the input types don't need splitting, just accumulate into the
3674 // low part of the accumulator.
3675 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3676 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3677 Hi = AccHi;
3678 return;
3679 }
3680
3681 SDValue Input1Lo, Input1Hi;
3682 SDValue Input2Lo, Input2Hi;
3683 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3684 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3685 EVT ResultVT = AccLo.getValueType();
3686
3687 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3688 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3689}
3690
3691void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3692 SDValue &Hi) {
3693 SDLoc DL(N);
3694 SDValue Op0 = N->getOperand(Num: 0);
3695 SDValue Op1 = N->getOperand(Num: 1);
3696 EVT OpVT = Op0.getValueType();
3697
3698 EVT LoVT, HiVT;
3699 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3700
3701 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3702 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3703 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3704 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3705}
3706
3707void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3708 SDValue &Hi) {
3709 SDValue SourceLo, SourceHi;
3710 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SourceLo, Hi&: SourceHi);
3711 SDValue MaskLo, MaskHi;
3712 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
3713 SDLoc DL(N);
3714
3715 Lo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskLo.getValueType(), N1: SourceLo,
3716 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
3717 Hi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskHi.getValueType(), N1: SourceHi,
3718 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
3719}
3720
3721void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3722 unsigned Factor = N->getNumOperands();
3723
3724 SmallVector<SDValue, 8> Ops(Factor * 2);
3725 for (unsigned i = 0; i != Factor; ++i) {
3726 SDValue OpLo, OpHi;
3727 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3728 Ops[i * 2] = OpLo;
3729 Ops[i * 2 + 1] = OpHi;
3730 }
3731
3732 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3733
3734 SDLoc DL(N);
3735 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3736 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3737 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3738 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3739
3740 for (unsigned i = 0; i != Factor; ++i)
3741 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3742}
3743
3744void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3745 unsigned Factor = N->getNumOperands();
3746
3747 SmallVector<SDValue, 8> Ops(Factor * 2);
3748 for (unsigned i = 0; i != Factor; ++i) {
3749 SDValue OpLo, OpHi;
3750 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3751 Ops[i] = OpLo;
3752 Ops[i + Factor] = OpHi;
3753 }
3754
3755 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3756
3757 SDLoc DL(N);
3758 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3759 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3760 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3761 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3762
3763 for (unsigned i = 0; i != Factor; ++i) {
3764 unsigned IdxLo = 2 * i;
3765 unsigned IdxHi = 2 * i + 1;
3766 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3767 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3768 }
3769}
3770
3771//===----------------------------------------------------------------------===//
3772// Operand Vector Splitting
3773//===----------------------------------------------------------------------===//
3774
3775/// This method is called when the specified operand of the specified node is
3776/// found to need vector splitting. At this point, all of the result types of
3777/// the node are known to be legal, but other operands of the node may need
3778/// legalization as well as the specified one.
3779bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3780 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3781 SDValue Res = SDValue();
3782
3783 // See if the target wants to custom split this node.
3784 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3785 return false;
3786
3787 switch (N->getOpcode()) {
3788 default:
3789#ifndef NDEBUG
3790 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3791 N->dump(&DAG);
3792 dbgs() << "\n";
3793#endif
3794 report_fatal_error(reason: "Do not know how to split this operator's "
3795 "operand!\n");
3796
3797 case ISD::STRICT_FSETCC:
3798 case ISD::STRICT_FSETCCS:
3799 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3800 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3801 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3802 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3803 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3804 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3805 case ISD::VECTOR_FIND_LAST_ACTIVE:
3806 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3807 break;
3808 case ISD::TRUNCATE:
3809 Res = SplitVecOp_TruncateHelper(N);
3810 break;
3811 case ISD::STRICT_FP_ROUND:
3812 case ISD::FP_ROUND:
3813 case ISD::CONVERT_FROM_ARBITRARY_FP:
3814 case ISD::CONVERT_TO_ARBITRARY_FP:
3815 Res = SplitVecOp_FP_ROUND(N);
3816 break;
3817 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3818 case ISD::STORE:
3819 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3820 break;
3821 case ISD::ATOMIC_STORE:
3822 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3823 break;
3824 case ISD::VP_STORE:
3825 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3826 break;
3827 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3828 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3829 break;
3830 case ISD::MSTORE:
3831 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3832 break;
3833 case ISD::MSCATTER:
3834 case ISD::VP_SCATTER:
3835 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3836 break;
3837 case ISD::MGATHER:
3838 case ISD::VP_GATHER:
3839 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3840 break;
3841 case ISD::VSELECT:
3842 Res = SplitVecOp_VSELECT(N, OpNo);
3843 break;
3844 case ISD::MASKED_UDIV:
3845 case ISD::MASKED_SDIV:
3846 case ISD::MASKED_UREM:
3847 case ISD::MASKED_SREM:
3848 Res = SplitVecOp_MaskedBinOp(N, OpNo);
3849 break;
3850 case ISD::VECTOR_COMPRESS:
3851 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3852 break;
3853 case ISD::STRICT_SINT_TO_FP:
3854 case ISD::STRICT_UINT_TO_FP:
3855 case ISD::SINT_TO_FP:
3856 case ISD::UINT_TO_FP:
3857 if (N->getValueType(ResNo: 0).bitsLT(
3858 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3859 Res = SplitVecOp_TruncateHelper(N);
3860 else
3861 Res = SplitVecOp_UnaryOp(N);
3862 break;
3863 case ISD::FP_TO_SINT_SAT:
3864 case ISD::FP_TO_UINT_SAT:
3865 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3866 break;
3867 case ISD::FP_TO_SINT:
3868 case ISD::FP_TO_UINT:
3869 case ISD::STRICT_FP_TO_SINT:
3870 case ISD::STRICT_FP_TO_UINT:
3871 case ISD::STRICT_FP_EXTEND:
3872 case ISD::FP_EXTEND:
3873 case ISD::SIGN_EXTEND:
3874 case ISD::ZERO_EXTEND:
3875 case ISD::ANY_EXTEND:
3876 case ISD::FTRUNC:
3877 case ISD::LROUND:
3878 case ISD::LLROUND:
3879 case ISD::LRINT:
3880 case ISD::LLRINT:
3881 Res = SplitVecOp_UnaryOp(N);
3882 break;
3883 case ISD::FLDEXP:
3884 Res = SplitVecOp_FPOpDifferentTypes(N);
3885 break;
3886
3887 case ISD::SCMP:
3888 case ISD::UCMP:
3889 Res = SplitVecOp_CMP(N);
3890 break;
3891
3892 case ISD::FAKE_USE:
3893 Res = SplitVecOp_FAKE_USE(N);
3894 break;
3895 case ISD::ANY_EXTEND_VECTOR_INREG:
3896 case ISD::SIGN_EXTEND_VECTOR_INREG:
3897 case ISD::ZERO_EXTEND_VECTOR_INREG:
3898 Res = SplitVecOp_ExtVecInRegOp(N);
3899 break;
3900
3901 case ISD::VECREDUCE_FADD:
3902 case ISD::VECREDUCE_FMUL:
3903 case ISD::VECREDUCE_ADD:
3904 case ISD::VECREDUCE_MUL:
3905 case ISD::VECREDUCE_AND:
3906 case ISD::VECREDUCE_OR:
3907 case ISD::VECREDUCE_XOR:
3908 case ISD::VECREDUCE_SMAX:
3909 case ISD::VECREDUCE_SMIN:
3910 case ISD::VECREDUCE_UMAX:
3911 case ISD::VECREDUCE_UMIN:
3912 case ISD::VECREDUCE_FMAX:
3913 case ISD::VECREDUCE_FMIN:
3914 case ISD::VECREDUCE_FMAXIMUM:
3915 case ISD::VECREDUCE_FMINIMUM:
3916 case ISD::VECREDUCE_FMAXIMUMNUM:
3917 case ISD::VECREDUCE_FMINIMUMNUM:
3918 Res = SplitVecOp_VECREDUCE(N, OpNo);
3919 break;
3920 case ISD::VECREDUCE_SEQ_FADD:
3921 case ISD::VECREDUCE_SEQ_FMUL:
3922 Res = SplitVecOp_VECREDUCE_SEQ(N);
3923 break;
3924 case ISD::VP_REDUCE_FADD:
3925 case ISD::VP_REDUCE_SEQ_FADD:
3926 case ISD::VP_REDUCE_FMUL:
3927 case ISD::VP_REDUCE_SEQ_FMUL:
3928 case ISD::VP_REDUCE_ADD:
3929 case ISD::VP_REDUCE_MUL:
3930 case ISD::VP_REDUCE_AND:
3931 case ISD::VP_REDUCE_OR:
3932 case ISD::VP_REDUCE_XOR:
3933 case ISD::VP_REDUCE_SMAX:
3934 case ISD::VP_REDUCE_SMIN:
3935 case ISD::VP_REDUCE_UMAX:
3936 case ISD::VP_REDUCE_UMIN:
3937 case ISD::VP_REDUCE_FMAX:
3938 case ISD::VP_REDUCE_FMIN:
3939 case ISD::VP_REDUCE_FMAXIMUM:
3940 case ISD::VP_REDUCE_FMINIMUM:
3941 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3942 break;
3943 case ISD::CTTZ_ELTS:
3944 case ISD::CTTZ_ELTS_ZERO_POISON:
3945 Res = SplitVecOp_CttzElts(N);
3946 break;
3947 case ISD::VP_CTTZ_ELTS:
3948 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
3949 Res = SplitVecOp_VP_CttzElements(N);
3950 break;
3951 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3952 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3953 break;
3954 case ISD::PARTIAL_REDUCE_UMLA:
3955 case ISD::PARTIAL_REDUCE_SMLA:
3956 case ISD::PARTIAL_REDUCE_SUMLA:
3957 case ISD::PARTIAL_REDUCE_FMLA:
3958 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3959 break;
3960 case ISD::VECTOR_MATCH:
3961 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
3962 break;
3963 }
3964
3965 // If the result is null, the sub-method took care of registering results etc.
3966 if (!Res.getNode()) return false;
3967
3968 // If the result is N, the sub-method updated N in place. Tell the legalizer
3969 // core about this.
3970 if (Res.getNode() == N)
3971 return true;
3972
3973 if (N->isStrictFPOpcode())
3974 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3975 "Invalid operand expansion");
3976 else
3977 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3978 "Invalid operand expansion");
3979
3980 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3981 return false;
3982}
3983
3984SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3985 SDLoc DL(N);
3986
3987 SDValue LoMask, HiMask;
3988 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
3989
3990 EVT VT = N->getValueType(ResNo: 0);
3991 EVT SplitVT = LoMask.getValueType();
3992 ElementCount SplitEC = SplitVT.getVectorElementCount();
3993
3994 // Find the last active in both the low and the high masks.
3995 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
3996 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
3997
3998 // Check if any lane is active in the high mask.
3999 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4000 // sentinel value for "none active".
4001 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4002 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4003 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4004
4005 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4006 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4007 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4008 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4009 N3: LoFind);
4010}
4011
4012SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4013 // The only possibility for an illegal operand is the mask, since result type
4014 // legalization would have handled this node already otherwise.
4015 assert(OpNo == 0 && "Illegal operand must be mask");
4016
4017 SDValue Mask = N->getOperand(Num: 0);
4018 SDValue Src0 = N->getOperand(Num: 1);
4019 SDValue Src1 = N->getOperand(Num: 2);
4020 EVT Src0VT = Src0.getValueType();
4021 SDLoc DL(N);
4022 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4023
4024 SDValue Lo, Hi;
4025 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4026 assert(Lo.getValueType() == Hi.getValueType() &&
4027 "Lo and Hi have differing types");
4028
4029 EVT LoOpVT, HiOpVT;
4030 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4031 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4032
4033 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4034 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4035 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4036 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4037
4038 SDValue LoSelect =
4039 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4040 SDValue HiSelect =
4041 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4042
4043 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4044}
4045
4046SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4047 assert(OpNo == 2 && "Illegal operand must be mask");
4048
4049 SDLoc DL(N);
4050 auto [LHSLo, LHSHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
4051 auto [RHSLo, RHSHi] = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4052 SDValue MaskLo, MaskHi;
4053 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
4054
4055 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo,
4056 N2: RHSLo, N3: MaskLo, Flags: N->getFlags());
4057 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi,
4058 N2: RHSHi, N3: MaskHi, Flags: N->getFlags());
4059 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4060}
4061
4062SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4063 // The only possibility for an illegal operand is the mask, since result type
4064 // legalization would have handled this node already otherwise.
4065 assert(OpNo == 1 && "Illegal operand must be mask");
4066
4067 // To split the mask, we need to split the result type too, so we can just
4068 // reuse that logic here.
4069 SDValue Lo, Hi;
4070 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4071
4072 EVT VecVT = N->getValueType(ResNo: 0);
4073 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4074}
4075
4076SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4077 EVT ResVT = N->getValueType(ResNo: 0);
4078 SDValue Lo, Hi;
4079 SDLoc dl(N);
4080
4081 SDValue VecOp = N->getOperand(Num: OpNo);
4082 EVT VecVT = VecOp.getValueType();
4083 assert(VecVT.isVector() && "Can only split reduce vector operand");
4084 GetSplitVector(Op: VecOp, Lo, Hi);
4085 EVT LoOpVT, HiOpVT;
4086 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4087
4088 // Use the appropriate scalar instruction on the split subvectors before
4089 // reducing the now partially reduced smaller vector.
4090 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4091 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4092 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4093}
4094
4095SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4096 EVT ResVT = N->getValueType(ResNo: 0);
4097 SDValue Lo, Hi;
4098 SDLoc dl(N);
4099
4100 SDValue AccOp = N->getOperand(Num: 0);
4101 SDValue VecOp = N->getOperand(Num: 1);
4102 SDNodeFlags Flags = N->getFlags();
4103
4104 EVT VecVT = VecOp.getValueType();
4105 assert(VecVT.isVector() && "Can only split reduce vector operand");
4106 GetSplitVector(Op: VecOp, Lo, Hi);
4107 EVT LoOpVT, HiOpVT;
4108 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4109
4110 // Reduce low half.
4111 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4112
4113 // Reduce high half, using low half result as initial value.
4114 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4115}
4116
4117SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4118 assert(N->isVPOpcode() && "Expected VP opcode");
4119 assert(OpNo == 1 && "Can only split reduce vector operand");
4120
4121 unsigned Opc = N->getOpcode();
4122 EVT ResVT = N->getValueType(ResNo: 0);
4123 SDValue Lo, Hi;
4124 SDLoc dl(N);
4125
4126 SDValue VecOp = N->getOperand(Num: OpNo);
4127 EVT VecVT = VecOp.getValueType();
4128 assert(VecVT.isVector() && "Can only split reduce vector operand");
4129 GetSplitVector(Op: VecOp, Lo, Hi);
4130
4131 SDValue MaskLo, MaskHi;
4132 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4133
4134 SDValue EVLLo, EVLHi;
4135 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4136
4137 const SDNodeFlags Flags = N->getFlags();
4138
4139 SDValue ResLo =
4140 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4141 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4142}
4143
4144SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4145 // The result has a legal vector type, but the input needs splitting.
4146 EVT ResVT = N->getValueType(ResNo: 0);
4147 SDValue Lo, Hi;
4148 SDLoc dl(N);
4149 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4150 EVT InVT = Lo.getValueType();
4151
4152 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4153 EC: InVT.getVectorElementCount());
4154
4155 if (N->isStrictFPOpcode()) {
4156 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4157 Ops: {N->getOperand(Num: 0), Lo});
4158 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4159 Ops: {N->getOperand(Num: 0), Hi});
4160
4161 // Build a factor node to remember that this operation is independent
4162 // of the other one.
4163 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4164 N2: Hi.getValue(R: 1));
4165
4166 // Legalize the chain result - switch anything that used the old chain to
4167 // use the new one.
4168 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4169 } else {
4170 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4171 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4172 }
4173
4174 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4175}
4176
4177// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4178SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4179 SDValue Lo, Hi;
4180 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4181 SDValue Chain =
4182 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4183 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4184}
4185
4186SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4187 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4188 // end up being split all the way down to individual components. Convert the
4189 // split pieces into integers and reassemble.
4190 EVT ResVT = N->getValueType(ResNo: 0);
4191 SDValue Lo, Hi;
4192 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4193 SDLoc dl(N);
4194
4195 if (ResVT.isScalableVector()) {
4196 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4197 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4198 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4199 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4200 }
4201
4202 Lo = BitConvertToInteger(Op: Lo);
4203 Hi = BitConvertToInteger(Op: Hi);
4204
4205 if (DAG.getDataLayout().isBigEndian())
4206 std::swap(a&: Lo, b&: Hi);
4207
4208 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4209}
4210
4211SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4212 unsigned OpNo) {
4213 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4214 // We know that the result type is legal.
4215 EVT ResVT = N->getValueType(ResNo: 0);
4216
4217 SDValue Vec = N->getOperand(Num: 0);
4218 SDValue SubVec = N->getOperand(Num: 1);
4219 SDValue Idx = N->getOperand(Num: 2);
4220 SDLoc dl(N);
4221
4222 SDValue Lo, Hi;
4223 GetSplitVector(Op: SubVec, Lo, Hi);
4224
4225 uint64_t IdxVal = Idx->getAsZExtVal();
4226 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4227
4228 SDValue FirstInsertion =
4229 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4230 SDValue SecondInsertion =
4231 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4232 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4233
4234 return SecondInsertion;
4235}
4236
4237SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4238 // We know that the extracted result type is legal.
4239 EVT SubVT = N->getValueType(ResNo: 0);
4240 SDValue Idx = N->getOperand(Num: 1);
4241 SDLoc dl(N);
4242 SDValue Lo, Hi;
4243
4244 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4245
4246 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4247 // Note: For scalable vectors, the index is scaled by vscale.
4248 ElementCount IdxVal =
4249 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4250 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4251
4252 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4253 ElementCount NumResultElts = SubVT.getVectorElementCount();
4254
4255 // If the extracted elements are all in the low half, do a simple extract.
4256 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4257 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4258
4259 unsigned LoEltsMin = LoElts.getKnownMinValue();
4260 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4261 SrcVT.isFixedLengthVector()) {
4262 // Extracted subvector crosses vector split, so we need to blend the two
4263 // halves.
4264 // TODO: May be able to emit partial extract_subvector.
4265 SmallVector<SDValue, 8> Elts;
4266 Elts.reserve(N: NumResultElts.getFixedValue());
4267
4268 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4269 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4270 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4271 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4272 /*Count=*/LoEltsMin - IdxValMin);
4273 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4274 /*Count=*/SubVT.getVectorNumElements() -
4275 Elts.size());
4276 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4277 }
4278
4279 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4280 ElementCount ExtractIdx = IdxVal - LoElts;
4281 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4282 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4283 Idx: ExtractIdx.getKnownMinValue());
4284
4285 EVT HiVT = Hi.getValueType();
4286 assert(HiVT.isFixedLengthVector() &&
4287 "Only fixed-vector extracts are supported in this case");
4288
4289 // We cannot create an extract_subvector that isn't a multiple of the
4290 // result size, which may go out of bounds for the last elements. Shuffle
4291 // the desired elements down to 0 and do a simple 0 extract.
4292 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4293 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4294 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4295
4296 SDValue Shuffle =
4297 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4298 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4299 }
4300
4301 // After this point the DAG node only permits extracting fixed-width
4302 // subvectors from scalable vectors.
4303 assert(SubVT.isFixedLengthVector() &&
4304 "Extracting scalable subvector from fixed-width unsupported");
4305
4306 // If the element type is i1 and we're not promoting the result, then we may
4307 // end up loading the wrong data since the bits are packed tightly into
4308 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4309 // type at index 4, then we will load a byte starting at index 0.
4310 if (SubVT.getScalarType() == MVT::i1)
4311 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4312 "subvector from a scalable predicate vector");
4313
4314 // Spill the vector to the stack. We should use the alignment for
4315 // the smallest part.
4316 SDValue Vec = N->getOperand(Num: 0);
4317 EVT VecVT = Vec.getValueType();
4318 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4319 SDValue StackPtr =
4320 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4321 auto &MF = DAG.getMachineFunction();
4322 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4323 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4324
4325 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4326 Alignment: SmallestAlign);
4327
4328 // Extract the subvector by loading the correct part.
4329 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4330
4331 return DAG.getLoad(
4332 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4333 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4334}
4335
4336SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4337 SDValue Vec = N->getOperand(Num: 0);
4338 SDValue Idx = N->getOperand(Num: 1);
4339 EVT VecVT = Vec.getValueType();
4340
4341 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4342 uint64_t IdxVal = Index->getZExtValue();
4343
4344 SDValue Lo, Hi;
4345 GetSplitVector(Op: Vec, Lo, Hi);
4346
4347 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4348
4349 if (IdxVal < LoElts)
4350 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4351 else if (!Vec.getValueType().isScalableVector())
4352 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4353 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4354 VT: Idx.getValueType())), 0);
4355 }
4356
4357 // See if the target wants to custom expand this node.
4358 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4359 return SDValue();
4360
4361 // Make the vector elements byte-addressable if they aren't already.
4362 SDLoc dl(N);
4363 EVT EltVT = VecVT.getVectorElementType();
4364 if (!EltVT.isByteSized()) {
4365 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4366 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4367 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4368 SDValue NewExtract =
4369 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4370 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4371 }
4372
4373 // Store the vector to the stack.
4374 // In cases where the vector is illegal it will be broken down into parts
4375 // and stored in parts - we should use the alignment for the smallest part.
4376 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4377 SDValue StackPtr =
4378 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4379 auto &MF = DAG.getMachineFunction();
4380 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4381 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4382 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4383 Alignment: SmallestAlign);
4384
4385 // Load back the required element.
4386 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4387
4388 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4389 // type, leaving the high bits undefined. But it can't truncate.
4390 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4391
4392 return DAG.getExtLoad(
4393 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4394 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4395 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4396}
4397
4398SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4399 SDValue Lo, Hi;
4400
4401 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4402 // splitting the result has the same effect as splitting the input operand.
4403 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4404
4405 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4406}
4407
4408SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4409 (void)OpNo;
4410 SDValue Lo, Hi;
4411 SplitVecRes_Gather(N, Lo, Hi);
4412
4413 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4414 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4415 return SDValue();
4416}
4417
4418SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4419 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4420 SDValue Ch = N->getChain();
4421 SDValue Ptr = N->getBasePtr();
4422 SDValue Offset = N->getOffset();
4423 assert(Offset.isUndef() && "Unexpected VP store offset");
4424 SDValue Mask = N->getMask();
4425 SDValue EVL = N->getVectorLength();
4426 SDValue Data = N->getValue();
4427 Align Alignment = N->getBaseAlign();
4428 SDLoc DL(N);
4429
4430 SDValue DataLo, DataHi;
4431 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4432 // Split Data operand
4433 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4434 else
4435 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4436
4437 // Split Mask operand
4438 SDValue MaskLo, MaskHi;
4439 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4440 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4441 } else {
4442 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4443 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4444 else
4445 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4446 }
4447
4448 EVT MemoryVT = N->getMemoryVT();
4449 EVT LoMemVT, HiMemVT;
4450 bool HiIsEmpty = false;
4451 std::tie(args&: LoMemVT, args&: HiMemVT) =
4452 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4453
4454 // Split EVL
4455 SDValue EVLLo, EVLHi;
4456 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4457
4458 SDValue Lo, Hi;
4459 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4460 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4461 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4462 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4463
4464 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4465 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4466 IsCompressing: N->isCompressingStore());
4467
4468 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4469 if (HiIsEmpty)
4470 return Lo;
4471
4472 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4473 IsCompressedMemory: N->isCompressingStore());
4474
4475 MachinePointerInfo MPI;
4476 if (LoMemVT.isScalableVector()) {
4477 Alignment = commonAlignment(A: Alignment,
4478 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4479 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4480 } else
4481 MPI = N->getPointerInfo().getWithOffset(
4482 O: LoMemVT.getStoreSize().getFixedValue());
4483
4484 MMO = DAG.getMachineFunction().getMachineMemOperand(
4485 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4486 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4487
4488 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4489 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4490 IsCompressing: N->isCompressingStore());
4491
4492 // Build a factor node to remember that this store is independent of the
4493 // other one.
4494 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4495}
4496
4497SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4498 unsigned OpNo) {
4499 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4500 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4501
4502 SDLoc DL(N);
4503
4504 SDValue Data = N->getValue();
4505 SDValue LoData, HiData;
4506 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4507 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4508 else
4509 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4510
4511 EVT LoMemVT, HiMemVT;
4512 bool HiIsEmpty = false;
4513 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4514 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4515
4516 SDValue Mask = N->getMask();
4517 SDValue LoMask, HiMask;
4518 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4519 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4520 else if (getTypeAction(VT: Mask.getValueType()) ==
4521 TargetLowering::TypeSplitVector)
4522 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4523 else
4524 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4525
4526 SDValue LoEVL, HiEVL;
4527 std::tie(args&: LoEVL, args&: HiEVL) =
4528 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4529
4530 // Generate the low vp_strided_store
4531 SDValue Lo = DAG.getStridedStoreVP(
4532 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4533 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4534 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4535
4536 // If the high vp_strided_store has zero storage size, only the low
4537 // vp_strided_store is needed.
4538 if (HiIsEmpty)
4539 return Lo;
4540
4541 // Generate the high vp_strided_store.
4542 // To calculate the high base address, we need to sum to the low base
4543 // address stride number of bytes for each element already stored by low,
4544 // that is: Ptr = Ptr + (LoEVL * Stride)
4545 EVT PtrVT = N->getBasePtr().getValueType();
4546 SDValue Increment =
4547 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4548 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4549 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4550
4551 Align Alignment = N->getBaseAlign();
4552 if (LoMemVT.isScalableVector())
4553 Alignment = commonAlignment(A: Alignment,
4554 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4555
4556 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4557 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4558 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4559 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4560
4561 SDValue Hi = DAG.getStridedStoreVP(
4562 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4563 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4564 IsCompressing: N->isCompressingStore());
4565
4566 // Build a factor node to remember that this store is independent of the
4567 // other one.
4568 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4569}
4570
4571SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4572 unsigned OpNo) {
4573 assert(N->isUnindexed() && "Indexed masked store of vector?");
4574 SDValue Ch = N->getChain();
4575 SDValue Ptr = N->getBasePtr();
4576 SDValue Offset = N->getOffset();
4577 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4578 SDValue Mask = N->getMask();
4579 SDValue Data = N->getValue();
4580 Align Alignment = N->getBaseAlign();
4581 SDLoc DL(N);
4582
4583 SDValue DataLo, DataHi;
4584 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4585 // Split Data operand
4586 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4587 else
4588 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4589
4590 // Split Mask operand
4591 SDValue MaskLo, MaskHi;
4592 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4593 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4594 } else {
4595 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4596 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4597 else
4598 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4599 }
4600
4601 EVT MemoryVT = N->getMemoryVT();
4602 EVT LoMemVT, HiMemVT;
4603 bool HiIsEmpty = false;
4604 std::tie(args&: LoMemVT, args&: HiMemVT) =
4605 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4606
4607 SDValue Lo, Hi, Res;
4608 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4609 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4610 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4611 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4612
4613 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4614 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4615 IsCompressing: N->isCompressingStore());
4616
4617 if (HiIsEmpty) {
4618 // The hi masked store has zero storage size.
4619 // Only the lo masked store is needed.
4620 Res = Lo;
4621 } else {
4622
4623 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4624 IsCompressedMemory: N->isCompressingStore());
4625
4626 MachinePointerInfo MPI;
4627 if (LoMemVT.isScalableVector()) {
4628 Alignment = commonAlignment(
4629 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4630 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4631 } else
4632 MPI = N->getPointerInfo().getWithOffset(
4633 O: LoMemVT.getStoreSize().getFixedValue());
4634
4635 MMO = DAG.getMachineFunction().getMachineMemOperand(
4636 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4637 BaseAlignment: Alignment,
4638 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4639
4640 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4641 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4642 IsCompressing: N->isCompressingStore());
4643
4644 // Build a factor node to remember that this store is independent of the
4645 // other one.
4646 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4647 }
4648
4649 return Res;
4650}
4651
4652SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4653 SDValue Ch = N->getChain();
4654 SDValue Ptr = N->getBasePtr();
4655 EVT MemoryVT = N->getMemoryVT();
4656 Align Alignment = N->getBaseAlign();
4657 SDLoc DL(N);
4658 struct Operands {
4659 SDValue Mask;
4660 SDValue Index;
4661 SDValue Scale;
4662 SDValue Data;
4663 } Ops = [&]() -> Operands {
4664 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4665 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4666 .Data: MSC->getValue()};
4667 }
4668 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4669 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4670 .Data: VPSC->getValue()};
4671 }();
4672 // Split all operands
4673
4674 EVT LoMemVT, HiMemVT;
4675 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4676
4677 SDValue DataLo, DataHi;
4678 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4679 // Split Data operand
4680 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4681 else
4682 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4683
4684 // Split Mask operand
4685 SDValue MaskLo, MaskHi;
4686 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4687 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4688 } else {
4689 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4690 }
4691
4692 SDValue IndexHi, IndexLo;
4693 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4694 TargetLowering::TypeSplitVector)
4695 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4696 else
4697 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4698
4699 SDValue Lo;
4700 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4701 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4702 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4703 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4704
4705 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4706 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4707 Lo =
4708 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4709 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4710
4711 // The order of the Scatter operation after split is well defined. The "Hi"
4712 // part comes after the "Lo". So these two operations should be chained one
4713 // after another.
4714 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4715 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4716 MMO, IndexType: MSC->getIndexType(),
4717 IsTruncating: MSC->isTruncatingStore());
4718 }
4719 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4720 SDValue EVLLo, EVLHi;
4721 std::tie(args&: EVLLo, args&: EVLHi) =
4722 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4723
4724 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4725 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4726 IndexType: VPSC->getIndexType());
4727
4728 // The order of the Scatter operation after split is well defined. The "Hi"
4729 // part comes after the "Lo". So these two operations should be chained one
4730 // after another.
4731 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4732 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4733 IndexType: VPSC->getIndexType());
4734}
4735
4736SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4737 assert(N->isUnindexed() && "Indexed store of vector?");
4738 assert(OpNo == 1 && "Can only split the stored value");
4739 SDLoc DL(N);
4740
4741 bool isTruncating = N->isTruncatingStore();
4742 SDValue Ch = N->getChain();
4743 SDValue Ptr = N->getBasePtr();
4744 EVT MemoryVT = N->getMemoryVT();
4745 Align Alignment = N->getBaseAlign();
4746 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4747 AAMDNodes AAInfo = N->getAAInfo();
4748 SDValue Lo, Hi;
4749 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4750
4751 EVT LoMemVT, HiMemVT;
4752 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4753
4754 // Scalarize if the split halves are not byte-sized.
4755 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4756 return TLI.scalarizeVectorStore(ST: N, DAG);
4757
4758 if (isTruncating)
4759 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4760 Alignment, MMOFlags, Metadata: AAInfo);
4761 else
4762 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4763 Metadata: AAInfo);
4764
4765 MachinePointerInfo MPI;
4766 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4767
4768 if (isTruncating)
4769 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4770 SVT: HiMemVT, Alignment, MMOFlags, Metadata: AAInfo);
4771 else
4772 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, Metadata: AAInfo);
4773
4774 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4775}
4776
4777SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4778 SDLoc DL(N);
4779 LLVMContext &Ctx = *DAG.getContext();
4780 SDValue StVal = N->getVal();
4781 EVT VT = StVal.getValueType();
4782 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4783
4784 // The store needs a single value spanning the full memory width. If the
4785 // value can be held in a legal vector register, keep it there and extract
4786 // the low integer element of the memory width. This lets the store be issued
4787 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4788 // bitcasting the split vector straight to a scalar integer, which would
4789 // reassemble the value element by element in GPRs.
4790 //
4791 // Reinterpret the value as a same-shaped integer vector first: an FP element
4792 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4793 // integer-of-element-size form does. Ask the target which legal vector type
4794 // it widens to.
4795 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4796 EVT IntEltVT = IntVecVT.getVectorElementType();
4797 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4798 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4799 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4800 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4801 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4802 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4803 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4804 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4805 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4806 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4807 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4808 Val: N->getBasePtr(), MMO: N->getMemOperand());
4809 }
4810
4811 // Otherwise issue a single atomic store of an integer that spans the full
4812 // memory width. Bitcasting the (illegal) vector value to that integer lets
4813 // the type legalizer further legalize the BITCAST input as needed, while the
4814 // ATOMIC_STORE itself uses only the legal integer type.
4815 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4816 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4817 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4818 Val: N->getBasePtr(), MMO: N->getMemOperand());
4819}
4820
4821SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4822 SDLoc DL(N);
4823
4824 // The input operands all must have the same type, and we know the result
4825 // type is valid. Convert this to a buildvector which extracts all the
4826 // input elements.
4827 // TODO: If the input elements are power-two vectors, we could convert this to
4828 // a new CONCAT_VECTORS node with elements that are half-wide.
4829 SmallVector<SDValue, 32> Elts;
4830 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4831 for (const SDValue &Op : N->op_values()) {
4832 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4833 i != e; ++i) {
4834 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4835 }
4836 }
4837
4838 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4839}
4840
4841SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4842 // The result type is legal, but the input type is illegal. If splitting
4843 // ends up with the result type of each half still being legal, just
4844 // do that. If, however, that would result in an illegal result type,
4845 // we can try to get more clever with power-two vectors. Specifically,
4846 // split the input type, but also widen the result element size, then
4847 // concatenate the halves and truncate again. For example, consider a target
4848 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4849 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4850 // %inlo = v4i32 extract_subvector %in, 0
4851 // %inhi = v4i32 extract_subvector %in, 4
4852 // %lo16 = v4i16 trunc v4i32 %inlo
4853 // %hi16 = v4i16 trunc v4i32 %inhi
4854 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4855 // %res = v8i8 trunc v8i16 %in16
4856 //
4857 // Without this transform, the original truncate would end up being
4858 // scalarized, which is pretty much always a last resort.
4859 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4860 SDValue InVec = N->getOperand(Num: OpNo);
4861 EVT InVT = InVec->getValueType(ResNo: 0);
4862 EVT OutVT = N->getValueType(ResNo: 0);
4863 ElementCount NumElements = OutVT.getVectorElementCount();
4864 bool IsFloat = OutVT.isFloatingPoint();
4865
4866 unsigned InElementSize = InVT.getScalarSizeInBits();
4867 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4868
4869 // Determine the split output VT. If its legal we can just split dirctly.
4870 EVT LoOutVT, HiOutVT;
4871 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4872 assert(LoOutVT == HiOutVT && "Unequal split?");
4873
4874 // If the input elements are only 1/2 the width of the result elements,
4875 // just use the normal splitting. Our trick only work if there's room
4876 // to split more than once.
4877 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4878 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4879 return SplitVecOp_UnaryOp(N);
4880 SDLoc DL(N);
4881
4882 // Don't touch if this will be scalarized.
4883 EVT FinalVT = InVT;
4884 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4885 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4886
4887 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4888 return SplitVecOp_UnaryOp(N);
4889
4890 // Get the split input vector.
4891 SDValue InLoVec, InHiVec;
4892 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4893
4894 // Truncate them to 1/2 the element size.
4895 //
4896 // This assumes the number of elements is a power of two; any vector that
4897 // isn't should be widened, not split.
4898 EVT HalfElementVT = IsFloat ?
4899 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4900 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4901 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4902 EC: NumElements.divideCoefficientBy(RHS: 2));
4903
4904 SDValue HalfLo;
4905 SDValue HalfHi;
4906 SDValue Chain;
4907 if (N->isStrictFPOpcode()) {
4908 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4909 Ops: {N->getOperand(Num: 0), InLoVec});
4910 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4911 Ops: {N->getOperand(Num: 0), InHiVec});
4912 // Legalize the chain result - switch anything that used the old chain to
4913 // use the new one.
4914 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4915 N2: HalfHi.getValue(R: 1));
4916 } else {
4917 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4918 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4919 }
4920
4921 // Concatenate them to get the full intermediate truncation result.
4922 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4923 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4924 N2: HalfHi);
4925 // Now finish up by truncating all the way down to the original result
4926 // type. This should normally be something that ends up being legal directly,
4927 // but in theory if a target has very wide vectors and an annoyingly
4928 // restricted set of legal types, this split can chain to build things up.
4929
4930 if (N->isStrictFPOpcode()) {
4931 SDValue Res = DAG.getNode(
4932 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4933 Ops: {Chain, InterVec,
4934 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4935 // Relink the chain
4936 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4937 return Res;
4938 }
4939
4940 return IsFloat
4941 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4942 N2: DAG.getTargetConstant(
4943 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4944 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4945}
4946
4947SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4948 unsigned Opc = N->getOpcode();
4949 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4950 assert(N->getValueType(0).isVector() &&
4951 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4952 "Operand types must be vectors");
4953 // The result has a legal vector type, but the input needs splitting.
4954 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4955 SDLoc DL(N);
4956 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4957 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4958
4959 EVT VT = N->getValueType(ResNo: 0);
4960 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
4961
4962 if (Opc == ISD::SETCC) {
4963 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4964 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4965 } else {
4966 assert(isStrict && "unexpected node");
4967 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4968 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4969 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4970 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4971 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4972 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4973 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4974 }
4975
4976 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4977 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
4978 if (VT == ConcatVT)
4979 return Con;
4980
4981 EVT OpVT = N->getOperand(Num: 0).getValueType();
4982 ISD::NodeType ExtendCode =
4983 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
4984 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
4985}
4986
4987
4988SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4989 // The result has a legal vector type, but the input needs splitting.
4990 EVT ResVT = N->getValueType(ResNo: 0);
4991 SDValue Lo, Hi;
4992 SDLoc DL(N);
4993 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4994 EVT InVT = Lo.getValueType();
4995
4996 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4997 EC: InVT.getVectorElementCount());
4998
4999 if (N->isStrictFPOpcode()) {
5000 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5001 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5002 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5003 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5004 // Legalize the chain result - switch anything that used the old chain to
5005 // use the new one.
5006 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5007 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5008 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5009 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5010 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5011 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5012 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5013 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5014 } else {
5015 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5016 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5017 }
5018
5019 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5020}
5021
5022// Split a vector type in an FP binary operation where the second operand has a
5023// different type from the first.
5024//
5025// The result (and the first input) has a legal vector type, but the second
5026// input needs splitting.
5027SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5028 SDLoc DL(N);
5029
5030 EVT LHSLoVT, LHSHiVT;
5031 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5032
5033 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5034 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5035
5036 SDValue LHSLo, LHSHi;
5037 std::tie(args&: LHSLo, args&: LHSHi) =
5038 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5039
5040 SDValue RHSLo, RHSHi;
5041 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5042
5043 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5044 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5045
5046 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5047}
5048
5049SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5050 LLVMContext &Ctxt = *DAG.getContext();
5051 SDLoc dl(N);
5052
5053 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5054 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5055 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5056
5057 EVT ResVT = N->getValueType(ResNo: 0);
5058 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5059 EVT NewResVT =
5060 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5061
5062 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5063 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5064
5065 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5066}
5067
5068SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5069 EVT ResVT = N->getValueType(ResNo: 0);
5070 SDValue Lo, Hi;
5071 SDLoc dl(N);
5072 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5073 EVT InVT = Lo.getValueType();
5074
5075 EVT NewResVT =
5076 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5077 EC: InVT.getVectorElementCount());
5078
5079 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5080 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5081
5082 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5083}
5084
5085SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5086 SDLoc DL(N);
5087 EVT ResVT = N->getValueType(ResNo: 0);
5088
5089 SDValue Lo, Hi;
5090 SDValue VecOp = N->getOperand(Num: 0);
5091 GetSplitVector(Op: VecOp, Lo, Hi);
5092
5093 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5094 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5095 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5096 SDValue VL =
5097 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5098 SDValue ResLoNotVL =
5099 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5100 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5101 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5102 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5103}
5104
5105SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5106 SDLoc DL(N);
5107 EVT ResVT = N->getValueType(ResNo: 0);
5108
5109 SDValue Lo, Hi;
5110 SDValue VecOp = N->getOperand(Num: 0);
5111 GetSplitVector(Op: VecOp, Lo, Hi);
5112
5113 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5114 auto [EVLLo, EVLHi] =
5115 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5116 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5117
5118 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5119 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5120 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5121 SDValue ResLoNotEVL =
5122 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5123 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5124 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5125 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5126}
5127
5128SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5129 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5130 SDLoc DL(HG);
5131 SDValue Inc = HG->getInc();
5132 SDValue Ptr = HG->getBasePtr();
5133 SDValue Scale = HG->getScale();
5134 SDValue IntID = HG->getIntID();
5135 EVT MemVT = HG->getMemoryVT();
5136 MachineMemOperand *MMO = HG->getMemOperand();
5137 ISD::MemIndexType IndexType = HG->getIndexType();
5138
5139 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5140 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5141 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5142 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5143 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5144 Ops: OpsLo, MMO, IndexType);
5145 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5146 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5147 MMO, IndexType);
5148}
5149
5150SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5151 SDLoc DL(N);
5152
5153 if (OpNo == 0) {
5154 EVT LoResVT, HiResVT;
5155 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5156 SDValue SourceLo, SourceHi;
5157 std::tie(args&: SourceLo, args&: SourceHi) = DAG.SplitVectorOperand(N, OpNo: 0);
5158 SDValue MaskLo, MaskHi;
5159 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVectorOperand(N, OpNo: 2);
5160
5161 SDValue MatchLo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: LoResVT, N1: SourceLo,
5162 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
5163 SDValue MatchHi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: HiResVT, N1: SourceHi,
5164 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
5165 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo,
5166 N2: MatchHi);
5167 }
5168
5169 // Note: The Mask (OpNo == 2) should be widened with the result.
5170 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5171
5172 SDValue NeedleLo, NeedleHi;
5173 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: NeedleLo, Hi&: NeedleHi);
5174
5175 SDValue MatchLo =
5176 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5177 N2: NeedleLo, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5178 SDValue MatchHi =
5179 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5180 N2: NeedleHi, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5181 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo, N2: MatchHi);
5182}
5183
5184SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5185 SDValue Acc = N->getOperand(Num: 0);
5186 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5187 "Accumulator should already be a legal type, and shouldn't need "
5188 "further splitting");
5189
5190 SDLoc DL(N);
5191 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5192 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5193 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5194 unsigned Opcode = N->getOpcode();
5195 EVT ResultVT = Acc.getValueType();
5196
5197 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5198 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5199}
5200
5201//===----------------------------------------------------------------------===//
5202// Result Vector Widening
5203//===----------------------------------------------------------------------===//
5204
5205void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5206 unsigned WidenResNo) {
5207 unsigned NumResults = N->getNumValues();
5208 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5209 if (ResNo == WidenResNo)
5210 continue;
5211 EVT ResVT = N->getValueType(ResNo);
5212 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5213 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5214 } else {
5215 SDLoc DL(N);
5216 SDValue ResVal =
5217 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5218 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5219 }
5220 }
5221}
5222
5223void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5224 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5225
5226 // See if the target wants to custom widen this node.
5227 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5228 return;
5229
5230 SDValue Res = SDValue();
5231
5232 auto unrollExpandedOp = [&]() {
5233 // We're going to widen this vector op to a legal type by padding with undef
5234 // elements. If the wide vector op is eventually going to be expanded to
5235 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5236 // libcalls on the undef elements.
5237 EVT VT = N->getValueType(ResNo: 0);
5238 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5239 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5240 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
5241 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5242 if (N->getNumValues() > 1)
5243 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
5244 return true;
5245 }
5246 return false;
5247 };
5248
5249 switch (N->getOpcode()) {
5250 default:
5251#ifndef NDEBUG
5252 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5253 N->dump(&DAG);
5254 dbgs() << "\n";
5255#endif
5256 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
5257
5258 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5259 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5260 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5261 break;
5262 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5263 case ISD::ADDRSPACECAST:
5264 Res = WidenVecRes_ADDRSPACECAST(N);
5265 break;
5266 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5267 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5268 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5269 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5270 case ISD::INSERT_SUBVECTOR:
5271 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5272 break;
5273 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5274 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5275 case ISD::ATOMIC_LOAD:
5276 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5277 break;
5278 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5279 case ISD::STEP_VECTOR:
5280 case ISD::SPLAT_VECTOR:
5281 case ISD::SCALAR_TO_VECTOR:
5282 Res = WidenVecRes_ScalarOp(N);
5283 break;
5284 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5285 case ISD::VSELECT:
5286 case ISD::SELECT:
5287 case ISD::VP_MERGE:
5288 Res = WidenVecRes_Select(N);
5289 break;
5290 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5291 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5292 case ISD::POISON:
5293 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5294 case ISD::VECTOR_SHUFFLE:
5295 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5296 break;
5297 case ISD::VP_LOAD:
5298 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5299 break;
5300 case ISD::VP_LOAD_FF:
5301 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5302 break;
5303 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5304 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5305 break;
5306 case ISD::VECTOR_COMPRESS:
5307 Res = WidenVecRes_VECTOR_COMPRESS(N);
5308 break;
5309 case ISD::MLOAD:
5310 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5311 break;
5312 case ISD::MGATHER:
5313 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5314 break;
5315 case ISD::VP_GATHER:
5316 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5317 break;
5318 case ISD::VECTOR_REVERSE:
5319 Res = WidenVecRes_VECTOR_REVERSE(N);
5320 break;
5321 case ISD::GET_ACTIVE_LANE_MASK:
5322 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5323 break;
5324 case ISD::VECTOR_INTERLEAVE:
5325 WidenVecRes_VECTOR_INTERLEAVE(N);
5326 break;
5327 case ISD::VECTOR_MATCH:
5328 Res = WidenVecRes_VECTOR_MATCH(N);
5329 break;
5330 case ISD::VECTOR_DEINTERLEAVE:
5331 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5332 break;
5333
5334 case ISD::ADD:
5335 case ISD::AND:
5336 case ISD::MUL:
5337 case ISD::MULHS:
5338 case ISD::MULHU:
5339 case ISD::ABDS:
5340 case ISD::ABDU:
5341 case ISD::OR:
5342 case ISD::SUB:
5343 case ISD::XOR:
5344 case ISD::SHL:
5345 case ISD::SRA:
5346 case ISD::SRL:
5347 case ISD::CLMUL:
5348 case ISD::CLMULR:
5349 case ISD::CLMULH:
5350 case ISD::PEXT:
5351 case ISD::PDEP:
5352 case ISD::FMINNUM:
5353 case ISD::FMINNUM_IEEE:
5354 case ISD::FMAXNUM:
5355 case ISD::FMAXNUM_IEEE:
5356 case ISD::FMINIMUM:
5357 case ISD::FMAXIMUM:
5358 case ISD::FMINIMUMNUM:
5359 case ISD::FMAXIMUMNUM:
5360 case ISD::SMIN:
5361 case ISD::SMAX:
5362 case ISD::UMIN:
5363 case ISD::UMAX:
5364 case ISD::UADDSAT:
5365 case ISD::SADDSAT:
5366 case ISD::USUBSAT:
5367 case ISD::SSUBSAT:
5368 case ISD::SSHLSAT:
5369 case ISD::USHLSAT:
5370 case ISD::ROTL:
5371 case ISD::ROTR:
5372 case ISD::AVGFLOORS:
5373 case ISD::AVGFLOORU:
5374 case ISD::AVGCEILS:
5375 case ISD::AVGCEILU:
5376 // Vector-predicated binary op widening. Note that -- unlike the
5377 // unpredicated versions -- we don't have to worry about trapping on
5378 // operations like UDIV, FADD, etc., as we pass on the original vector
5379 // length parameter. This means the widened elements containing garbage
5380 // aren't active.
5381 case ISD::VP_SDIV:
5382 case ISD::VP_UDIV:
5383 case ISD::VP_SREM:
5384 case ISD::VP_UREM:
5385 Res = WidenVecRes_Binary(N);
5386 break;
5387
5388 case ISD::MASKED_UDIV:
5389 case ISD::MASKED_SDIV:
5390 case ISD::MASKED_UREM:
5391 case ISD::MASKED_SREM:
5392 Res = WidenVecRes_MaskedBinary(N);
5393 break;
5394
5395 case ISD::SCMP:
5396 case ISD::UCMP:
5397 Res = WidenVecRes_CMP(N);
5398 break;
5399
5400 case ISD::FPOW:
5401 case ISD::FATAN2:
5402 case ISD::FREM:
5403 if (unrollExpandedOp())
5404 break;
5405 // If the target has custom/legal support for the scalar FP intrinsic ops
5406 // (they are probably not destined to become libcalls), then widen those
5407 // like any other binary ops.
5408 [[fallthrough]];
5409
5410 case ISD::FADD:
5411 case ISD::FMUL:
5412 case ISD::FSUB:
5413 case ISD::FDIV:
5414 case ISD::SDIV:
5415 case ISD::UDIV:
5416 case ISD::SREM:
5417 case ISD::UREM:
5418 Res = WidenVecRes_BinaryCanTrap(N);
5419 break;
5420
5421 case ISD::SMULFIX:
5422 case ISD::SMULFIXSAT:
5423 case ISD::UMULFIX:
5424 case ISD::UMULFIXSAT:
5425 // These are binary operations, but with an extra operand that shouldn't
5426 // be widened (the scale).
5427 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5428 break;
5429
5430#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5431 case ISD::STRICT_##DAGN:
5432#include "llvm/IR/ConstrainedOps.def"
5433 Res = WidenVecRes_StrictFP(N);
5434 break;
5435
5436 case ISD::UADDO:
5437 case ISD::SADDO:
5438 case ISD::USUBO:
5439 case ISD::SSUBO:
5440 case ISD::UMULO:
5441 case ISD::SMULO:
5442 Res = WidenVecRes_OverflowOp(N, ResNo);
5443 break;
5444
5445 case ISD::FCOPYSIGN:
5446 Res = WidenVecRes_FCOPYSIGN(N);
5447 break;
5448
5449 case ISD::IS_FPCLASS:
5450 case ISD::FPTRUNC_ROUND:
5451 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5452 break;
5453
5454 case ISD::FLDEXP:
5455 case ISD::FPOWI:
5456 if (!unrollExpandedOp())
5457 Res = WidenVecRes_ExpOp(N);
5458 break;
5459
5460 case ISD::ANY_EXTEND_VECTOR_INREG:
5461 case ISD::SIGN_EXTEND_VECTOR_INREG:
5462 case ISD::ZERO_EXTEND_VECTOR_INREG:
5463 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5464 break;
5465
5466 case ISD::ANY_EXTEND:
5467 case ISD::FP_EXTEND:
5468 case ISD::FP_ROUND:
5469 case ISD::FP_TO_SINT:
5470 case ISD::FP_TO_UINT:
5471 case ISD::SIGN_EXTEND:
5472 case ISD::SINT_TO_FP:
5473 case ISD::TRUNCATE:
5474 case ISD::UINT_TO_FP:
5475 case ISD::ZERO_EXTEND:
5476 case ISD::CONVERT_FROM_ARBITRARY_FP:
5477 case ISD::CONVERT_TO_ARBITRARY_FP:
5478 Res = WidenVecRes_Convert(N);
5479 break;
5480
5481 case ISD::FP_TO_SINT_SAT:
5482 case ISD::FP_TO_UINT_SAT:
5483 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5484 break;
5485
5486 case ISD::LRINT:
5487 case ISD::LLRINT:
5488 case ISD::LROUND:
5489 case ISD::LLROUND:
5490 Res = WidenVecRes_XROUND(N);
5491 break;
5492
5493 case ISD::FACOS:
5494 case ISD::FASIN:
5495 case ISD::FATAN:
5496 case ISD::FCEIL:
5497 case ISD::FCOS:
5498 case ISD::FCOSH:
5499 case ISD::FEXP:
5500 case ISD::FEXP2:
5501 case ISD::FEXP10:
5502 case ISD::FFLOOR:
5503 case ISD::FLOG:
5504 case ISD::FLOG10:
5505 case ISD::FLOG2:
5506 case ISD::FNEARBYINT:
5507 case ISD::FRINT:
5508 case ISD::FROUND:
5509 case ISD::FROUNDEVEN:
5510 case ISD::FSIN:
5511 case ISD::FSINH:
5512 case ISD::FSQRT:
5513 case ISD::FTAN:
5514 case ISD::FTANH:
5515 case ISD::FTRUNC:
5516 if (unrollExpandedOp())
5517 break;
5518 // If the target has custom/legal support for the scalar FP intrinsic ops
5519 // (they are probably not destined to become libcalls), then widen those
5520 // like any other unary ops.
5521 [[fallthrough]];
5522
5523 case ISD::ABS:
5524 case ISD::ABS_MIN_POISON:
5525 case ISD::BITREVERSE:
5526 case ISD::BSWAP:
5527 case ISD::CTLZ:
5528 case ISD::CTLZ_ZERO_POISON:
5529 case ISD::CTPOP:
5530 case ISD::CTTZ:
5531 case ISD::CTTZ_ZERO_POISON:
5532 case ISD::FNEG:
5533 case ISD::FABS:
5534 case ISD::FREEZE:
5535 case ISD::ARITH_FENCE:
5536 case ISD::FCANONICALIZE:
5537 case ISD::AssertNoFPClass:
5538 Res = WidenVecRes_Unary(N);
5539 break;
5540 case ISD::FMA:
5541 case ISD::FSHL:
5542 case ISD::FSHR:
5543 Res = WidenVecRes_Ternary(N);
5544 break;
5545 case ISD::FMODF:
5546 case ISD::FFREXP:
5547 case ISD::FSINCOS:
5548 case ISD::FSINCOSPI: {
5549 if (!unrollExpandedOp())
5550 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5551 break;
5552 }
5553 case ISD::PARTIAL_REDUCE_UMLA:
5554 case ISD::PARTIAL_REDUCE_SMLA:
5555 case ISD::PARTIAL_REDUCE_SUMLA:
5556 case ISD::PARTIAL_REDUCE_FMLA:
5557 Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
5558 break;
5559 }
5560
5561 // If Res is null, the sub-method took care of registering the result.
5562 if (Res.getNode())
5563 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5564}
5565
5566SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5567 // Ternary op widening.
5568 SDLoc dl(N);
5569 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5570 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5571 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5572 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5573 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5574}
5575
5576SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5577 // Binary op widening.
5578 SDLoc dl(N);
5579 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5580 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5581 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5582 if (N->getNumOperands() == 2)
5583 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5584 Flags: N->getFlags());
5585
5586 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5587 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5588 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5589 "Expected VP opcode");
5590
5591 SDValue Mask =
5592 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5593 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5594 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5595}
5596
5597SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5598 SDLoc dl(N);
5599 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5600 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5601 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5602 SDValue Mask = N->getOperand(Num: 2);
5603 EVT WideMaskVT = WidenVT.changeVectorElementType(
5604 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5605 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5606 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5607 Flags: N->getFlags());
5608}
5609
5610SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5611 LLVMContext &Ctxt = *DAG.getContext();
5612 SDLoc dl(N);
5613
5614 SDValue LHS = N->getOperand(Num: 0);
5615 SDValue RHS = N->getOperand(Num: 1);
5616 EVT OpVT = LHS.getValueType();
5617 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5618 LHS = GetWidenedVector(Op: LHS);
5619 RHS = GetWidenedVector(Op: RHS);
5620 OpVT = LHS.getValueType();
5621 }
5622
5623 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5624 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5625 if (WidenResEC == OpVT.getVectorElementCount()) {
5626 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5627 }
5628
5629 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5630}
5631
5632SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5633 // Binary op widening, but with an extra operand that shouldn't be widened.
5634 SDLoc dl(N);
5635 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5636 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5637 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5638 SDValue InOp3 = N->getOperand(Num: 2);
5639 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5640 Flags: N->getFlags());
5641}
5642
5643// Given a vector of operations that have been broken up to widen, see
5644// if we can collect them together into the next widest legal VT. This
5645// implementation is trap-safe.
5646static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5647 SmallVectorImpl<SDValue> &ConcatOps,
5648 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5649 EVT WidenVT) {
5650 // Check to see if we have a single operation with the widen type.
5651 if (ConcatEnd == 1) {
5652 VT = ConcatOps[0].getValueType();
5653 if (VT == WidenVT)
5654 return ConcatOps[0];
5655 }
5656
5657 SDLoc dl(ConcatOps[0]);
5658 EVT WidenEltVT = WidenVT.getVectorElementType();
5659
5660 // while (Some element of ConcatOps is not of type MaxVT) {
5661 // From the end of ConcatOps, collect elements of the same type and put
5662 // them into an op of the next larger supported type
5663 // }
5664 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5665 int Idx = ConcatEnd - 1;
5666 VT = ConcatOps[Idx--].getValueType();
5667 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5668 Idx--;
5669
5670 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5671 EVT NextVT;
5672 do {
5673 NextSize *= 2;
5674 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5675 } while (!TLI.isTypeLegal(VT: NextVT));
5676
5677 if (!VT.isVector()) {
5678 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5679 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5680 unsigned NumToInsert = ConcatEnd - Idx - 1;
5681 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5682 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5683 ConcatOps[Idx+1] = VecOp;
5684 ConcatEnd = Idx + 2;
5685 } else {
5686 // Vector type, create a CONCAT_VECTORS of type NextVT
5687 SDValue undefVec = DAG.getPOISON(VT);
5688 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5689 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5690 unsigned RealVals = ConcatEnd - Idx - 1;
5691 unsigned SubConcatEnd = 0;
5692 unsigned SubConcatIdx = Idx + 1;
5693 while (SubConcatEnd < RealVals)
5694 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5695 while (SubConcatEnd < OpsToConcat)
5696 SubConcatOps[SubConcatEnd++] = undefVec;
5697 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5698 VT: NextVT, Ops: SubConcatOps);
5699 ConcatEnd = SubConcatIdx + 1;
5700 }
5701 }
5702
5703 // Check to see if we have a single operation with the widen type.
5704 if (ConcatEnd == 1) {
5705 VT = ConcatOps[0].getValueType();
5706 if (VT == WidenVT)
5707 return ConcatOps[0];
5708 }
5709
5710 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5711 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5712 if (NumOps != ConcatEnd ) {
5713 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5714 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5715 ConcatOps[j] = UndefVal;
5716 }
5717 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5718 Ops: ArrayRef(ConcatOps.data(), NumOps));
5719}
5720
5721SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5722 // Binary op widening for operations that can trap.
5723 unsigned Opcode = N->getOpcode();
5724 SDLoc dl(N);
5725 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5726 EVT WidenEltVT = WidenVT.getVectorElementType();
5727 EVT VT = WidenVT;
5728 unsigned NumElts = VT.getVectorMinNumElements();
5729 const SDNodeFlags Flags = N->getFlags();
5730 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5731 NumElts = NumElts / 2;
5732 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5733 }
5734
5735 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5736 // Operation doesn't trap so just widen as normal.
5737 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5738 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5739 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5740 }
5741
5742 // Generate a vp.op if it is custom/legal for the target. This avoids need
5743 // to split and tile the subvectors (below), because the inactive lanes can
5744 // simply be disabled. To avoid possible recursion, only do this if the
5745 // widened mask type is legal.
5746 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5747 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5748 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5749 EC: WidenVT.getVectorElementCount());
5750 TLI.isTypeLegal(VT: WideMaskVT)) {
5751 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5752 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5753 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5754 SDValue EVL =
5755 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5756 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5757 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5758 Flags);
5759 }
5760 }
5761
5762 // FIXME: Improve support for scalable vectors.
5763 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5764
5765 // No legal vector version so unroll the vector operation and then widen.
5766 if (NumElts == 1)
5767 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5768
5769 // Since the operation can trap, apply operation on the original vector.
5770 EVT MaxVT = VT;
5771 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5772 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5773 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5774
5775 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5776 unsigned ConcatEnd = 0; // Current ConcatOps index.
5777 int Idx = 0; // Current Idx into input vectors.
5778
5779 // NumElts := greatest legal vector size (at most WidenVT)
5780 // while (orig. vector has unhandled elements) {
5781 // take munches of size NumElts from the beginning and add to ConcatOps
5782 // NumElts := next smaller supported vector size or 1
5783 // }
5784 while (CurNumElts != 0) {
5785 while (CurNumElts >= NumElts) {
5786 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5787 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5788 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5789 Idx += NumElts;
5790 CurNumElts -= NumElts;
5791 }
5792 do {
5793 NumElts = NumElts / 2;
5794 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5795 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5796
5797 if (NumElts == 1) {
5798 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5799 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5800 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5801 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5802 N1: EOp1, N2: EOp2, Flags);
5803 }
5804 CurNumElts = 0;
5805 }
5806 }
5807
5808 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5809}
5810
5811SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5812 switch (N->getOpcode()) {
5813 case ISD::STRICT_FSETCC:
5814 case ISD::STRICT_FSETCCS:
5815 return WidenVecRes_STRICT_FSETCC(N);
5816 case ISD::STRICT_FP_EXTEND:
5817 case ISD::STRICT_FP_ROUND:
5818 case ISD::STRICT_FP_TO_SINT:
5819 case ISD::STRICT_FP_TO_UINT:
5820 case ISD::STRICT_SINT_TO_FP:
5821 case ISD::STRICT_UINT_TO_FP:
5822 return WidenVecRes_Convert_StrictFP(N);
5823 default:
5824 break;
5825 }
5826
5827 // StrictFP op widening for operations that can trap.
5828 unsigned NumOpers = N->getNumOperands();
5829 unsigned Opcode = N->getOpcode();
5830 SDLoc dl(N);
5831 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5832 EVT WidenEltVT = WidenVT.getVectorElementType();
5833 EVT VT = WidenVT;
5834 unsigned NumElts = VT.getVectorNumElements();
5835 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5836 NumElts = NumElts / 2;
5837 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5838 }
5839
5840 // No legal vector version so unroll the vector operation and then widen.
5841 if (NumElts == 1)
5842 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5843
5844 // Since the operation can trap, apply operation on the original vector.
5845 EVT MaxVT = VT;
5846 SmallVector<SDValue, 4> InOps;
5847 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5848
5849 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5850 SmallVector<SDValue, 16> Chains;
5851 unsigned ConcatEnd = 0; // Current ConcatOps index.
5852 int Idx = 0; // Current Idx into input vectors.
5853
5854 // The Chain is the first operand.
5855 InOps.push_back(Elt: N->getOperand(Num: 0));
5856
5857 // Now process the remaining operands.
5858 for (unsigned i = 1; i < NumOpers; ++i) {
5859 SDValue Oper = N->getOperand(Num: i);
5860
5861 EVT OpVT = Oper.getValueType();
5862 if (OpVT.isVector()) {
5863 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5864 Oper = GetWidenedVector(Op: Oper);
5865 else {
5866 EVT WideOpVT =
5867 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5868 EC: WidenVT.getVectorElementCount());
5869 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5870 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5871 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5872 }
5873 }
5874
5875 InOps.push_back(Elt: Oper);
5876 }
5877
5878 // NumElts := greatest legal vector size (at most WidenVT)
5879 // while (orig. vector has unhandled elements) {
5880 // take munches of size NumElts from the beginning and add to ConcatOps
5881 // NumElts := next smaller supported vector size or 1
5882 // }
5883 while (CurNumElts != 0) {
5884 while (CurNumElts >= NumElts) {
5885 SmallVector<SDValue, 4> EOps;
5886
5887 for (unsigned i = 0; i < NumOpers; ++i) {
5888 SDValue Op = InOps[i];
5889
5890 EVT OpVT = Op.getValueType();
5891 if (OpVT.isVector()) {
5892 EVT OpExtractVT =
5893 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5894 EC: VT.getVectorElementCount());
5895 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5896 }
5897
5898 EOps.push_back(Elt: Op);
5899 }
5900
5901 EVT OperVT[] = {VT, MVT::Other};
5902 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5903 ConcatOps[ConcatEnd++] = Oper;
5904 Chains.push_back(Elt: Oper.getValue(R: 1));
5905 Idx += NumElts;
5906 CurNumElts -= NumElts;
5907 }
5908 do {
5909 NumElts = NumElts / 2;
5910 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5911 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5912
5913 if (NumElts == 1) {
5914 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5915 SmallVector<SDValue, 4> EOps;
5916
5917 for (unsigned i = 0; i < NumOpers; ++i) {
5918 SDValue Op = InOps[i];
5919
5920 EVT OpVT = Op.getValueType();
5921 if (OpVT.isVector())
5922 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5923 Idx);
5924
5925 EOps.push_back(Elt: Op);
5926 }
5927
5928 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5929 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5930 ConcatOps[ConcatEnd++] = Oper;
5931 Chains.push_back(Elt: Oper.getValue(R: 1));
5932 }
5933 CurNumElts = 0;
5934 }
5935 }
5936
5937 // Build a factor node to remember all the Ops that have been created.
5938 SDValue NewChain;
5939 if (Chains.size() == 1)
5940 NewChain = Chains[0];
5941 else
5942 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5943 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5944
5945 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5946}
5947
5948SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5949 SDLoc DL(N);
5950 EVT ResVT = N->getValueType(ResNo: 0);
5951 EVT OvVT = N->getValueType(ResNo: 1);
5952 EVT WideResVT, WideOvVT;
5953 SDValue WideLHS, WideRHS;
5954
5955 // TODO: This might result in a widen/split loop.
5956 if (ResNo == 0) {
5957 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5958 WideOvVT = EVT::getVectorVT(
5959 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5960 NumElements: WideResVT.getVectorNumElements());
5961
5962 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5963 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5964 } else {
5965 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5966 WideResVT = EVT::getVectorVT(
5967 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5968 NumElements: WideOvVT.getVectorNumElements());
5969
5970 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5971 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5972
5973 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5974 N2: N->getOperand(Num: 0), N3: Zero);
5975 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5976 N2: N->getOperand(Num: 1), N3: Zero);
5977 }
5978
5979 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5980 SDNode *WideNode = DAG.getNode(
5981 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5982
5983 // Replace the other vector result not being explicitly widened here.
5984 unsigned OtherNo = 1 - ResNo;
5985 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5986 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5987 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5988 } else {
5989 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5990 SDValue OtherVal = DAG.getNode(
5991 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5992 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5993 }
5994
5995 return SDValue(WideNode, ResNo);
5996}
5997
5998SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5999 LLVMContext &Ctx = *DAG.getContext();
6000 SDValue InOp = N->getOperand(Num: 0);
6001 SDLoc DL(N);
6002
6003 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6004 ElementCount WidenEC = WidenVT.getVectorElementCount();
6005
6006 EVT InVT = InOp.getValueType();
6007
6008 unsigned Opcode = N->getOpcode();
6009 const SDNodeFlags Flags = N->getFlags();
6010
6011 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6012 // not equal that of WidenVT.
6013 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6014 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6015 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6016 WidenVT.getScalarSizeInBits()) {
6017 InOp = ZExtPromotedInteger(Op: InOp);
6018 InVT = InOp.getValueType();
6019 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6020 Opcode = ISD::TRUNCATE;
6021 }
6022
6023 EVT InEltVT = InVT.getVectorElementType();
6024 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6025 ElementCount InVTEC = InVT.getVectorElementCount();
6026
6027 // Helper to build node with all scalar trailing operands.
6028 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6029 if (N->getNumOperands() == 1)
6030 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6031 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6032 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6033 N4: N->getOperand(Num: 3), Flags);
6034 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6035 };
6036
6037 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6038 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6039 InVT = InOp.getValueType();
6040 InVTEC = InVT.getVectorElementCount();
6041 if (InVTEC == WidenEC)
6042 return MakeConvertNode(WidenVT, InOp);
6043 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6044 // If both input and result vector types are of same width, extend
6045 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6046 // accepts fewer elements in the result than in the input.
6047 if (Opcode == ISD::ANY_EXTEND)
6048 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6049 if (Opcode == ISD::SIGN_EXTEND)
6050 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6051 if (Opcode == ISD::ZERO_EXTEND)
6052 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6053 }
6054
6055 // For TRUNCATE, try to widen using the legal EC of the input type instead
6056 // if the legalisation action for that intermediate type is not widening.
6057 // E.g. for trunc nxv1i64 -> nxv1i8 where
6058 // - nxv1i64 input gets widened to nxv2i64
6059 // - nxv1i8 output gets widened to nxv16i8
6060 // Then one can try widening the result to nxv2i8 (instead of going all the
6061 // way to nxv16i8) if this later allows type promotion.
6062 EVT MidResVT =
6063 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6064 if (N->getOpcode() == ISD::TRUNCATE &&
6065 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6066 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6067 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6068 }
6069 }
6070
6071 if (TLI.isTypeLegal(VT: InWidenVT)) {
6072 // Because the result and the input are different vector types, widening
6073 // the result could create a legal type but widening the input might make
6074 // it an illegal type that might lead to repeatedly splitting the input
6075 // and then widening it. To avoid this, we widen the input only if
6076 // it results in a legal type.
6077 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6078 // Widen the input and call convert on the widened input vector.
6079 unsigned NumConcat =
6080 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6081 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6082 Ops[0] = InOp;
6083 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6084 return MakeConvertNode(WidenVT, InVec);
6085 }
6086
6087 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6088 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6089 // Extract the input and convert the shorten input vector.
6090 return MakeConvertNode(WidenVT, InVal);
6091 }
6092 }
6093
6094 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6095 EVT EltVT = WidenVT.getVectorElementType();
6096 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6097 // Use the original element count so we don't do more scalar opts than
6098 // necessary.
6099 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6100 for (unsigned i=0; i < MinElts; ++i) {
6101 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6102 Ops[i] = MakeConvertNode(EltVT, Val);
6103 }
6104
6105 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6106}
6107
6108SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6109 SDLoc dl(N);
6110 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6111 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6112
6113 SDValue Src = N->getOperand(Num: 0);
6114 EVT SrcVT = Src.getValueType();
6115
6116 // Also widen the input.
6117 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6118 Src = GetWidenedVector(Op: Src);
6119 SrcVT = Src.getValueType();
6120 }
6121
6122 // Input and output not widened to the same size, give up.
6123 if (WidenNumElts != SrcVT.getVectorElementCount())
6124 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6125
6126 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6127}
6128
6129SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6130 SDLoc dl(N);
6131 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6132 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6133
6134 SDValue Src = N->getOperand(Num: 0);
6135 EVT SrcVT = Src.getValueType();
6136
6137 // Also widen the input.
6138 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6139 Src = GetWidenedVector(Op: Src);
6140 SrcVT = Src.getValueType();
6141 }
6142
6143 // Input and output not widened to the same size, give up.
6144 if (WidenNumElts != SrcVT.getVectorElementCount())
6145 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6146
6147 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6148}
6149
6150SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6151 SDValue InOp = N->getOperand(Num: 1);
6152 SDLoc DL(N);
6153 SmallVector<SDValue, 4> NewOps(N->ops());
6154
6155 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6156 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6157
6158 EVT InVT = InOp.getValueType();
6159 EVT InEltVT = InVT.getVectorElementType();
6160
6161 unsigned Opcode = N->getOpcode();
6162
6163 // FIXME: Optimizations need to be implemented here.
6164
6165 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6166 EVT EltVT = WidenVT.getVectorElementType();
6167 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6168 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6169 SmallVector<SDValue, 32> OpChains;
6170 // Use the original element count so we don't do more scalar opts than
6171 // necessary.
6172 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6173 for (unsigned i=0; i < MinElts; ++i) {
6174 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6175 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6176 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6177 }
6178 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6179 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6180
6181 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6182}
6183
6184SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6185 unsigned Opcode = N->getOpcode();
6186 SDValue InOp = N->getOperand(Num: 0);
6187 SDLoc DL(N);
6188
6189 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6190 EVT WidenSVT = WidenVT.getVectorElementType();
6191 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6192
6193 EVT InVT = InOp.getValueType();
6194 EVT InSVT = InVT.getVectorElementType();
6195 unsigned InVTNumElts = InVT.getVectorNumElements();
6196
6197 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6198 InOp = GetWidenedVector(Op: InOp);
6199 InVT = InOp.getValueType();
6200 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6201 switch (Opcode) {
6202 case ISD::ANY_EXTEND_VECTOR_INREG:
6203 case ISD::SIGN_EXTEND_VECTOR_INREG:
6204 case ISD::ZERO_EXTEND_VECTOR_INREG:
6205 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6206 }
6207 }
6208 }
6209
6210 // Unroll, extend the scalars and rebuild the vector.
6211 SmallVector<SDValue, 16> Ops;
6212 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6213 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6214 switch (Opcode) {
6215 case ISD::ANY_EXTEND_VECTOR_INREG:
6216 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6217 break;
6218 case ISD::SIGN_EXTEND_VECTOR_INREG:
6219 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6220 break;
6221 case ISD::ZERO_EXTEND_VECTOR_INREG:
6222 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6223 break;
6224 default:
6225 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6226 }
6227 Ops.push_back(Elt: Val);
6228 }
6229
6230 while (Ops.size() != WidenNumElts)
6231 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6232
6233 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6234}
6235
6236SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6237 // If this is an FCOPYSIGN with same input types, we can treat it as a
6238 // normal (can trap) binary op.
6239 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6240 return WidenVecRes_BinaryCanTrap(N);
6241
6242 // If the types are different, fall back to unrolling.
6243 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6244 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6245}
6246
6247/// Result and first source operand are different scalar types, but must have
6248/// the same number of elements. There is an additional control argument which
6249/// should be passed through unchanged.
6250SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6251 SDValue FpValue = N->getOperand(Num: 0);
6252 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6253 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6254 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6255 SDValue Arg = GetWidenedVector(Op: FpValue);
6256 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6257 Flags: N->getFlags());
6258}
6259
6260SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6261 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6262 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6263 SDValue RHS = N->getOperand(Num: 1);
6264 EVT ExpVT = RHS.getValueType();
6265 SDValue ExpOp = RHS;
6266 if (ExpVT.isVector()) {
6267 EVT WideExpVT = WidenVT.changeVectorElementType(
6268 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6269 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6270 }
6271
6272 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6273}
6274
6275SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6276 // Unary op widening.
6277 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6278 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6279 if (N->getNumOperands() == 1)
6280 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6281 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6282 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1),
6283 Flags: N->getFlags());
6284}
6285
6286SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6287 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6288 EVT ExtVT = EVT::getVectorVT(
6289 Context&: *DAG.getContext(),
6290 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6291 EC: WidenVT.getVectorElementCount());
6292 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6293 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6294 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6295}
6296
6297SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6298 unsigned ResNo) {
6299 EVT VT0 = N->getValueType(ResNo: 0);
6300 EVT VT1 = N->getValueType(ResNo: 1);
6301
6302 assert(VT0.isVector() && VT1.isVector() &&
6303 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6304 "expected both results to be vectors of matching element count");
6305
6306 LLVMContext &Ctx = *DAG.getContext();
6307 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6308
6309 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6310 ElementCount WidenEC = WidenVT.getVectorElementCount();
6311
6312 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6313 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6314
6315 SDNode *WidenNode =
6316 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6317 .getNode();
6318
6319 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6320 return SDValue(WidenNode, ResNo);
6321}
6322
6323SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6324 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6325 return GetWidenedVector(Op: WidenVec);
6326}
6327
6328SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6329 SDLoc DL(N);
6330 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6331 ElementCount WidenEC = WidenVT.getVectorElementCount();
6332 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6333
6334 // The source has the same number of elements as the result, so widen it to
6335 // match WidenVT. It only lives in the widened-vector map if it is itself
6336 // widened; otherwise pad it up to the widened element count.
6337 SDValue InOp = N->getOperand(Num: 0);
6338 EVT InVT = InOp.getValueType();
6339 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6340 InOp = GetWidenedVector(Op: InOp);
6341 } else {
6342 EVT InWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6343 VT: InVT.getVectorElementType(), EC: WidenEC);
6344 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InWidenVT), SubVec: InOp, Idx: 0);
6345 }
6346
6347 return DAG.getAddrSpaceCast(
6348 dl: DL, VT: WidenVT, Ptr: InOp, SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6349 DestAS: AddrSpaceCastN->getDestAddressSpace(), Flags: AddrSpaceCastN->getFlags());
6350}
6351
6352SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6353 SDValue InOp = N->getOperand(Num: 0);
6354 EVT InVT = InOp.getValueType();
6355 EVT VT = N->getValueType(ResNo: 0);
6356 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6357 SDLoc dl(N);
6358
6359 switch (getTypeAction(VT: InVT)) {
6360 case TargetLowering::TypeLegal:
6361 break;
6362 case TargetLowering::TypeScalarizeScalableVector:
6363 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6364 case TargetLowering::TypePromoteInteger: {
6365 // If the incoming type is a vector that is being promoted, then
6366 // we know that the elements are arranged differently and that we
6367 // must perform the conversion using a stack slot.
6368 if (InVT.isVector())
6369 break;
6370
6371 // If the InOp is promoted to the same size, convert it. Otherwise,
6372 // fall out of the switch and widen the promoted input.
6373 SDValue NInOp = GetPromotedInteger(Op: InOp);
6374 EVT NInVT = NInOp.getValueType();
6375 if (WidenVT.bitsEq(VT: NInVT)) {
6376 // For big endian targets we need to shift the input integer or the
6377 // interesting bits will end up at the wrong place.
6378 if (DAG.getDataLayout().isBigEndian()) {
6379 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6380 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6381 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6382 }
6383 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6384 }
6385 InOp = NInOp;
6386 InVT = NInVT;
6387 break;
6388 }
6389 case TargetLowering::TypeSoftenFloat:
6390 case TargetLowering::TypeSoftPromoteHalf:
6391 case TargetLowering::TypeExpandInteger:
6392 case TargetLowering::TypeExpandFloat:
6393 case TargetLowering::TypeScalarizeVector:
6394 case TargetLowering::TypeSplitVector:
6395 break;
6396 case TargetLowering::TypeWidenVector:
6397 // If the InOp is widened to the same size, convert it. Otherwise, fall
6398 // out of the switch and widen the widened input.
6399 InOp = GetWidenedVector(Op: InOp);
6400 InVT = InOp.getValueType();
6401 if (WidenVT.bitsEq(VT: InVT))
6402 // The input widens to the same size. Convert to the widen value.
6403 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6404 break;
6405 }
6406
6407 unsigned WidenSize = WidenVT.getSizeInBits();
6408 unsigned InSize = InVT.getSizeInBits();
6409 unsigned InScalarSize = InVT.getScalarSizeInBits();
6410 // x86mmx is not an acceptable vector element type, so don't try.
6411 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6412 // Determine new input vector type. The new input vector type will use
6413 // the same element type (if its a vector) or use the input type as a
6414 // vector. It is the same size as the type to widen to.
6415 EVT NewInVT;
6416 unsigned NewNumParts = WidenSize / InSize;
6417 if (InVT.isVector()) {
6418 EVT InEltVT = InVT.getVectorElementType();
6419 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6420 NumElements: WidenSize / InEltVT.getSizeInBits());
6421 } else {
6422 // For big endian systems, using the promoted input scalar type
6423 // to produce the scalar_to_vector would put the desired bits into
6424 // the least significant byte(s) of the wider element zero. This
6425 // will mean that the users of the result vector are using incorrect
6426 // bits. Use the original input type instead. Although either input
6427 // type can be used on little endian systems, for consistency we
6428 // use the original type there as well.
6429 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6430 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6431 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6432 }
6433
6434 if (TLI.isTypeLegal(VT: NewInVT)) {
6435 SDValue NewVec;
6436 if (InVT.isVector()) {
6437 // Because the result and the input are different vector types, widening
6438 // the result could create a legal type but widening the input might
6439 // make it an illegal type that might lead to repeatedly splitting the
6440 // input and then widening it. To avoid this, we widen the input only if
6441 // it results in a legal type.
6442 if (WidenSize % InSize == 0) {
6443 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6444 Ops[0] = InOp;
6445
6446 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6447 } else {
6448 SmallVector<SDValue, 16> Ops;
6449 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6450 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6451 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6452
6453 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6454 }
6455 } else {
6456 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6457 }
6458 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6459 }
6460 }
6461
6462 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6463}
6464
6465SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6466 return DAG.getNode(
6467 Opcode: N->getOpcode(), DL: SDLoc(N),
6468 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6469 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6470}
6471
6472SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6473 SDLoc dl(N);
6474 // Build a vector with poison for the new nodes.
6475 EVT VT = N->getValueType(ResNo: 0);
6476
6477 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6478 // type. The POISONs need to have the same type as the existing operands.
6479 EVT EltVT = N->getOperand(Num: 0).getValueType();
6480 unsigned NumElts = VT.getVectorNumElements();
6481
6482 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6483 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6484
6485 SmallVector<SDValue, 16> NewOps(N->ops());
6486 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6487 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6488
6489 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6490}
6491
6492SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6493 EVT InVT = N->getOperand(Num: 0).getValueType();
6494 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6495 SDLoc dl(N);
6496 unsigned NumOperands = N->getNumOperands();
6497
6498 bool InputWidened = false; // Indicates we need to widen the input.
6499 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6500 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6501 unsigned NumInElts = InVT.getVectorMinNumElements();
6502 if (WidenNumElts % NumInElts == 0) {
6503 // Add undef vectors to widen to correct length.
6504 unsigned NumConcat = WidenNumElts / NumInElts;
6505 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6506 SmallVector<SDValue, 16> Ops(NumConcat);
6507 for (unsigned i=0; i < NumOperands; ++i)
6508 Ops[i] = N->getOperand(Num: i);
6509 for (unsigned i = NumOperands; i != NumConcat; ++i)
6510 Ops[i] = UndefVal;
6511 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6512 }
6513 } else {
6514 InputWidened = true;
6515 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6516 // The inputs and the result are widen to the same value.
6517 unsigned i;
6518 for (i=1; i < NumOperands; ++i)
6519 if (!N->getOperand(Num: i).isUndef())
6520 break;
6521
6522 if (i == NumOperands)
6523 // Everything but the first operand is an UNDEF so just return the
6524 // widened first operand.
6525 return GetWidenedVector(Op: N->getOperand(Num: 0));
6526
6527 if (NumOperands == 2) {
6528 assert(!WidenVT.isScalableVector() &&
6529 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6530 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6531 unsigned NumInElts = InVT.getVectorNumElements();
6532
6533 // Replace concat of two operands with a shuffle.
6534 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6535 for (unsigned i = 0; i < NumInElts; ++i) {
6536 MaskOps[i] = i;
6537 MaskOps[i + NumInElts] = i + WidenNumElts;
6538 }
6539 return DAG.getVectorShuffle(VT: WidenVT, dl,
6540 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6541 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6542 Mask: MaskOps);
6543 }
6544 }
6545 }
6546
6547 if (WidenVT.isScalableVector()) {
6548 SDValue WideVec = DAG.getPOISON(VT: WidenVT);
6549 unsigned NumInElts = InVT.getVectorMinNumElements();
6550 for (unsigned I = 0; I < NumOperands; ++I)
6551 WideVec =
6552 DAG.getInsertSubvector(DL: dl, Vec: WideVec, SubVec: N->getOperand(Num: I), Idx: I * NumInElts);
6553 return WideVec;
6554 }
6555
6556 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6557 unsigned NumInElts = InVT.getVectorNumElements();
6558
6559 // Fall back to use extracts and build vector.
6560 EVT EltVT = WidenVT.getVectorElementType();
6561 SmallVector<SDValue, 16> Ops(WidenNumElts);
6562 unsigned Idx = 0;
6563 for (unsigned i=0; i < NumOperands; ++i) {
6564 SDValue InOp = N->getOperand(Num: i);
6565 if (InputWidened)
6566 InOp = GetWidenedVector(Op: InOp);
6567 for (unsigned j = 0; j < NumInElts; ++j)
6568 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6569 }
6570 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6571 for (; Idx < WidenNumElts; ++Idx)
6572 Ops[Idx] = UndefVal;
6573 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6574}
6575
6576SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6577 EVT VT = N->getValueType(ResNo: 0);
6578 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6579 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6580 SDValue InOp2 = N->getOperand(Num: 1);
6581 SDValue Idx = N->getOperand(Num: 2);
6582 SDLoc dl(N);
6583 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6584}
6585
6586SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6587 EVT VT = N->getValueType(ResNo: 0);
6588 EVT EltVT = VT.getVectorElementType();
6589 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6590 SDValue InOp = N->getOperand(Num: 0);
6591 SDValue Idx = N->getOperand(Num: 1);
6592 SDLoc dl(N);
6593
6594 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6595 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6596 InOp = GetWidenedVector(Op: InOp);
6597
6598 EVT InVT = InOp.getValueType();
6599
6600 // Check if we can just return the input vector after widening.
6601 uint64_t IdxVal = Idx->getAsZExtVal();
6602 if (IdxVal == 0 && InVT == WidenVT)
6603 return InOp;
6604
6605 // Check if we can extract from the vector.
6606 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6607 unsigned InNumElts = InVT.getVectorMinNumElements();
6608 unsigned VTNumElts = VT.getVectorMinNumElements();
6609 assert(IdxVal % VTNumElts == 0 &&
6610 "Expected Idx to be a multiple of subvector minimum vector length");
6611 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6612 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6613
6614 if (VT.isScalableVector()) {
6615 // Try to split the operation up into smaller extracts and concat the
6616 // results together, e.g.
6617 // nxv6i64 extract_subvector(nxv12i64, 6)
6618 // <->
6619 // nxv8i64 concat(
6620 // nxv2i64 extract_subvector(nxv16i64, 6)
6621 // nxv2i64 extract_subvector(nxv16i64, 8)
6622 // nxv2i64 extract_subvector(nxv16i64, 10)
6623 // undef)
6624 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6625 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6626 "down type's element count");
6627 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6628 EC: ElementCount::getScalable(MinVal: GCD));
6629 // Avoid recursion around e.g. nxv1i8.
6630 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6631 SmallVector<SDValue> Parts;
6632 unsigned I = 0;
6633 for (; I < VTNumElts / GCD; ++I)
6634 Parts.push_back(
6635 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6636 for (; I < WidenNumElts / GCD; ++I)
6637 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6638
6639 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6640 }
6641
6642 // Fallback to extracting through memory.
6643
6644 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6645 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6646 MachineFunction &MF = DAG.getMachineFunction();
6647 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6648 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6649
6650 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6651 PtrInfo, F: MachineMemOperand::MOStore,
6652 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6653 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6654 PtrInfo, F: MachineMemOperand::MOLoad,
6655 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6656
6657 // Write out the input vector.
6658 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6659
6660 // Build a mask to match the length of the non-widened result.
6661 SDValue Mask =
6662 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6663
6664 // Read back the sub-vector setting the remaining lanes to poison.
6665 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6666 return DAG.getMaskedLoad(
6667 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6668 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6669 }
6670
6671 // We could try widening the input to the right length but for now, extract
6672 // the original elements, fill the rest with undefs and build a vector.
6673 SmallVector<SDValue, 16> Ops(WidenNumElts);
6674 unsigned i;
6675 for (i = 0; i < VTNumElts; ++i)
6676 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6677
6678 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6679 for (; i < WidenNumElts; ++i)
6680 Ops[i] = UndefVal;
6681 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6682}
6683
6684SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6685 SDValue InOp = ModifyToType(
6686 InOp: N->getOperand(Num: 0),
6687 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6688 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6689 N2: N->getOperand(Num: 1));
6690}
6691
6692SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6693 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6694 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6695 VT: InOp.getValueType(), N1: InOp,
6696 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6697}
6698
6699/// Either return the same load or provide appropriate casts
6700/// from the load and return that.
6701static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6702 TypeSize LdWidth, TypeSize FirstVTWidth,
6703 SDLoc dl, SelectionDAG &DAG) {
6704 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6705 "Load width must be less than or equal to first value type width");
6706 TypeSize WidenWidth = WidenVT.getSizeInBits();
6707 if (!FirstVT.isVector()) {
6708 unsigned NumElts =
6709 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6710 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6711 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6712 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6713 }
6714 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6715 return LdOp;
6716}
6717
6718/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6719/// widened value so it can be issued in a single atomic store.
6720static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6721 TypeSize FirstVTWidth, const SDLoc &dl,
6722 SelectionDAG &DAG) {
6723 TypeSize WidenWidth = WidenVT.getSizeInBits();
6724 if (!FirstVT.isVector()) {
6725 unsigned NumElts =
6726 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6727 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6728 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6729 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6730 }
6731 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6732 return StVal;
6733}
6734
6735static std::optional<EVT> findMemType(SelectionDAG &DAG,
6736 const TargetLowering &TLI, unsigned Width,
6737 EVT WidenVT, unsigned Align,
6738 unsigned WidenEx);
6739
6740SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6741 EVT WidenVT =
6742 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6743 EVT LdVT = LD->getMemoryVT();
6744 SDLoc dl(LD);
6745
6746 // Load information
6747 SDValue Chain = LD->getChain();
6748 SDValue BasePtr = LD->getBasePtr();
6749
6750 TypeSize LdWidth = LdVT.getSizeInBits();
6751 TypeSize WidenWidth = WidenVT.getSizeInBits();
6752 TypeSize WidthDiff = WidenWidth - LdWidth;
6753
6754 // Find the vector type that can load from.
6755 std::optional<EVT> FirstVT =
6756 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6757 WidenEx: WidthDiff.getKnownMinValue());
6758
6759 if (!FirstVT)
6760 return SDValue();
6761
6762 SmallVector<EVT, 8> MemVTs;
6763 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6764
6765 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6766 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6767
6768 // Load the element with one instruction.
6769 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6770 FirstVTWidth, dl, DAG);
6771
6772 // Modified the chain - switch anything that used the old chain to use
6773 // the new one.
6774 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6775 return Result;
6776}
6777
6778SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6779 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6780 ISD::LoadExtType ExtType = LD->getExtensionType();
6781
6782 // A vector must always be stored in memory as-is, i.e. without any padding
6783 // between the elements, since various code depend on it, e.g. in the
6784 // handling of a bitcast of a vector type to int, which may be done with a
6785 // vector store followed by an integer load. A vector that does not have
6786 // elements that are byte-sized must therefore be stored as an integer
6787 // built out of the extracted vector elements.
6788 if (!LD->getMemoryVT().isByteSized()) {
6789 SDValue Value, NewChain;
6790 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6791 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6792 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6793 return SDValue();
6794 }
6795
6796 // Generate a vector-predicated load if it is custom/legal on the target. To
6797 // avoid possible recursion, only do this if the widened mask type is legal.
6798 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6799 // removed from the IR by the ExpandVectorPredication pass but we're
6800 // reintroducing them here.
6801 EVT VT = LD->getValueType(ResNo: 0);
6802 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6803 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6804
6805 if (ExtType == ISD::NON_EXTLOAD &&
6806 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6807 TLI.isTypeLegal(VT: WideMaskVT)) {
6808 SDLoc DL(N);
6809 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6810 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6811 EC: VT.getVectorElementCount());
6812 SDValue NewLoad =
6813 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6814 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6815 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6816
6817 // Modified the chain - switch anything that used the old chain to use
6818 // the new one.
6819 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6820
6821 return NewLoad;
6822 }
6823
6824 SDValue Result;
6825 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6826 if (ExtType != ISD::NON_EXTLOAD)
6827 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6828 else
6829 Result = GenWidenVectorLoads(LdChain, LD);
6830
6831 if (Result) {
6832 // If we generate a single load, we can use that for the chain. Otherwise,
6833 // build a factor node to remember the multiple loads are independent and
6834 // chain to that.
6835 SDValue NewChain;
6836 if (LdChain.size() == 1)
6837 NewChain = LdChain[0];
6838 else
6839 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6840
6841 // Modified the chain - switch anything that used the old chain to use
6842 // the new one.
6843 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6844
6845 return Result;
6846 }
6847
6848 if (VT.isVector()) {
6849 // If all else fails replace the load with a wide masked load.
6850 SDLoc DL(N);
6851 SDValue Mask =
6852 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6853
6854 SDValue NewLoad = DAG.getMaskedLoad(
6855 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6856 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6857 AM: LD->getAddressingMode(), LD->getExtensionType());
6858
6859 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6860 return NewLoad;
6861 }
6862
6863 report_fatal_error(reason: "Unable to widen vector load");
6864}
6865
6866SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6867 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6868 SDValue Mask = N->getMask();
6869 SDValue EVL = N->getVectorLength();
6870 ISD::LoadExtType ExtType = N->getExtensionType();
6871 SDLoc dl(N);
6872
6873 // The mask should be widened as well
6874 assert(getTypeAction(Mask.getValueType()) ==
6875 TargetLowering::TypeWidenVector &&
6876 "Unable to widen binary VP op");
6877 Mask = GetWidenedVector(Op: Mask);
6878 assert(Mask.getValueType().getVectorElementCount() ==
6879 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6880 .getVectorElementCount() &&
6881 "Unable to widen vector load");
6882
6883 SDValue Res =
6884 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6885 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6886 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6887 // Legalize the chain result - switch anything that used the old chain to
6888 // use the new one.
6889 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6890 return Res;
6891}
6892
6893SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6894 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6895 SDValue Mask = N->getMask();
6896 SDValue EVL = N->getVectorLength();
6897 SDLoc dl(N);
6898
6899 // The mask should be widened as well
6900 assert(getTypeAction(Mask.getValueType()) ==
6901 TargetLowering::TypeWidenVector &&
6902 "Unable to widen binary VP op");
6903 Mask = GetWidenedVector(Op: Mask);
6904 assert(Mask.getValueType().getVectorElementCount() ==
6905 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6906 .getVectorElementCount() &&
6907 "Unable to widen vector load");
6908
6909 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6910 Mask, EVL, MMO: N->getMemOperand());
6911 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6912 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6913 return Res;
6914}
6915
6916SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6917 SDLoc DL(N);
6918
6919 // The mask should be widened as well
6920 SDValue Mask = N->getMask();
6921 assert(getTypeAction(Mask.getValueType()) ==
6922 TargetLowering::TypeWidenVector &&
6923 "Unable to widen VP strided load");
6924 Mask = GetWidenedVector(Op: Mask);
6925
6926 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6927 assert(Mask.getValueType().getVectorElementCount() ==
6928 WidenVT.getVectorElementCount() &&
6929 "Data and mask vectors should have the same number of elements");
6930
6931 SDValue Res = DAG.getStridedLoadVP(
6932 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6933 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6934 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6935 IsExpanding: N->isExpandingLoad());
6936
6937 // Legalize the chain result - switch anything that used the old chain to
6938 // use the new one.
6939 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6940 return Res;
6941}
6942
6943SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6944 SDValue Vec = N->getOperand(Num: 0);
6945 SDValue Mask = N->getOperand(Num: 1);
6946 SDValue Passthru = N->getOperand(Num: 2);
6947 EVT WideVecVT =
6948 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6949 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6950 VT: Mask.getValueType().getVectorElementType(),
6951 EC: WideVecVT.getVectorElementCount());
6952
6953 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6954 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6955 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6956 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6957 N2: WideMask, N3: WidePassthru);
6958}
6959
6960SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6961 EVT VT = N->getValueType(ResNo: 0);
6962 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6963 SDValue Mask = N->getMask();
6964 EVT MaskVT = Mask.getValueType();
6965 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6966 ISD::LoadExtType ExtType = N->getExtensionType();
6967 SDLoc dl(N);
6968
6969 EVT WideMaskVT =
6970 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6971 EC: WidenVT.getVectorElementCount());
6972
6973 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
6974 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6975 TLI.isTypeLegal(VT: WideMaskVT) &&
6976 // If there is a passthru, we shouldn't use vp.load. However,
6977 // type legalizer will struggle on masked.load with
6978 // scalable vectors, so for scalable vectors, we still use vp.load
6979 // but manually merge the load result with the passthru using vp.select.
6980 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6981 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6982 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6983 EC: VT.getVectorElementCount());
6984 SDValue NewLoad =
6985 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6986 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6987 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6988 SDValue NewVal = NewLoad;
6989
6990 // Manually merge with vselect
6991 if (!N->getPassThru()->isUndef()) {
6992 assert(WidenVT.isScalableVector());
6993 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
6994 // The lanes past EVL are poison.
6995 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
6996 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
6997 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
6998 }
6999
7000 // Modified the chain - switch anything that used the old chain to use
7001 // the new one.
7002 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7003
7004 return NewVal;
7005 }
7006
7007 // The mask should be widened as well
7008 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7009
7010 SDValue Res = DAG.getMaskedLoad(
7011 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7012 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7013 ExtType, IsExpanding: N->isExpandingLoad());
7014 // Legalize the chain result - switch anything that used the old chain to
7015 // use the new one.
7016 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7017 return Res;
7018}
7019
7020SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7021
7022 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7023 SDValue Mask = N->getMask();
7024 EVT MaskVT = Mask.getValueType();
7025 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7026 SDValue Scale = N->getScale();
7027 ElementCount WideEC = WideVT.getVectorElementCount();
7028 SDLoc dl(N);
7029
7030 // The mask should be widened as well
7031 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7032 VT: MaskVT.getVectorElementType(), EC: WideEC);
7033 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7034
7035 // Widen the Index operand
7036 SDValue Index = N->getIndex();
7037 EVT WideIndexVT = EVT::getVectorVT(
7038 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7039 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7040 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7041 Scale };
7042
7043 // Widen the MemoryType
7044 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7045 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7046 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7047 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7048 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7049
7050 // Legalize the chain result - switch anything that used the old chain to
7051 // use the new one.
7052 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7053 return Res;
7054}
7055
7056SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7057 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7058 SDValue Mask = N->getMask();
7059 SDValue Scale = N->getScale();
7060 ElementCount WideEC = WideVT.getVectorElementCount();
7061 SDLoc dl(N);
7062
7063 SDValue Index = GetWidenedVector(Op: N->getIndex());
7064 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7065 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7066 Mask = GetWidenedMask(Mask, EC: WideEC);
7067
7068 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7069 Mask, N->getVectorLength()};
7070 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7071 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7072
7073 // Legalize the chain result - switch anything that used the old chain to
7074 // use the new one.
7075 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7076 return Res;
7077}
7078
7079SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7080 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7081 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7082}
7083
7084// Return true is this is a SETCC node or a strict version of it.
7085static inline bool isSETCCOp(unsigned Opcode) {
7086 switch (Opcode) {
7087 case ISD::SETCC:
7088 case ISD::STRICT_FSETCC:
7089 case ISD::STRICT_FSETCCS:
7090 return true;
7091 }
7092 return false;
7093}
7094
7095// Return true if this is a node that could have two SETCCs as operands.
7096static inline bool isLogicalMaskOp(unsigned Opcode) {
7097 switch (Opcode) {
7098 case ISD::AND:
7099 case ISD::OR:
7100 case ISD::XOR:
7101 return true;
7102 }
7103 return false;
7104}
7105
7106// If N is a SETCC or a strict variant of it, return the type
7107// of the compare operands.
7108static inline EVT getSETCCOperandType(SDValue N) {
7109 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7110 return N->getOperand(Num: OpNo).getValueType();
7111}
7112
7113// This is used just for the assert in convertMask(). Check that this either
7114// a SETCC or a previously handled SETCC by convertMask().
7115#ifndef NDEBUG
7116static inline bool isSETCCorConvertedSETCC(SDValue N) {
7117 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7118 N = N.getOperand(0);
7119 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7120 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7121 if (!N->getOperand(i)->isUndef())
7122 return false;
7123 N = N.getOperand(0);
7124 }
7125
7126 if (N.getOpcode() == ISD::TRUNCATE)
7127 N = N.getOperand(0);
7128 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7129 N = N.getOperand(0);
7130
7131 if (isLogicalMaskOp(N.getOpcode()))
7132 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7133 isSETCCorConvertedSETCC(N.getOperand(1));
7134
7135 return (isSETCCOp(N.getOpcode()) ||
7136 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7137}
7138#endif
7139
7140// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7141// to ToMaskVT if needed with vector extension or truncation.
7142SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7143 EVT ToMaskVT) {
7144 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
7145 // FIXME: This code seems to be too restrictive, we might consider
7146 // generalizing it or dropping it.
7147 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7148
7149 // Make a new Mask node, with a legal result VT.
7150 SDValue Mask;
7151 SmallVector<SDValue, 4> Ops;
7152 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7153 Ops.push_back(Elt: InMask->getOperand(Num: i));
7154 if (InMask->isStrictFPOpcode()) {
7155 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7156 ResultTys: { MaskVT, MVT::Other }, Ops);
7157 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7158 }
7159 else
7160 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7161 Flags: InMask->getFlags());
7162
7163 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
7164 // extend or truncate is needed.
7165 LLVMContext &Ctx = *DAG.getContext();
7166 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7167 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7168 if (MaskScalarBits < ToMaskScalBits) {
7169 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7170 NumElements: MaskVT.getVectorNumElements());
7171 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7172 } else if (MaskScalarBits > ToMaskScalBits) {
7173 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7174 NumElements: MaskVT.getVectorNumElements());
7175 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7176 }
7177
7178 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7179 ToMaskVT.getScalarSizeInBits() &&
7180 "Mask should have the right element size by now.");
7181
7182 // Adjust Mask to the right number of elements.
7183 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7184 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7185 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7186 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7187 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7188 EVT SubVT = Mask->getValueType(ResNo: 0);
7189 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7190 SubOps[0] = Mask;
7191 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7192 }
7193
7194 assert((Mask->getValueType(0) == ToMaskVT) &&
7195 "A mask of ToMaskVT should have been produced by now.");
7196
7197 return Mask;
7198}
7199
7200// This method tries to handle some special cases for the vselect mask
7201// and if needed adjusting the mask vector type to match that of the VSELECT.
7202// Without it, many cases end up with scalarization of the SETCC, with many
7203// unnecessary instructions.
7204SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7205 LLVMContext &Ctx = *DAG.getContext();
7206 SDValue Cond = N->getOperand(Num: 0);
7207
7208 if (N->getOpcode() != ISD::VSELECT)
7209 return SDValue();
7210
7211 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
7212 return SDValue();
7213
7214 // If this is a splitted VSELECT that was previously already handled, do
7215 // nothing.
7216 EVT CondVT = Cond->getValueType(ResNo: 0);
7217 if (CondVT.getScalarSizeInBits() != 1)
7218 return SDValue();
7219
7220 EVT VSelVT = N->getValueType(ResNo: 0);
7221
7222 // This method can't handle scalable vector types.
7223 // FIXME: This support could be added in the future.
7224 if (VSelVT.isScalableVector())
7225 return SDValue();
7226
7227 // Only handle vector types which are a power of 2.
7228 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7229 return SDValue();
7230
7231 // Don't touch if this will be scalarized.
7232 EVT FinalVT = VSelVT;
7233 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7234 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7235
7236 if (FinalVT.getVectorNumElements() == 1)
7237 return SDValue();
7238
7239 // If there is support for an i1 vector mask, don't touch.
7240 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7241 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7242 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7243 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7244 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7245 if (SetCCResVT.getScalarSizeInBits() == 1)
7246 return SDValue();
7247 } else if (CondVT.getScalarType() == MVT::i1) {
7248 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7249 // don't touch.
7250 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7251 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7252
7253 if (CondVT.getScalarType() == MVT::i1)
7254 return SDValue();
7255 }
7256
7257 // Widen the vselect result type if needed.
7258 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7259 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7260
7261 // The mask of the VSELECT should have integer elements.
7262 EVT ToMaskVT = VSelVT;
7263 if (!ToMaskVT.getScalarType().isInteger())
7264 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7265
7266 SDValue Mask;
7267 if (isSETCCOp(Opcode: Cond->getOpcode())) {
7268 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
7269 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7270 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
7271 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
7272 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
7273 // Cond is (AND/OR/XOR (SETCC, SETCC))
7274 SDValue SETCC0 = Cond->getOperand(Num: 0);
7275 SDValue SETCC1 = Cond->getOperand(Num: 1);
7276 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
7277 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
7278 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
7279 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
7280 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
7281 EVT MaskVT;
7282 // If the two SETCCs have different VTs, either extend/truncate one of
7283 // them to the other "towards" ToMaskVT, or truncate one and extend the
7284 // other to ToMaskVT.
7285 if (ScalarBits0 != ScalarBits1) {
7286 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
7287 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
7288 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
7289 MaskVT = WideVT;
7290 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
7291 MaskVT = NarrowVT;
7292 else
7293 MaskVT = ToMaskVT;
7294 } else
7295 // If the two SETCCs have the same VT, don't change it.
7296 MaskVT = VT0;
7297
7298 // Make new SETCCs and logical nodes.
7299 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
7300 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
7301 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
7302
7303 // Convert the logical op for VSELECT if needed.
7304 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7305 } else
7306 return SDValue();
7307
7308 return Mask;
7309}
7310
7311SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7312 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7313 ElementCount WidenEC = WidenVT.getVectorElementCount();
7314
7315 SDValue Cond1 = N->getOperand(Num: 0);
7316 EVT CondVT = Cond1.getValueType();
7317 unsigned Opcode = N->getOpcode();
7318 if (CondVT.isVector()) {
7319 if (SDValue WideCond = WidenVSELECTMask(N)) {
7320 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7321 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7322 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7323 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7324 }
7325
7326 EVT CondEltVT = CondVT.getVectorElementType();
7327 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7328 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7329 Cond1 = GetWidenedVector(Op: Cond1);
7330
7331 // If we have to split the condition there is no point in widening the
7332 // select. This would result in an cycle of widening the select ->
7333 // widening the condition operand -> splitting the condition operand ->
7334 // splitting the select -> widening the select. Instead split this select
7335 // further and widen the resulting type.
7336 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7337 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7338 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7339 return Res;
7340 }
7341
7342 if (Cond1.getValueType() != CondWidenVT)
7343 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7344 }
7345
7346 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7347 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7348 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7349 if (Opcode == ISD::VP_MERGE)
7350 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7351 N4: N->getOperand(Num: 3));
7352 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7353}
7354
7355SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7356 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7357 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7358 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7359 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7360 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7361}
7362
7363SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7364 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7365 return DAG.getUNDEF(VT: WidenVT);
7366}
7367
7368SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7369 EVT VT = N->getValueType(ResNo: 0);
7370 SDLoc dl(N);
7371
7372 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7373 unsigned NumElts = VT.getVectorNumElements();
7374 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7375
7376 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7377 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7378
7379 // Adjust mask based on new input vector length.
7380 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7381 for (unsigned i = 0; i != NumElts; ++i) {
7382 int Idx = N->getMaskElt(Idx: i);
7383 if (Idx < (int)NumElts)
7384 NewMask[i] = Idx;
7385 else
7386 NewMask[i] = Idx - NumElts + WidenNumElts;
7387 }
7388 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7389}
7390
7391SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7392 EVT VT = N->getValueType(ResNo: 0);
7393 EVT EltVT = VT.getVectorElementType();
7394 SDLoc dl(N);
7395
7396 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7397 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7398 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7399
7400 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7401 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7402 unsigned VTNumElts = VT.getVectorMinNumElements();
7403 unsigned IdxVal = WidenNumElts - VTNumElts;
7404
7405 if (VT.isScalableVector()) {
7406 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7407 // results together, e.g.(nxv6i64 -> nxv8i64)
7408 // nxv8i64 vector_reverse
7409 // <->
7410 // nxv8i64 concat(
7411 // nxv2i64 extract_subvector(nxv8i64, 2)
7412 // nxv2i64 extract_subvector(nxv8i64, 4)
7413 // nxv2i64 extract_subvector(nxv8i64, 6)
7414 // nxv2i64 undef)
7415
7416 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7417 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7418 EC: ElementCount::getScalable(MinVal: GCD));
7419 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7420 "down type's element count");
7421 SmallVector<SDValue> Parts;
7422 unsigned i = 0;
7423 for (; i < VTNumElts / GCD; ++i)
7424 Parts.push_back(
7425 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7426 for (; i < WidenNumElts / GCD; ++i)
7427 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7428
7429 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7430 }
7431
7432 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7433 // fixed-vectors.
7434 SmallVector<int, 16> Mask(WidenNumElts, -1);
7435 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7436
7437 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7438 Mask);
7439}
7440
7441SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7442 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7443 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7444}
7445
7446void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7447 EVT VT = N->getValueType(ResNo: 0);
7448 EVT EltVT = VT.getVectorElementType();
7449 ElementCount OrigEC = VT.getVectorElementCount();
7450 unsigned Factor = N->getNumOperands();
7451 SDLoc DL(N);
7452
7453 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7454 ElementCount WidenEC = WidenVT.getVectorElementCount();
7455
7456 SmallVector<SDValue, 8> WidenOps(Factor);
7457 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7458 WidenOps[Idx] = GetWidenedVector(Op: N->getOperand(Num: Idx));
7459
7460 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7461 SDValue Interleaved =
7462 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: WidenVTs, Ops: WidenOps);
7463
7464 EVT PackedWidenVT =
7465 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7466 SmallVector<SDValue, 8> Slices(Factor);
7467 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7468 Slices[Idx] = Interleaved.getValue(R: Idx);
7469
7470 SDValue Packed = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedWidenVT, Ops: Slices);
7471
7472 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7473 SDValue Narrow = DAG.getExtractSubvector(DL, VT, Vec: Packed,
7474 Idx: OrigEC.getKnownMinValue() * Idx);
7475 SDValue Wide =
7476 DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: Narrow, /*Idx=*/0U);
7477 SetWidenedVector(Op: SDValue(N, Idx), Result: Wide);
7478 }
7479}
7480
7481SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7482 SDLoc DL(N);
7483 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7484 EVT SourceVT = N->getOperand(Num: 0).getValueType();
7485 EVT WideSourceVT =
7486 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceVT.getVectorElementType(),
7487 EC: WidenVT.getVectorElementCount());
7488
7489 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
7490 SubVec: N->getOperand(Num: 0), Idx: 0);
7491 SDValue WideMask = DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT),
7492 SubVec: N->getOperand(Num: 2), Idx: 0);
7493 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
7494 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
7495}
7496
7497void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7498 EVT VT = N->getValueType(ResNo: 0);
7499 EVT EltVT = VT.getVectorElementType();
7500 ElementCount OrigEC = VT.getVectorElementCount();
7501 unsigned Factor = N->getNumOperands();
7502 SDLoc DL(N);
7503
7504 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7505 ElementCount WidenEC = WidenVT.getVectorElementCount();
7506 // We cannot just use the widened operands directly: since they might be
7507 // individually widened, using them directly will result in de-interleaving
7508 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7509 // not concat the widened operands but the original ones to effectively
7510 // generate a "packed" concated and widened vector, before extracting new
7511 // operand vectors with the widened type.
7512 EVT PackedWidenVT =
7513 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7514 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: OrigEC * Factor);
7515 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7516 SDValue PackedWidenVec = DAG.getInsertSubvector(
7517 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7518
7519 // Extract the new widened operand vectors.
7520 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7521 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7522 NewOps[Idx] = DAG.getExtractSubvector(DL, VT: WidenVT, Vec: PackedWidenVec,
7523 Idx: WidenEC.getKnownMinValue() * Idx);
7524 }
7525
7526 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7527 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7528 // Set the widened results manually.
7529 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7530 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7531}
7532
7533SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7534 assert(N->getValueType(0).isVector() &&
7535 N->getOperand(0).getValueType().isVector() &&
7536 "Operands must be vectors");
7537 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7538 ElementCount WidenEC = WidenVT.getVectorElementCount();
7539
7540 SDValue InOp1 = N->getOperand(Num: 0);
7541 EVT InVT = InOp1.getValueType();
7542 assert(InVT.isVector() && "can not widen non-vector type");
7543 EVT WidenInVT =
7544 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7545
7546 // The input and output types often differ here, and it could be that while
7547 // we'd prefer to widen the result type, the input operands have been split.
7548 // In this case, we also need to split the result of this node as well.
7549 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7550 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7551 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7552 return Res;
7553 }
7554
7555 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7556 SDValue InOp2 = N->getOperand(Num: 1);
7557 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7558 InOp1 = GetWidenedVector(Op: InOp1);
7559 InOp2 = GetWidenedVector(Op: InOp2);
7560 } else {
7561 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7562 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7563 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7564 N2: InOp1, N3: ZeroIdx);
7565 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7566 N2: InOp2, N3: ZeroIdx);
7567 }
7568
7569 // Assume that the input and output will be widen appropriately. If not,
7570 // we will have to unroll it at some point.
7571 assert(InOp1.getValueType() == WidenInVT &&
7572 InOp2.getValueType() == WidenInVT &&
7573 "Input not widened to expected type!");
7574 (void)WidenInVT;
7575 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7576 N3: N->getOperand(Num: 2));
7577}
7578
7579SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7580 assert(N->getValueType(0).isVector() &&
7581 N->getOperand(1).getValueType().isVector() &&
7582 "Operands must be vectors");
7583 EVT VT = N->getValueType(ResNo: 0);
7584 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7585 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7586 unsigned NumElts = VT.getVectorNumElements();
7587 EVT EltVT = VT.getVectorElementType();
7588
7589 SDLoc dl(N);
7590 SDValue Chain = N->getOperand(Num: 0);
7591 SDValue LHS = N->getOperand(Num: 1);
7592 SDValue RHS = N->getOperand(Num: 2);
7593 SDValue CC = N->getOperand(Num: 3);
7594 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7595
7596 // Fully unroll and reassemble.
7597 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7598 SmallVector<SDValue, 8> Chains(NumElts);
7599 for (unsigned i = 0; i != NumElts; ++i) {
7600 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7601 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7602
7603 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7604 Ops: {Chain, LHSElem, RHSElem, CC});
7605 Chains[i] = Scalars[i].getValue(R: 1);
7606 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7607 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7608 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7609 }
7610
7611 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7612 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7613
7614 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7615}
7616
7617SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
7618 SDLoc DL(N);
7619 EVT VT = N->getValueType(ResNo: 0);
7620
7621 // Expand, then widen the result.
7622 SDValue Expanded = TLI.expandPartialReduceMLA(Node: N, DAG);
7623 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7624 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WideVT), SubVec: Expanded, Idx: 0);
7625}
7626
7627//===----------------------------------------------------------------------===//
7628// Widen Vector Operand
7629//===----------------------------------------------------------------------===//
7630bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7631 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7632 SDValue Res = SDValue();
7633
7634 // See if the target wants to custom widen this node.
7635 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7636 return false;
7637
7638 switch (N->getOpcode()) {
7639 default:
7640#ifndef NDEBUG
7641 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7642 N->dump(&DAG);
7643 dbgs() << "\n";
7644#endif
7645 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7646
7647 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7648 case ISD::FAKE_USE:
7649 Res = WidenVecOp_FAKE_USE(N);
7650 break;
7651 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7652 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7653 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7654 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7655 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7656 case ISD::ATOMIC_STORE:
7657 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7658 break;
7659 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7660 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7661 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7662 break;
7663 case ISD::ANY_EXTEND_VECTOR_INREG:
7664 case ISD::SIGN_EXTEND_VECTOR_INREG:
7665 case ISD::ZERO_EXTEND_VECTOR_INREG:
7666 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7667 break;
7668 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7669 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7670 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7671 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7672 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7673 case ISD::STRICT_FSETCC:
7674 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7675 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7676 case ISD::FLDEXP:
7677 case ISD::FCOPYSIGN:
7678 case ISD::LROUND:
7679 case ISD::LLROUND:
7680 case ISD::LRINT:
7681 case ISD::LLRINT:
7682 Res = WidenVecOp_UnrollVectorOp(N);
7683 break;
7684 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7685
7686 case ISD::ANY_EXTEND:
7687 case ISD::SIGN_EXTEND:
7688 case ISD::ZERO_EXTEND:
7689 Res = WidenVecOp_EXTEND(N);
7690 break;
7691
7692 case ISD::SCMP:
7693 case ISD::UCMP:
7694 Res = WidenVecOp_CMP(N);
7695 break;
7696
7697 case ISD::FP_EXTEND:
7698 case ISD::STRICT_FP_EXTEND:
7699 case ISD::FP_ROUND:
7700 case ISD::STRICT_FP_ROUND:
7701 case ISD::FP_TO_SINT:
7702 case ISD::STRICT_FP_TO_SINT:
7703 case ISD::FP_TO_UINT:
7704 case ISD::STRICT_FP_TO_UINT:
7705 case ISD::SINT_TO_FP:
7706 case ISD::STRICT_SINT_TO_FP:
7707 case ISD::UINT_TO_FP:
7708 case ISD::STRICT_UINT_TO_FP:
7709 case ISD::TRUNCATE:
7710 case ISD::CONVERT_FROM_ARBITRARY_FP:
7711 case ISD::CONVERT_TO_ARBITRARY_FP:
7712 Res = WidenVecOp_Convert(N);
7713 break;
7714
7715 case ISD::FP_TO_SINT_SAT:
7716 case ISD::FP_TO_UINT_SAT:
7717 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7718 break;
7719
7720 case ISD::VECREDUCE_FADD:
7721 case ISD::VECREDUCE_FMUL:
7722 case ISD::VECREDUCE_ADD:
7723 case ISD::VECREDUCE_MUL:
7724 case ISD::VECREDUCE_AND:
7725 case ISD::VECREDUCE_OR:
7726 case ISD::VECREDUCE_XOR:
7727 case ISD::VECREDUCE_SMAX:
7728 case ISD::VECREDUCE_SMIN:
7729 case ISD::VECREDUCE_UMAX:
7730 case ISD::VECREDUCE_UMIN:
7731 case ISD::VECREDUCE_FMAX:
7732 case ISD::VECREDUCE_FMIN:
7733 case ISD::VECREDUCE_FMAXIMUM:
7734 case ISD::VECREDUCE_FMINIMUM:
7735 case ISD::VECREDUCE_FMAXIMUMNUM:
7736 case ISD::VECREDUCE_FMINIMUMNUM:
7737 Res = WidenVecOp_VECREDUCE(N);
7738 break;
7739 case ISD::VECREDUCE_SEQ_FADD:
7740 case ISD::VECREDUCE_SEQ_FMUL:
7741 Res = WidenVecOp_VECREDUCE_SEQ(N);
7742 break;
7743 case ISD::VP_REDUCE_FADD:
7744 case ISD::VP_REDUCE_SEQ_FADD:
7745 case ISD::VP_REDUCE_FMUL:
7746 case ISD::VP_REDUCE_SEQ_FMUL:
7747 case ISD::VP_REDUCE_ADD:
7748 case ISD::VP_REDUCE_MUL:
7749 case ISD::VP_REDUCE_AND:
7750 case ISD::VP_REDUCE_OR:
7751 case ISD::VP_REDUCE_XOR:
7752 case ISD::VP_REDUCE_SMAX:
7753 case ISD::VP_REDUCE_SMIN:
7754 case ISD::VP_REDUCE_UMAX:
7755 case ISD::VP_REDUCE_UMIN:
7756 case ISD::VP_REDUCE_FMAX:
7757 case ISD::VP_REDUCE_FMIN:
7758 case ISD::VP_REDUCE_FMAXIMUM:
7759 case ISD::VP_REDUCE_FMINIMUM:
7760 Res = WidenVecOp_VP_REDUCE(N);
7761 break;
7762 case ISD::CTTZ_ELTS:
7763 case ISD::CTTZ_ELTS_ZERO_POISON:
7764 Res = WidenVecOp_CttzElements(N);
7765 break;
7766 case ISD::VP_CTTZ_ELTS:
7767 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
7768 Res = WidenVecOp_VP_CttzElements(N);
7769 break;
7770 case ISD::VECTOR_FIND_LAST_ACTIVE:
7771 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7772 break;
7773 case ISD::VECTOR_MATCH:
7774 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
7775 break;
7776 }
7777
7778 // If Res is null, the sub-method took care of registering the result.
7779 if (!Res.getNode()) return false;
7780
7781 // If the result is N, the sub-method updated N in place. Tell the legalizer
7782 // core about this.
7783 if (Res.getNode() == N)
7784 return true;
7785
7786
7787 if (N->isStrictFPOpcode())
7788 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7789 "Invalid operand expansion");
7790 else
7791 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7792 "Invalid operand expansion");
7793
7794 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7795 return false;
7796}
7797
7798SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7799 SDLoc DL(N);
7800 EVT VT = N->getValueType(ResNo: 0);
7801
7802 SDValue InOp = N->getOperand(Num: 0);
7803 assert(getTypeAction(InOp.getValueType()) ==
7804 TargetLowering::TypeWidenVector &&
7805 "Unexpected type action");
7806 InOp = GetWidenedVector(Op: InOp);
7807 assert(VT.getVectorNumElements() <
7808 InOp.getValueType().getVectorNumElements() &&
7809 "Input wasn't widened!");
7810
7811 // We may need to further widen the operand until it has the same total
7812 // vector size as the result.
7813 EVT InVT = InOp.getValueType();
7814 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7815 EVT InEltVT = InVT.getVectorElementType();
7816 for (EVT FixedVT : MVT::vector_valuetypes()) {
7817 EVT FixedEltVT = FixedVT.getVectorElementType();
7818 if (TLI.isTypeLegal(VT: FixedVT) &&
7819 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7820 FixedEltVT == InEltVT) {
7821 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7822 "Not enough elements in the fixed type for the operand!");
7823 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7824 "We can't have the same type as we started with!");
7825 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7826 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7827 else
7828 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7829 break;
7830 }
7831 }
7832 InVT = InOp.getValueType();
7833 if (InVT.getSizeInBits() != VT.getSizeInBits())
7834 // We couldn't find a legal vector type that was a widening of the input
7835 // and could be extended in-register to the result type, so we have to
7836 // scalarize.
7837 return WidenVecOp_Convert(N);
7838 }
7839
7840 // Use special DAG nodes to represent the operation of extending the
7841 // low lanes.
7842 switch (N->getOpcode()) {
7843 default:
7844 llvm_unreachable("Extend legalization on extend operation!");
7845 case ISD::ANY_EXTEND:
7846 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7847 case ISD::SIGN_EXTEND:
7848 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7849 case ISD::ZERO_EXTEND:
7850 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7851 }
7852}
7853
7854SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7855 SDLoc dl(N);
7856
7857 EVT OpVT = N->getOperand(Num: 0).getValueType();
7858 EVT ResVT = N->getValueType(ResNo: 0);
7859 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7860 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7861
7862 // 1. EXTRACT_SUBVECTOR
7863 // 2. SIGN_EXTEND/ZERO_EXTEND
7864 // 3. CMP
7865 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7866 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7867
7868 // At this point the result type is guaranteed to be valid, so we can use it
7869 // as the operand type by extending it appropriately
7870 ISD::NodeType ExtendOpcode =
7871 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7872 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7873 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7874
7875 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7876}
7877
7878SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7879 // The result (and first input) is legal, but the second input is illegal.
7880 // We can't do much to fix that, so just unroll and let the extracts off of
7881 // the second input be widened as needed later.
7882 return DAG.UnrollVectorOp(N);
7883}
7884
7885SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7886 SDLoc DL(N);
7887 EVT ResultVT = N->getValueType(ResNo: 0);
7888 SDValue Test = N->getOperand(Num: 1);
7889 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7890
7891 // Process this node similarly to SETCC.
7892 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7893 if (ResultVT.getScalarType() == MVT::i1)
7894 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7895 NumElements: WideResultVT.getVectorNumElements());
7896
7897 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7898 Ops: {WideArg, Test}, Flags: N->getFlags());
7899
7900 // Extract the needed results from the result vector.
7901 EVT ResVT =
7902 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7903 NumElements: ResultVT.getVectorNumElements());
7904 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7905
7906 EVT OpVT = N->getOperand(Num: 0).getValueType();
7907 ISD::NodeType ExtendCode =
7908 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7909 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7910}
7911
7912SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7913 // Since the result is legal and the input is illegal.
7914 EVT VT = N->getValueType(ResNo: 0);
7915 EVT EltVT = VT.getVectorElementType();
7916 SDLoc dl(N);
7917 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7918 assert(getTypeAction(InOp.getValueType()) ==
7919 TargetLowering::TypeWidenVector &&
7920 "Unexpected type action");
7921 InOp = GetWidenedVector(Op: InOp);
7922 EVT InVT = InOp.getValueType();
7923 unsigned Opcode = N->getOpcode();
7924
7925 // Helper to build a convert node with all scalar trailing operands.
7926 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
7927 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
7928 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
7929 N4: N->getOperand(Num: 3));
7930 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7931 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
7932 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
7933 };
7934
7935 // See if a widened result type would be legal, if so widen the node.
7936 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7937 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7938 EC: InVT.getVectorElementCount());
7939 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7940 SDValue Res;
7941 if (N->isStrictFPOpcode()) {
7942 if (Opcode == ISD::STRICT_FP_ROUND)
7943 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7944 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7945 else
7946 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7947 Ops: { N->getOperand(Num: 0), InOp });
7948 // Legalize the chain result - switch anything that used the old chain to
7949 // use the new one.
7950 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7951 } else {
7952 Res = MakeConvertNode(WideVT, InOp);
7953 }
7954 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7955 }
7956
7957 EVT InEltVT = InVT.getVectorElementType();
7958
7959 // Unroll the convert into some scalar code and create a nasty build vector.
7960 unsigned NumElts = VT.getVectorNumElements();
7961 SmallVector<SDValue, 16> Ops(NumElts);
7962 if (N->isStrictFPOpcode()) {
7963 SmallVector<SDValue, 4> NewOps(N->ops());
7964 SmallVector<SDValue, 32> OpChains;
7965 for (unsigned i=0; i < NumElts; ++i) {
7966 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7967 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7968 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7969 }
7970 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7971 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7972 } else {
7973 for (unsigned i = 0; i < NumElts; ++i) {
7974 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7975 Ops[i] = MakeConvertNode(EltVT, Elt);
7976 }
7977 }
7978
7979 return DAG.getBuildVector(VT, DL: dl, Ops);
7980}
7981
7982SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7983 EVT DstVT = N->getValueType(ResNo: 0);
7984 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7985 EVT SrcVT = Src.getValueType();
7986 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7987 SDLoc dl(N);
7988
7989 // See if a widened result type would be legal, if so widen the node.
7990 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7991 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7992 if (TLI.isTypeLegal(VT: WideDstVT)) {
7993 SDValue Res =
7994 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7995 return DAG.getNode(
7996 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7997 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7998 }
7999
8000 // Give up and unroll.
8001 return DAG.UnrollVectorOp(N);
8002}
8003
8004SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
8005 EVT VT = N->getValueType(ResNo: 0);
8006 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8007 EVT InWidenVT = InOp.getValueType();
8008 SDLoc dl(N);
8009
8010 // Check if we can convert between two legal vector types and extract.
8011 TypeSize InWidenSize = InWidenVT.getSizeInBits();
8012 TypeSize Size = VT.getSizeInBits();
8013 // x86mmx is not an acceptable vector element type, so don't try.
8014 if (!VT.isVector() && VT != MVT::x86mmx &&
8015 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
8016 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
8017 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
8018 if (TLI.isTypeLegal(VT: NewVT)) {
8019 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8020 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
8021 }
8022 }
8023
8024 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8025 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8026 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8027 // having to copy via memory.
8028 if (VT.isVector()) {
8029 EVT EltVT = VT.getVectorElementType();
8030 unsigned EltSize = EltVT.getFixedSizeInBits();
8031 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
8032 ElementCount NewNumElts =
8033 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8034 .divideCoefficientBy(RHS: EltSize);
8035 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8036 if (TLI.isTypeLegal(VT: NewVT)) {
8037 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8038 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8039 }
8040 }
8041 }
8042
8043 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8044}
8045
8046// Vectors with sizes that are not powers of 2 need to be widened to the
8047// next largest power of 2. For example, we may get a vector of 3 32-bit
8048// integers or of 6 16-bit integers, both of which have to be widened to a
8049// 128-bit vector.
8050SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8051 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8052 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8053 N2: WidenedOp);
8054}
8055
8056SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8057 EVT VT = N->getValueType(ResNo: 0);
8058 EVT EltVT = VT.getVectorElementType();
8059 EVT InVT = N->getOperand(Num: 0).getValueType();
8060 SDLoc dl(N);
8061
8062 // If the widen width for this operand is the same as the width of the concat
8063 // and all but the first operand is undef, just use the widened operand.
8064 unsigned NumOperands = N->getNumOperands();
8065 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8066 unsigned i;
8067 for (i = 1; i < NumOperands; ++i)
8068 if (!N->getOperand(Num: i).isUndef())
8069 break;
8070
8071 if (i == NumOperands)
8072 return GetWidenedVector(Op: N->getOperand(Num: 0));
8073 }
8074
8075 if (VT.isScalableVector()) {
8076 SDValue Result = DAG.getPOISON(VT);
8077 unsigned NumInElts = InVT.getVectorMinNumElements();
8078 for (unsigned i = 0; i < NumOperands; ++i) {
8079 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: i));
8080 if (InOp.getValueType() != InVT)
8081 InOp = DAG.getExtractSubvector(DL: dl, VT: InVT, Vec: InOp, Idx: 0);
8082 Result = DAG.getInsertSubvector(DL: dl, Vec: Result, SubVec: InOp, Idx: i * NumInElts);
8083 }
8084 return Result;
8085 }
8086
8087 // Otherwise, fall back to a nasty build vector.
8088 unsigned NumElts = VT.getVectorNumElements();
8089 SmallVector<SDValue, 16> Ops(NumElts);
8090
8091 unsigned NumInElts = InVT.getVectorNumElements();
8092
8093 unsigned Idx = 0;
8094 for (unsigned i=0; i < NumOperands; ++i) {
8095 SDValue InOp = N->getOperand(Num: i);
8096 assert(getTypeAction(InOp.getValueType()) ==
8097 TargetLowering::TypeWidenVector &&
8098 "Unexpected type action");
8099 InOp = GetWidenedVector(Op: InOp);
8100 for (unsigned j = 0; j < NumInElts; ++j)
8101 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8102 }
8103 return DAG.getBuildVector(VT, DL: dl, Ops);
8104}
8105
8106SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8107 EVT VT = N->getValueType(ResNo: 0);
8108 SDValue SubVec = N->getOperand(Num: 1);
8109 SDValue InVec = N->getOperand(Num: 0);
8110
8111 EVT OrigVT = SubVec.getValueType();
8112 SubVec = GetWidenedVector(Op: SubVec);
8113 EVT SubVT = SubVec.getValueType();
8114
8115 // Whether or not all the elements of the widened SubVec will be inserted into
8116 // valid indices of VT.
8117 bool IndicesValid = false;
8118 // If we statically know that VT can fit SubVT, the indices are valid.
8119 if (VT.knownBitsGE(VT: SubVT))
8120 IndicesValid = true;
8121 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8122 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8123 // we know the minimum vscale we can work out if it's valid ourselves.
8124 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8125 Kind: Attribute::VScaleRange);
8126 if (Attr.isValid()) {
8127 unsigned VScaleMin = Attr.getVScaleRangeMin();
8128 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8129 SubVT.getFixedSizeInBits())
8130 IndicesValid = true;
8131 }
8132 }
8133
8134 if (!IndicesValid)
8135 report_fatal_error(
8136 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8137
8138 SDLoc DL(N);
8139
8140 // We need to make sure that the indices are still valid, otherwise we might
8141 // widen what was previously well-defined to something undefined.
8142 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8143 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8144 N3: N->getOperand(Num: 2));
8145
8146 if (OrigVT.isScalableVector()) {
8147 // When the widened types match, overwriting the start of a vector is
8148 // effectively a merge operation that can be implement as a vselect.
8149 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8150 SDValue Mask =
8151 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8152 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8153 }
8154
8155 // Fallback to inserting through memory.
8156 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8157 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8158 MachineFunction &MF = DAG.getMachineFunction();
8159 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8160 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8161
8162 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8163 PtrInfo, F: MachineMemOperand::MOStore,
8164 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8165 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8166 PtrInfo, F: MachineMemOperand::MOLoad,
8167 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8168
8169 // Write out the vector being inserting into.
8170 SDValue Ch =
8171 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8172
8173 // Build a mask to match the length of the sub-vector.
8174 SDValue Mask =
8175 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8176
8177 // Overwrite the sub-vector at the required offset.
8178 SDValue SubVecPtr =
8179 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8180 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8181 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8182 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8183
8184 // Read back the result.
8185 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8186 }
8187
8188 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8189 // with a series of INSERT_VECTOR_ELT
8190 unsigned Idx = N->getConstantOperandVal(Num: 2);
8191
8192 SDValue InsertElt = InVec;
8193 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8194 SDValue ExtractElt =
8195 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8196 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8197 }
8198
8199 return InsertElt;
8200}
8201
8202SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8203 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8204 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8205 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8206}
8207
8208SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8209 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8210 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8211 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8212}
8213
8214SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8215 SDLoc DL(N);
8216 EVT ResVT = N->getValueType(ResNo: 0);
8217
8218 // Widen the input as requested by the legalizer.
8219 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8220 EVT WideInVT = WideInOp.getValueType();
8221
8222 // Simple case: if widened input is still smaller than or equal to result,
8223 // just use it directly.
8224 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8225 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8226
8227 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8228 // If widening makes the input larger than the original result, widen the
8229 // result to match, then extract back down.
8230 EVT ResEltVT = ResVT.getVectorElementType();
8231 unsigned EltBits = ResEltVT.getSizeInBits();
8232 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8233 "Widened input size must be a multiple of result element size");
8234
8235 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8236 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8237
8238 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8239 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8240}
8241
8242SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8243 // We have to widen the value, but we want only to store the original
8244 // vector type.
8245 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8246
8247 if (!ST->getMemoryVT().getScalarType().isByteSized())
8248 return TLI.scalarizeVectorStore(ST, DAG);
8249
8250 if (ST->isTruncatingStore())
8251 return TLI.scalarizeVectorStore(ST, DAG);
8252
8253 // Generate a vector-predicated store if it is custom/legal on the target.
8254 // To avoid possible recursion, only do this if the widened mask type is
8255 // legal.
8256 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8257 // removed from the IR by the ExpandVectorPredication pass but we're
8258 // reintroducing them here.
8259 SDValue StVal = ST->getValue();
8260 EVT StVT = StVal.getValueType();
8261 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8262 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8263
8264 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8265 TLI.isTypeLegal(VT: WideMaskVT)) {
8266 // Widen the value.
8267 SDLoc DL(N);
8268 StVal = GetWidenedVector(Op: StVal);
8269 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8270 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8271 EC: StVT.getVectorElementCount());
8272 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8273 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8274 AM: ST->getAddressingMode());
8275 }
8276
8277 SmallVector<SDValue, 16> StChain;
8278 if (GenWidenVectorStores(StChain, ST)) {
8279 if (StChain.size() == 1)
8280 return StChain[0];
8281
8282 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8283 }
8284
8285 if (StVT.isVector()) {
8286 // If all else fails replace the store with a wide masked store.
8287 SDLoc DL(N);
8288 SDValue WideStVal = GetWidenedVector(Op: StVal);
8289 SDValue Mask =
8290 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8291
8292 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8293 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8294 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8295 IsTruncating: ST->isTruncatingStore());
8296 }
8297
8298 report_fatal_error(reason: "Unable to widen vector store");
8299}
8300
8301SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8302 EVT StVT = ST->getMemoryVT();
8303 SDLoc dl(ST);
8304
8305 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8306 EVT WidenVT = StVal.getValueType();
8307
8308 TypeSize StWidth = StVT.getSizeInBits();
8309 TypeSize WidenWidth = WidenVT.getSizeInBits();
8310 TypeSize WidthDiff = WidenWidth - StWidth;
8311
8312 // Find the vector type that can store the original memory width in one
8313 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8314 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8315 // implies a 4-byte movl, writing undef bytes past its object).
8316 std::optional<EVT> FirstVT =
8317 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8318 WidenEx: WidthDiff.getKnownMinValue());
8319 if (!FirstVT)
8320 return SDValue();
8321
8322 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8323
8324 SDValue StOp =
8325 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8326
8327 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8328 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8329}
8330
8331SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8332 assert((OpNo == 1 || OpNo == 3) &&
8333 "Can widen only data or mask operand of vp_store");
8334 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8335 SDValue Mask = ST->getMask();
8336 SDValue StVal = ST->getValue();
8337 SDLoc dl(N);
8338
8339 if (OpNo == 1) {
8340 // Widen the value.
8341 StVal = GetWidenedVector(Op: StVal);
8342
8343 // We only handle the case where the mask needs widening to an
8344 // identically-sized type as the vector inputs.
8345 assert(getTypeAction(Mask.getValueType()) ==
8346 TargetLowering::TypeWidenVector &&
8347 "Unable to widen VP store");
8348 Mask = GetWidenedVector(Op: Mask);
8349 } else {
8350 Mask = GetWidenedVector(Op: Mask);
8351
8352 // We only handle the case where the stored value needs widening to an
8353 // identically-sized type as the mask.
8354 assert(getTypeAction(StVal.getValueType()) ==
8355 TargetLowering::TypeWidenVector &&
8356 "Unable to widen VP store");
8357 StVal = GetWidenedVector(Op: StVal);
8358 }
8359
8360 assert(Mask.getValueType().getVectorElementCount() ==
8361 StVal.getValueType().getVectorElementCount() &&
8362 "Mask and data vectors should have the same number of elements");
8363 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8364 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8365 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8366 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8367 IsCompressing: ST->isCompressingStore());
8368}
8369
8370SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8371 unsigned OpNo) {
8372 assert((OpNo == 1 || OpNo == 4) &&
8373 "Can widen only data or mask operand of vp_strided_store");
8374 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8375 SDValue Mask = SST->getMask();
8376 SDValue StVal = SST->getValue();
8377 SDLoc DL(N);
8378
8379 if (OpNo == 1)
8380 assert(getTypeAction(Mask.getValueType()) ==
8381 TargetLowering::TypeWidenVector &&
8382 "Unable to widen VP strided store");
8383 else
8384 assert(getTypeAction(StVal.getValueType()) ==
8385 TargetLowering::TypeWidenVector &&
8386 "Unable to widen VP strided store");
8387
8388 StVal = GetWidenedVector(Op: StVal);
8389 Mask = GetWidenedVector(Op: Mask);
8390
8391 assert(StVal.getValueType().getVectorElementCount() ==
8392 Mask.getValueType().getVectorElementCount() &&
8393 "Data and mask vectors should have the same number of elements");
8394
8395 return DAG.getStridedStoreVP(
8396 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8397 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8398 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8399 IsCompressing: SST->isCompressingStore());
8400}
8401
8402SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8403 assert((OpNo == 1 || OpNo == 4) &&
8404 "Can widen only data or mask operand of mstore");
8405 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8406 SDValue Mask = MST->getMask();
8407 EVT MaskVT = Mask.getValueType();
8408 SDValue StVal = MST->getValue();
8409 EVT VT = StVal.getValueType();
8410 SDLoc dl(N);
8411
8412 EVT WideVT, WideMaskVT;
8413 if (OpNo == 1) {
8414 // Widen the value.
8415 StVal = GetWidenedVector(Op: StVal);
8416
8417 WideVT = StVal.getValueType();
8418 WideMaskVT =
8419 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8420 EC: WideVT.getVectorElementCount());
8421 } else {
8422 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8423
8424 EVT ValueVT = StVal.getValueType();
8425 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8426 EC: WideMaskVT.getVectorElementCount());
8427 }
8428
8429 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8430 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8431 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8432 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8433 EC: VT.getVectorElementCount());
8434 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8435 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8436 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8437 }
8438
8439 if (OpNo == 1) {
8440 // The mask should be widened as well.
8441 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8442 } else {
8443 // Widen the mask.
8444 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8445
8446 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8447 }
8448
8449 assert(Mask.getValueType().getVectorElementCount() ==
8450 StVal.getValueType().getVectorElementCount() &&
8451 "Mask and data vectors should have the same number of elements");
8452 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8453 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8454 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8455 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8456}
8457
8458SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8459 assert(OpNo == 4 && "Can widen only the index of mgather");
8460 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8461 SDValue DataOp = MG->getPassThru();
8462 SDValue Mask = MG->getMask();
8463 SDValue Scale = MG->getScale();
8464
8465 // Just widen the index. It's allowed to have extra elements.
8466 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8467
8468 SDLoc dl(N);
8469 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8470 Scale};
8471 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8472 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8473 ExtTy: MG->getExtensionType());
8474 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8475 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8476 return SDValue();
8477}
8478
8479SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8480 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8481 SDValue DataOp = MSC->getValue();
8482 SDValue Mask = MSC->getMask();
8483 SDValue Index = MSC->getIndex();
8484 SDValue Scale = MSC->getScale();
8485 EVT WideMemVT = MSC->getMemoryVT();
8486
8487 if (OpNo == 1) {
8488 DataOp = GetWidenedVector(Op: DataOp);
8489 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8490
8491 // Widen index.
8492 EVT IndexVT = Index.getValueType();
8493 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8494 VT: IndexVT.getVectorElementType(), EC: WideEC);
8495 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8496
8497 // The mask should be widened as well.
8498 EVT MaskVT = Mask.getValueType();
8499 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8500 VT: MaskVT.getVectorElementType(), EC: WideEC);
8501 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8502
8503 // Widen the MemoryType
8504 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8505 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8506 } else if (OpNo == 4) {
8507 // Just widen the index. It's allowed to have extra elements.
8508 Index = GetWidenedVector(Op: Index);
8509 } else
8510 llvm_unreachable("Can't widen this operand of mscatter");
8511
8512 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8513 Scale};
8514 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8515 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8516 IsTruncating: MSC->isTruncatingStore());
8517}
8518
8519SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8520 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8521 SDValue DataOp = VPSC->getValue();
8522 SDValue Mask = VPSC->getMask();
8523 SDValue Index = VPSC->getIndex();
8524 SDValue Scale = VPSC->getScale();
8525 EVT WideMemVT = VPSC->getMemoryVT();
8526
8527 if (OpNo == 1) {
8528 DataOp = GetWidenedVector(Op: DataOp);
8529 Index = GetWidenedVector(Op: Index);
8530 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8531 Mask = GetWidenedMask(Mask, EC: WideEC);
8532 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8533 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8534 } else if (OpNo == 3) {
8535 // Just widen the index. It's allowed to have extra elements.
8536 Index = GetWidenedVector(Op: Index);
8537 } else
8538 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8539
8540 SDValue Ops[] = {
8541 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8542 VPSC->getVectorLength()};
8543 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8544 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8545}
8546
8547SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8548 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8549 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8550 SDLoc dl(N);
8551 EVT VT = N->getValueType(ResNo: 0);
8552
8553 // WARNING: In this code we widen the compare instruction with garbage.
8554 // This garbage may contain denormal floats which may be slow. Is this a real
8555 // concern ? Should we zero the unused lanes if this is a float compare ?
8556
8557 // Get a new SETCC node to compare the newly widened operands.
8558 // Only some of the compared elements are legal.
8559 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8560 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8561 if (VT.getScalarType() == MVT::i1)
8562 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8563 EC: SVT.getVectorElementCount());
8564
8565 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8566 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8567
8568 // Extract the needed results from the result vector.
8569 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8570 VT: SVT.getVectorElementType(),
8571 EC: VT.getVectorElementCount());
8572 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8573
8574 EVT OpVT = N->getOperand(Num: 0).getValueType();
8575 ISD::NodeType ExtendCode =
8576 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8577 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8578}
8579
8580SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8581 SDValue Chain = N->getOperand(Num: 0);
8582 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8583 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8584 SDValue CC = N->getOperand(Num: 3);
8585 SDLoc dl(N);
8586
8587 EVT VT = N->getValueType(ResNo: 0);
8588 EVT EltVT = VT.getVectorElementType();
8589 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8590 unsigned NumElts = VT.getVectorNumElements();
8591
8592 // Unroll into a build vector.
8593 SmallVector<SDValue, 8> Scalars(NumElts);
8594 SmallVector<SDValue, 8> Chains(NumElts);
8595
8596 for (unsigned i = 0; i != NumElts; ++i) {
8597 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8598 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8599
8600 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8601 Ops: {Chain, LHSElem, RHSElem, CC});
8602 Chains[i] = Scalars[i].getValue(R: 1);
8603 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8604 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8605 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8606 }
8607
8608 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8609 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8610
8611 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8612}
8613
8614static unsigned getExtendForIntVecReduction(unsigned Opc) {
8615 switch (Opc) {
8616 default:
8617 llvm_unreachable("Expected integer vector reduction");
8618 case ISD::VECREDUCE_ADD:
8619 case ISD::VECREDUCE_MUL:
8620 case ISD::VECREDUCE_AND:
8621 case ISD::VECREDUCE_OR:
8622 case ISD::VECREDUCE_XOR:
8623 return ISD::ANY_EXTEND;
8624 case ISD::VECREDUCE_SMAX:
8625 case ISD::VECREDUCE_SMIN:
8626 return ISD::SIGN_EXTEND;
8627 case ISD::VECREDUCE_UMAX:
8628 case ISD::VECREDUCE_UMIN:
8629 return ISD::ZERO_EXTEND;
8630 }
8631}
8632
8633SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8634 SDLoc dl(N);
8635 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8636 EVT VT = N->getValueType(ResNo: 0);
8637 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8638 EVT WideVT = Op.getValueType();
8639 EVT ElemVT = OrigVT.getVectorElementType();
8640 SDNodeFlags Flags = N->getFlags();
8641
8642 unsigned Opc = N->getOpcode();
8643 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8644 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8645 assert(NeutralElem && "Neutral element must exist");
8646
8647 // Pad the vector with the neutral element.
8648 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8649 unsigned WideElts = WideVT.getVectorMinNumElements();
8650
8651 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8652 // needing to pad the source vector, because the inactive lanes can simply be
8653 // disabled and not contribute to the result.
8654 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8655 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8656 SDValue Start = NeutralElem;
8657 if (VT.isInteger())
8658 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8659 assert(Start.getValueType() == VT);
8660 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8661 EC: WideVT.getVectorElementCount());
8662 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8663 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8664 EC: OrigVT.getVectorElementCount());
8665 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8666 }
8667
8668 if (WideVT.isScalableVector()) {
8669 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8670 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8671 EC: ElementCount::getScalable(MinVal: GCD));
8672 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8673 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8674 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8675 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8676 }
8677
8678 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8679 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8680
8681 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8682}
8683
8684SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8685 SDLoc dl(N);
8686 SDValue AccOp = N->getOperand(Num: 0);
8687 SDValue VecOp = N->getOperand(Num: 1);
8688 SDValue Op = GetWidenedVector(Op: VecOp);
8689
8690 EVT VT = N->getValueType(ResNo: 0);
8691 EVT OrigVT = VecOp.getValueType();
8692 EVT WideVT = Op.getValueType();
8693 EVT ElemVT = OrigVT.getVectorElementType();
8694 SDNodeFlags Flags = N->getFlags();
8695
8696 unsigned Opc = N->getOpcode();
8697 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8698 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8699
8700 // Pad the vector with the neutral element.
8701 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8702 unsigned WideElts = WideVT.getVectorMinNumElements();
8703
8704 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8705 // needing to pad the source vector, because the inactive lanes can simply be
8706 // disabled and not contribute to the result.
8707 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8708 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8709 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8710 EC: WideVT.getVectorElementCount());
8711 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8712 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8713 EC: OrigVT.getVectorElementCount());
8714 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8715 }
8716
8717 if (WideVT.isScalableVector()) {
8718 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8719 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8720 EC: ElementCount::getScalable(MinVal: GCD));
8721 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8722 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8723 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8724 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8725 }
8726
8727 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8728 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8729
8730 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8731}
8732
8733SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8734 assert(N->isVPOpcode() && "Expected VP opcode");
8735
8736 SDLoc dl(N);
8737 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8738 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8739 EC: Op.getValueType().getVectorElementCount());
8740
8741 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8742 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8743 Flags: N->getFlags());
8744}
8745
8746SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8747 // This only gets called in the case that the left and right inputs and
8748 // result are of a legal odd vector type, and the condition is illegal i1 of
8749 // the same odd width that needs widening.
8750 EVT VT = N->getValueType(ResNo: 0);
8751 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8752
8753 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8754 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8755 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8756 SDLoc DL(N);
8757
8758 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8759 N2: LeftIn, N3: RightIn);
8760 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8761}
8762
8763SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
8764 SDLoc DL(N);
8765 SDValue Source = N->getOperand(Num: 0);
8766 EVT SourceVT = Source.getValueType();
8767 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
8768
8769 SDValue WideSource;
8770 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
8771 WideSource = GetWidenedVector(Op: Source);
8772 } else {
8773 // Pad the widened portion with all-ones so the extra lanes appear as
8774 // active (non-zero) elements and do not contribute trailing zeros.
8775 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
8776 if (WideVT.isFixedLengthVector() &&
8777 getTypeAction(VT: WideVT) == TargetLowering::TypeSplitVector) {
8778 WideSource = GetWidenedVector(Op: Source);
8779 unsigned WideElts = WideVT.getVectorNumElements();
8780 SmallVector<int> Mask(WideElts);
8781 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
8782 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
8783 Mask[I] += WideElts;
8784 WideSource = DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: WideSource, N2: AllOnes, Mask);
8785 } else {
8786 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
8787 }
8788 }
8789
8790 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
8791 Flags: N->getFlags());
8792}
8793
8794SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8795 SDLoc DL(N);
8796 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8797 EVT SrcVT = Source.getValueType();
8798 SDValue Mask =
8799 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8800
8801 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8802 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8803}
8804
8805SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8806 SDLoc DL(N);
8807 SDValue Mask = N->getOperand(Num: 0);
8808 EVT OrigMaskVT = Mask.getValueType();
8809 SDValue WideMask = GetWidenedVector(Op: Mask);
8810 EVT WideMaskVT = WideMask.getValueType();
8811
8812 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8813 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8814 unsigned WideElts = WideMaskVT.getVectorNumElements();
8815 if (OrigElts != WideElts) {
8816 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8817 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8818 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8819 }
8820
8821 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8822 Operand: WideMask);
8823}
8824
8825SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
8826 if (OpNo == 0) {
8827 SDLoc DL(N);
8828 EVT ResVT = N->getValueType(ResNo: 0);
8829 EVT SourceVT = N->getOperand(Num: 0).getValueType();
8830 EVT WideSourceVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
8831 EVT WidenVT =
8832 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
8833 EC: WideSourceVT.getVectorElementCount());
8834
8835 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
8836 SubVec: N->getOperand(Num: 0), Idx: 0);
8837 SDValue WideMask = DAG.getInsertSubvector(
8838 DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT), SubVec: N->getOperand(Num: 2), Idx: 0);
8839 SDValue WideMatch = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
8840 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
8841 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideMatch, Idx: 0);
8842 }
8843
8844 // Note: The Mask (OpNo == 2) should be widened with the result.
8845 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
8846
8847 SDLoc DL(N);
8848 SDValue Needle = N->getOperand(Num: 1);
8849 EVT NeedleVT = Needle.getValueType();
8850 if (NeedleVT.getVectorNumElements() == 1)
8851 return TLI.expandVectorMatch(N, DAG);
8852
8853 EVT WidenNeedleVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: NeedleVT);
8854
8855 SDValue Fill =
8856 DAG.getExtractVectorElt(DL, VT: NeedleVT.getVectorElementType(), Vec: Needle, Idx: 0);
8857 SDValue WideNeedle = DAG.getSplatVector(VT: WidenNeedleVT, DL, Op: Fill);
8858 WideNeedle = DAG.getInsertSubvector(DL, Vec: WideNeedle, SubVec: Needle, Idx: 0);
8859
8860 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0),
8861 N1: N->getOperand(Num: 0), N2: WideNeedle, N3: N->getOperand(Num: 2),
8862 Flags: N->getFlags());
8863}
8864
8865//===----------------------------------------------------------------------===//
8866// Vector Widening Utilities
8867//===----------------------------------------------------------------------===//
8868
8869// Utility function to find the type to chop up a widen vector for load/store
8870// TLI: Target lowering used to determine legal types.
8871// Width: Width left need to load/store.
8872// WidenVT: The widen vector type to load to/store from
8873// Align: If 0, don't allow use of a wider type
8874// WidenEx: If Align is not 0, the amount additional we can load/store from.
8875
8876static std::optional<EVT> findMemType(SelectionDAG &DAG,
8877 const TargetLowering &TLI, unsigned Width,
8878 EVT WidenVT, unsigned Align = 0,
8879 unsigned WidenEx = 0) {
8880 EVT WidenEltVT = WidenVT.getVectorElementType();
8881 const bool Scalable = WidenVT.isScalableVector();
8882 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8883 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8884 unsigned AlignInBits = Align*8;
8885
8886 EVT RetVT = WidenEltVT;
8887 // Don't bother looking for an integer type if the vector is scalable, skip
8888 // to vector types.
8889 if (!Scalable) {
8890 // If we have one element to load/store, return it.
8891 if (Width == WidenEltWidth)
8892 return RetVT;
8893
8894 // See if there is larger legal integer than the element type to load/store.
8895 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8896 unsigned MemVTWidth = MemVT.getSizeInBits();
8897 if (MemVT.getSizeInBits() <= WidenEltWidth)
8898 break;
8899 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8900 if ((Action == TargetLowering::TypeLegal ||
8901 Action == TargetLowering::TypePromoteInteger) &&
8902 (WidenWidth % MemVTWidth) == 0 &&
8903 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8904 (MemVTWidth <= Width ||
8905 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8906 if (MemVTWidth == WidenWidth)
8907 return MemVT;
8908 RetVT = MemVT;
8909 break;
8910 }
8911 }
8912 }
8913
8914 // See if there is a larger vector type to load/store that has the same vector
8915 // element type and is evenly divisible with the WidenVT.
8916 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8917 // Skip vector MVTs which don't match the scalable property of WidenVT.
8918 if (Scalable != MemVT.isScalableVector())
8919 continue;
8920 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8921 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8922 if ((Action == TargetLowering::TypeLegal ||
8923 Action == TargetLowering::TypePromoteInteger) &&
8924 WidenEltVT == MemVT.getVectorElementType() &&
8925 (WidenWidth % MemVTWidth) == 0 &&
8926 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8927 (MemVTWidth <= Width ||
8928 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8929 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8930 return MemVT;
8931 }
8932 }
8933
8934 // Using element-wise loads and stores for widening operations is not
8935 // supported for scalable vectors
8936 if (Scalable)
8937 return std::nullopt;
8938
8939 return RetVT;
8940}
8941
8942// Builds a vector type from scalar loads
8943// VecTy: Resulting Vector type
8944// LDOps: Load operators to build a vector type
8945// [Start,End) the list of loads to use.
8946static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8947 SmallVectorImpl<SDValue> &LdOps,
8948 unsigned Start, unsigned End) {
8949 SDLoc dl(LdOps[Start]);
8950 EVT LdTy = LdOps[Start].getValueType();
8951 unsigned Width = VecTy.getSizeInBits();
8952 unsigned NumElts = Width / LdTy.getSizeInBits();
8953 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8954
8955 unsigned Idx = 1;
8956 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8957
8958 for (unsigned i = Start + 1; i != End; ++i) {
8959 EVT NewLdTy = LdOps[i].getValueType();
8960 if (NewLdTy != LdTy) {
8961 NumElts = Width / NewLdTy.getSizeInBits();
8962 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8963 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8964 // Readjust position and vector position based on new load type.
8965 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8966 LdTy = NewLdTy;
8967 }
8968 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8969 }
8970 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8971}
8972
8973SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8974 LoadSDNode *LD) {
8975 // The strategy assumes that we can efficiently load power-of-two widths.
8976 // The routine chops the vector into the largest vector loads with the same
8977 // element type or scalar loads and then recombines it to the widen vector
8978 // type.
8979 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8980 EVT LdVT = LD->getMemoryVT();
8981 SDLoc dl(LD);
8982 assert(LdVT.isVector() && WidenVT.isVector());
8983 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8984 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8985
8986 // Load information
8987 SDValue Chain = LD->getChain();
8988 SDValue BasePtr = LD->getBasePtr();
8989 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8990 AAMDNodes AAInfo = LD->getAAInfo();
8991
8992 TypeSize LdWidth = LdVT.getSizeInBits();
8993 TypeSize WidenWidth = WidenVT.getSizeInBits();
8994 TypeSize WidthDiff = WidenWidth - LdWidth;
8995 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8996 // and if the original load is simple.
8997 unsigned LdAlign =
8998 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8999
9000 // Find the vector type that can load from.
9001 std::optional<EVT> FirstVT =
9002 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
9003 WidenEx: WidthDiff.getKnownMinValue());
9004
9005 if (!FirstVT)
9006 return SDValue();
9007
9008 SmallVector<EVT, 8> MemVTs;
9009 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9010
9011 // Unless we're able to load in one instruction we must work out how to load
9012 // the remainder.
9013 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
9014 std::optional<EVT> NewVT = FirstVT;
9015 TypeSize RemainingWidth = LdWidth;
9016 TypeSize NewVTWidth = FirstVTWidth;
9017 do {
9018 RemainingWidth -= NewVTWidth;
9019 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
9020 // The current type we are using is too large. Find a better size.
9021 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
9022 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
9023 if (!NewVT)
9024 return SDValue();
9025 NewVTWidth = NewVT->getSizeInBits();
9026 }
9027 MemVTs.push_back(Elt: *NewVT);
9028 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
9029 }
9030
9031 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9032 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9033 LdChain.push_back(Elt: LdOp.getValue(R: 1));
9034
9035 // Check if we can load the element with one instruction.
9036 if (MemVTs.empty())
9037 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
9038 DAG);
9039
9040 // Load vector by using multiple loads from largest vector to scalar.
9041 SmallVector<SDValue, 16> LdOps;
9042 LdOps.push_back(Elt: LdOp);
9043
9044 uint64_t ScaledOffset = 0;
9045 MachinePointerInfo MPI = LD->getPointerInfo();
9046
9047 // First incremement past the first load.
9048 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
9049 ScaledOffset: &ScaledOffset);
9050
9051 for (EVT MemVT : MemVTs) {
9052 Align NewAlign = ScaledOffset == 0
9053 ? LD->getBaseAlign()
9054 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
9055 SDValue L =
9056 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, Metadata: AAInfo);
9057
9058 LdOps.push_back(Elt: L);
9059 LdChain.push_back(Elt: L.getValue(R: 1));
9060 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
9061 }
9062
9063 // Build the vector from the load operations.
9064 unsigned End = LdOps.size();
9065 if (!LdOps[0].getValueType().isVector())
9066 // All the loads are scalar loads.
9067 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
9068
9069 // If the load contains vectors, build the vector using concat vector.
9070 // All of the vectors used to load are power-of-2, and the scalar loads can be
9071 // combined to make a power-of-2 vector.
9072 SmallVector<SDValue, 16> ConcatOps(End);
9073 int i = End - 1;
9074 int Idx = End;
9075 EVT LdTy = LdOps[i].getValueType();
9076 // First, combine the scalar loads to a vector.
9077 if (!LdTy.isVector()) {
9078 for (--i; i >= 0; --i) {
9079 LdTy = LdOps[i].getValueType();
9080 if (LdTy.isVector())
9081 break;
9082 }
9083 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
9084 }
9085
9086 ConcatOps[--Idx] = LdOps[i];
9087 for (--i; i >= 0; --i) {
9088 EVT NewLdTy = LdOps[i].getValueType();
9089 if (NewLdTy != LdTy) {
9090 // Create a larger vector.
9091 TypeSize LdTySize = LdTy.getSizeInBits();
9092 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9093 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9094 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9095 unsigned NumOps =
9096 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9097 SmallVector<SDValue, 16> WidenOps(NumOps);
9098 unsigned j = 0;
9099 for (; j != End-Idx; ++j)
9100 WidenOps[j] = ConcatOps[Idx+j];
9101 for (; j != NumOps; ++j)
9102 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9103
9104 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9105 Ops: WidenOps);
9106 Idx = End - 1;
9107 LdTy = NewLdTy;
9108 }
9109 ConcatOps[--Idx] = LdOps[i];
9110 }
9111
9112 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9113 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9114 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9115
9116 // We need to fill the rest with undefs to build the vector.
9117 unsigned NumOps =
9118 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9119 SmallVector<SDValue, 16> WidenOps(NumOps);
9120 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9121 {
9122 unsigned i = 0;
9123 for (; i != End-Idx; ++i)
9124 WidenOps[i] = ConcatOps[Idx+i];
9125 for (; i != NumOps; ++i)
9126 WidenOps[i] = UndefVal;
9127 }
9128 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9129}
9130
9131SDValue
9132DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9133 LoadSDNode *LD,
9134 ISD::LoadExtType ExtType) {
9135 // For extension loads, it may not be more efficient to chop up the vector
9136 // and then extend it. Instead, we unroll the load and build a new vector.
9137 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9138 EVT LdVT = LD->getMemoryVT();
9139 SDLoc dl(LD);
9140 assert(LdVT.isVector() && WidenVT.isVector());
9141 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9142
9143 // Load information
9144 SDValue Chain = LD->getChain();
9145 SDValue BasePtr = LD->getBasePtr();
9146 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9147 AAMDNodes AAInfo = LD->getAAInfo();
9148
9149 if (LdVT.isScalableVector())
9150 return SDValue();
9151
9152 EVT EltVT = WidenVT.getVectorElementType();
9153 EVT LdEltVT = LdVT.getVectorElementType();
9154 unsigned NumElts = LdVT.getVectorNumElements();
9155
9156 // Load each element and widen.
9157 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9158 SmallVector<SDValue, 16> Ops(WidenNumElts);
9159 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9160 Ops[0] =
9161 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9162 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9163 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9164 unsigned i = 0, Offset = Increment;
9165 for (i=1; i < NumElts; ++i, Offset += Increment) {
9166 SDValue NewBasePtr =
9167 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9168 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9169 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9170 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9171 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9172 }
9173
9174 // Fill the rest with undefs.
9175 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9176 for (; i != WidenNumElts; ++i)
9177 Ops[i] = UndefVal;
9178
9179 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9180}
9181
9182bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9183 StoreSDNode *ST) {
9184 // The strategy assumes that we can efficiently store power-of-two widths.
9185 // The routine chops the vector into the largest vector stores with the same
9186 // element type or scalar stores.
9187 SDValue Chain = ST->getChain();
9188 SDValue BasePtr = ST->getBasePtr();
9189 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9190 AAMDNodes AAInfo = ST->getAAInfo();
9191 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9192 SDLoc dl(ST);
9193
9194 EVT StVT = ST->getMemoryVT();
9195 TypeSize StWidth = StVT.getSizeInBits();
9196 EVT ValVT = ValOp.getValueType();
9197 TypeSize ValWidth = ValVT.getSizeInBits();
9198 EVT ValEltVT = ValVT.getVectorElementType();
9199 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9200 assert(StVT.getVectorElementType() == ValEltVT);
9201 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9202 "Mismatch between store and value types");
9203
9204 int Idx = 0; // current index to store
9205
9206 MachinePointerInfo MPI = ST->getPointerInfo();
9207 uint64_t ScaledOffset = 0;
9208
9209 // A breakdown of how to widen this vector store. Each element of the vector
9210 // is a memory VT combined with the number of times it is to be stored to,
9211 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9212 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9213
9214 while (StWidth.isNonZero()) {
9215 // Find the largest vector type we can store with.
9216 std::optional<EVT> NewVT =
9217 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9218 if (!NewVT)
9219 return false;
9220 MemVTs.push_back(Elt: {*NewVT, 0});
9221 TypeSize NewVTWidth = NewVT->getSizeInBits();
9222
9223 do {
9224 StWidth -= NewVTWidth;
9225 MemVTs.back().second++;
9226 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9227 }
9228
9229 for (const auto &Pair : MemVTs) {
9230 EVT NewVT = Pair.first;
9231 unsigned Count = Pair.second;
9232 TypeSize NewVTWidth = NewVT.getSizeInBits();
9233
9234 if (NewVT.isVector()) {
9235 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9236 do {
9237 Align NewAlign = ScaledOffset == 0
9238 ? ST->getBaseAlign()
9239 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9240 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9241 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9242 MMOFlags, Metadata: AAInfo);
9243 StChain.push_back(Elt: PartStore);
9244
9245 Idx += NumVTElts;
9246 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9247 ScaledOffset: &ScaledOffset);
9248 } while (--Count);
9249 } else {
9250 // Cast the vector to the scalar type we can store.
9251 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9252 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9253 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9254 // Readjust index position based on new vector type.
9255 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9256 do {
9257 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9258 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
9259 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9260 StChain.push_back(Elt: PartStore);
9261
9262 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9263 } while (--Count);
9264 // Restore index back to be relative to the original widen element type.
9265 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9266 }
9267 }
9268
9269 return true;
9270}
9271
9272/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9273/// input vector must have the same element type as NVT.
9274/// FillWithZeroes specifies that the vector should be widened with zeroes.
9275SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9276 bool FillWithZeroes) {
9277 // Note that InOp might have been widened so it might already have
9278 // the right width or it might need be narrowed.
9279 EVT InVT = InOp.getValueType();
9280 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9281 "input and widen element type must match");
9282 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9283 "cannot modify scalable vectors in this way");
9284 SDLoc dl(InOp);
9285
9286 // Check if InOp already has the right width.
9287 if (InVT == NVT)
9288 return InOp;
9289
9290 ElementCount InEC = InVT.getVectorElementCount();
9291 ElementCount WidenEC = NVT.getVectorElementCount();
9292 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9293 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9294 SmallVector<SDValue, 16> Ops(NumConcat);
9295 SDValue FillVal =
9296 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9297 Ops[0] = InOp;
9298 for (unsigned i = 1; i != NumConcat; ++i)
9299 Ops[i] = FillVal;
9300
9301 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9302 }
9303
9304 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9305 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9306
9307 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9308 // Split the input into the largest equal-sized scalable subvectors.
9309 unsigned InNumElts = InVT.getVectorMinNumElements();
9310 unsigned NewNumElts = NVT.getVectorMinNumElements();
9311 unsigned CommonFactor = std::gcd(m: InNumElts, n: NewNumElts);
9312 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
9313 EC: ElementCount::getScalable(MinVal: CommonFactor));
9314
9315 SmallVector<SDValue, 16> Ops;
9316 unsigned NumCopiedParts = std::min(a: InNumElts, b: NewNumElts) / CommonFactor;
9317 for (unsigned I = 0; I != NumCopiedParts; ++I)
9318 Ops.push_back(
9319 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: I * CommonFactor));
9320
9321 unsigned NumResultParts = NewNumElts / CommonFactor;
9322 if (NumResultParts > NumCopiedParts) {
9323 SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: PartVT)
9324 : DAG.getPOISON(VT: PartVT);
9325 Ops.append(NumInputs: NumResultParts - NumCopiedParts, Elt: FillVal);
9326 }
9327
9328 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9329 }
9330
9331 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9332 "Scalable vectors should have been handled already.");
9333
9334 unsigned InNumElts = InEC.getFixedValue();
9335 unsigned WidenNumElts = WidenEC.getFixedValue();
9336
9337 // Fall back to extract and build (+ mask, if padding with zeros).
9338 SmallVector<SDValue, 16> Ops(WidenNumElts);
9339 EVT EltVT = NVT.getVectorElementType();
9340 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9341 unsigned Idx;
9342 for (Idx = 0; Idx < MinNumElts; ++Idx)
9343 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9344
9345 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9346 for (; Idx < WidenNumElts; ++Idx)
9347 Ops[Idx] = UndefVal;
9348
9349 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9350 if (!FillWithZeroes)
9351 return Widened;
9352
9353 assert(NVT.isInteger() &&
9354 "We expect to never want to FillWithZeroes for non-integral types.");
9355
9356 SmallVector<SDValue, 16> MaskOps;
9357 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9358 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9359
9360 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9361 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9362}
9363