1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error(reason: "Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::LOOP_DEPENDENCE_WAR_MASK:
57 case ISD::LOOP_DEPENDENCE_RAW_MASK:
58 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
59 break;
60 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
61 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
62 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
63 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
64 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
65 case ISD::CONVERT_FROM_ARBITRARY_FP:
66 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
67 break;
68 case ISD::AssertZext:
69 case ISD::AssertSext:
70 case ISD::FPOWI:
71 case ISD::AssertNoFPClass:
72 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
73 break;
74 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
75 case ISD::ATOMIC_LOAD:
76 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
77 break;
78 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
79 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
80 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
81 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
82 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
83 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
84 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
85 case ISD::POISON:
86 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
87 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
88 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
89 case ISD::ANY_EXTEND_VECTOR_INREG:
90 case ISD::SIGN_EXTEND_VECTOR_INREG:
91 case ISD::ZERO_EXTEND_VECTOR_INREG:
92 R = ScalarizeVecRes_VecInregOp(N);
93 break;
94 case ISD::ABS:
95 case ISD::ANY_EXTEND:
96 case ISD::BITREVERSE:
97 case ISD::BSWAP:
98 case ISD::CTLZ:
99 case ISD::CTLZ_ZERO_UNDEF:
100 case ISD::CTPOP:
101 case ISD::CTTZ:
102 case ISD::CTTZ_ZERO_UNDEF:
103 case ISD::FABS:
104 case ISD::FACOS:
105 case ISD::FASIN:
106 case ISD::FATAN:
107 case ISD::FCEIL:
108 case ISD::FCOS:
109 case ISD::FCOSH:
110 case ISD::FEXP:
111 case ISD::FEXP2:
112 case ISD::FEXP10:
113 case ISD::FFLOOR:
114 case ISD::FLOG:
115 case ISD::FLOG10:
116 case ISD::FLOG2:
117 case ISD::FNEARBYINT:
118 case ISD::FNEG:
119 case ISD::FREEZE:
120 case ISD::ARITH_FENCE:
121 case ISD::FP_EXTEND:
122 case ISD::FP_TO_SINT:
123 case ISD::FP_TO_UINT:
124 case ISD::FRINT:
125 case ISD::LRINT:
126 case ISD::LLRINT:
127 case ISD::FROUND:
128 case ISD::FROUNDEVEN:
129 case ISD::LROUND:
130 case ISD::LLROUND:
131 case ISD::FSIN:
132 case ISD::FSINH:
133 case ISD::FSQRT:
134 case ISD::FTAN:
135 case ISD::FTANH:
136 case ISD::FTRUNC:
137 case ISD::SIGN_EXTEND:
138 case ISD::SINT_TO_FP:
139 case ISD::TRUNCATE:
140 case ISD::UINT_TO_FP:
141 case ISD::ZERO_EXTEND:
142 case ISD::FCANONICALIZE:
143 R = ScalarizeVecRes_UnaryOp(N);
144 break;
145 case ISD::ADDRSPACECAST:
146 R = ScalarizeVecRes_ADDRSPACECAST(N);
147 break;
148 case ISD::FMODF:
149 case ISD::FFREXP:
150 case ISD::FSINCOS:
151 case ISD::FSINCOSPI:
152 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
153 break;
154 case ISD::ADD:
155 case ISD::AND:
156 case ISD::AVGCEILS:
157 case ISD::AVGCEILU:
158 case ISD::AVGFLOORS:
159 case ISD::AVGFLOORU:
160 case ISD::FADD:
161 case ISD::FCOPYSIGN:
162 case ISD::FDIV:
163 case ISD::FMUL:
164 case ISD::FMINNUM:
165 case ISD::FMAXNUM:
166 case ISD::FMINNUM_IEEE:
167 case ISD::FMAXNUM_IEEE:
168 case ISD::FMINIMUM:
169 case ISD::FMAXIMUM:
170 case ISD::FMINIMUMNUM:
171 case ISD::FMAXIMUMNUM:
172 case ISD::FLDEXP:
173 case ISD::ABDS:
174 case ISD::ABDU:
175 case ISD::SMIN:
176 case ISD::SMAX:
177 case ISD::UMIN:
178 case ISD::UMAX:
179
180 case ISD::SADDSAT:
181 case ISD::UADDSAT:
182 case ISD::SSUBSAT:
183 case ISD::USUBSAT:
184 case ISD::SSHLSAT:
185 case ISD::USHLSAT:
186
187 case ISD::FPOW:
188 case ISD::FATAN2:
189 case ISD::FREM:
190 case ISD::FSUB:
191 case ISD::MUL:
192 case ISD::MULHS:
193 case ISD::MULHU:
194 case ISD::OR:
195 case ISD::SDIV:
196 case ISD::SREM:
197 case ISD::SUB:
198 case ISD::UDIV:
199 case ISD::UREM:
200 case ISD::XOR:
201 case ISD::SHL:
202 case ISD::SRA:
203 case ISD::SRL:
204 case ISD::ROTL:
205 case ISD::ROTR:
206 case ISD::CLMUL:
207 case ISD::CLMULR:
208 case ISD::CLMULH:
209 R = ScalarizeVecRes_BinOp(N);
210 break;
211
212 case ISD::SCMP:
213 case ISD::UCMP:
214 R = ScalarizeVecRes_CMP(N);
215 break;
216
217 case ISD::FMA:
218 case ISD::FSHL:
219 case ISD::FSHR:
220 R = ScalarizeVecRes_TernaryOp(N);
221 break;
222
223#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
224 case ISD::STRICT_##DAGN:
225#include "llvm/IR/ConstrainedOps.def"
226 R = ScalarizeVecRes_StrictFPOp(N);
227 break;
228
229 case ISD::FP_TO_UINT_SAT:
230 case ISD::FP_TO_SINT_SAT:
231 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
232 break;
233
234 case ISD::UADDO:
235 case ISD::SADDO:
236 case ISD::USUBO:
237 case ISD::SSUBO:
238 case ISD::UMULO:
239 case ISD::SMULO:
240 R = ScalarizeVecRes_OverflowOp(N, ResNo);
241 break;
242 case ISD::SMULFIX:
243 case ISD::SMULFIXSAT:
244 case ISD::UMULFIX:
245 case ISD::UMULFIXSAT:
246 case ISD::SDIVFIX:
247 case ISD::SDIVFIXSAT:
248 case ISD::UDIVFIX:
249 case ISD::UDIVFIXSAT:
250 R = ScalarizeVecRes_FIX(N);
251 break;
252 }
253
254 // If R is null, the sub-method took care of registering the result.
255 if (R.getNode())
256 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
257}
258
259SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
260 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
261 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
262 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
263 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
264}
265
266SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
267 SDLoc DL(N);
268
269 SDValue LHS = N->getOperand(Num: 0);
270 SDValue RHS = N->getOperand(Num: 1);
271 if (getTypeAction(VT: LHS.getValueType()) ==
272 TargetLowering::TypeScalarizeVector) {
273 LHS = GetScalarizedVector(Op: LHS);
274 RHS = GetScalarizedVector(Op: RHS);
275 } else {
276 EVT VT = LHS.getValueType().getVectorElementType();
277 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
278 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
279 }
280
281 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
282 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
283}
284
285SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
286 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
287 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
288 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
289 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
290 N3: Op2, Flags: N->getFlags());
291}
292
293SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
294 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
295 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
296 SDValue Op2 = N->getOperand(Num: 2);
297 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
298 N3: Op2, Flags: N->getFlags());
299}
300
301SDValue
302DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
303 unsigned ResNo) {
304 assert(N->getValueType(0).getVectorNumElements() == 1 &&
305 "Unexpected vector type!");
306 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
307
308 EVT VT0 = N->getValueType(ResNo: 0);
309 EVT VT1 = N->getValueType(ResNo: 1);
310 SDLoc dl(N);
311
312 SDNode *ScalarNode =
313 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
314 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
315 .getNode();
316
317 // Replace the other vector result not being explicitly scalarized here.
318 unsigned OtherNo = 1 - ResNo;
319 EVT OtherVT = N->getValueType(ResNo: OtherNo);
320 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
321 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
322 } else {
323 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
324 Operand: SDValue(ScalarNode, OtherNo));
325 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
326 }
327
328 return SDValue(ScalarNode, ResNo);
329}
330
331SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
332 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
333 unsigned NumOpers = N->getNumOperands();
334 SDValue Chain = N->getOperand(Num: 0);
335 EVT ValueVTs[] = {VT, MVT::Other};
336 SDLoc dl(N);
337
338 SmallVector<SDValue, 4> Opers(NumOpers);
339
340 // The Chain is the first operand.
341 Opers[0] = Chain;
342
343 // Now process the remaining operands.
344 for (unsigned i = 1; i < NumOpers; ++i) {
345 SDValue Oper = N->getOperand(Num: i);
346 EVT OperVT = Oper.getValueType();
347
348 if (OperVT.isVector()) {
349 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
350 Oper = GetScalarizedVector(Op: Oper);
351 else
352 Oper =
353 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
354 }
355
356 Opers[i] = Oper;
357 }
358
359 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
360 Ops: Opers, Flags: N->getFlags());
361
362 // Legalize the chain result - switch anything that used the old chain to
363 // use the new one.
364 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
365 return Result;
366}
367
368SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
369 unsigned ResNo) {
370 SDLoc DL(N);
371 EVT ResVT = N->getValueType(ResNo: 0);
372 EVT OvVT = N->getValueType(ResNo: 1);
373
374 SDValue ScalarLHS, ScalarRHS;
375 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
376 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
377 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
378 } else {
379 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
380 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
381 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
382 ScalarLHS = ElemsLHS[0];
383 ScalarRHS = ElemsRHS[0];
384 }
385
386 SDVTList ScalarVTs = DAG.getVTList(
387 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
388 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
389 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
390 .getNode();
391
392 // Replace the other vector result not being explicitly scalarized here.
393 unsigned OtherNo = 1 - ResNo;
394 EVT OtherVT = N->getValueType(ResNo: OtherNo);
395 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
396 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
397 } else {
398 SDValue OtherVal = DAG.getNode(
399 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
400 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
401 }
402
403 return SDValue(ScalarNode, ResNo);
404}
405
406SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
407 unsigned ResNo) {
408 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
409 return GetScalarizedVector(Op);
410}
411
412SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
413 SDLoc DL(N);
414 SDValue SourceValue = N->getOperand(Num: 0);
415 SDValue SinkValue = N->getOperand(Num: 1);
416 SDValue EltSizeInBytes = N->getOperand(Num: 2);
417 SDValue LaneOffset = N->getOperand(Num: 3);
418
419 EVT PtrVT = SourceValue->getValueType(ResNo: 0);
420 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
421
422 // Take the difference between the pointers and divided by the element size,
423 // to see how many lanes separate them.
424 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: SinkValue, N2: SourceValue);
425 if (IsReadAfterWrite)
426 Diff = DAG.getNode(Opcode: ISD::ABS, DL, VT: PtrVT, Operand: Diff);
427 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: PtrVT, N1: Diff, N2: EltSizeInBytes);
428
429 // The pointers do not alias if:
430 // * Diff <= 0 || LaneOffset < Diff (WAR_MASK)
431 // * Diff == 0 || LaneOffset < abs(Diff) (RAW_MASK)
432 // Note: If LaneOffset is zero, both cases will fold to "true".
433 EVT CmpVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
434 VT: Diff.getValueType());
435 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
436 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Diff, RHS: Zero,
437 Cond: IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
438 return DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Cmp,
439 N2: DAG.getSetCC(DL, VT: CmpVT, LHS: LaneOffset, RHS: Diff, Cond: ISD::SETULT));
440}
441
442SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
443 SDValue Op = N->getOperand(Num: 0);
444 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
445 Op = GetScalarizedVector(Op);
446 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
447 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
448 VT: NewVT, Operand: Op);
449}
450
451SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
452 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
453 SDValue InOp = N->getOperand(Num: 0);
454 // The BUILD_VECTOR operands may be of wider element types and
455 // we may need to truncate them back to the requested return type.
456 if (EltVT.isInteger())
457 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
458 return InOp;
459}
460
461SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
462 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
463 VT: N->getValueType(ResNo: 0).getVectorElementType(),
464 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
465}
466
467SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
468 SDLoc DL(N);
469 SDValue Op = N->getOperand(Num: 0);
470 EVT OpVT = Op.getValueType();
471 // The result needs scalarizing, but it's not a given that the source does.
472 // See similar logic in ScalarizeVecRes_UnaryOp.
473 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
474 Op = GetScalarizedVector(Op);
475 } else {
476 EVT VT = OpVT.getVectorElementType();
477 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
478 }
479 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
480 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
481 N2: N->getOperand(Num: 1));
482}
483
484SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
485 SDLoc DL(N);
486 SDValue Op = N->getOperand(Num: 0);
487 EVT OpVT = Op.getValueType();
488 // The result needs scalarizing, but it's not a given that the source does.
489 // See similar logic in ScalarizeVecRes_UnaryOp.
490 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
491 Op = GetScalarizedVector(Op);
492 } else {
493 EVT VT = OpVT.getVectorElementType();
494 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
495 }
496 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
497 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
498 N2: N->getOperand(Num: 1));
499}
500
501SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
502 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
503 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
504 N2: N->getOperand(Num: 1));
505}
506
507SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
508 // The value to insert may have a wider type than the vector element type,
509 // so be sure to truncate it to the element type if necessary.
510 SDValue Op = N->getOperand(Num: 1);
511 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
512 if (Op.getValueType() != EltVT)
513 // FIXME: Can this happen for floating point types?
514 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
515 return Op;
516}
517
518SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
519 SDValue Result = DAG.getAtomicLoad(
520 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
521 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
522 MMO: N->getMemOperand());
523
524 // Legalize the chain result - switch anything that used the old chain to
525 // use the new one.
526 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
527 return Result;
528}
529
530SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
531 assert(N->isUnindexed() && "Indexed vector load?");
532
533 SDValue Result = DAG.getLoad(
534 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
535 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
536 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
537 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
538 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
539
540 // Legalize the chain result - switch anything that used the old chain to
541 // use the new one.
542 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
543 return Result;
544}
545
546SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
547 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
548 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
549 SDValue Op = N->getOperand(Num: 0);
550 EVT OpVT = Op.getValueType();
551 SDLoc DL(N);
552 // The result needs scalarizing, but it's not a given that the source does.
553 // This is a workaround for targets where it's impossible to scalarize the
554 // result of a conversion, because the source type is legal.
555 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
556 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
557 // legal and was not scalarized.
558 // See the similar logic in ScalarizeVecRes_SETCC
559 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
560 Op = GetScalarizedVector(Op);
561 } else {
562 EVT VT = OpVT.getVectorElementType();
563 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
564 }
565 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
566}
567
568SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
569 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
570 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
571 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
572 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
573 N1: LHS, N2: DAG.getValueType(ExtVT));
574}
575
576SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
577 SDLoc DL(N);
578 SDValue Op = N->getOperand(Num: 0);
579
580 EVT OpVT = Op.getValueType();
581 EVT OpEltVT = OpVT.getVectorElementType();
582 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
583
584 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
585 Op = GetScalarizedVector(Op);
586 } else {
587 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
588 }
589
590 switch (N->getOpcode()) {
591 case ISD::ANY_EXTEND_VECTOR_INREG:
592 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
593 case ISD::SIGN_EXTEND_VECTOR_INREG:
594 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
595 case ISD::ZERO_EXTEND_VECTOR_INREG:
596 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
597 }
598
599 llvm_unreachable("Illegal extend_vector_inreg opcode");
600}
601
602SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
603 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
604 SDValue Op = N->getOperand(Num: 0);
605 EVT OpVT = Op.getValueType();
606 SDLoc DL(N);
607 // The result needs scalarizing, but it's not a given that the source does.
608 // This is a workaround for targets where it's impossible to scalarize the
609 // result of a conversion, because the source type is legal.
610 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
611 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
612 // legal and was not scalarized.
613 // See the similar logic in ScalarizeVecRes_SETCC
614 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
615 Op = GetScalarizedVector(Op);
616 } else {
617 EVT VT = OpVT.getVectorElementType();
618 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
619 }
620 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
621 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
622 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
623 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
624}
625
626SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
627 // If the operand is wider than the vector element type then it is implicitly
628 // truncated. Make that explicit here.
629 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
630 SDValue InOp = N->getOperand(Num: 0);
631 if (InOp.getValueType() != EltVT)
632 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
633 return InOp;
634}
635
636SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
637 SDValue Cond = N->getOperand(Num: 0);
638 EVT OpVT = Cond.getValueType();
639 SDLoc DL(N);
640 // The vselect result and true/value operands needs scalarizing, but it's
641 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
642 // See the similar logic in ScalarizeVecRes_SETCC
643 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
644 Cond = GetScalarizedVector(Op: Cond);
645 } else {
646 EVT VT = OpVT.getVectorElementType();
647 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
648 }
649
650 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
651 TargetLowering::BooleanContent ScalarBool =
652 TLI.getBooleanContents(isVec: false, isFloat: false);
653 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
654
655 // If integer and float booleans have different contents then we can't
656 // reliably optimize in all cases. There is a full explanation for this in
657 // DAGCombiner::visitSELECT() where the same issue affects folding
658 // (select C, 0, 1) to (xor C, 1).
659 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
660 TLI.getBooleanContents(isVec: false, isFloat: true)) {
661 // At least try the common case where the boolean is generated by a
662 // comparison.
663 if (Cond->getOpcode() == ISD::SETCC) {
664 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
665 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
666 VecBool = TLI.getBooleanContents(Type: OpVT);
667 } else
668 ScalarBool = TargetLowering::UndefinedBooleanContent;
669 }
670
671 EVT CondVT = Cond.getValueType();
672 if (ScalarBool != VecBool) {
673 switch (ScalarBool) {
674 case TargetLowering::UndefinedBooleanContent:
675 break;
676 case TargetLowering::ZeroOrOneBooleanContent:
677 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
678 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
679 // Vector read from all ones, scalar expects a single 1 so mask.
680 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
681 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
682 break;
683 case TargetLowering::ZeroOrNegativeOneBooleanContent:
684 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
685 VecBool == TargetLowering::ZeroOrOneBooleanContent);
686 // Vector reads from a one, scalar from all ones so sign extend.
687 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
688 N1: Cond, N2: DAG.getValueType(MVT::i1));
689 break;
690 }
691 }
692
693 // Truncate the condition if needed
694 auto BoolVT = getSetCCResultType(VT: CondVT);
695 if (BoolVT.bitsLT(VT: CondVT))
696 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
697
698 return DAG.getSelect(DL: SDLoc(N),
699 VT: LHS.getValueType(), Cond, LHS,
700 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
701}
702
703SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
704 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
705 return DAG.getSelect(DL: SDLoc(N),
706 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
707 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
708}
709
710SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
711 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
712 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
713 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
714 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
715 N5: N->getOperand(Num: 4));
716}
717
718SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
719 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
720}
721
722SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
723 // Figure out if the scalar is the LHS or RHS and return it.
724 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
725 if (Arg.isUndef())
726 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
727 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
728 return GetScalarizedVector(Op: N->getOperand(Num: Op));
729}
730
731SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
732 SDValue Src = N->getOperand(Num: 0);
733 EVT SrcVT = Src.getValueType();
734 SDLoc dl(N);
735
736 // Handle case where result is scalarized but operand is not
737 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
738 Src = GetScalarizedVector(Op: Src);
739 else
740 Src = DAG.getNode(
741 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
742 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
743
744 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
745 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
746}
747
748SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
749 assert(N->getValueType(0).isVector() &&
750 N->getOperand(0).getValueType().isVector() &&
751 "Operand types must be vectors");
752 SDValue LHS = N->getOperand(Num: 0);
753 SDValue RHS = N->getOperand(Num: 1);
754 EVT OpVT = LHS.getValueType();
755 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
756 SDLoc DL(N);
757
758 // The result needs scalarizing, but it's not a given that the source does.
759 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
760 LHS = GetScalarizedVector(Op: LHS);
761 RHS = GetScalarizedVector(Op: RHS);
762 } else {
763 EVT VT = OpVT.getVectorElementType();
764 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
765 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
766 }
767
768 // Turn it into a scalar SETCC.
769 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
770 N3: N->getOperand(Num: 2));
771 // Vectors may have a different boolean contents to scalars. Promote the
772 // value appropriately.
773 ISD::NodeType ExtendCode =
774 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
775 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
776}
777
778SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
779 SDLoc DL(N);
780 SDValue Arg = N->getOperand(Num: 0);
781 SDValue Test = N->getOperand(Num: 1);
782 EVT ArgVT = Arg.getValueType();
783 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
784
785 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
786 Arg = GetScalarizedVector(Op: Arg);
787 } else {
788 EVT VT = ArgVT.getVectorElementType();
789 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
790 }
791
792 SDValue Res =
793 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
794 // Vectors may have a different boolean contents to scalars. Promote the
795 // value appropriately.
796 ISD::NodeType ExtendCode =
797 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
798 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
799}
800
801//===----------------------------------------------------------------------===//
802// Operand Vector Scalarization <1 x ty> -> ty.
803//===----------------------------------------------------------------------===//
804
805bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
806 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
807 N->dump(&DAG));
808 SDValue Res = SDValue();
809
810 switch (N->getOpcode()) {
811 default:
812#ifndef NDEBUG
813 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
814 N->dump(&DAG);
815 dbgs() << "\n";
816#endif
817 report_fatal_error(reason: "Do not know how to scalarize this operator's "
818 "operand!\n");
819 case ISD::BITCAST:
820 Res = ScalarizeVecOp_BITCAST(N);
821 break;
822 case ISD::FAKE_USE:
823 Res = ScalarizeVecOp_FAKE_USE(N);
824 break;
825 case ISD::ANY_EXTEND:
826 case ISD::ZERO_EXTEND:
827 case ISD::SIGN_EXTEND:
828 case ISD::TRUNCATE:
829 case ISD::FP_TO_SINT:
830 case ISD::FP_TO_UINT:
831 case ISD::SINT_TO_FP:
832 case ISD::UINT_TO_FP:
833 case ISD::LROUND:
834 case ISD::LLROUND:
835 case ISD::LRINT:
836 case ISD::LLRINT:
837 Res = ScalarizeVecOp_UnaryOp(N);
838 break;
839 case ISD::FP_TO_SINT_SAT:
840 case ISD::FP_TO_UINT_SAT:
841 case ISD::CONVERT_FROM_ARBITRARY_FP:
842 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
843 break;
844 case ISD::STRICT_SINT_TO_FP:
845 case ISD::STRICT_UINT_TO_FP:
846 case ISD::STRICT_FP_TO_SINT:
847 case ISD::STRICT_FP_TO_UINT:
848 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
849 break;
850 case ISD::CONCAT_VECTORS:
851 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
852 break;
853 case ISD::INSERT_SUBVECTOR:
854 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
855 break;
856 case ISD::EXTRACT_VECTOR_ELT:
857 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
858 break;
859 case ISD::VSELECT:
860 Res = ScalarizeVecOp_VSELECT(N);
861 break;
862 case ISD::SETCC:
863 Res = ScalarizeVecOp_VSETCC(N);
864 break;
865 case ISD::STRICT_FSETCC:
866 case ISD::STRICT_FSETCCS:
867 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
868 break;
869 case ISD::STORE:
870 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
871 break;
872 case ISD::STRICT_FP_ROUND:
873 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
874 break;
875 case ISD::FP_ROUND:
876 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
877 break;
878 case ISD::STRICT_FP_EXTEND:
879 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
880 break;
881 case ISD::FP_EXTEND:
882 Res = ScalarizeVecOp_FP_EXTEND(N);
883 break;
884 case ISD::VECREDUCE_FADD:
885 case ISD::VECREDUCE_FMUL:
886 case ISD::VECREDUCE_ADD:
887 case ISD::VECREDUCE_MUL:
888 case ISD::VECREDUCE_AND:
889 case ISD::VECREDUCE_OR:
890 case ISD::VECREDUCE_XOR:
891 case ISD::VECREDUCE_SMAX:
892 case ISD::VECREDUCE_SMIN:
893 case ISD::VECREDUCE_UMAX:
894 case ISD::VECREDUCE_UMIN:
895 case ISD::VECREDUCE_FMAX:
896 case ISD::VECREDUCE_FMIN:
897 case ISD::VECREDUCE_FMAXIMUM:
898 case ISD::VECREDUCE_FMINIMUM:
899 Res = ScalarizeVecOp_VECREDUCE(N);
900 break;
901 case ISD::VECREDUCE_SEQ_FADD:
902 case ISD::VECREDUCE_SEQ_FMUL:
903 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
904 break;
905 case ISD::SCMP:
906 case ISD::UCMP:
907 Res = ScalarizeVecOp_CMP(N);
908 break;
909 case ISD::VECTOR_FIND_LAST_ACTIVE:
910 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
911 break;
912 case ISD::CTTZ_ELTS:
913 case ISD::CTTZ_ELTS_ZERO_POISON:
914 Res = ScalarizeVecOp_CTTZ_ELTS(N);
915 break;
916 }
917
918 // If the result is null, the sub-method took care of registering results etc.
919 if (!Res.getNode()) return false;
920
921 // If the result is N, the sub-method updated N in place. Tell the legalizer
922 // core about this.
923 if (Res.getNode() == N)
924 return true;
925
926 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
927 "Invalid operand expansion");
928
929 ReplaceValueWith(From: SDValue(N, 0), To: Res);
930 return false;
931}
932
933/// If the value to convert is a vector that needs to be scalarized, it must be
934/// <1 x ty>. Convert the element instead.
935SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
936 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
937 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
938 VT: N->getValueType(ResNo: 0), Operand: Elt);
939}
940
941// Need to legalize vector operands of fake uses. Must be <1 x ty>.
942SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
943 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
944 "Fake Use: Unexpected vector type!");
945 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
946 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
947}
948
949/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
950/// Do the operation on the element instead.
951SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
952 assert(N->getValueType(0).getVectorNumElements() == 1 &&
953 "Unexpected vector type!");
954 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
955 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
956 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
957 // Revectorize the result so the types line up with what the uses of this
958 // expression expect.
959 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
960}
961
962/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
963/// typesize).
964SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
965 assert(N->getValueType(0).getVectorNumElements() == 1 &&
966 "Unexpected vector type!");
967 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
968 SDValue Op =
969 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
970 N1: Elt, N2: N->getOperand(Num: 1));
971 // Revectorize the result so the types line up with what the uses of this
972 // expression expect.
973 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
974}
975
976/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
977/// Do the strict FP operation on the element instead.
978SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
979 assert(N->getValueType(0).getVectorNumElements() == 1 &&
980 "Unexpected vector type!");
981 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
982 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
983 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
984 Ops: { N->getOperand(Num: 0), Elt });
985 // Legalize the chain result - switch anything that used the old chain to
986 // use the new one.
987 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
988 // Revectorize the result so the types line up with what the uses of this
989 // expression expect.
990 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
991
992 // Do our own replacement and return SDValue() to tell the caller that we
993 // handled all replacements since caller can only handle a single result.
994 ReplaceValueWith(From: SDValue(N, 0), To: Res);
995 return SDValue();
996}
997
998/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
999SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1000 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1001 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1002 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1003 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1004}
1005
1006/// The inserted subvector is to be scalarized - use insert vector element
1007/// instead.
1008SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1009 unsigned OpNo) {
1010 // We should not be attempting to scalarize the containing vector
1011 assert(OpNo == 1);
1012 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1013 SDValue ContainingVec = N->getOperand(Num: 0);
1014 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1015 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1016 N3: N->getOperand(Num: 2));
1017}
1018
1019/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1020/// so just return the element, ignoring the index.
1021SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1022 EVT VT = N->getValueType(ResNo: 0);
1023 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1024 if (Res.getValueType() != VT)
1025 Res = VT.isFloatingPoint()
1026 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1027 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1028 return Res;
1029}
1030
1031/// If the input condition is a vector that needs to be scalarized, it must be
1032/// <1 x i1>, so just convert to a normal ISD::SELECT
1033/// (still with vector output type since that was acceptable if we got here).
1034SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1035 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1036 EVT VT = N->getValueType(ResNo: 0);
1037
1038 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1039 N3: N->getOperand(Num: 2));
1040}
1041
1042/// If the operand is a vector that needs to be scalarized then the
1043/// result must be v1i1, so just convert to a scalar SETCC and wrap
1044/// with a scalar_to_vector since the res type is legal if we got here
1045SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1046 assert(N->getValueType(0).isVector() &&
1047 N->getOperand(0).getValueType().isVector() &&
1048 "Operand types must be vectors");
1049 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1050
1051 EVT VT = N->getValueType(ResNo: 0);
1052 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1053 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1054
1055 EVT OpVT = N->getOperand(Num: 0).getValueType();
1056 EVT NVT = VT.getVectorElementType();
1057 SDLoc DL(N);
1058 // Turn it into a scalar SETCC.
1059 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1060 N3: N->getOperand(Num: 2));
1061
1062 // Vectors may have a different boolean contents to scalars. Promote the
1063 // value appropriately.
1064 ISD::NodeType ExtendCode =
1065 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1066
1067 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1068
1069 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1070}
1071
1072// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1073SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1074 unsigned OpNo) {
1075 assert(OpNo == 1 && "Wrong operand for scalarization!");
1076 assert(N->getValueType(0).isVector() &&
1077 N->getOperand(1).getValueType().isVector() &&
1078 "Operand types must be vectors");
1079 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1080
1081 EVT VT = N->getValueType(ResNo: 0);
1082 SDValue Ch = N->getOperand(Num: 0);
1083 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1084 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1085 SDValue CC = N->getOperand(Num: 3);
1086
1087 EVT OpVT = N->getOperand(Num: 1).getValueType();
1088 EVT NVT = VT.getVectorElementType();
1089 SDLoc DL(N);
1090 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1091 Ops: {Ch, LHS, RHS, CC});
1092
1093 // Legalize the chain result - switch anything that used the old chain to
1094 // use the new one.
1095 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1096
1097 ISD::NodeType ExtendCode =
1098 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1099
1100 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1101 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1102
1103 // Do our own replacement and return SDValue() to tell the caller that we
1104 // handled all replacements since caller can only handle a single result.
1105 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1106 return SDValue();
1107}
1108
1109/// If the value to store is a vector that needs to be scalarized, it must be
1110/// <1 x ty>. Just store the element.
1111SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1112 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1113 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1114 SDLoc dl(N);
1115
1116 if (N->isTruncatingStore())
1117 return DAG.getTruncStore(
1118 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1119 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1120 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1121 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1122
1123 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1124 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1125 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1126}
1127
1128/// If the value to round is a vector that needs to be scalarized, it must be
1129/// <1 x ty>. Convert the element instead.
1130SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1131 assert(OpNo == 0 && "Wrong operand for scalarization!");
1132 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1133 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1134 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1135 N2: N->getOperand(Num: 1));
1136 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1137}
1138
1139SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1140 unsigned OpNo) {
1141 assert(OpNo == 1 && "Wrong operand for scalarization!");
1142 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1143 SDValue Res =
1144 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1145 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1146 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1147 // Legalize the chain result - switch anything that used the old chain to
1148 // use the new one.
1149 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1150
1151 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1152
1153 // Do our own replacement and return SDValue() to tell the caller that we
1154 // handled all replacements since caller can only handle a single result.
1155 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1156 return SDValue();
1157}
1158
1159/// If the value to extend is a vector that needs to be scalarized, it must be
1160/// <1 x ty>. Convert the element instead.
1161SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1162 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1163 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1164 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1165 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1166}
1167
1168/// If the value to extend is a vector that needs to be scalarized, it must be
1169/// <1 x ty>. Convert the element instead.
1170SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1171 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1172 SDValue Res =
1173 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1174 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1175 Ops: {N->getOperand(Num: 0), Elt});
1176 // Legalize the chain result - switch anything that used the old chain to
1177 // use the new one.
1178 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1179
1180 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1181
1182 // Do our own replacement and return SDValue() to tell the caller that we
1183 // handled all replacements since caller can only handle a single result.
1184 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1185 return SDValue();
1186}
1187
1188SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1189 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1190 // Result type may be wider than element type.
1191 if (Res.getValueType() != N->getValueType(ResNo: 0))
1192 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1193 return Res;
1194}
1195
1196SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1197 SDValue AccOp = N->getOperand(Num: 0);
1198 SDValue VecOp = N->getOperand(Num: 1);
1199
1200 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1201
1202 SDValue Op = GetScalarizedVector(Op: VecOp);
1203 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1204 N1: AccOp, N2: Op, Flags: N->getFlags());
1205}
1206
1207SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1208 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1209 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1210
1211 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1212 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1213 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1214}
1215
1216SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1217 // Since there is no "none-active" result, the only valid return for <1 x ty>
1218 // is 0. Note: Since we check the high mask during splitting this is safe.
1219 // As e.g., a <2 x ty> operation would split to:
1220 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1221 // : `last_active(%lo_mask)`
1222 // Which then scalarizes to:
1223 // %mask[1] ? 1 : 0
1224 EVT VT = N->getValueType(ResNo: 0);
1225 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1226}
1227
1228SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1229 // The number of trailing zero elements is 1 if the element is 0, and 0
1230 // otherwise.
1231 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1232 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1233 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1234 SDValue SetCC =
1235 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1236 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1237 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1238}
1239
1240//===----------------------------------------------------------------------===//
1241// Result Vector Splitting
1242//===----------------------------------------------------------------------===//
1243
1244/// This method is called when the specified result of the specified node is
1245/// found to need vector splitting. At this point, the node may also have
1246/// invalid operands or may have other results that need legalization, we just
1247/// know that (at least) one result needs vector splitting.
1248void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1249 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1250 SDValue Lo, Hi;
1251
1252 // See if the target wants to custom expand this node.
1253 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1254 return;
1255
1256 switch (N->getOpcode()) {
1257 default:
1258#ifndef NDEBUG
1259 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1260 N->dump(&DAG);
1261 dbgs() << "\n";
1262#endif
1263 report_fatal_error(reason: "Do not know how to split the result of this "
1264 "operator!\n");
1265
1266 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1267 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1268 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1269 break;
1270 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1271 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1272 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1273 case ISD::VSELECT:
1274 case ISD::SELECT:
1275 case ISD::VP_MERGE:
1276 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1277 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1278 case ISD::POISON:
1279 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1280 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1281 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1282 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1283 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1284 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1285 case ISD::FPOWI:
1286 case ISD::FLDEXP:
1287 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1288 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1289 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1290 case ISD::SPLAT_VECTOR:
1291 case ISD::SCALAR_TO_VECTOR:
1292 SplitVecRes_ScalarOp(N, Lo, Hi);
1293 break;
1294 case ISD::STEP_VECTOR:
1295 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1296 break;
1297 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1298 case ISD::LOAD:
1299 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1300 break;
1301 case ISD::VP_LOAD:
1302 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1303 break;
1304 case ISD::VP_LOAD_FF:
1305 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1306 break;
1307 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1308 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1309 break;
1310 case ISD::MLOAD:
1311 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1312 break;
1313 case ISD::MGATHER:
1314 case ISD::VP_GATHER:
1315 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1316 break;
1317 case ISD::VECTOR_COMPRESS:
1318 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1319 break;
1320 case ISD::SETCC:
1321 case ISD::VP_SETCC:
1322 SplitVecRes_SETCC(N, Lo, Hi);
1323 break;
1324 case ISD::VECTOR_REVERSE:
1325 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1326 break;
1327 case ISD::VECTOR_SHUFFLE:
1328 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1329 break;
1330 case ISD::VECTOR_SPLICE_LEFT:
1331 case ISD::VECTOR_SPLICE_RIGHT:
1332 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1333 break;
1334 case ISD::VECTOR_DEINTERLEAVE:
1335 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1336 return;
1337 case ISD::VECTOR_INTERLEAVE:
1338 SplitVecRes_VECTOR_INTERLEAVE(N);
1339 return;
1340 case ISD::VAARG:
1341 SplitVecRes_VAARG(N, Lo, Hi);
1342 break;
1343
1344 case ISD::ANY_EXTEND_VECTOR_INREG:
1345 case ISD::SIGN_EXTEND_VECTOR_INREG:
1346 case ISD::ZERO_EXTEND_VECTOR_INREG:
1347 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1348 break;
1349
1350 case ISD::ABS:
1351 case ISD::VP_ABS:
1352 case ISD::BITREVERSE:
1353 case ISD::VP_BITREVERSE:
1354 case ISD::BSWAP:
1355 case ISD::VP_BSWAP:
1356 case ISD::CTLZ:
1357 case ISD::VP_CTLZ:
1358 case ISD::CTTZ:
1359 case ISD::VP_CTTZ:
1360 case ISD::CTLZ_ZERO_UNDEF:
1361 case ISD::VP_CTLZ_ZERO_UNDEF:
1362 case ISD::CTTZ_ZERO_UNDEF:
1363 case ISD::VP_CTTZ_ZERO_UNDEF:
1364 case ISD::CTPOP:
1365 case ISD::VP_CTPOP:
1366 case ISD::FABS: case ISD::VP_FABS:
1367 case ISD::FACOS:
1368 case ISD::FASIN:
1369 case ISD::FATAN:
1370 case ISD::FCEIL:
1371 case ISD::VP_FCEIL:
1372 case ISD::FCOS:
1373 case ISD::FCOSH:
1374 case ISD::FEXP:
1375 case ISD::FEXP2:
1376 case ISD::FEXP10:
1377 case ISD::FFLOOR:
1378 case ISD::VP_FFLOOR:
1379 case ISD::FLOG:
1380 case ISD::FLOG10:
1381 case ISD::FLOG2:
1382 case ISD::FNEARBYINT:
1383 case ISD::VP_FNEARBYINT:
1384 case ISD::FNEG: case ISD::VP_FNEG:
1385 case ISD::FREEZE:
1386 case ISD::ARITH_FENCE:
1387 case ISD::FP_EXTEND:
1388 case ISD::VP_FP_EXTEND:
1389 case ISD::FP_ROUND:
1390 case ISD::VP_FP_ROUND:
1391 case ISD::FP_TO_SINT:
1392 case ISD::VP_FP_TO_SINT:
1393 case ISD::FP_TO_UINT:
1394 case ISD::VP_FP_TO_UINT:
1395 case ISD::FRINT:
1396 case ISD::VP_FRINT:
1397 case ISD::LRINT:
1398 case ISD::VP_LRINT:
1399 case ISD::LLRINT:
1400 case ISD::VP_LLRINT:
1401 case ISD::FROUND:
1402 case ISD::VP_FROUND:
1403 case ISD::FROUNDEVEN:
1404 case ISD::VP_FROUNDEVEN:
1405 case ISD::LROUND:
1406 case ISD::LLROUND:
1407 case ISD::FSIN:
1408 case ISD::FSINH:
1409 case ISD::FSQRT: case ISD::VP_SQRT:
1410 case ISD::FTAN:
1411 case ISD::FTANH:
1412 case ISD::FTRUNC:
1413 case ISD::VP_FROUNDTOZERO:
1414 case ISD::SINT_TO_FP:
1415 case ISD::VP_SINT_TO_FP:
1416 case ISD::TRUNCATE:
1417 case ISD::VP_TRUNCATE:
1418 case ISD::UINT_TO_FP:
1419 case ISD::VP_UINT_TO_FP:
1420 case ISD::FCANONICALIZE:
1421 case ISD::AssertNoFPClass:
1422 case ISD::CONVERT_FROM_ARBITRARY_FP:
1423 SplitVecRes_UnaryOp(N, Lo, Hi);
1424 break;
1425 case ISD::ADDRSPACECAST:
1426 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1427 break;
1428 case ISD::FMODF:
1429 case ISD::FFREXP:
1430 case ISD::FSINCOS:
1431 case ISD::FSINCOSPI:
1432 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1433 break;
1434
1435 case ISD::ANY_EXTEND:
1436 case ISD::SIGN_EXTEND:
1437 case ISD::ZERO_EXTEND:
1438 case ISD::VP_SIGN_EXTEND:
1439 case ISD::VP_ZERO_EXTEND:
1440 SplitVecRes_ExtendOp(N, Lo, Hi);
1441 break;
1442
1443 case ISD::ADD: case ISD::VP_ADD:
1444 case ISD::SUB: case ISD::VP_SUB:
1445 case ISD::MUL: case ISD::VP_MUL:
1446 case ISD::CLMUL:
1447 case ISD::CLMULR:
1448 case ISD::CLMULH:
1449 case ISD::MULHS:
1450 case ISD::MULHU:
1451 case ISD::ABDS:
1452 case ISD::ABDU:
1453 case ISD::AVGCEILS:
1454 case ISD::AVGCEILU:
1455 case ISD::AVGFLOORS:
1456 case ISD::AVGFLOORU:
1457 case ISD::FADD: case ISD::VP_FADD:
1458 case ISD::FSUB: case ISD::VP_FSUB:
1459 case ISD::FMUL: case ISD::VP_FMUL:
1460 case ISD::FMINNUM:
1461 case ISD::FMINNUM_IEEE:
1462 case ISD::VP_FMINNUM:
1463 case ISD::FMAXNUM:
1464 case ISD::FMAXNUM_IEEE:
1465 case ISD::VP_FMAXNUM:
1466 case ISD::FMINIMUM:
1467 case ISD::VP_FMINIMUM:
1468 case ISD::FMAXIMUM:
1469 case ISD::VP_FMAXIMUM:
1470 case ISD::FMINIMUMNUM:
1471 case ISD::FMAXIMUMNUM:
1472 case ISD::SDIV: case ISD::VP_SDIV:
1473 case ISD::UDIV: case ISD::VP_UDIV:
1474 case ISD::FDIV: case ISD::VP_FDIV:
1475 case ISD::FPOW:
1476 case ISD::FATAN2:
1477 case ISD::AND: case ISD::VP_AND:
1478 case ISD::OR: case ISD::VP_OR:
1479 case ISD::XOR: case ISD::VP_XOR:
1480 case ISD::SHL: case ISD::VP_SHL:
1481 case ISD::SRA: case ISD::VP_SRA:
1482 case ISD::SRL: case ISD::VP_SRL:
1483 case ISD::UREM: case ISD::VP_UREM:
1484 case ISD::SREM: case ISD::VP_SREM:
1485 case ISD::FREM: case ISD::VP_FREM:
1486 case ISD::SMIN: case ISD::VP_SMIN:
1487 case ISD::SMAX: case ISD::VP_SMAX:
1488 case ISD::UMIN: case ISD::VP_UMIN:
1489 case ISD::UMAX: case ISD::VP_UMAX:
1490 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1491 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1492 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1493 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1494 case ISD::SSHLSAT:
1495 case ISD::USHLSAT:
1496 case ISD::ROTL:
1497 case ISD::ROTR:
1498 case ISD::VP_FCOPYSIGN:
1499 SplitVecRes_BinOp(N, Lo, Hi);
1500 break;
1501 case ISD::FMA: case ISD::VP_FMA:
1502 case ISD::FSHL:
1503 case ISD::VP_FSHL:
1504 case ISD::FSHR:
1505 case ISD::VP_FSHR:
1506 SplitVecRes_TernaryOp(N, Lo, Hi);
1507 break;
1508
1509 case ISD::SCMP: case ISD::UCMP:
1510 SplitVecRes_CMP(N, Lo, Hi);
1511 break;
1512
1513#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1514 case ISD::STRICT_##DAGN:
1515#include "llvm/IR/ConstrainedOps.def"
1516 SplitVecRes_StrictFPOp(N, Lo, Hi);
1517 break;
1518
1519 case ISD::FP_TO_UINT_SAT:
1520 case ISD::FP_TO_SINT_SAT:
1521 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1522 break;
1523
1524 case ISD::UADDO:
1525 case ISD::SADDO:
1526 case ISD::USUBO:
1527 case ISD::SSUBO:
1528 case ISD::UMULO:
1529 case ISD::SMULO:
1530 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1531 break;
1532 case ISD::SMULFIX:
1533 case ISD::SMULFIXSAT:
1534 case ISD::UMULFIX:
1535 case ISD::UMULFIXSAT:
1536 case ISD::SDIVFIX:
1537 case ISD::SDIVFIXSAT:
1538 case ISD::UDIVFIX:
1539 case ISD::UDIVFIXSAT:
1540 SplitVecRes_FIX(N, Lo, Hi);
1541 break;
1542 case ISD::EXPERIMENTAL_VP_SPLICE:
1543 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1544 break;
1545 case ISD::EXPERIMENTAL_VP_REVERSE:
1546 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1547 break;
1548 case ISD::PARTIAL_REDUCE_UMLA:
1549 case ISD::PARTIAL_REDUCE_SMLA:
1550 case ISD::PARTIAL_REDUCE_SUMLA:
1551 case ISD::PARTIAL_REDUCE_FMLA:
1552 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1553 break;
1554 case ISD::GET_ACTIVE_LANE_MASK:
1555 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1556 break;
1557 }
1558
1559 // If Lo/Hi is null, the sub-method took care of registering results etc.
1560 if (Lo.getNode())
1561 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1565 MachinePointerInfo &MPI, SDValue &Ptr,
1566 uint64_t *ScaledOffset) {
1567 SDLoc DL(N);
1568 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1569
1570 if (MemVT.isScalableVector()) {
1571 SDValue BytesIncrement = DAG.getVScale(
1572 DL, VT: Ptr.getValueType(),
1573 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1574 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1575 if (ScaledOffset)
1576 *ScaledOffset += IncrementSize;
1577 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1578 Flags: SDNodeFlags::NoUnsignedWrap);
1579 } else {
1580 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1581 // Increment the pointer to the other half.
1582 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1583 }
1584}
1585
1586std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1587 return SplitMask(Mask, DL: SDLoc(Mask));
1588}
1589
1590std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1591 const SDLoc &DL) {
1592 SDValue MaskLo, MaskHi;
1593 EVT MaskVT = Mask.getValueType();
1594 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1595 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1596 else
1597 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1598 return std::make_pair(x&: MaskLo, y&: MaskHi);
1599}
1600
1601void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1602 SDValue LHSLo, LHSHi;
1603 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1604 SDValue RHSLo, RHSHi;
1605 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1606 SDLoc dl(N);
1607
1608 const SDNodeFlags Flags = N->getFlags();
1609 unsigned Opcode = N->getOpcode();
1610 if (N->getNumOperands() == 2) {
1611 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1612 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1613 return;
1614 }
1615
1616 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1617 assert(N->isVPOpcode() && "Expected VP opcode");
1618
1619 SDValue MaskLo, MaskHi;
1620 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1621
1622 SDValue EVLLo, EVLHi;
1623 std::tie(args&: EVLLo, args&: EVLHi) =
1624 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1625
1626 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1627 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1628 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1629 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1630}
1631
1632void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1633 SDValue &Hi) {
1634 SDValue Op0Lo, Op0Hi;
1635 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1636 SDValue Op1Lo, Op1Hi;
1637 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1638 SDValue Op2Lo, Op2Hi;
1639 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1640 SDLoc dl(N);
1641
1642 const SDNodeFlags Flags = N->getFlags();
1643 unsigned Opcode = N->getOpcode();
1644 if (N->getNumOperands() == 3) {
1645 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1646 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1647 return;
1648 }
1649
1650 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1651 assert(N->isVPOpcode() && "Expected VP opcode");
1652
1653 SDValue MaskLo, MaskHi;
1654 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1655
1656 SDValue EVLLo, EVLHi;
1657 std::tie(args&: EVLLo, args&: EVLHi) =
1658 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1659
1660 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1661 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1662 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1663 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1664}
1665
1666void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1667 LLVMContext &Ctxt = *DAG.getContext();
1668 SDLoc dl(N);
1669
1670 SDValue LHS = N->getOperand(Num: 0);
1671 SDValue RHS = N->getOperand(Num: 1);
1672
1673 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1674 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1675 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1676 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1677 } else {
1678 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1679 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1680 }
1681
1682 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1683 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1684 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1685}
1686
1687void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1688 SDValue LHSLo, LHSHi;
1689 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1690 SDValue RHSLo, RHSHi;
1691 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1692 SDLoc dl(N);
1693 SDValue Op2 = N->getOperand(Num: 2);
1694
1695 unsigned Opcode = N->getOpcode();
1696 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1697 Flags: N->getFlags());
1698 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1699 Flags: N->getFlags());
1700}
1701
1702void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1703 SDValue &Hi) {
1704 // We know the result is a vector. The input may be either a vector or a
1705 // scalar value.
1706 EVT LoVT, HiVT;
1707 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1708 SDLoc dl(N);
1709
1710 SDValue InOp = N->getOperand(Num: 0);
1711 EVT InVT = InOp.getValueType();
1712
1713 // Handle some special cases efficiently.
1714 switch (getTypeAction(VT: InVT)) {
1715 case TargetLowering::TypeLegal:
1716 case TargetLowering::TypePromoteInteger:
1717 case TargetLowering::TypeSoftPromoteHalf:
1718 case TargetLowering::TypeSoftenFloat:
1719 case TargetLowering::TypeScalarizeVector:
1720 case TargetLowering::TypeWidenVector:
1721 break;
1722 case TargetLowering::TypeExpandInteger:
1723 case TargetLowering::TypeExpandFloat:
1724 // A scalar to vector conversion, where the scalar needs expansion.
1725 // If the vector is being split in two then we can just convert the
1726 // expanded pieces.
1727 if (LoVT == HiVT) {
1728 GetExpandedOp(Op: InOp, Lo, Hi);
1729 if (DAG.getDataLayout().isBigEndian())
1730 std::swap(a&: Lo, b&: Hi);
1731 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1732 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1733 return;
1734 }
1735 break;
1736 case TargetLowering::TypeSplitVector:
1737 // If the input is a vector that needs to be split, convert each split
1738 // piece of the input now.
1739 GetSplitVector(Op: InOp, Lo, Hi);
1740 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1741 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1742 return;
1743 case TargetLowering::TypeScalarizeScalableVector:
1744 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1745 }
1746
1747 if (LoVT.isScalableVector()) {
1748 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1749 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1750 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1751 return;
1752 }
1753
1754 // In the general case, convert the input to an integer and split it by hand.
1755 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1756 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1757 if (DAG.getDataLayout().isBigEndian())
1758 std::swap(a&: LoIntVT, b&: HiIntVT);
1759
1760 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1761
1762 if (DAG.getDataLayout().isBigEndian())
1763 std::swap(a&: Lo, b&: Hi);
1764 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1765 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1766}
1767
1768void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1769 SDValue &Hi) {
1770 SDLoc DL(N);
1771 EVT LoVT, HiVT;
1772 SDValue PtrA = N->getOperand(Num: 0);
1773 SDValue PtrB = N->getOperand(Num: 1);
1774 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1775
1776 // The lane offset for the "Lo" half of the mask is unchanged.
1777 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1778 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1779 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1780 // The lane offset for the "Hi" half of the mask is incremented by the number
1781 // of elements in the "Lo" half.
1782 unsigned LaneOffset =
1783 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1784 // Note: The lane offset is implicitly scalable for scalable masks.
1785 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1786 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1787 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1788}
1789
1790void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1791 SDValue &Hi) {
1792 EVT LoVT, HiVT;
1793 SDLoc dl(N);
1794 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1795 unsigned LoNumElts = LoVT.getVectorNumElements();
1796 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1797 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1798
1799 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1800 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1801}
1802
1803void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1804 SDValue &Hi) {
1805 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1806 SDLoc dl(N);
1807 unsigned NumSubvectors = N->getNumOperands() / 2;
1808 if (NumSubvectors == 1) {
1809 Lo = N->getOperand(Num: 0);
1810 Hi = N->getOperand(Num: 1);
1811 return;
1812 }
1813
1814 EVT LoVT, HiVT;
1815 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1816
1817 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1818 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1819
1820 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1821 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1822}
1823
1824void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1825 SDValue &Hi) {
1826 SDValue Vec = N->getOperand(Num: 0);
1827 SDValue Idx = N->getOperand(Num: 1);
1828 SDLoc dl(N);
1829
1830 EVT LoVT, HiVT;
1831 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1832
1833 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1834 uint64_t IdxVal = Idx->getAsZExtVal();
1835 Hi = DAG.getNode(
1836 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1837 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1838}
1839
1840void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1841 SDValue &Hi) {
1842 SDValue Vec = N->getOperand(Num: 0);
1843 SDValue SubVec = N->getOperand(Num: 1);
1844 SDValue Idx = N->getOperand(Num: 2);
1845 SDLoc dl(N);
1846 GetSplitVector(Op: Vec, Lo, Hi);
1847
1848 EVT VecVT = Vec.getValueType();
1849 EVT LoVT = Lo.getValueType();
1850 EVT SubVecVT = SubVec.getValueType();
1851 unsigned VecElems = VecVT.getVectorMinNumElements();
1852 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1853 unsigned LoElems = LoVT.getVectorMinNumElements();
1854
1855 // If we know the index is in the first half, and we know the subvector
1856 // doesn't cross the boundary between the halves, we can avoid spilling the
1857 // vector, and insert into the lower half of the split vector directly.
1858 unsigned IdxVal = Idx->getAsZExtVal();
1859 if (IdxVal + SubElems <= LoElems) {
1860 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1861 return;
1862 }
1863 // Similarly if the subvector is fully in the high half, but mind that we
1864 // can't tell whether a fixed-length subvector is fully within the high half
1865 // of a scalable vector.
1866 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1867 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1868 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1869 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1870 return;
1871 }
1872
1873 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1874 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1875 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1876 if (WideSubVec.getValueType() == VecVT) {
1877 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1878 return;
1879 }
1880 }
1881
1882 // Spill the vector to the stack.
1883 // In cases where the vector is illegal it will be broken down into parts
1884 // and stored in parts - we should use the alignment for the smallest part.
1885 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1886 SDValue StackPtr =
1887 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1888 auto &MF = DAG.getMachineFunction();
1889 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1890 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1891
1892 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1893 Alignment: SmallestAlign);
1894
1895 // Store the new subvector into the specified index.
1896 SDValue SubVecPtr =
1897 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1898 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1899 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1900
1901 // Load the Lo part from the stack slot.
1902 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1903 Alignment: SmallestAlign);
1904
1905 // Increment the pointer to the other part.
1906 auto *Load = cast<LoadSDNode>(Val&: Lo);
1907 MachinePointerInfo MPI = Load->getPointerInfo();
1908 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1909
1910 // Load the Hi part from the stack slot.
1911 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1912}
1913
1914// Handle splitting an FP where the second operand does not match the first
1915// type. The second operand may be a scalar, or a vector that has exactly as
1916// many elements as the first
1917void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1918 SDValue &Hi) {
1919 SDValue LHSLo, LHSHi;
1920 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1921 SDLoc DL(N);
1922
1923 SDValue RHSLo, RHSHi;
1924 SDValue RHS = N->getOperand(Num: 1);
1925 EVT RHSVT = RHS.getValueType();
1926 if (RHSVT.isVector()) {
1927 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1928 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1929 else
1930 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1931
1932 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1933 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1934 } else {
1935 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1936 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1937 }
1938}
1939
1940void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1941 SDValue &Hi) {
1942 SDLoc DL(N);
1943 SDValue ArgLo, ArgHi;
1944 SDValue Test = N->getOperand(Num: 1);
1945 SDValue FpValue = N->getOperand(Num: 0);
1946 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1947 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1948 else
1949 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1950 EVT LoVT, HiVT;
1951 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1952
1953 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1954 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1955}
1956
1957void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1958 SDValue &Hi) {
1959 SDValue LHSLo, LHSHi;
1960 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1961 SDLoc dl(N);
1962
1963 EVT LoVT, HiVT;
1964 std::tie(args&: LoVT, args&: HiVT) =
1965 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1966
1967 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1968 N2: DAG.getValueType(LoVT));
1969 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1970 N2: DAG.getValueType(HiVT));
1971}
1972
1973void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1974 SDValue &Hi) {
1975 unsigned Opcode = N->getOpcode();
1976 SDValue N0 = N->getOperand(Num: 0);
1977
1978 SDLoc dl(N);
1979 SDValue InLo, InHi;
1980
1981 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1982 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1983 else
1984 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1985
1986 EVT InLoVT = InLo.getValueType();
1987 unsigned InNumElements = InLoVT.getVectorNumElements();
1988
1989 EVT OutLoVT, OutHiVT;
1990 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1991 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1992 assert((2 * OutNumElements) <= InNumElements &&
1993 "Illegal extend vector in reg split");
1994
1995 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1996 // input vector (i.e. we only use InLo):
1997 // OutLo will extend the first OutNumElements from InLo.
1998 // OutHi will extend the next OutNumElements from InLo.
1999
2000 // Shuffle the elements from InLo for OutHi into the bottom elements to
2001 // create a 'fake' InHi.
2002 SmallVector<int, 8> SplitHi(InNumElements, -1);
2003 for (unsigned i = 0; i != OutNumElements; ++i)
2004 SplitHi[i] = i + OutNumElements;
2005 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2006
2007 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2008 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2009}
2010
2011void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2012 SDValue &Hi) {
2013 unsigned NumOps = N->getNumOperands();
2014 SDValue Chain = N->getOperand(Num: 0);
2015 EVT LoVT, HiVT;
2016 SDLoc dl(N);
2017 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2018
2019 SmallVector<SDValue, 4> OpsLo(NumOps);
2020 SmallVector<SDValue, 4> OpsHi(NumOps);
2021
2022 // The Chain is the first operand.
2023 OpsLo[0] = Chain;
2024 OpsHi[0] = Chain;
2025
2026 // Now process the remaining operands.
2027 for (unsigned i = 1; i < NumOps; ++i) {
2028 SDValue Op = N->getOperand(Num: i);
2029 SDValue OpLo = Op;
2030 SDValue OpHi = Op;
2031
2032 EVT InVT = Op.getValueType();
2033 if (InVT.isVector()) {
2034 // If the input also splits, handle it directly for a
2035 // compile time speedup. Otherwise split it by hand.
2036 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2037 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2038 else
2039 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2040 }
2041
2042 OpsLo[i] = OpLo;
2043 OpsHi[i] = OpHi;
2044 }
2045
2046 EVT LoValueVTs[] = {LoVT, MVT::Other};
2047 EVT HiValueVTs[] = {HiVT, MVT::Other};
2048 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2049 Flags: N->getFlags());
2050 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2051 Flags: N->getFlags());
2052
2053 // Build a factor node to remember that this Op is independent of the
2054 // other one.
2055 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2056 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2057
2058 // Legalize the chain result - switch anything that used the old chain to
2059 // use the new one.
2060 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2061}
2062
2063SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2064 SDValue Chain = N->getOperand(Num: 0);
2065 EVT VT = N->getValueType(ResNo: 0);
2066 unsigned NE = VT.getVectorNumElements();
2067 EVT EltVT = VT.getVectorElementType();
2068 SDLoc dl(N);
2069
2070 SmallVector<SDValue, 8> Scalars;
2071 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2072
2073 // If ResNE is 0, fully unroll the vector op.
2074 if (ResNE == 0)
2075 ResNE = NE;
2076 else if (NE > ResNE)
2077 NE = ResNE;
2078
2079 //The results of each unrolled operation, including the chain.
2080 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2081 SmallVector<SDValue, 8> Chains;
2082
2083 unsigned i;
2084 for (i = 0; i != NE; ++i) {
2085 Operands[0] = Chain;
2086 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2087 SDValue Operand = N->getOperand(Num: j);
2088 EVT OperandVT = Operand.getValueType();
2089 if (OperandVT.isVector()) {
2090 EVT OperandEltVT = OperandVT.getVectorElementType();
2091 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2092 } else {
2093 Operands[j] = Operand;
2094 }
2095 }
2096 SDValue Scalar =
2097 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2098
2099 //Add in the scalar as well as its chain value to the
2100 //result vectors.
2101 Scalars.push_back(Elt: Scalar);
2102 Chains.push_back(Elt: Scalar.getValue(R: 1));
2103 }
2104
2105 for (; i < ResNE; ++i)
2106 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2107
2108 // Build a new factor node to connect the chain back together.
2109 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2110 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2111
2112 // Create a new BUILD_VECTOR node
2113 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2114 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2115}
2116
2117void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2118 SDValue &Lo, SDValue &Hi) {
2119 SDLoc dl(N);
2120 EVT ResVT = N->getValueType(ResNo: 0);
2121 EVT OvVT = N->getValueType(ResNo: 1);
2122 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2123 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2124 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2125
2126 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2127 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2128 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2129 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2130 } else {
2131 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2132 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2133 }
2134
2135 unsigned Opcode = N->getOpcode();
2136 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2137 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2138 SDNode *LoNode =
2139 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2140 SDNode *HiNode =
2141 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2142
2143 Lo = SDValue(LoNode, ResNo);
2144 Hi = SDValue(HiNode, ResNo);
2145
2146 // Replace the other vector result not being explicitly split here.
2147 unsigned OtherNo = 1 - ResNo;
2148 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2149 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2150 SetSplitVector(Op: SDValue(N, OtherNo),
2151 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2152 } else {
2153 SDValue OtherVal = DAG.getNode(
2154 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2155 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2156 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2157 }
2158}
2159
2160void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2161 SDValue &Hi) {
2162 SDValue Vec = N->getOperand(Num: 0);
2163 SDValue Elt = N->getOperand(Num: 1);
2164 SDValue Idx = N->getOperand(Num: 2);
2165 SDLoc dl(N);
2166 GetSplitVector(Op: Vec, Lo, Hi);
2167
2168 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2169 unsigned IdxVal = CIdx->getZExtValue();
2170 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2171 if (IdxVal < LoNumElts) {
2172 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2173 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2174 return;
2175 } else if (!Vec.getValueType().isScalableVector()) {
2176 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2177 return;
2178 }
2179 }
2180
2181 // Make the vector elements byte-addressable if they aren't already.
2182 EVT VecVT = Vec.getValueType();
2183 EVT EltVT = VecVT.getVectorElementType();
2184 if (!EltVT.isByteSized()) {
2185 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2186 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2187 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2188 // Extend the element type to match if needed.
2189 if (EltVT.bitsGT(VT: Elt.getValueType()))
2190 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2191 }
2192
2193 // Spill the vector to the stack.
2194 // In cases where the vector is illegal it will be broken down into parts
2195 // and stored in parts - we should use the alignment for the smallest part.
2196 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2197 SDValue StackPtr =
2198 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2199 auto &MF = DAG.getMachineFunction();
2200 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2201 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2202
2203 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2204 Alignment: SmallestAlign);
2205
2206 // Store the new element. This may be larger than the vector element type,
2207 // so use a truncating store.
2208 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2209 Store = DAG.getTruncStore(
2210 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2211 Alignment: commonAlignment(A: SmallestAlign,
2212 Offset: EltVT.getFixedSizeInBits() / 8));
2213
2214 EVT LoVT, HiVT;
2215 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2216
2217 // Load the Lo part from the stack slot.
2218 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2219
2220 // Increment the pointer to the other part.
2221 auto Load = cast<LoadSDNode>(Val&: Lo);
2222 MachinePointerInfo MPI = Load->getPointerInfo();
2223 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2224
2225 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2226
2227 // If we adjusted the original type, we need to truncate the results.
2228 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2229 if (LoVT != Lo.getValueType())
2230 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2231 if (HiVT != Hi.getValueType())
2232 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2233}
2234
2235void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2236 SDValue &Hi) {
2237 EVT LoVT, HiVT;
2238 SDLoc dl(N);
2239 assert(N->getValueType(0).isScalableVector() &&
2240 "Only scalable vectors are supported for STEP_VECTOR");
2241 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2242 SDValue Step = N->getOperand(Num: 0);
2243
2244 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2245
2246 // Hi = Lo + (EltCnt * Step)
2247 EVT EltVT = Step.getValueType();
2248 APInt StepVal = Step->getAsAPIntVal();
2249 SDValue StartOfHi =
2250 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2251 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2252 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2253
2254 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2255 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2256}
2257
2258void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2259 SDValue &Hi) {
2260 EVT LoVT, HiVT;
2261 SDLoc dl(N);
2262 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2263 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2264 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2265 Hi = DAG.getPOISON(VT: HiVT);
2266 } else {
2267 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2268 Hi = Lo;
2269 }
2270}
2271
2272void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2273 SDValue &Hi) {
2274 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2275 EVT LoVT, HiVT;
2276 SDLoc dl(LD);
2277 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2278
2279 ISD::LoadExtType ExtType = LD->getExtensionType();
2280 SDValue Ch = LD->getChain();
2281 SDValue Ptr = LD->getBasePtr();
2282 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2283 EVT MemoryVT = LD->getMemoryVT();
2284 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2285 AAMDNodes AAInfo = LD->getAAInfo();
2286
2287 EVT LoMemVT, HiMemVT;
2288 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2289
2290 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2291 SDValue Value, NewChain;
2292 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2293 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2294 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2295 return;
2296 }
2297
2298 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2299 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2300 AAInfo);
2301
2302 MachinePointerInfo MPI;
2303 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2304
2305 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2306 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2307
2308 // Build a factor node to remember that this load is independent of the
2309 // other one.
2310 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2311 N2: Hi.getValue(R: 1));
2312
2313 // Legalize the chain result - switch anything that used the old chain to
2314 // use the new one.
2315 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2316}
2317
2318void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2319 SDValue &Hi) {
2320 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2321 EVT LoVT, HiVT;
2322 SDLoc dl(LD);
2323 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2324
2325 ISD::LoadExtType ExtType = LD->getExtensionType();
2326 SDValue Ch = LD->getChain();
2327 SDValue Ptr = LD->getBasePtr();
2328 SDValue Offset = LD->getOffset();
2329 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2330 Align Alignment = LD->getBaseAlign();
2331 SDValue Mask = LD->getMask();
2332 SDValue EVL = LD->getVectorLength();
2333 EVT MemoryVT = LD->getMemoryVT();
2334
2335 EVT LoMemVT, HiMemVT;
2336 bool HiIsEmpty = false;
2337 std::tie(args&: LoMemVT, args&: HiMemVT) =
2338 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2339
2340 // Split Mask operand
2341 SDValue MaskLo, MaskHi;
2342 if (Mask.getOpcode() == ISD::SETCC) {
2343 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2344 } else {
2345 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2346 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2347 else
2348 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2349 }
2350
2351 // Split EVL operand
2352 SDValue EVLLo, EVLHi;
2353 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2354
2355 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2356 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2357 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2358 Ranges: LD->getRanges());
2359
2360 Lo =
2361 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2362 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2363
2364 if (HiIsEmpty) {
2365 // The hi vp_load has zero storage size. We therefore simply set it to
2366 // the low vp_load and rely on subsequent removal from the chain.
2367 Hi = Lo;
2368 } else {
2369 // Generate hi vp_load.
2370 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2371 IsCompressedMemory: LD->isExpandingLoad());
2372
2373 MachinePointerInfo MPI;
2374 if (LoMemVT.isScalableVector())
2375 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2376 else
2377 MPI = LD->getPointerInfo().getWithOffset(
2378 O: LoMemVT.getStoreSize().getFixedValue());
2379
2380 MMO = DAG.getMachineFunction().getMachineMemOperand(
2381 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2382 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2383
2384 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2385 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2386 IsExpanding: LD->isExpandingLoad());
2387 }
2388
2389 // Build a factor node to remember that this load is independent of the
2390 // other one.
2391 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2392 N2: Hi.getValue(R: 1));
2393
2394 // Legalize the chain result - switch anything that used the old chain to
2395 // use the new one.
2396 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2397}
2398
2399void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2400 SDValue &Hi) {
2401 SDLoc dl(LD);
2402 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2403
2404 SDValue Ch = LD->getChain();
2405 SDValue Ptr = LD->getBasePtr();
2406 Align Alignment = LD->getBaseAlign();
2407 SDValue Mask = LD->getMask();
2408 SDValue EVL = LD->getVectorLength();
2409
2410 // Split Mask operand
2411 SDValue MaskLo, MaskHi;
2412 if (Mask.getOpcode() == ISD::SETCC) {
2413 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2414 } else {
2415 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2416 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2417 else
2418 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2419 }
2420
2421 // Split EVL operand
2422 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2423
2424 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2425 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2426 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2427 Ranges: LD->getRanges());
2428
2429 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2430
2431 // Fill the upper half with poison.
2432 Hi = DAG.getPOISON(VT: HiVT);
2433
2434 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2435 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2436}
2437
2438void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2439 SDValue &Lo, SDValue &Hi) {
2440 assert(SLD->isUnindexed() &&
2441 "Indexed VP strided load during type legalization!");
2442 assert(SLD->getOffset().isUndef() &&
2443 "Unexpected indexed variable-length load offset");
2444
2445 SDLoc DL(SLD);
2446
2447 EVT LoVT, HiVT;
2448 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2449
2450 EVT LoMemVT, HiMemVT;
2451 bool HiIsEmpty = false;
2452 std::tie(args&: LoMemVT, args&: HiMemVT) =
2453 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2454
2455 SDValue Mask = SLD->getMask();
2456 SDValue LoMask, HiMask;
2457 if (Mask.getOpcode() == ISD::SETCC) {
2458 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2459 } else {
2460 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2461 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2462 else
2463 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2464 }
2465
2466 SDValue LoEVL, HiEVL;
2467 std::tie(args&: LoEVL, args&: HiEVL) =
2468 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2469
2470 // Generate the low vp_strided_load
2471 Lo = DAG.getStridedLoadVP(
2472 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2473 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2474 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2475
2476 if (HiIsEmpty) {
2477 // The high vp_strided_load has zero storage size. We therefore simply set
2478 // it to the low vp_strided_load and rely on subsequent removal from the
2479 // chain.
2480 Hi = Lo;
2481 } else {
2482 // Generate the high vp_strided_load.
2483 // To calculate the high base address, we need to sum to the low base
2484 // address stride number of bytes for each element already loaded by low,
2485 // that is: Ptr = Ptr + (LoEVL * Stride)
2486 EVT PtrVT = SLD->getBasePtr().getValueType();
2487 SDValue Increment =
2488 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2489 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2490 SDValue Ptr =
2491 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2492
2493 Align Alignment = SLD->getBaseAlign();
2494 if (LoMemVT.isScalableVector())
2495 Alignment = commonAlignment(
2496 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2497
2498 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2499 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2500 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2501 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2502
2503 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2504 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2505 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2506 IsExpanding: SLD->isExpandingLoad());
2507 }
2508
2509 // Build a factor node to remember that this load is independent of the
2510 // other one.
2511 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2512 N2: Hi.getValue(R: 1));
2513
2514 // Legalize the chain result - switch anything that used the old chain to
2515 // use the new one.
2516 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2517}
2518
2519void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2520 SDValue &Lo, SDValue &Hi) {
2521 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2522 EVT LoVT, HiVT;
2523 SDLoc dl(MLD);
2524 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2525
2526 SDValue Ch = MLD->getChain();
2527 SDValue Ptr = MLD->getBasePtr();
2528 SDValue Offset = MLD->getOffset();
2529 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2530 SDValue Mask = MLD->getMask();
2531 SDValue PassThru = MLD->getPassThru();
2532 Align Alignment = MLD->getBaseAlign();
2533 ISD::LoadExtType ExtType = MLD->getExtensionType();
2534 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2535
2536 // Split Mask operand
2537 SDValue MaskLo, MaskHi;
2538 if (Mask.getOpcode() == ISD::SETCC) {
2539 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2540 } else {
2541 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2542 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2543 else
2544 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2545 }
2546
2547 EVT MemoryVT = MLD->getMemoryVT();
2548 EVT LoMemVT, HiMemVT;
2549 bool HiIsEmpty = false;
2550 std::tie(args&: LoMemVT, args&: HiMemVT) =
2551 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2552
2553 SDValue PassThruLo, PassThruHi;
2554 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2555 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2556 else
2557 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2558
2559 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2560 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2561 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2562
2563 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2564 MMO, AM: MLD->getAddressingMode(), ExtType,
2565 IsExpanding: MLD->isExpandingLoad());
2566
2567 if (HiIsEmpty) {
2568 // The hi masked load has zero storage size. We therefore simply set it to
2569 // the low masked load and rely on subsequent removal from the chain.
2570 Hi = Lo;
2571 } else {
2572 // Generate hi masked load.
2573 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2574 IsCompressedMemory: MLD->isExpandingLoad());
2575
2576 MachinePointerInfo MPI;
2577 if (LoMemVT.isScalableVector())
2578 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2579 else
2580 MPI = MLD->getPointerInfo().getWithOffset(
2581 O: LoMemVT.getStoreSize().getFixedValue());
2582
2583 MMO = DAG.getMachineFunction().getMachineMemOperand(
2584 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2585 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2586
2587 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2588 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2589 IsExpanding: MLD->isExpandingLoad());
2590 }
2591
2592 // Build a factor node to remember that this load is independent of the
2593 // other one.
2594 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2595 N2: Hi.getValue(R: 1));
2596
2597 // Legalize the chain result - switch anything that used the old chain to
2598 // use the new one.
2599 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2600
2601}
2602
2603void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2604 SDValue &Hi, bool SplitSETCC) {
2605 EVT LoVT, HiVT;
2606 SDLoc dl(N);
2607 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2608
2609 SDValue Ch = N->getChain();
2610 SDValue Ptr = N->getBasePtr();
2611 struct Operands {
2612 SDValue Mask;
2613 SDValue Index;
2614 SDValue Scale;
2615 } Ops = [&]() -> Operands {
2616 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2617 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2618 }
2619 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2620 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2621 }();
2622
2623 EVT MemoryVT = N->getMemoryVT();
2624 Align Alignment = N->getBaseAlign();
2625
2626 // Split Mask operand
2627 SDValue MaskLo, MaskHi;
2628 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2629 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2630 } else {
2631 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2632 }
2633
2634 EVT LoMemVT, HiMemVT;
2635 // Split MemoryVT
2636 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2637
2638 SDValue IndexHi, IndexLo;
2639 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2640 TargetLowering::TypeSplitVector)
2641 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2642 else
2643 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2644
2645 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2646 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2647 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2648 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2649
2650 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2651 SDValue PassThru = MGT->getPassThru();
2652 SDValue PassThruLo, PassThruHi;
2653 if (getTypeAction(VT: PassThru.getValueType()) ==
2654 TargetLowering::TypeSplitVector)
2655 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2656 else
2657 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2658
2659 ISD::LoadExtType ExtType = MGT->getExtensionType();
2660 ISD::MemIndexType IndexTy = MGT->getIndexType();
2661
2662 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2663 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2664 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2665
2666 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2667 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2668 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2669 } else {
2670 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2671 SDValue EVLLo, EVLHi;
2672 std::tie(args&: EVLLo, args&: EVLHi) =
2673 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2674
2675 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2676 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2677 MMO, IndexType: VPGT->getIndexType());
2678
2679 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2680 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2681 MMO, IndexType: VPGT->getIndexType());
2682 }
2683
2684 // Build a factor node to remember that this load is independent of the
2685 // other one.
2686 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2687 N2: Hi.getValue(R: 1));
2688
2689 // Legalize the chain result - switch anything that used the old chain to
2690 // use the new one.
2691 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2692}
2693
2694void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2695 SDValue &Hi) {
2696 // This is not "trivial", as there is a dependency between the two subvectors.
2697 // Depending on the number of 1s in the mask, the elements from the Hi vector
2698 // need to be moved to the Lo vector. Passthru values make this even harder.
2699 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2700 // smaller types and passthru is undef, as it is most likely faster than the
2701 // fully expand path. Otherwise, just do the full expansion as one "big"
2702 // operation and then extract the Lo and Hi vectors from that. This gets
2703 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2704 SDLoc DL(N);
2705 EVT VecVT = N->getValueType(ResNo: 0);
2706
2707 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2708 bool HasCustomLowering = false;
2709 EVT CheckVT = LoVT;
2710 while (CheckVT.getVectorMinNumElements() > 1) {
2711 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2712 // custom lowering for illegal types. So we do the checks separately.
2713 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2714 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2715 HasCustomLowering = true;
2716 break;
2717 }
2718 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2719 }
2720
2721 SDValue Passthru = N->getOperand(Num: 2);
2722 if (!HasCustomLowering) {
2723 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2724 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2725 return;
2726 }
2727
2728 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2729 SDValue Mask = N->getOperand(Num: 1);
2730 SDValue LoMask, HiMask;
2731 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2732 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2733
2734 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2735 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2736 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2737
2738 SDValue StackPtr = DAG.CreateStackTemporary(
2739 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2740 MachineFunction &MF = DAG.getMachineFunction();
2741 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2742 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2743
2744 EVT MaskVT = LoMask.getValueType();
2745 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2746
2747 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2748 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2749 EC: MaskVT.getVectorElementCount());
2750 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2751 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2752 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2753
2754 SDValue Chain = DAG.getEntryNode();
2755 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2756 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2757 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2758
2759 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2760 if (!Passthru.isUndef()) {
2761 Compressed =
2762 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2763 }
2764 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2765}
2766
2767void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2768 assert(N->getValueType(0).isVector() &&
2769 N->getOperand(0).getValueType().isVector() &&
2770 "Operand types must be vectors");
2771
2772 EVT LoVT, HiVT;
2773 SDLoc DL(N);
2774 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2775
2776 // If the input also splits, handle it directly. Otherwise split it by hand.
2777 SDValue LL, LH, RL, RH;
2778 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2779 TargetLowering::TypeSplitVector)
2780 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2781 else
2782 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2783
2784 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2785 TargetLowering::TypeSplitVector)
2786 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2787 else
2788 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2789
2790 if (N->getOpcode() == ISD::SETCC) {
2791 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2792 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2793 } else {
2794 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2795 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2796 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2797 std::tie(args&: EVLLo, args&: EVLHi) =
2798 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2799 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2800 N5: EVLLo);
2801 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2802 N5: EVLHi);
2803 }
2804}
2805
2806void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2807 SDValue &Hi) {
2808 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2809 EVT LoVT, HiVT;
2810 SDLoc dl(N);
2811 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2812
2813 // If the input also splits, handle it directly for a compile time speedup.
2814 // Otherwise split it by hand.
2815 EVT InVT = N->getOperand(Num: 0).getValueType();
2816 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2817 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2818 else
2819 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2820
2821 const SDNodeFlags Flags = N->getFlags();
2822 unsigned Opcode = N->getOpcode();
2823 if (N->getNumOperands() <= 2) {
2824 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2825 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2826 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2827 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2828 } else {
2829 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2830 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2831 }
2832 return;
2833 }
2834
2835 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2836 assert(N->isVPOpcode() && "Expected VP opcode");
2837
2838 SDValue MaskLo, MaskHi;
2839 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2840
2841 SDValue EVLLo, EVLHi;
2842 std::tie(args&: EVLLo, args&: EVLHi) =
2843 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2844
2845 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2846 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2847}
2848
2849void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2850 SDValue &Hi) {
2851 SDLoc dl(N);
2852 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2853
2854 // If the input also splits, handle it directly for a compile time speedup.
2855 // Otherwise split it by hand.
2856 EVT InVT = N->getOperand(Num: 0).getValueType();
2857 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2858 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2859 else
2860 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2861
2862 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
2863 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2864 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2865 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
2866 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
2867}
2868
2869void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
2870 unsigned ResNo,
2871 SDValue &Lo,
2872 SDValue &Hi) {
2873 SDLoc dl(N);
2874 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2875 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2876
2877 // If the input also splits, handle it directly for a compile time speedup.
2878 // Otherwise split it by hand.
2879 EVT InVT = N->getOperand(Num: 0).getValueType();
2880 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2881 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2882 else
2883 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2884
2885 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
2886 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
2887
2888 SDNode *HiNode = Hi.getNode();
2889 SDNode *LoNode = Lo.getNode();
2890
2891 // Replace the other vector result not being explicitly split here.
2892 unsigned OtherNo = 1 - ResNo;
2893 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2894 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2895 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2896 Hi: SDValue(HiNode, OtherNo));
2897 } else {
2898 SDValue OtherVal =
2899 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2900 N2: SDValue(HiNode, OtherNo));
2901 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2902 }
2903}
2904
2905void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2906 SDValue &Hi) {
2907 SDLoc dl(N);
2908 EVT SrcVT = N->getOperand(Num: 0).getValueType();
2909 EVT DestVT = N->getValueType(ResNo: 0);
2910 EVT LoVT, HiVT;
2911 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2912
2913 // We can do better than a generic split operation if the extend is doing
2914 // more than just doubling the width of the elements and the following are
2915 // true:
2916 // - The number of vector elements is even,
2917 // - the source type is legal,
2918 // - the type of a split source is illegal,
2919 // - the type of an extended (by doubling element size) source is legal, and
2920 // - the type of that extended source when split is legal.
2921 //
2922 // This won't necessarily completely legalize the operation, but it will
2923 // more effectively move in the right direction and prevent falling down
2924 // to scalarization in many cases due to the input vector being split too
2925 // far.
2926 if (SrcVT.getVectorElementCount().isKnownEven() &&
2927 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2928 LLVMContext &Ctx = *DAG.getContext();
2929 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2930 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2931
2932 EVT SplitLoVT, SplitHiVT;
2933 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2934 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2935 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2936 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2937 N->dump(&DAG); dbgs() << "\n");
2938 if (!N->isVPOpcode()) {
2939 // Extend the source vector by one step.
2940 SDValue NewSrc =
2941 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2942 // Get the low and high halves of the new, extended one step, vector.
2943 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2944 // Extend those vector halves the rest of the way.
2945 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2946 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2947 return;
2948 }
2949
2950 // Extend the source vector by one step.
2951 SDValue NewSrc =
2952 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2953 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2954 // Get the low and high halves of the new, extended one step, vector.
2955 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2956
2957 SDValue MaskLo, MaskHi;
2958 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2959
2960 SDValue EVLLo, EVLHi;
2961 std::tie(args&: EVLLo, args&: EVLHi) =
2962 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2963 // Extend those vector halves the rest of the way.
2964 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2965 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2966 return;
2967 }
2968 }
2969 // Fall back to the generic unary operator splitting otherwise.
2970 SplitVecRes_UnaryOp(N, Lo, Hi);
2971}
2972
2973void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2974 SDValue &Lo, SDValue &Hi) {
2975 // The low and high parts of the original input give four input vectors.
2976 SDValue Inputs[4];
2977 SDLoc DL(N);
2978 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2979 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2980 EVT NewVT = Inputs[0].getValueType();
2981 unsigned NewElts = NewVT.getVectorNumElements();
2982
2983 auto &&IsConstant = [](const SDValue &N) {
2984 APInt SplatValue;
2985 return N.getResNo() == 0 &&
2986 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2987 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2988 };
2989 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2990 SDValue &Input2,
2991 ArrayRef<int> Mask) {
2992 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2993 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2994 "Expected build vector node.");
2995 EVT EltVT = NewVT.getVectorElementType();
2996 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
2997 for (unsigned I = 0; I < NewElts; ++I) {
2998 if (Mask[I] == PoisonMaskElem)
2999 continue;
3000 unsigned Idx = Mask[I];
3001 if (Idx >= NewElts)
3002 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3003 else
3004 Ops[I] = Input1.getOperand(i: Idx);
3005 // Make the type of all elements the same as the element type.
3006 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3007 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3008 }
3009 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3010 };
3011
3012 // If Lo or Hi uses elements from at most two of the four input vectors, then
3013 // express it as a vector shuffle of those two inputs. Otherwise extract the
3014 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3015 SmallVector<int> OrigMask(N->getMask());
3016 // Try to pack incoming shuffles/inputs.
3017 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3018 &DL](SmallVectorImpl<int> &Mask) {
3019 // Check if all inputs are shuffles of the same operands or non-shuffles.
3020 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3021 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3022 SDValue Input = Inputs[Idx];
3023 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3024 if (!Shuffle ||
3025 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3026 continue;
3027 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3028 .push_back(Elt: Idx);
3029 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3030 .push_back(Elt: Idx);
3031 }
3032 for (auto &P : ShufflesIdxs) {
3033 if (P.second.size() < 2)
3034 continue;
3035 // Use shuffles operands instead of shuffles themselves.
3036 // 1. Adjust mask.
3037 for (int &Idx : Mask) {
3038 if (Idx == PoisonMaskElem)
3039 continue;
3040 unsigned SrcRegIdx = Idx / NewElts;
3041 if (Inputs[SrcRegIdx].isUndef()) {
3042 Idx = PoisonMaskElem;
3043 continue;
3044 }
3045 auto *Shuffle =
3046 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3047 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3048 continue;
3049 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3050 if (MaskElt == PoisonMaskElem) {
3051 Idx = PoisonMaskElem;
3052 continue;
3053 }
3054 Idx = MaskElt % NewElts +
3055 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3056 ? 0
3057 : 1] *
3058 NewElts;
3059 }
3060 // 2. Update inputs.
3061 Inputs[P.second[0]] = P.first.first;
3062 Inputs[P.second[1]] = P.first.second;
3063 // Clear the pair data.
3064 P.second.clear();
3065 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3066 }
3067 // Check if any concat_vectors can be simplified.
3068 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3069 for (int &Idx : Mask) {
3070 if (Idx == PoisonMaskElem)
3071 continue;
3072 unsigned SrcRegIdx = Idx / NewElts;
3073 if (Inputs[SrcRegIdx].isUndef()) {
3074 Idx = PoisonMaskElem;
3075 continue;
3076 }
3077 TargetLowering::LegalizeTypeAction TypeAction =
3078 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3079 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3080 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3081 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3082 (TypeAction == TargetLowering::TypeLegal ||
3083 TypeAction == TargetLowering::TypeWidenVector))
3084 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3085 }
3086 if (UsedSubVector.count() > 1) {
3087 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3088 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3089 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3090 continue;
3091 if (Pairs.empty() || Pairs.back().size() == 2)
3092 Pairs.emplace_back();
3093 if (UsedSubVector.test(Idx: 2 * I)) {
3094 Pairs.back().emplace_back(Args&: I, Args: 0);
3095 } else {
3096 assert(UsedSubVector.test(2 * I + 1) &&
3097 "Expected to be used one of the subvectors.");
3098 Pairs.back().emplace_back(Args&: I, Args: 1);
3099 }
3100 }
3101 if (!Pairs.empty() && Pairs.front().size() > 1) {
3102 // Adjust mask.
3103 for (int &Idx : Mask) {
3104 if (Idx == PoisonMaskElem)
3105 continue;
3106 unsigned SrcRegIdx = Idx / NewElts;
3107 auto *It = find_if(
3108 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3109 return Idxs.front().first == SrcRegIdx ||
3110 Idxs.back().first == SrcRegIdx;
3111 });
3112 if (It == Pairs.end())
3113 continue;
3114 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3115 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3116 }
3117 // Adjust inputs.
3118 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3119 Inputs[Idxs.front().first] = DAG.getNode(
3120 Opcode: ISD::CONCAT_VECTORS, DL,
3121 VT: Inputs[Idxs.front().first].getValueType(),
3122 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3123 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3124 }
3125 }
3126 }
3127 bool Changed;
3128 do {
3129 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3130 // reused operands.
3131 Changed = false;
3132 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3133 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3134 if (!Shuffle)
3135 continue;
3136 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3137 continue;
3138 int Op = -1;
3139 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3140 !Shuffle->isSplat()) {
3141 Op = 0;
3142 } else if (!Inputs[I].hasOneUse() &&
3143 !Shuffle->getOperand(Num: 1).isUndef()) {
3144 // Find the only used operand, if possible.
3145 for (int &Idx : Mask) {
3146 if (Idx == PoisonMaskElem)
3147 continue;
3148 unsigned SrcRegIdx = Idx / NewElts;
3149 if (SrcRegIdx != I)
3150 continue;
3151 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3152 if (MaskElt == PoisonMaskElem) {
3153 Idx = PoisonMaskElem;
3154 continue;
3155 }
3156 int OpIdx = MaskElt / NewElts;
3157 if (Op == -1) {
3158 Op = OpIdx;
3159 continue;
3160 }
3161 if (Op != OpIdx) {
3162 Op = -1;
3163 break;
3164 }
3165 }
3166 }
3167 if (Op < 0) {
3168 // Try to check if one of the shuffle operands is used already.
3169 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3170 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3171 continue;
3172 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3173 if (It == std::end(arr&: Inputs))
3174 continue;
3175 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3176 // Found that operand is used already.
3177 // 1. Fix the mask for the reused operand.
3178 for (int &Idx : Mask) {
3179 if (Idx == PoisonMaskElem)
3180 continue;
3181 unsigned SrcRegIdx = Idx / NewElts;
3182 if (SrcRegIdx != I)
3183 continue;
3184 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3185 if (MaskElt == PoisonMaskElem) {
3186 Idx = PoisonMaskElem;
3187 continue;
3188 }
3189 int MaskIdx = MaskElt / NewElts;
3190 if (OpIdx == MaskIdx)
3191 Idx = MaskElt % NewElts + FoundOp * NewElts;
3192 }
3193 // 2. Set Op to the unused OpIdx.
3194 Op = (OpIdx + 1) % 2;
3195 break;
3196 }
3197 }
3198 if (Op >= 0) {
3199 Changed = true;
3200 Inputs[I] = Shuffle->getOperand(Num: Op);
3201 // Adjust mask.
3202 for (int &Idx : Mask) {
3203 if (Idx == PoisonMaskElem)
3204 continue;
3205 unsigned SrcRegIdx = Idx / NewElts;
3206 if (SrcRegIdx != I)
3207 continue;
3208 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3209 int OpIdx = MaskElt / NewElts;
3210 if (OpIdx != Op)
3211 continue;
3212 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3213 }
3214 }
3215 }
3216 } while (Changed);
3217 };
3218 TryPeekThroughShufflesInputs(OrigMask);
3219 // Proces unique inputs.
3220 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3221 NewElts](SmallVectorImpl<int> &Mask) {
3222 SetVector<SDValue> UniqueInputs;
3223 SetVector<SDValue> UniqueConstantInputs;
3224 for (const auto &I : Inputs) {
3225 if (IsConstant(I))
3226 UniqueConstantInputs.insert(X: I);
3227 else if (!I.isUndef())
3228 UniqueInputs.insert(X: I);
3229 }
3230 // Adjust mask in case of reused inputs. Also, need to insert constant
3231 // inputs at first, otherwise it affects the final outcome.
3232 if (UniqueInputs.size() != std::size(Inputs)) {
3233 auto &&UniqueVec = UniqueInputs.takeVector();
3234 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3235 unsigned ConstNum = UniqueConstantVec.size();
3236 for (int &Idx : Mask) {
3237 if (Idx == PoisonMaskElem)
3238 continue;
3239 unsigned SrcRegIdx = Idx / NewElts;
3240 if (Inputs[SrcRegIdx].isUndef()) {
3241 Idx = PoisonMaskElem;
3242 continue;
3243 }
3244 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3245 if (It != UniqueConstantVec.end()) {
3246 Idx = (Idx % NewElts) +
3247 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3248 assert(Idx >= 0 && "Expected defined mask idx.");
3249 continue;
3250 }
3251 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3252 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3253 Idx = (Idx % NewElts) +
3254 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3255 assert(Idx >= 0 && "Expected defined mask idx.");
3256 }
3257 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3258 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3259 }
3260 };
3261 MakeUniqueInputs(OrigMask);
3262 SDValue OrigInputs[4];
3263 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3264 for (unsigned High = 0; High < 2; ++High) {
3265 SDValue &Output = High ? Hi : Lo;
3266
3267 // Build a shuffle mask for the output, discovering on the fly which
3268 // input vectors to use as shuffle operands.
3269 unsigned FirstMaskIdx = High * NewElts;
3270 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3271 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3272 assert(!Output && "Expected default initialized initial value.");
3273 TryPeekThroughShufflesInputs(Mask);
3274 MakeUniqueInputs(Mask);
3275 SDValue TmpInputs[4];
3276 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3277 // Track changes in the output registers.
3278 int UsedIdx = -1;
3279 bool SecondIteration = false;
3280 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3281 if (UsedIdx < 0) {
3282 UsedIdx = Idx;
3283 return false;
3284 }
3285 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3286 SecondIteration = true;
3287 return SecondIteration;
3288 };
3289 processShuffleMasks(
3290 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3291 /*NumOfUsedRegs=*/1,
3292 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3293 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3294 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3295 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3296 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3297 else
3298 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3299 N2: DAG.getPOISON(VT: NewVT), Mask);
3300 Inputs[Idx] = Output;
3301 },
3302 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3303 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3304 unsigned Idx2, bool /*Unused*/) {
3305 if (AccumulateResults(Idx1)) {
3306 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3307 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3308 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3309 else
3310 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3311 N2: Inputs[Idx2], Mask);
3312 } else {
3313 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3314 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3315 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3316 else
3317 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3318 N2: TmpInputs[Idx2], Mask);
3319 }
3320 Inputs[Idx1] = Output;
3321 });
3322 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3323 }
3324}
3325
3326void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3327 EVT OVT = N->getValueType(ResNo: 0);
3328 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3329 SDValue Chain = N->getOperand(Num: 0);
3330 SDValue Ptr = N->getOperand(Num: 1);
3331 SDValue SV = N->getOperand(Num: 2);
3332 SDLoc dl(N);
3333
3334 const Align Alignment =
3335 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3336
3337 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3338 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3339 Chain = Hi.getValue(R: 1);
3340
3341 // Modified the chain - switch anything that used the old chain to use
3342 // the new one.
3343 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3344}
3345
3346void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3347 SDValue &Hi) {
3348 EVT DstVTLo, DstVTHi;
3349 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3350 SDLoc dl(N);
3351
3352 SDValue SrcLo, SrcHi;
3353 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3354 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3355 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3356 else
3357 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3358
3359 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3360 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3361}
3362
3363void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3364 SDValue &Hi) {
3365 SDValue InLo, InHi;
3366 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3367 SDLoc DL(N);
3368
3369 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3370 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3371}
3372
3373void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3374 SDValue &Hi) {
3375 SDLoc DL(N);
3376
3377 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3378 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3379}
3380
3381void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3382 SDValue &Hi) {
3383 EVT VT = N->getValueType(ResNo: 0);
3384 SDValue Val = N->getOperand(Num: 0);
3385 SDValue Mask = N->getOperand(Num: 1);
3386 SDValue EVL = N->getOperand(Num: 2);
3387 SDLoc DL(N);
3388
3389 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3390 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3391
3392 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3393 EC: VT.getVectorElementCount());
3394 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3395 EVT PtrVT = StackPtr.getValueType();
3396 auto &MF = DAG.getMachineFunction();
3397 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3398 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3399
3400 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3401 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3402 BaseAlignment: Alignment);
3403 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3404 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3405 BaseAlignment: Alignment);
3406
3407 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3408 SDValue NumElemMinus1 =
3409 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3410 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3411 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3412 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3413 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3414 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3415
3416 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3417 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3418 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3419 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3420
3421 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3422
3423 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3424}
3425
3426void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3427 SDValue &Hi) {
3428 EVT VT = N->getValueType(ResNo: 0);
3429 SDValue V1 = N->getOperand(Num: 0);
3430 SDValue V2 = N->getOperand(Num: 1);
3431 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3432 SDValue Mask = N->getOperand(Num: 3);
3433 SDValue EVL1 = N->getOperand(Num: 4);
3434 SDValue EVL2 = N->getOperand(Num: 5);
3435 SDLoc DL(N);
3436
3437 // Since EVL2 is considered the real VL it gets promoted during
3438 // SelectionDAGBuilder. Promote EVL1 here if needed.
3439 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3440 EVL1 = ZExtPromotedInteger(Op: EVL1);
3441
3442 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3443
3444 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3445 EC: VT.getVectorElementCount() * 2);
3446 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3447 EVT PtrVT = StackPtr.getValueType();
3448 auto &MF = DAG.getMachineFunction();
3449 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3450 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3451
3452 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3453 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3454 BaseAlignment: Alignment);
3455 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3456 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3457 BaseAlignment: Alignment);
3458
3459 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: EVL1);
3460 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3461
3462 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3463 SDValue StoreV1 =
3464 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3465 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3466
3467 SDValue StoreV2 =
3468 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3469 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3470
3471 SDValue Load;
3472 if (Imm >= 0) {
3473 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3474 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3475 } else {
3476 uint64_t TrailingElts = -Imm;
3477 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3478 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3479
3480 // Make sure TrailingBytes doesn't exceed the size of vec1.
3481 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3482 TrailingBytes =
3483 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3484
3485 // Calculate the start address of the spliced result.
3486 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3487 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3488 }
3489
3490 EVT LoVT, HiVT;
3491 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
3492 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3493 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3494 Hi =
3495 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3496 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3497}
3498
3499void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3500 SDValue &Hi) {
3501 SDLoc DL(N);
3502 SDValue Acc = N->getOperand(Num: 0);
3503 SDValue Input1 = N->getOperand(Num: 1);
3504 SDValue Input2 = N->getOperand(Num: 2);
3505
3506 SDValue AccLo, AccHi;
3507 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3508 unsigned Opcode = N->getOpcode();
3509
3510 // If the input types don't need splitting, just accumulate into the
3511 // low part of the accumulator.
3512 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3513 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3514 Hi = AccHi;
3515 return;
3516 }
3517
3518 SDValue Input1Lo, Input1Hi;
3519 SDValue Input2Lo, Input2Hi;
3520 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3521 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3522 EVT ResultVT = AccLo.getValueType();
3523
3524 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3525 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3526}
3527
3528void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3529 SDValue &Hi) {
3530 SDLoc DL(N);
3531 SDValue Op0 = N->getOperand(Num: 0);
3532 SDValue Op1 = N->getOperand(Num: 1);
3533 EVT OpVT = Op0.getValueType();
3534
3535 EVT LoVT, HiVT;
3536 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3537
3538 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3539 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3540 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3541 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3542}
3543
3544void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3545 unsigned Factor = N->getNumOperands();
3546
3547 SmallVector<SDValue, 8> Ops(Factor * 2);
3548 for (unsigned i = 0; i != Factor; ++i) {
3549 SDValue OpLo, OpHi;
3550 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3551 Ops[i * 2] = OpLo;
3552 Ops[i * 2 + 1] = OpHi;
3553 }
3554
3555 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3556
3557 SDLoc DL(N);
3558 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3559 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3560 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3561 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3562
3563 for (unsigned i = 0; i != Factor; ++i)
3564 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3565}
3566
3567void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3568 unsigned Factor = N->getNumOperands();
3569
3570 SmallVector<SDValue, 8> Ops(Factor * 2);
3571 for (unsigned i = 0; i != Factor; ++i) {
3572 SDValue OpLo, OpHi;
3573 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3574 Ops[i] = OpLo;
3575 Ops[i + Factor] = OpHi;
3576 }
3577
3578 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3579
3580 SDLoc DL(N);
3581 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3582 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3583 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3584 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3585
3586 for (unsigned i = 0; i != Factor; ++i) {
3587 unsigned IdxLo = 2 * i;
3588 unsigned IdxHi = 2 * i + 1;
3589 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3590 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3591 }
3592}
3593
3594//===----------------------------------------------------------------------===//
3595// Operand Vector Splitting
3596//===----------------------------------------------------------------------===//
3597
3598/// This method is called when the specified operand of the specified node is
3599/// found to need vector splitting. At this point, all of the result types of
3600/// the node are known to be legal, but other operands of the node may need
3601/// legalization as well as the specified one.
3602bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3603 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3604 SDValue Res = SDValue();
3605
3606 // See if the target wants to custom split this node.
3607 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3608 return false;
3609
3610 switch (N->getOpcode()) {
3611 default:
3612#ifndef NDEBUG
3613 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3614 N->dump(&DAG);
3615 dbgs() << "\n";
3616#endif
3617 report_fatal_error(reason: "Do not know how to split this operator's "
3618 "operand!\n");
3619
3620 case ISD::VP_SETCC:
3621 case ISD::STRICT_FSETCC:
3622 case ISD::STRICT_FSETCCS:
3623 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3624 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3625 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3626 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3627 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3628 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3629 case ISD::VECTOR_FIND_LAST_ACTIVE:
3630 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3631 break;
3632 case ISD::VP_TRUNCATE:
3633 case ISD::TRUNCATE:
3634 Res = SplitVecOp_TruncateHelper(N);
3635 break;
3636 case ISD::STRICT_FP_ROUND:
3637 case ISD::VP_FP_ROUND:
3638 case ISD::FP_ROUND:
3639 case ISD::CONVERT_FROM_ARBITRARY_FP:
3640 Res = SplitVecOp_FP_ROUND(N);
3641 break;
3642 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3643 case ISD::STORE:
3644 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3645 break;
3646 case ISD::VP_STORE:
3647 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3648 break;
3649 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3650 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3651 break;
3652 case ISD::MSTORE:
3653 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3654 break;
3655 case ISD::MSCATTER:
3656 case ISD::VP_SCATTER:
3657 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3658 break;
3659 case ISD::MGATHER:
3660 case ISD::VP_GATHER:
3661 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3662 break;
3663 case ISD::VSELECT:
3664 Res = SplitVecOp_VSELECT(N, OpNo);
3665 break;
3666 case ISD::VECTOR_COMPRESS:
3667 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3668 break;
3669 case ISD::STRICT_SINT_TO_FP:
3670 case ISD::STRICT_UINT_TO_FP:
3671 case ISD::SINT_TO_FP:
3672 case ISD::UINT_TO_FP:
3673 case ISD::VP_SINT_TO_FP:
3674 case ISD::VP_UINT_TO_FP:
3675 if (N->getValueType(ResNo: 0).bitsLT(
3676 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3677 Res = SplitVecOp_TruncateHelper(N);
3678 else
3679 Res = SplitVecOp_UnaryOp(N);
3680 break;
3681 case ISD::FP_TO_SINT_SAT:
3682 case ISD::FP_TO_UINT_SAT:
3683 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3684 break;
3685 case ISD::FP_TO_SINT:
3686 case ISD::FP_TO_UINT:
3687 case ISD::VP_FP_TO_SINT:
3688 case ISD::VP_FP_TO_UINT:
3689 case ISD::STRICT_FP_TO_SINT:
3690 case ISD::STRICT_FP_TO_UINT:
3691 case ISD::STRICT_FP_EXTEND:
3692 case ISD::FP_EXTEND:
3693 case ISD::SIGN_EXTEND:
3694 case ISD::ZERO_EXTEND:
3695 case ISD::ANY_EXTEND:
3696 case ISD::FTRUNC:
3697 case ISD::LROUND:
3698 case ISD::LLROUND:
3699 case ISD::LRINT:
3700 case ISD::LLRINT:
3701 Res = SplitVecOp_UnaryOp(N);
3702 break;
3703 case ISD::FLDEXP:
3704 Res = SplitVecOp_FPOpDifferentTypes(N);
3705 break;
3706
3707 case ISD::SCMP:
3708 case ISD::UCMP:
3709 Res = SplitVecOp_CMP(N);
3710 break;
3711
3712 case ISD::FAKE_USE:
3713 Res = SplitVecOp_FAKE_USE(N);
3714 break;
3715 case ISD::ANY_EXTEND_VECTOR_INREG:
3716 case ISD::SIGN_EXTEND_VECTOR_INREG:
3717 case ISD::ZERO_EXTEND_VECTOR_INREG:
3718 Res = SplitVecOp_ExtVecInRegOp(N);
3719 break;
3720
3721 case ISD::VECREDUCE_FADD:
3722 case ISD::VECREDUCE_FMUL:
3723 case ISD::VECREDUCE_ADD:
3724 case ISD::VECREDUCE_MUL:
3725 case ISD::VECREDUCE_AND:
3726 case ISD::VECREDUCE_OR:
3727 case ISD::VECREDUCE_XOR:
3728 case ISD::VECREDUCE_SMAX:
3729 case ISD::VECREDUCE_SMIN:
3730 case ISD::VECREDUCE_UMAX:
3731 case ISD::VECREDUCE_UMIN:
3732 case ISD::VECREDUCE_FMAX:
3733 case ISD::VECREDUCE_FMIN:
3734 case ISD::VECREDUCE_FMAXIMUM:
3735 case ISD::VECREDUCE_FMINIMUM:
3736 Res = SplitVecOp_VECREDUCE(N, OpNo);
3737 break;
3738 case ISD::VECREDUCE_SEQ_FADD:
3739 case ISD::VECREDUCE_SEQ_FMUL:
3740 Res = SplitVecOp_VECREDUCE_SEQ(N);
3741 break;
3742 case ISD::VP_REDUCE_FADD:
3743 case ISD::VP_REDUCE_SEQ_FADD:
3744 case ISD::VP_REDUCE_FMUL:
3745 case ISD::VP_REDUCE_SEQ_FMUL:
3746 case ISD::VP_REDUCE_ADD:
3747 case ISD::VP_REDUCE_MUL:
3748 case ISD::VP_REDUCE_AND:
3749 case ISD::VP_REDUCE_OR:
3750 case ISD::VP_REDUCE_XOR:
3751 case ISD::VP_REDUCE_SMAX:
3752 case ISD::VP_REDUCE_SMIN:
3753 case ISD::VP_REDUCE_UMAX:
3754 case ISD::VP_REDUCE_UMIN:
3755 case ISD::VP_REDUCE_FMAX:
3756 case ISD::VP_REDUCE_FMIN:
3757 case ISD::VP_REDUCE_FMAXIMUM:
3758 case ISD::VP_REDUCE_FMINIMUM:
3759 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3760 break;
3761 case ISD::CTTZ_ELTS:
3762 case ISD::CTTZ_ELTS_ZERO_POISON:
3763 Res = SplitVecOp_CttzElts(N);
3764 break;
3765 case ISD::VP_CTTZ_ELTS:
3766 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
3767 Res = SplitVecOp_VP_CttzElements(N);
3768 break;
3769 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3770 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3771 break;
3772 case ISD::PARTIAL_REDUCE_UMLA:
3773 case ISD::PARTIAL_REDUCE_SMLA:
3774 case ISD::PARTIAL_REDUCE_SUMLA:
3775 case ISD::PARTIAL_REDUCE_FMLA:
3776 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3777 break;
3778 }
3779
3780 // If the result is null, the sub-method took care of registering results etc.
3781 if (!Res.getNode()) return false;
3782
3783 // If the result is N, the sub-method updated N in place. Tell the legalizer
3784 // core about this.
3785 if (Res.getNode() == N)
3786 return true;
3787
3788 if (N->isStrictFPOpcode())
3789 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3790 "Invalid operand expansion");
3791 else
3792 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3793 "Invalid operand expansion");
3794
3795 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3796 return false;
3797}
3798
3799SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3800 SDLoc DL(N);
3801
3802 SDValue LoMask, HiMask;
3803 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
3804
3805 EVT VT = N->getValueType(ResNo: 0);
3806 EVT SplitVT = LoMask.getValueType();
3807 ElementCount SplitEC = SplitVT.getVectorElementCount();
3808
3809 // Find the last active in both the low and the high masks.
3810 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
3811 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
3812
3813 // Check if any lane is active in the high mask.
3814 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
3815 // sentinel value for "none active".
3816 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
3817 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
3818 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
3819
3820 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
3821 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
3822 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
3823 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
3824 N3: LoFind);
3825}
3826
3827SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3828 // The only possibility for an illegal operand is the mask, since result type
3829 // legalization would have handled this node already otherwise.
3830 assert(OpNo == 0 && "Illegal operand must be mask");
3831
3832 SDValue Mask = N->getOperand(Num: 0);
3833 SDValue Src0 = N->getOperand(Num: 1);
3834 SDValue Src1 = N->getOperand(Num: 2);
3835 EVT Src0VT = Src0.getValueType();
3836 SDLoc DL(N);
3837 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3838
3839 SDValue Lo, Hi;
3840 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3841 assert(Lo.getValueType() == Hi.getValueType() &&
3842 "Lo and Hi have differing types");
3843
3844 EVT LoOpVT, HiOpVT;
3845 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3846 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3847
3848 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3849 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3850 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3851 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3852
3853 SDValue LoSelect =
3854 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3855 SDValue HiSelect =
3856 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3857
3858 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3859}
3860
3861SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
3862 // The only possibility for an illegal operand is the mask, since result type
3863 // legalization would have handled this node already otherwise.
3864 assert(OpNo == 1 && "Illegal operand must be mask");
3865
3866 // To split the mask, we need to split the result type too, so we can just
3867 // reuse that logic here.
3868 SDValue Lo, Hi;
3869 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
3870
3871 EVT VecVT = N->getValueType(ResNo: 0);
3872 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
3873}
3874
3875SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3876 EVT ResVT = N->getValueType(ResNo: 0);
3877 SDValue Lo, Hi;
3878 SDLoc dl(N);
3879
3880 SDValue VecOp = N->getOperand(Num: OpNo);
3881 EVT VecVT = VecOp.getValueType();
3882 assert(VecVT.isVector() && "Can only split reduce vector operand");
3883 GetSplitVector(Op: VecOp, Lo, Hi);
3884 EVT LoOpVT, HiOpVT;
3885 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3886
3887 // Use the appropriate scalar instruction on the split subvectors before
3888 // reducing the now partially reduced smaller vector.
3889 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3890 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3891 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3892}
3893
3894SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3895 EVT ResVT = N->getValueType(ResNo: 0);
3896 SDValue Lo, Hi;
3897 SDLoc dl(N);
3898
3899 SDValue AccOp = N->getOperand(Num: 0);
3900 SDValue VecOp = N->getOperand(Num: 1);
3901 SDNodeFlags Flags = N->getFlags();
3902
3903 EVT VecVT = VecOp.getValueType();
3904 assert(VecVT.isVector() && "Can only split reduce vector operand");
3905 GetSplitVector(Op: VecOp, Lo, Hi);
3906 EVT LoOpVT, HiOpVT;
3907 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3908
3909 // Reduce low half.
3910 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3911
3912 // Reduce high half, using low half result as initial value.
3913 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3914}
3915
3916SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3917 assert(N->isVPOpcode() && "Expected VP opcode");
3918 assert(OpNo == 1 && "Can only split reduce vector operand");
3919
3920 unsigned Opc = N->getOpcode();
3921 EVT ResVT = N->getValueType(ResNo: 0);
3922 SDValue Lo, Hi;
3923 SDLoc dl(N);
3924
3925 SDValue VecOp = N->getOperand(Num: OpNo);
3926 EVT VecVT = VecOp.getValueType();
3927 assert(VecVT.isVector() && "Can only split reduce vector operand");
3928 GetSplitVector(Op: VecOp, Lo, Hi);
3929
3930 SDValue MaskLo, MaskHi;
3931 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3932
3933 SDValue EVLLo, EVLHi;
3934 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3935
3936 const SDNodeFlags Flags = N->getFlags();
3937
3938 SDValue ResLo =
3939 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3940 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3941}
3942
3943SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3944 // The result has a legal vector type, but the input needs splitting.
3945 EVT ResVT = N->getValueType(ResNo: 0);
3946 SDValue Lo, Hi;
3947 SDLoc dl(N);
3948 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3949 EVT InVT = Lo.getValueType();
3950
3951 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3952 EC: InVT.getVectorElementCount());
3953
3954 if (N->isStrictFPOpcode()) {
3955 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3956 Ops: {N->getOperand(Num: 0), Lo});
3957 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3958 Ops: {N->getOperand(Num: 0), Hi});
3959
3960 // Build a factor node to remember that this operation is independent
3961 // of the other one.
3962 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
3963 N2: Hi.getValue(R: 1));
3964
3965 // Legalize the chain result - switch anything that used the old chain to
3966 // use the new one.
3967 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3968 } else if (N->getNumOperands() == 3) {
3969 assert(N->isVPOpcode() && "Expected VP opcode");
3970 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3971 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3972 std::tie(args&: EVLLo, args&: EVLHi) =
3973 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3974 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3975 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3976 } else {
3977 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3978 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3979 }
3980
3981 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3982}
3983
3984// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
3985SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
3986 SDValue Lo, Hi;
3987 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3988 SDValue Chain =
3989 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
3990 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
3991}
3992
3993SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3994 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3995 // end up being split all the way down to individual components. Convert the
3996 // split pieces into integers and reassemble.
3997 EVT ResVT = N->getValueType(ResNo: 0);
3998 SDValue Lo, Hi;
3999 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4000 SDLoc dl(N);
4001
4002 if (ResVT.isScalableVector()) {
4003 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4004 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4005 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4006 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4007 }
4008
4009 Lo = BitConvertToInteger(Op: Lo);
4010 Hi = BitConvertToInteger(Op: Hi);
4011
4012 if (DAG.getDataLayout().isBigEndian())
4013 std::swap(a&: Lo, b&: Hi);
4014
4015 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4016}
4017
4018SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4019 unsigned OpNo) {
4020 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4021 // We know that the result type is legal.
4022 EVT ResVT = N->getValueType(ResNo: 0);
4023
4024 SDValue Vec = N->getOperand(Num: 0);
4025 SDValue SubVec = N->getOperand(Num: 1);
4026 SDValue Idx = N->getOperand(Num: 2);
4027 SDLoc dl(N);
4028
4029 SDValue Lo, Hi;
4030 GetSplitVector(Op: SubVec, Lo, Hi);
4031
4032 uint64_t IdxVal = Idx->getAsZExtVal();
4033 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4034
4035 SDValue FirstInsertion =
4036 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4037 SDValue SecondInsertion =
4038 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4039 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4040
4041 return SecondInsertion;
4042}
4043
4044SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4045 // We know that the extracted result type is legal.
4046 EVT SubVT = N->getValueType(ResNo: 0);
4047 SDValue Idx = N->getOperand(Num: 1);
4048 SDLoc dl(N);
4049 SDValue Lo, Hi;
4050
4051 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4052
4053 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4054 // Note: For scalable vectors, the index is scaled by vscale.
4055 ElementCount IdxVal =
4056 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4057 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4058
4059 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4060 ElementCount NumResultElts = SubVT.getVectorElementCount();
4061
4062 // If the extracted elements are all in the low half, do a simple extract.
4063 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4064 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4065
4066 unsigned LoEltsMin = LoElts.getKnownMinValue();
4067 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4068 SrcVT.isFixedLengthVector()) {
4069 // Extracted subvector crosses vector split, so we need to blend the two
4070 // halves.
4071 // TODO: May be able to emit partial extract_subvector.
4072 SmallVector<SDValue, 8> Elts;
4073 Elts.reserve(N: NumResultElts.getFixedValue());
4074
4075 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4076 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4077 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4078 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4079 /*Count=*/LoEltsMin - IdxValMin);
4080 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4081 /*Count=*/SubVT.getVectorNumElements() -
4082 Elts.size());
4083 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4084 }
4085
4086 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4087 ElementCount ExtractIdx = IdxVal - LoElts;
4088 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4089 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4090 Idx: ExtractIdx.getKnownMinValue());
4091
4092 EVT HiVT = Hi.getValueType();
4093 assert(HiVT.isFixedLengthVector() &&
4094 "Only fixed-vector extracts are supported in this case");
4095
4096 // We cannot create an extract_subvector that isn't a multiple of the
4097 // result size, which may go out of bounds for the last elements. Shuffle
4098 // the desired elements down to 0 and do a simple 0 extract.
4099 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4100 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4101 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4102
4103 SDValue Shuffle =
4104 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4105 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4106 }
4107
4108 // After this point the DAG node only permits extracting fixed-width
4109 // subvectors from scalable vectors.
4110 assert(SubVT.isFixedLengthVector() &&
4111 "Extracting scalable subvector from fixed-width unsupported");
4112
4113 // If the element type is i1 and we're not promoting the result, then we may
4114 // end up loading the wrong data since the bits are packed tightly into
4115 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4116 // type at index 4, then we will load a byte starting at index 0.
4117 if (SubVT.getScalarType() == MVT::i1)
4118 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4119 "subvector from a scalable predicate vector");
4120
4121 // Spill the vector to the stack. We should use the alignment for
4122 // the smallest part.
4123 SDValue Vec = N->getOperand(Num: 0);
4124 EVT VecVT = Vec.getValueType();
4125 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4126 SDValue StackPtr =
4127 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4128 auto &MF = DAG.getMachineFunction();
4129 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4130 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4131
4132 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4133 Alignment: SmallestAlign);
4134
4135 // Extract the subvector by loading the correct part.
4136 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4137
4138 return DAG.getLoad(
4139 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4140 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4141}
4142
4143SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4144 SDValue Vec = N->getOperand(Num: 0);
4145 SDValue Idx = N->getOperand(Num: 1);
4146 EVT VecVT = Vec.getValueType();
4147
4148 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4149 uint64_t IdxVal = Index->getZExtValue();
4150
4151 SDValue Lo, Hi;
4152 GetSplitVector(Op: Vec, Lo, Hi);
4153
4154 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4155
4156 if (IdxVal < LoElts)
4157 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4158 else if (!Vec.getValueType().isScalableVector())
4159 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4160 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4161 VT: Idx.getValueType())), 0);
4162 }
4163
4164 // See if the target wants to custom expand this node.
4165 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4166 return SDValue();
4167
4168 // Make the vector elements byte-addressable if they aren't already.
4169 SDLoc dl(N);
4170 EVT EltVT = VecVT.getVectorElementType();
4171 if (!EltVT.isByteSized()) {
4172 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4173 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4174 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4175 SDValue NewExtract =
4176 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4177 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4178 }
4179
4180 // Store the vector to the stack.
4181 // In cases where the vector is illegal it will be broken down into parts
4182 // and stored in parts - we should use the alignment for the smallest part.
4183 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4184 SDValue StackPtr =
4185 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4186 auto &MF = DAG.getMachineFunction();
4187 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4188 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4189 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4190 Alignment: SmallestAlign);
4191
4192 // Load back the required element.
4193 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4194
4195 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4196 // type, leaving the high bits undefined. But it can't truncate.
4197 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4198
4199 return DAG.getExtLoad(
4200 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4201 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4202 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4203}
4204
4205SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4206 SDValue Lo, Hi;
4207
4208 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4209 // splitting the result has the same effect as splitting the input operand.
4210 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4211
4212 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4213}
4214
4215SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4216 (void)OpNo;
4217 SDValue Lo, Hi;
4218 SplitVecRes_Gather(N, Lo, Hi);
4219
4220 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4221 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4222 return SDValue();
4223}
4224
4225SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4226 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4227 SDValue Ch = N->getChain();
4228 SDValue Ptr = N->getBasePtr();
4229 SDValue Offset = N->getOffset();
4230 assert(Offset.isUndef() && "Unexpected VP store offset");
4231 SDValue Mask = N->getMask();
4232 SDValue EVL = N->getVectorLength();
4233 SDValue Data = N->getValue();
4234 Align Alignment = N->getBaseAlign();
4235 SDLoc DL(N);
4236
4237 SDValue DataLo, DataHi;
4238 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4239 // Split Data operand
4240 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4241 else
4242 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4243
4244 // Split Mask operand
4245 SDValue MaskLo, MaskHi;
4246 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4247 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4248 } else {
4249 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4250 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4251 else
4252 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4253 }
4254
4255 EVT MemoryVT = N->getMemoryVT();
4256 EVT LoMemVT, HiMemVT;
4257 bool HiIsEmpty = false;
4258 std::tie(args&: LoMemVT, args&: HiMemVT) =
4259 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4260
4261 // Split EVL
4262 SDValue EVLLo, EVLHi;
4263 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4264
4265 SDValue Lo, Hi;
4266 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4267 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4268 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4269 Ranges: N->getRanges());
4270
4271 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4272 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4273 IsCompressing: N->isCompressingStore());
4274
4275 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4276 if (HiIsEmpty)
4277 return Lo;
4278
4279 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4280 IsCompressedMemory: N->isCompressingStore());
4281
4282 MachinePointerInfo MPI;
4283 if (LoMemVT.isScalableVector()) {
4284 Alignment = commonAlignment(A: Alignment,
4285 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4286 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4287 } else
4288 MPI = N->getPointerInfo().getWithOffset(
4289 O: LoMemVT.getStoreSize().getFixedValue());
4290
4291 MMO = DAG.getMachineFunction().getMachineMemOperand(
4292 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4293 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4294
4295 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4296 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4297 IsCompressing: N->isCompressingStore());
4298
4299 // Build a factor node to remember that this store is independent of the
4300 // other one.
4301 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4302}
4303
4304SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4305 unsigned OpNo) {
4306 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4307 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4308
4309 SDLoc DL(N);
4310
4311 SDValue Data = N->getValue();
4312 SDValue LoData, HiData;
4313 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4314 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4315 else
4316 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4317
4318 EVT LoMemVT, HiMemVT;
4319 bool HiIsEmpty = false;
4320 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4321 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4322
4323 SDValue Mask = N->getMask();
4324 SDValue LoMask, HiMask;
4325 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4326 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4327 else if (getTypeAction(VT: Mask.getValueType()) ==
4328 TargetLowering::TypeSplitVector)
4329 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4330 else
4331 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4332
4333 SDValue LoEVL, HiEVL;
4334 std::tie(args&: LoEVL, args&: HiEVL) =
4335 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4336
4337 // Generate the low vp_strided_store
4338 SDValue Lo = DAG.getStridedStoreVP(
4339 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4340 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4341 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4342
4343 // If the high vp_strided_store has zero storage size, only the low
4344 // vp_strided_store is needed.
4345 if (HiIsEmpty)
4346 return Lo;
4347
4348 // Generate the high vp_strided_store.
4349 // To calculate the high base address, we need to sum to the low base
4350 // address stride number of bytes for each element already stored by low,
4351 // that is: Ptr = Ptr + (LoEVL * Stride)
4352 EVT PtrVT = N->getBasePtr().getValueType();
4353 SDValue Increment =
4354 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4355 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4356 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4357
4358 Align Alignment = N->getBaseAlign();
4359 if (LoMemVT.isScalableVector())
4360 Alignment = commonAlignment(A: Alignment,
4361 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4362
4363 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4364 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4365 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4366 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4367
4368 SDValue Hi = DAG.getStridedStoreVP(
4369 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4370 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4371 IsCompressing: N->isCompressingStore());
4372
4373 // Build a factor node to remember that this store is independent of the
4374 // other one.
4375 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4376}
4377
4378SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4379 unsigned OpNo) {
4380 assert(N->isUnindexed() && "Indexed masked store of vector?");
4381 SDValue Ch = N->getChain();
4382 SDValue Ptr = N->getBasePtr();
4383 SDValue Offset = N->getOffset();
4384 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4385 SDValue Mask = N->getMask();
4386 SDValue Data = N->getValue();
4387 Align Alignment = N->getBaseAlign();
4388 SDLoc DL(N);
4389
4390 SDValue DataLo, DataHi;
4391 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4392 // Split Data operand
4393 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4394 else
4395 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4396
4397 // Split Mask operand
4398 SDValue MaskLo, MaskHi;
4399 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4400 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4401 } else {
4402 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4403 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4404 else
4405 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4406 }
4407
4408 EVT MemoryVT = N->getMemoryVT();
4409 EVT LoMemVT, HiMemVT;
4410 bool HiIsEmpty = false;
4411 std::tie(args&: LoMemVT, args&: HiMemVT) =
4412 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4413
4414 SDValue Lo, Hi, Res;
4415 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4416 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4417 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4418 Ranges: N->getRanges());
4419
4420 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4421 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4422 IsCompressing: N->isCompressingStore());
4423
4424 if (HiIsEmpty) {
4425 // The hi masked store has zero storage size.
4426 // Only the lo masked store is needed.
4427 Res = Lo;
4428 } else {
4429
4430 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4431 IsCompressedMemory: N->isCompressingStore());
4432
4433 MachinePointerInfo MPI;
4434 if (LoMemVT.isScalableVector()) {
4435 Alignment = commonAlignment(
4436 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4437 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4438 } else
4439 MPI = N->getPointerInfo().getWithOffset(
4440 O: LoMemVT.getStoreSize().getFixedValue());
4441
4442 MMO = DAG.getMachineFunction().getMachineMemOperand(
4443 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4444 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4445
4446 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4447 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4448 IsCompressing: N->isCompressingStore());
4449
4450 // Build a factor node to remember that this store is independent of the
4451 // other one.
4452 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4453 }
4454
4455 return Res;
4456}
4457
4458SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4459 SDValue Ch = N->getChain();
4460 SDValue Ptr = N->getBasePtr();
4461 EVT MemoryVT = N->getMemoryVT();
4462 Align Alignment = N->getBaseAlign();
4463 SDLoc DL(N);
4464 struct Operands {
4465 SDValue Mask;
4466 SDValue Index;
4467 SDValue Scale;
4468 SDValue Data;
4469 } Ops = [&]() -> Operands {
4470 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4471 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4472 .Data: MSC->getValue()};
4473 }
4474 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4475 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4476 .Data: VPSC->getValue()};
4477 }();
4478 // Split all operands
4479
4480 EVT LoMemVT, HiMemVT;
4481 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4482
4483 SDValue DataLo, DataHi;
4484 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4485 // Split Data operand
4486 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4487 else
4488 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4489
4490 // Split Mask operand
4491 SDValue MaskLo, MaskHi;
4492 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4493 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4494 } else {
4495 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4496 }
4497
4498 SDValue IndexHi, IndexLo;
4499 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4500 TargetLowering::TypeSplitVector)
4501 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4502 else
4503 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4504
4505 SDValue Lo;
4506 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4507 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4508 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4509 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4510
4511 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4512 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4513 Lo =
4514 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4515 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4516
4517 // The order of the Scatter operation after split is well defined. The "Hi"
4518 // part comes after the "Lo". So these two operations should be chained one
4519 // after another.
4520 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4521 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4522 MMO, IndexType: MSC->getIndexType(),
4523 IsTruncating: MSC->isTruncatingStore());
4524 }
4525 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4526 SDValue EVLLo, EVLHi;
4527 std::tie(args&: EVLLo, args&: EVLHi) =
4528 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4529
4530 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4531 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4532 IndexType: VPSC->getIndexType());
4533
4534 // The order of the Scatter operation after split is well defined. The "Hi"
4535 // part comes after the "Lo". So these two operations should be chained one
4536 // after another.
4537 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4538 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4539 IndexType: VPSC->getIndexType());
4540}
4541
4542SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4543 assert(N->isUnindexed() && "Indexed store of vector?");
4544 assert(OpNo == 1 && "Can only split the stored value");
4545 SDLoc DL(N);
4546
4547 bool isTruncating = N->isTruncatingStore();
4548 SDValue Ch = N->getChain();
4549 SDValue Ptr = N->getBasePtr();
4550 EVT MemoryVT = N->getMemoryVT();
4551 Align Alignment = N->getBaseAlign();
4552 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4553 AAMDNodes AAInfo = N->getAAInfo();
4554 SDValue Lo, Hi;
4555 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4556
4557 EVT LoMemVT, HiMemVT;
4558 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4559
4560 // Scalarize if the split halves are not byte-sized.
4561 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4562 return TLI.scalarizeVectorStore(ST: N, DAG);
4563
4564 if (isTruncating)
4565 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4566 Alignment, MMOFlags, AAInfo);
4567 else
4568 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4569 AAInfo);
4570
4571 MachinePointerInfo MPI;
4572 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4573
4574 if (isTruncating)
4575 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4576 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4577 else
4578 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4579
4580 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4581}
4582
4583SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4584 SDLoc DL(N);
4585
4586 // The input operands all must have the same type, and we know the result
4587 // type is valid. Convert this to a buildvector which extracts all the
4588 // input elements.
4589 // TODO: If the input elements are power-two vectors, we could convert this to
4590 // a new CONCAT_VECTORS node with elements that are half-wide.
4591 SmallVector<SDValue, 32> Elts;
4592 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4593 for (const SDValue &Op : N->op_values()) {
4594 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4595 i != e; ++i) {
4596 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4597 }
4598 }
4599
4600 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4601}
4602
4603SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4604 // The result type is legal, but the input type is illegal. If splitting
4605 // ends up with the result type of each half still being legal, just
4606 // do that. If, however, that would result in an illegal result type,
4607 // we can try to get more clever with power-two vectors. Specifically,
4608 // split the input type, but also widen the result element size, then
4609 // concatenate the halves and truncate again. For example, consider a target
4610 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4611 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4612 // %inlo = v4i32 extract_subvector %in, 0
4613 // %inhi = v4i32 extract_subvector %in, 4
4614 // %lo16 = v4i16 trunc v4i32 %inlo
4615 // %hi16 = v4i16 trunc v4i32 %inhi
4616 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4617 // %res = v8i8 trunc v8i16 %in16
4618 //
4619 // Without this transform, the original truncate would end up being
4620 // scalarized, which is pretty much always a last resort.
4621 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4622 SDValue InVec = N->getOperand(Num: OpNo);
4623 EVT InVT = InVec->getValueType(ResNo: 0);
4624 EVT OutVT = N->getValueType(ResNo: 0);
4625 ElementCount NumElements = OutVT.getVectorElementCount();
4626 bool IsFloat = OutVT.isFloatingPoint();
4627
4628 unsigned InElementSize = InVT.getScalarSizeInBits();
4629 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4630
4631 // Determine the split output VT. If its legal we can just split dirctly.
4632 EVT LoOutVT, HiOutVT;
4633 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4634 assert(LoOutVT == HiOutVT && "Unequal split?");
4635
4636 // If the input elements are only 1/2 the width of the result elements,
4637 // just use the normal splitting. Our trick only work if there's room
4638 // to split more than once.
4639 if (isTypeLegal(VT: LoOutVT) ||
4640 InElementSize <= OutElementSize * 2)
4641 return SplitVecOp_UnaryOp(N);
4642 SDLoc DL(N);
4643
4644 // Don't touch if this will be scalarized.
4645 EVT FinalVT = InVT;
4646 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4647 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4648
4649 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4650 return SplitVecOp_UnaryOp(N);
4651
4652 // Get the split input vector.
4653 SDValue InLoVec, InHiVec;
4654 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4655
4656 // Truncate them to 1/2 the element size.
4657 //
4658 // This assumes the number of elements is a power of two; any vector that
4659 // isn't should be widened, not split.
4660 EVT HalfElementVT = IsFloat ?
4661 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4662 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4663 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4664 EC: NumElements.divideCoefficientBy(RHS: 2));
4665
4666 SDValue HalfLo;
4667 SDValue HalfHi;
4668 SDValue Chain;
4669 if (N->isStrictFPOpcode()) {
4670 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4671 Ops: {N->getOperand(Num: 0), InLoVec});
4672 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4673 Ops: {N->getOperand(Num: 0), InHiVec});
4674 // Legalize the chain result - switch anything that used the old chain to
4675 // use the new one.
4676 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4677 N2: HalfHi.getValue(R: 1));
4678 } else {
4679 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4680 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4681 }
4682
4683 // Concatenate them to get the full intermediate truncation result.
4684 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4685 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4686 N2: HalfHi);
4687 // Now finish up by truncating all the way down to the original result
4688 // type. This should normally be something that ends up being legal directly,
4689 // but in theory if a target has very wide vectors and an annoyingly
4690 // restricted set of legal types, this split can chain to build things up.
4691
4692 if (N->isStrictFPOpcode()) {
4693 SDValue Res = DAG.getNode(
4694 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4695 Ops: {Chain, InterVec,
4696 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4697 // Relink the chain
4698 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4699 return Res;
4700 }
4701
4702 return IsFloat
4703 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4704 N2: DAG.getTargetConstant(
4705 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4706 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4707}
4708
4709SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4710 unsigned Opc = N->getOpcode();
4711 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4712 assert(N->getValueType(0).isVector() &&
4713 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4714 "Operand types must be vectors");
4715 // The result has a legal vector type, but the input needs splitting.
4716 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4717 SDLoc DL(N);
4718 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4719 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4720
4721 EVT VT = N->getValueType(ResNo: 0);
4722 EVT PartResVT = Lo0.getValueType().changeElementType(Context&: *DAG.getContext(),
4723 EltVT: VT.getScalarType());
4724
4725 if (Opc == ISD::SETCC) {
4726 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4727 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4728 } else if (isStrict) {
4729 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4730 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4731 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4732 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4733 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4734 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4735 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4736 } else {
4737 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4738 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4739 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4740 std::tie(args&: EVLLo, args&: EVLHi) =
4741 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4742 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4743 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4744 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4745 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4746 }
4747
4748 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: LoRes, N2: HiRes);
4749}
4750
4751
4752SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4753 // The result has a legal vector type, but the input needs splitting.
4754 EVT ResVT = N->getValueType(ResNo: 0);
4755 SDValue Lo, Hi;
4756 SDLoc DL(N);
4757 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4758 EVT InVT = Lo.getValueType();
4759
4760 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4761 EC: InVT.getVectorElementCount());
4762
4763 if (N->isStrictFPOpcode()) {
4764 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4765 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
4766 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4767 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
4768 // Legalize the chain result - switch anything that used the old chain to
4769 // use the new one.
4770 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4771 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
4772 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4773 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4774 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4775 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4776 std::tie(args&: EVLLo, args&: EVLHi) =
4777 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
4778 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4779 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4780 } else {
4781 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4782 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4783 }
4784
4785 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4786}
4787
4788// Split a vector type in an FP binary operation where the second operand has a
4789// different type from the first.
4790//
4791// The result (and the first input) has a legal vector type, but the second
4792// input needs splitting.
4793SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4794 SDLoc DL(N);
4795
4796 EVT LHSLoVT, LHSHiVT;
4797 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4798
4799 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4800 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4801
4802 SDValue LHSLo, LHSHi;
4803 std::tie(args&: LHSLo, args&: LHSHi) =
4804 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4805
4806 SDValue RHSLo, RHSHi;
4807 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4808
4809 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4810 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4811
4812 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4813}
4814
4815SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
4816 LLVMContext &Ctxt = *DAG.getContext();
4817 SDLoc dl(N);
4818
4819 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4820 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
4821 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
4822
4823 EVT ResVT = N->getValueType(ResNo: 0);
4824 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
4825 EVT NewResVT =
4826 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
4827
4828 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
4829 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
4830
4831 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4832}
4833
4834SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4835 EVT ResVT = N->getValueType(ResNo: 0);
4836 SDValue Lo, Hi;
4837 SDLoc dl(N);
4838 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4839 EVT InVT = Lo.getValueType();
4840
4841 EVT NewResVT =
4842 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4843 EC: InVT.getVectorElementCount());
4844
4845 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4846 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4847
4848 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4849}
4850
4851SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
4852 SDLoc DL(N);
4853 EVT ResVT = N->getValueType(ResNo: 0);
4854
4855 SDValue Lo, Hi;
4856 SDValue VecOp = N->getOperand(Num: 0);
4857 GetSplitVector(Op: VecOp, Lo, Hi);
4858
4859 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
4860 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
4861 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
4862 SDValue VL =
4863 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
4864 SDValue ResLoNotVL =
4865 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
4866 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
4867 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
4868 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
4869}
4870
4871SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
4872 SDLoc DL(N);
4873 EVT ResVT = N->getValueType(ResNo: 0);
4874
4875 SDValue Lo, Hi;
4876 SDValue VecOp = N->getOperand(Num: 0);
4877 GetSplitVector(Op: VecOp, Lo, Hi);
4878
4879 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
4880 auto [EVLLo, EVLHi] =
4881 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
4882 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
4883
4884 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
4885 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_UNDEF(Hi)).
4886 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4887 SDValue ResLoNotEVL =
4888 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
4889 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4890 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
4891 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
4892}
4893
4894SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
4895 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
4896 SDLoc DL(HG);
4897 SDValue Inc = HG->getInc();
4898 SDValue Ptr = HG->getBasePtr();
4899 SDValue Scale = HG->getScale();
4900 SDValue IntID = HG->getIntID();
4901 EVT MemVT = HG->getMemoryVT();
4902 MachineMemOperand *MMO = HG->getMemOperand();
4903 ISD::MemIndexType IndexType = HG->getIndexType();
4904
4905 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
4906 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
4907 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
4908 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
4909 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
4910 Ops: OpsLo, MMO, IndexType);
4911 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
4912 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
4913 MMO, IndexType);
4914}
4915
4916SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
4917 SDValue Acc = N->getOperand(Num: 0);
4918 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
4919 "Accumulator should already be a legal type, and shouldn't need "
4920 "further splitting");
4921
4922 SDLoc DL(N);
4923 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
4924 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
4925 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
4926 unsigned Opcode = N->getOpcode();
4927 EVT ResultVT = Acc.getValueType();
4928
4929 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
4930 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
4931}
4932
4933//===----------------------------------------------------------------------===//
4934// Result Vector Widening
4935//===----------------------------------------------------------------------===//
4936
4937void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
4938 unsigned WidenResNo) {
4939 unsigned NumResults = N->getNumValues();
4940 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
4941 if (ResNo == WidenResNo)
4942 continue;
4943 EVT ResVT = N->getValueType(ResNo);
4944 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
4945 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
4946 } else {
4947 SDLoc DL(N);
4948 SDValue ResVal =
4949 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
4950 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
4951 }
4952 }
4953}
4954
4955void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4956 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4957
4958 // See if the target wants to custom widen this node.
4959 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4960 return;
4961
4962 SDValue Res = SDValue();
4963
4964 auto unrollExpandedOp = [&]() {
4965 // We're going to widen this vector op to a legal type by padding with undef
4966 // elements. If the wide vector op is eventually going to be expanded to
4967 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4968 // libcalls on the undef elements.
4969 EVT VT = N->getValueType(ResNo: 0);
4970 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4971 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
4972 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
4973 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4974 if (N->getNumValues() > 1)
4975 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
4976 return true;
4977 }
4978 return false;
4979 };
4980
4981 switch (N->getOpcode()) {
4982 default:
4983#ifndef NDEBUG
4984 dbgs() << "WidenVectorResult #" << ResNo << ": ";
4985 N->dump(&DAG);
4986 dbgs() << "\n";
4987#endif
4988 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4989
4990 case ISD::LOOP_DEPENDENCE_RAW_MASK:
4991 case ISD::LOOP_DEPENDENCE_WAR_MASK:
4992 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
4993 break;
4994 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4995 case ISD::ADDRSPACECAST:
4996 Res = WidenVecRes_ADDRSPACECAST(N);
4997 break;
4998 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4999 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5000 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5001 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5002 case ISD::INSERT_SUBVECTOR:
5003 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5004 break;
5005 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5006 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5007 case ISD::ATOMIC_LOAD:
5008 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5009 break;
5010 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5011 case ISD::STEP_VECTOR:
5012 case ISD::SPLAT_VECTOR:
5013 case ISD::SCALAR_TO_VECTOR:
5014 Res = WidenVecRes_ScalarOp(N);
5015 break;
5016 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5017 case ISD::VSELECT:
5018 case ISD::SELECT:
5019 case ISD::VP_SELECT:
5020 case ISD::VP_MERGE:
5021 Res = WidenVecRes_Select(N);
5022 break;
5023 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5024 case ISD::VP_SETCC:
5025 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5026 case ISD::POISON:
5027 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5028 case ISD::VECTOR_SHUFFLE:
5029 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5030 break;
5031 case ISD::VP_LOAD:
5032 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5033 break;
5034 case ISD::VP_LOAD_FF:
5035 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5036 break;
5037 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5038 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5039 break;
5040 case ISD::VECTOR_COMPRESS:
5041 Res = WidenVecRes_VECTOR_COMPRESS(N);
5042 break;
5043 case ISD::MLOAD:
5044 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5045 break;
5046 case ISD::MGATHER:
5047 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5048 break;
5049 case ISD::VP_GATHER:
5050 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5051 break;
5052 case ISD::VECTOR_REVERSE:
5053 Res = WidenVecRes_VECTOR_REVERSE(N);
5054 break;
5055 case ISD::GET_ACTIVE_LANE_MASK:
5056 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5057 break;
5058
5059 case ISD::ADD: case ISD::VP_ADD:
5060 case ISD::AND: case ISD::VP_AND:
5061 case ISD::MUL: case ISD::VP_MUL:
5062 case ISD::MULHS:
5063 case ISD::MULHU:
5064 case ISD::ABDS:
5065 case ISD::ABDU:
5066 case ISD::OR: case ISD::VP_OR:
5067 case ISD::SUB: case ISD::VP_SUB:
5068 case ISD::XOR: case ISD::VP_XOR:
5069 case ISD::SHL: case ISD::VP_SHL:
5070 case ISD::SRA: case ISD::VP_SRA:
5071 case ISD::SRL: case ISD::VP_SRL:
5072 case ISD::CLMUL:
5073 case ISD::CLMULR:
5074 case ISD::CLMULH:
5075 case ISD::FMINNUM:
5076 case ISD::FMINNUM_IEEE:
5077 case ISD::VP_FMINNUM:
5078 case ISD::FMAXNUM:
5079 case ISD::FMAXNUM_IEEE:
5080 case ISD::VP_FMAXNUM:
5081 case ISD::FMINIMUM:
5082 case ISD::VP_FMINIMUM:
5083 case ISD::FMAXIMUM:
5084 case ISD::VP_FMAXIMUM:
5085 case ISD::FMINIMUMNUM:
5086 case ISD::FMAXIMUMNUM:
5087 case ISD::SMIN: case ISD::VP_SMIN:
5088 case ISD::SMAX: case ISD::VP_SMAX:
5089 case ISD::UMIN: case ISD::VP_UMIN:
5090 case ISD::UMAX: case ISD::VP_UMAX:
5091 case ISD::UADDSAT: case ISD::VP_UADDSAT:
5092 case ISD::SADDSAT: case ISD::VP_SADDSAT:
5093 case ISD::USUBSAT: case ISD::VP_USUBSAT:
5094 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
5095 case ISD::SSHLSAT:
5096 case ISD::USHLSAT:
5097 case ISD::ROTL:
5098 case ISD::ROTR:
5099 case ISD::AVGFLOORS:
5100 case ISD::AVGFLOORU:
5101 case ISD::AVGCEILS:
5102 case ISD::AVGCEILU:
5103 // Vector-predicated binary op widening. Note that -- unlike the
5104 // unpredicated versions -- we don't have to worry about trapping on
5105 // operations like UDIV, FADD, etc., as we pass on the original vector
5106 // length parameter. This means the widened elements containing garbage
5107 // aren't active.
5108 case ISD::VP_SDIV:
5109 case ISD::VP_UDIV:
5110 case ISD::VP_SREM:
5111 case ISD::VP_UREM:
5112 case ISD::VP_FADD:
5113 case ISD::VP_FSUB:
5114 case ISD::VP_FMUL:
5115 case ISD::VP_FDIV:
5116 case ISD::VP_FREM:
5117 case ISD::VP_FCOPYSIGN:
5118 Res = WidenVecRes_Binary(N);
5119 break;
5120
5121 case ISD::SCMP:
5122 case ISD::UCMP:
5123 Res = WidenVecRes_CMP(N);
5124 break;
5125
5126 case ISD::FPOW:
5127 case ISD::FATAN2:
5128 case ISD::FREM:
5129 if (unrollExpandedOp())
5130 break;
5131 // If the target has custom/legal support for the scalar FP intrinsic ops
5132 // (they are probably not destined to become libcalls), then widen those
5133 // like any other binary ops.
5134 [[fallthrough]];
5135
5136 case ISD::FADD:
5137 case ISD::FMUL:
5138 case ISD::FSUB:
5139 case ISD::FDIV:
5140 case ISD::SDIV:
5141 case ISD::UDIV:
5142 case ISD::SREM:
5143 case ISD::UREM:
5144 Res = WidenVecRes_BinaryCanTrap(N);
5145 break;
5146
5147 case ISD::SMULFIX:
5148 case ISD::SMULFIXSAT:
5149 case ISD::UMULFIX:
5150 case ISD::UMULFIXSAT:
5151 // These are binary operations, but with an extra operand that shouldn't
5152 // be widened (the scale).
5153 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5154 break;
5155
5156#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5157 case ISD::STRICT_##DAGN:
5158#include "llvm/IR/ConstrainedOps.def"
5159 Res = WidenVecRes_StrictFP(N);
5160 break;
5161
5162 case ISD::UADDO:
5163 case ISD::SADDO:
5164 case ISD::USUBO:
5165 case ISD::SSUBO:
5166 case ISD::UMULO:
5167 case ISD::SMULO:
5168 Res = WidenVecRes_OverflowOp(N, ResNo);
5169 break;
5170
5171 case ISD::FCOPYSIGN:
5172 Res = WidenVecRes_FCOPYSIGN(N);
5173 break;
5174
5175 case ISD::IS_FPCLASS:
5176 case ISD::FPTRUNC_ROUND:
5177 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5178 break;
5179
5180 case ISD::FLDEXP:
5181 case ISD::FPOWI:
5182 if (!unrollExpandedOp())
5183 Res = WidenVecRes_ExpOp(N);
5184 break;
5185
5186 case ISD::ANY_EXTEND_VECTOR_INREG:
5187 case ISD::SIGN_EXTEND_VECTOR_INREG:
5188 case ISD::ZERO_EXTEND_VECTOR_INREG:
5189 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5190 break;
5191
5192 case ISD::ANY_EXTEND:
5193 case ISD::FP_EXTEND:
5194 case ISD::VP_FP_EXTEND:
5195 case ISD::FP_ROUND:
5196 case ISD::VP_FP_ROUND:
5197 case ISD::FP_TO_SINT:
5198 case ISD::VP_FP_TO_SINT:
5199 case ISD::FP_TO_UINT:
5200 case ISD::VP_FP_TO_UINT:
5201 case ISD::SIGN_EXTEND:
5202 case ISD::VP_SIGN_EXTEND:
5203 case ISD::SINT_TO_FP:
5204 case ISD::VP_SINT_TO_FP:
5205 case ISD::VP_TRUNCATE:
5206 case ISD::TRUNCATE:
5207 case ISD::UINT_TO_FP:
5208 case ISD::VP_UINT_TO_FP:
5209 case ISD::ZERO_EXTEND:
5210 case ISD::VP_ZERO_EXTEND:
5211 case ISD::CONVERT_FROM_ARBITRARY_FP:
5212 Res = WidenVecRes_Convert(N);
5213 break;
5214
5215 case ISD::FP_TO_SINT_SAT:
5216 case ISD::FP_TO_UINT_SAT:
5217 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5218 break;
5219
5220 case ISD::LRINT:
5221 case ISD::LLRINT:
5222 case ISD::VP_LRINT:
5223 case ISD::VP_LLRINT:
5224 case ISD::LROUND:
5225 case ISD::LLROUND:
5226 Res = WidenVecRes_XROUND(N);
5227 break;
5228
5229 case ISD::FACOS:
5230 case ISD::FASIN:
5231 case ISD::FATAN:
5232 case ISD::FCEIL:
5233 case ISD::FCOS:
5234 case ISD::FCOSH:
5235 case ISD::FEXP:
5236 case ISD::FEXP2:
5237 case ISD::FEXP10:
5238 case ISD::FFLOOR:
5239 case ISD::FLOG:
5240 case ISD::FLOG10:
5241 case ISD::FLOG2:
5242 case ISD::FNEARBYINT:
5243 case ISD::FRINT:
5244 case ISD::FROUND:
5245 case ISD::FROUNDEVEN:
5246 case ISD::FSIN:
5247 case ISD::FSINH:
5248 case ISD::FSQRT:
5249 case ISD::FTAN:
5250 case ISD::FTANH:
5251 case ISD::FTRUNC:
5252 if (unrollExpandedOp())
5253 break;
5254 // If the target has custom/legal support for the scalar FP intrinsic ops
5255 // (they are probably not destined to become libcalls), then widen those
5256 // like any other unary ops.
5257 [[fallthrough]];
5258
5259 case ISD::ABS:
5260 case ISD::VP_ABS:
5261 case ISD::BITREVERSE:
5262 case ISD::VP_BITREVERSE:
5263 case ISD::BSWAP:
5264 case ISD::VP_BSWAP:
5265 case ISD::CTLZ:
5266 case ISD::VP_CTLZ:
5267 case ISD::CTLZ_ZERO_UNDEF:
5268 case ISD::VP_CTLZ_ZERO_UNDEF:
5269 case ISD::CTPOP:
5270 case ISD::VP_CTPOP:
5271 case ISD::CTTZ:
5272 case ISD::VP_CTTZ:
5273 case ISD::CTTZ_ZERO_UNDEF:
5274 case ISD::VP_CTTZ_ZERO_UNDEF:
5275 case ISD::FNEG: case ISD::VP_FNEG:
5276 case ISD::FABS: case ISD::VP_FABS:
5277 case ISD::VP_SQRT:
5278 case ISD::VP_FCEIL:
5279 case ISD::VP_FFLOOR:
5280 case ISD::VP_FRINT:
5281 case ISD::VP_FNEARBYINT:
5282 case ISD::VP_FROUND:
5283 case ISD::VP_FROUNDEVEN:
5284 case ISD::VP_FROUNDTOZERO:
5285 case ISD::FREEZE:
5286 case ISD::ARITH_FENCE:
5287 case ISD::FCANONICALIZE:
5288 case ISD::AssertNoFPClass:
5289 Res = WidenVecRes_Unary(N);
5290 break;
5291 case ISD::FMA: case ISD::VP_FMA:
5292 case ISD::FSHL:
5293 case ISD::VP_FSHL:
5294 case ISD::FSHR:
5295 case ISD::VP_FSHR:
5296 Res = WidenVecRes_Ternary(N);
5297 break;
5298 case ISD::FMODF:
5299 case ISD::FFREXP:
5300 case ISD::FSINCOS:
5301 case ISD::FSINCOSPI: {
5302 if (!unrollExpandedOp())
5303 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5304 break;
5305 }
5306 }
5307
5308 // If Res is null, the sub-method took care of registering the result.
5309 if (Res.getNode())
5310 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5311}
5312
5313SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5314 // Ternary op widening.
5315 SDLoc dl(N);
5316 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5317 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5318 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5319 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5320 if (N->getNumOperands() == 3)
5321 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5322
5323 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5324 assert(N->isVPOpcode() && "Expected VP opcode");
5325
5326 SDValue Mask =
5327 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5328 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5329 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5330}
5331
5332SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5333 // Binary op widening.
5334 SDLoc dl(N);
5335 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5336 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5337 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5338 if (N->getNumOperands() == 2)
5339 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5340 Flags: N->getFlags());
5341
5342 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5343 assert(N->isVPOpcode() && "Expected VP opcode");
5344
5345 SDValue Mask =
5346 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5347 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5348 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5349}
5350
5351SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5352 LLVMContext &Ctxt = *DAG.getContext();
5353 SDLoc dl(N);
5354
5355 SDValue LHS = N->getOperand(Num: 0);
5356 SDValue RHS = N->getOperand(Num: 1);
5357 EVT OpVT = LHS.getValueType();
5358 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5359 LHS = GetWidenedVector(Op: LHS);
5360 RHS = GetWidenedVector(Op: RHS);
5361 OpVT = LHS.getValueType();
5362 }
5363
5364 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5365 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5366 if (WidenResEC == OpVT.getVectorElementCount()) {
5367 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5368 }
5369
5370 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5371}
5372
5373SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5374 // Binary op widening, but with an extra operand that shouldn't be widened.
5375 SDLoc dl(N);
5376 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5377 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5378 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5379 SDValue InOp3 = N->getOperand(Num: 2);
5380 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5381 Flags: N->getFlags());
5382}
5383
5384// Given a vector of operations that have been broken up to widen, see
5385// if we can collect them together into the next widest legal VT. This
5386// implementation is trap-safe.
5387static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5388 SmallVectorImpl<SDValue> &ConcatOps,
5389 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5390 EVT WidenVT) {
5391 // Check to see if we have a single operation with the widen type.
5392 if (ConcatEnd == 1) {
5393 VT = ConcatOps[0].getValueType();
5394 if (VT == WidenVT)
5395 return ConcatOps[0];
5396 }
5397
5398 SDLoc dl(ConcatOps[0]);
5399 EVT WidenEltVT = WidenVT.getVectorElementType();
5400
5401 // while (Some element of ConcatOps is not of type MaxVT) {
5402 // From the end of ConcatOps, collect elements of the same type and put
5403 // them into an op of the next larger supported type
5404 // }
5405 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5406 int Idx = ConcatEnd - 1;
5407 VT = ConcatOps[Idx--].getValueType();
5408 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5409 Idx--;
5410
5411 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5412 EVT NextVT;
5413 do {
5414 NextSize *= 2;
5415 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5416 } while (!TLI.isTypeLegal(VT: NextVT));
5417
5418 if (!VT.isVector()) {
5419 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5420 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5421 unsigned NumToInsert = ConcatEnd - Idx - 1;
5422 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5423 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5424 ConcatOps[Idx+1] = VecOp;
5425 ConcatEnd = Idx + 2;
5426 } else {
5427 // Vector type, create a CONCAT_VECTORS of type NextVT
5428 SDValue undefVec = DAG.getPOISON(VT);
5429 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5430 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5431 unsigned RealVals = ConcatEnd - Idx - 1;
5432 unsigned SubConcatEnd = 0;
5433 unsigned SubConcatIdx = Idx + 1;
5434 while (SubConcatEnd < RealVals)
5435 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5436 while (SubConcatEnd < OpsToConcat)
5437 SubConcatOps[SubConcatEnd++] = undefVec;
5438 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5439 VT: NextVT, Ops: SubConcatOps);
5440 ConcatEnd = SubConcatIdx + 1;
5441 }
5442 }
5443
5444 // Check to see if we have a single operation with the widen type.
5445 if (ConcatEnd == 1) {
5446 VT = ConcatOps[0].getValueType();
5447 if (VT == WidenVT)
5448 return ConcatOps[0];
5449 }
5450
5451 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5452 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5453 if (NumOps != ConcatEnd ) {
5454 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5455 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5456 ConcatOps[j] = UndefVal;
5457 }
5458 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5459 Ops: ArrayRef(ConcatOps.data(), NumOps));
5460}
5461
5462SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5463 // Binary op widening for operations that can trap.
5464 unsigned Opcode = N->getOpcode();
5465 SDLoc dl(N);
5466 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5467 EVT WidenEltVT = WidenVT.getVectorElementType();
5468 EVT VT = WidenVT;
5469 unsigned NumElts = VT.getVectorMinNumElements();
5470 const SDNodeFlags Flags = N->getFlags();
5471 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5472 NumElts = NumElts / 2;
5473 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5474 }
5475
5476 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5477 // Operation doesn't trap so just widen as normal.
5478 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5479 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5480 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5481 }
5482
5483 // Generate a vp.op if it is custom/legal for the target. This avoids need
5484 // to split and tile the subvectors (below), because the inactive lanes can
5485 // simply be disabled. To avoid possible recursion, only do this if the
5486 // widened mask type is legal.
5487 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5488 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5489 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5490 EC: WidenVT.getVectorElementCount());
5491 TLI.isTypeLegal(VT: WideMaskVT)) {
5492 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5493 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5494 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5495 SDValue EVL =
5496 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5497 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5498 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5499 Flags);
5500 }
5501 }
5502
5503 // FIXME: Improve support for scalable vectors.
5504 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5505
5506 // No legal vector version so unroll the vector operation and then widen.
5507 if (NumElts == 1)
5508 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5509
5510 // Since the operation can trap, apply operation on the original vector.
5511 EVT MaxVT = VT;
5512 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5513 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5514 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5515
5516 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5517 unsigned ConcatEnd = 0; // Current ConcatOps index.
5518 int Idx = 0; // Current Idx into input vectors.
5519
5520 // NumElts := greatest legal vector size (at most WidenVT)
5521 // while (orig. vector has unhandled elements) {
5522 // take munches of size NumElts from the beginning and add to ConcatOps
5523 // NumElts := next smaller supported vector size or 1
5524 // }
5525 while (CurNumElts != 0) {
5526 while (CurNumElts >= NumElts) {
5527 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5528 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5529 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5530 Idx += NumElts;
5531 CurNumElts -= NumElts;
5532 }
5533 do {
5534 NumElts = NumElts / 2;
5535 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5536 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5537
5538 if (NumElts == 1) {
5539 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5540 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5541 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5542 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5543 N1: EOp1, N2: EOp2, Flags);
5544 }
5545 CurNumElts = 0;
5546 }
5547 }
5548
5549 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5550}
5551
5552SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5553 switch (N->getOpcode()) {
5554 case ISD::STRICT_FSETCC:
5555 case ISD::STRICT_FSETCCS:
5556 return WidenVecRes_STRICT_FSETCC(N);
5557 case ISD::STRICT_FP_EXTEND:
5558 case ISD::STRICT_FP_ROUND:
5559 case ISD::STRICT_FP_TO_SINT:
5560 case ISD::STRICT_FP_TO_UINT:
5561 case ISD::STRICT_SINT_TO_FP:
5562 case ISD::STRICT_UINT_TO_FP:
5563 return WidenVecRes_Convert_StrictFP(N);
5564 default:
5565 break;
5566 }
5567
5568 // StrictFP op widening for operations that can trap.
5569 unsigned NumOpers = N->getNumOperands();
5570 unsigned Opcode = N->getOpcode();
5571 SDLoc dl(N);
5572 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5573 EVT WidenEltVT = WidenVT.getVectorElementType();
5574 EVT VT = WidenVT;
5575 unsigned NumElts = VT.getVectorNumElements();
5576 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5577 NumElts = NumElts / 2;
5578 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5579 }
5580
5581 // No legal vector version so unroll the vector operation and then widen.
5582 if (NumElts == 1)
5583 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5584
5585 // Since the operation can trap, apply operation on the original vector.
5586 EVT MaxVT = VT;
5587 SmallVector<SDValue, 4> InOps;
5588 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5589
5590 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5591 SmallVector<SDValue, 16> Chains;
5592 unsigned ConcatEnd = 0; // Current ConcatOps index.
5593 int Idx = 0; // Current Idx into input vectors.
5594
5595 // The Chain is the first operand.
5596 InOps.push_back(Elt: N->getOperand(Num: 0));
5597
5598 // Now process the remaining operands.
5599 for (unsigned i = 1; i < NumOpers; ++i) {
5600 SDValue Oper = N->getOperand(Num: i);
5601
5602 EVT OpVT = Oper.getValueType();
5603 if (OpVT.isVector()) {
5604 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5605 Oper = GetWidenedVector(Op: Oper);
5606 else {
5607 EVT WideOpVT =
5608 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5609 EC: WidenVT.getVectorElementCount());
5610 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5611 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5612 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5613 }
5614 }
5615
5616 InOps.push_back(Elt: Oper);
5617 }
5618
5619 // NumElts := greatest legal vector size (at most WidenVT)
5620 // while (orig. vector has unhandled elements) {
5621 // take munches of size NumElts from the beginning and add to ConcatOps
5622 // NumElts := next smaller supported vector size or 1
5623 // }
5624 while (CurNumElts != 0) {
5625 while (CurNumElts >= NumElts) {
5626 SmallVector<SDValue, 4> EOps;
5627
5628 for (unsigned i = 0; i < NumOpers; ++i) {
5629 SDValue Op = InOps[i];
5630
5631 EVT OpVT = Op.getValueType();
5632 if (OpVT.isVector()) {
5633 EVT OpExtractVT =
5634 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5635 EC: VT.getVectorElementCount());
5636 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5637 }
5638
5639 EOps.push_back(Elt: Op);
5640 }
5641
5642 EVT OperVT[] = {VT, MVT::Other};
5643 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5644 ConcatOps[ConcatEnd++] = Oper;
5645 Chains.push_back(Elt: Oper.getValue(R: 1));
5646 Idx += NumElts;
5647 CurNumElts -= NumElts;
5648 }
5649 do {
5650 NumElts = NumElts / 2;
5651 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5652 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5653
5654 if (NumElts == 1) {
5655 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5656 SmallVector<SDValue, 4> EOps;
5657
5658 for (unsigned i = 0; i < NumOpers; ++i) {
5659 SDValue Op = InOps[i];
5660
5661 EVT OpVT = Op.getValueType();
5662 if (OpVT.isVector())
5663 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5664 Idx);
5665
5666 EOps.push_back(Elt: Op);
5667 }
5668
5669 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5670 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5671 ConcatOps[ConcatEnd++] = Oper;
5672 Chains.push_back(Elt: Oper.getValue(R: 1));
5673 }
5674 CurNumElts = 0;
5675 }
5676 }
5677
5678 // Build a factor node to remember all the Ops that have been created.
5679 SDValue NewChain;
5680 if (Chains.size() == 1)
5681 NewChain = Chains[0];
5682 else
5683 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5684 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5685
5686 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5687}
5688
5689SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5690 SDLoc DL(N);
5691 EVT ResVT = N->getValueType(ResNo: 0);
5692 EVT OvVT = N->getValueType(ResNo: 1);
5693 EVT WideResVT, WideOvVT;
5694 SDValue WideLHS, WideRHS;
5695
5696 // TODO: This might result in a widen/split loop.
5697 if (ResNo == 0) {
5698 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5699 WideOvVT = EVT::getVectorVT(
5700 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5701 NumElements: WideResVT.getVectorNumElements());
5702
5703 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5704 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5705 } else {
5706 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5707 WideResVT = EVT::getVectorVT(
5708 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5709 NumElements: WideOvVT.getVectorNumElements());
5710
5711 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5712 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5713
5714 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5715 N2: N->getOperand(Num: 0), N3: Zero);
5716 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5717 N2: N->getOperand(Num: 1), N3: Zero);
5718 }
5719
5720 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5721 SDNode *WideNode = DAG.getNode(
5722 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5723
5724 // Replace the other vector result not being explicitly widened here.
5725 unsigned OtherNo = 1 - ResNo;
5726 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5727 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5728 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5729 } else {
5730 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5731 SDValue OtherVal = DAG.getNode(
5732 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5733 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5734 }
5735
5736 return SDValue(WideNode, ResNo);
5737}
5738
5739SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5740 LLVMContext &Ctx = *DAG.getContext();
5741 SDValue InOp = N->getOperand(Num: 0);
5742 SDLoc DL(N);
5743
5744 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
5745 ElementCount WidenEC = WidenVT.getVectorElementCount();
5746
5747 EVT InVT = InOp.getValueType();
5748
5749 unsigned Opcode = N->getOpcode();
5750 const SDNodeFlags Flags = N->getFlags();
5751
5752 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
5753 // not equal that of WidenVT.
5754 if (N->getOpcode() == ISD::ZERO_EXTEND &&
5755 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
5756 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
5757 WidenVT.getScalarSizeInBits()) {
5758 InOp = ZExtPromotedInteger(Op: InOp);
5759 InVT = InOp.getValueType();
5760 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
5761 Opcode = ISD::TRUNCATE;
5762 }
5763
5764 EVT InEltVT = InVT.getVectorElementType();
5765 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
5766 ElementCount InVTEC = InVT.getVectorElementCount();
5767
5768 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5769 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5770 InVT = InOp.getValueType();
5771 InVTEC = InVT.getVectorElementCount();
5772 if (InVTEC == WidenEC) {
5773 if (N->getNumOperands() == 1)
5774 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp, Flags);
5775 if (N->getNumOperands() == 3) {
5776 assert(N->isVPOpcode() && "Expected VP opcode");
5777 SDValue Mask =
5778 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5779 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
5780 }
5781 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
5782 }
5783 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
5784 // If both input and result vector types are of same width, extend
5785 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
5786 // accepts fewer elements in the result than in the input.
5787 if (Opcode == ISD::ANY_EXTEND)
5788 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5789 if (Opcode == ISD::SIGN_EXTEND)
5790 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5791 if (Opcode == ISD::ZERO_EXTEND)
5792 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5793 }
5794
5795 // For TRUNCATE, try to widen using the legal EC of the input type instead
5796 // if the legalisation action for that intermediate type is not widening.
5797 // E.g. for trunc nxv1i64 -> nxv1i8 where
5798 // - nxv1i64 input gets widened to nxv2i64
5799 // - nxv1i8 output gets widened to nxv16i8
5800 // Then one can try widening the result to nxv2i8 (instead of going all the
5801 // way to nxv16i8) if this later allows type promotion.
5802 EVT MidResVT =
5803 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
5804 if (N->getOpcode() == ISD::TRUNCATE &&
5805 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
5806 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
5807 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
5808 }
5809 }
5810
5811 if (TLI.isTypeLegal(VT: InWidenVT)) {
5812 // Because the result and the input are different vector types, widening
5813 // the result could create a legal type but widening the input might make
5814 // it an illegal type that might lead to repeatedly splitting the input
5815 // and then widening it. To avoid this, we widen the input only if
5816 // it results in a legal type.
5817 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
5818 // Widen the input and call convert on the widened input vector.
5819 unsigned NumConcat =
5820 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
5821 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
5822 Ops[0] = InOp;
5823 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
5824 if (N->getNumOperands() == 1)
5825 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec, Flags);
5826 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
5827 }
5828
5829 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
5830 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
5831 // Extract the input and convert the shorten input vector.
5832 if (N->getNumOperands() == 1)
5833 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal, Flags);
5834 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
5835 }
5836 }
5837
5838 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5839 EVT EltVT = WidenVT.getVectorElementType();
5840 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
5841 // Use the original element count so we don't do more scalar opts than
5842 // necessary.
5843 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5844 for (unsigned i=0; i < MinElts; ++i) {
5845 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5846 if (N->getNumOperands() == 1)
5847 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val, Flags);
5848 else
5849 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
5850 }
5851
5852 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5853}
5854
5855SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
5856 SDLoc dl(N);
5857 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5858 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5859
5860 SDValue Src = N->getOperand(Num: 0);
5861 EVT SrcVT = Src.getValueType();
5862
5863 // Also widen the input.
5864 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5865 Src = GetWidenedVector(Op: Src);
5866 SrcVT = Src.getValueType();
5867 }
5868
5869 // Input and output not widened to the same size, give up.
5870 if (WidenNumElts != SrcVT.getVectorElementCount())
5871 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5872
5873 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
5874}
5875
5876SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
5877 SDLoc dl(N);
5878 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5879 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5880
5881 SDValue Src = N->getOperand(Num: 0);
5882 EVT SrcVT = Src.getValueType();
5883
5884 // Also widen the input.
5885 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5886 Src = GetWidenedVector(Op: Src);
5887 SrcVT = Src.getValueType();
5888 }
5889
5890 // Input and output not widened to the same size, give up.
5891 if (WidenNumElts != SrcVT.getVectorElementCount())
5892 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5893
5894 if (N->getNumOperands() == 1)
5895 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
5896
5897 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5898 assert(N->isVPOpcode() && "Expected VP opcode");
5899
5900 SDValue Mask =
5901 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5902 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
5903}
5904
5905SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
5906 SDValue InOp = N->getOperand(Num: 1);
5907 SDLoc DL(N);
5908 SmallVector<SDValue, 4> NewOps(N->ops());
5909
5910 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5911 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5912
5913 EVT InVT = InOp.getValueType();
5914 EVT InEltVT = InVT.getVectorElementType();
5915
5916 unsigned Opcode = N->getOpcode();
5917
5918 // FIXME: Optimizations need to be implemented here.
5919
5920 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5921 EVT EltVT = WidenVT.getVectorElementType();
5922 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
5923 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
5924 SmallVector<SDValue, 32> OpChains;
5925 // Use the original element count so we don't do more scalar opts than
5926 // necessary.
5927 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5928 for (unsigned i=0; i < MinElts; ++i) {
5929 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5930 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
5931 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
5932 }
5933 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
5934 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5935
5936 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5937}
5938
5939SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
5940 unsigned Opcode = N->getOpcode();
5941 SDValue InOp = N->getOperand(Num: 0);
5942 SDLoc DL(N);
5943
5944 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5945 EVT WidenSVT = WidenVT.getVectorElementType();
5946 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5947
5948 EVT InVT = InOp.getValueType();
5949 EVT InSVT = InVT.getVectorElementType();
5950 unsigned InVTNumElts = InVT.getVectorNumElements();
5951
5952 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5953 InOp = GetWidenedVector(Op: InOp);
5954 InVT = InOp.getValueType();
5955 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
5956 switch (Opcode) {
5957 case ISD::ANY_EXTEND_VECTOR_INREG:
5958 case ISD::SIGN_EXTEND_VECTOR_INREG:
5959 case ISD::ZERO_EXTEND_VECTOR_INREG:
5960 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
5961 }
5962 }
5963 }
5964
5965 // Unroll, extend the scalars and rebuild the vector.
5966 SmallVector<SDValue, 16> Ops;
5967 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
5968 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
5969 switch (Opcode) {
5970 case ISD::ANY_EXTEND_VECTOR_INREG:
5971 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
5972 break;
5973 case ISD::SIGN_EXTEND_VECTOR_INREG:
5974 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
5975 break;
5976 case ISD::ZERO_EXTEND_VECTOR_INREG:
5977 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
5978 break;
5979 default:
5980 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
5981 }
5982 Ops.push_back(Elt: Val);
5983 }
5984
5985 while (Ops.size() != WidenNumElts)
5986 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
5987
5988 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5989}
5990
5991SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
5992 // If this is an FCOPYSIGN with same input types, we can treat it as a
5993 // normal (can trap) binary op.
5994 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
5995 return WidenVecRes_BinaryCanTrap(N);
5996
5997 // If the types are different, fall back to unrolling.
5998 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5999 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6000}
6001
6002/// Result and first source operand are different scalar types, but must have
6003/// the same number of elements. There is an additional control argument which
6004/// should be passed through unchanged.
6005SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6006 SDValue FpValue = N->getOperand(Num: 0);
6007 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6008 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6009 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6010 SDValue Arg = GetWidenedVector(Op: FpValue);
6011 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6012 Flags: N->getFlags());
6013}
6014
6015SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6016 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6017 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6018 SDValue RHS = N->getOperand(Num: 1);
6019 EVT ExpVT = RHS.getValueType();
6020 SDValue ExpOp = RHS;
6021 if (ExpVT.isVector()) {
6022 EVT WideExpVT = WidenVT.changeVectorElementType(
6023 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6024 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6025 }
6026
6027 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6028}
6029
6030SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6031 // Unary op widening.
6032 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6033 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6034 if (N->getNumOperands() == 1)
6035 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6036 if (N->getOpcode() == ISD::AssertNoFPClass)
6037 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
6038 N2: N->getOperand(Num: 1), Flags: N->getFlags());
6039
6040 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6041 assert(N->isVPOpcode() && "Expected VP opcode");
6042
6043 SDValue Mask =
6044 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6045 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
6046 Ops: {InOp, Mask, N->getOperand(Num: 2)});
6047}
6048
6049SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6050 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6051 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6052 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
6053 .getVectorElementType(),
6054 NumElements: WidenVT.getVectorNumElements());
6055 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6056 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6057 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6058}
6059
6060SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6061 unsigned ResNo) {
6062 EVT VT0 = N->getValueType(ResNo: 0);
6063 EVT VT1 = N->getValueType(ResNo: 1);
6064
6065 assert(VT0.isVector() && VT1.isVector() &&
6066 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6067 "expected both results to be vectors of matching element count");
6068
6069 LLVMContext &Ctx = *DAG.getContext();
6070 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6071
6072 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6073 ElementCount WidenEC = WidenVT.getVectorElementCount();
6074
6075 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6076 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6077
6078 SDNode *WidenNode =
6079 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6080 .getNode();
6081
6082 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6083 return SDValue(WidenNode, ResNo);
6084}
6085
6086SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6087 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6088 return GetWidenedVector(Op: WidenVec);
6089}
6090
6091SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6092 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6093 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6094 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6095
6096 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
6097 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6098 DestAS: AddrSpaceCastN->getDestAddressSpace());
6099}
6100
6101SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6102 SDValue InOp = N->getOperand(Num: 0);
6103 EVT InVT = InOp.getValueType();
6104 EVT VT = N->getValueType(ResNo: 0);
6105 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6106 SDLoc dl(N);
6107
6108 switch (getTypeAction(VT: InVT)) {
6109 case TargetLowering::TypeLegal:
6110 break;
6111 case TargetLowering::TypeScalarizeScalableVector:
6112 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6113 case TargetLowering::TypePromoteInteger: {
6114 // If the incoming type is a vector that is being promoted, then
6115 // we know that the elements are arranged differently and that we
6116 // must perform the conversion using a stack slot.
6117 if (InVT.isVector())
6118 break;
6119
6120 // If the InOp is promoted to the same size, convert it. Otherwise,
6121 // fall out of the switch and widen the promoted input.
6122 SDValue NInOp = GetPromotedInteger(Op: InOp);
6123 EVT NInVT = NInOp.getValueType();
6124 if (WidenVT.bitsEq(VT: NInVT)) {
6125 // For big endian targets we need to shift the input integer or the
6126 // interesting bits will end up at the wrong place.
6127 if (DAG.getDataLayout().isBigEndian()) {
6128 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6129 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6130 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6131 }
6132 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6133 }
6134 InOp = NInOp;
6135 InVT = NInVT;
6136 break;
6137 }
6138 case TargetLowering::TypeSoftenFloat:
6139 case TargetLowering::TypeSoftPromoteHalf:
6140 case TargetLowering::TypeExpandInteger:
6141 case TargetLowering::TypeExpandFloat:
6142 case TargetLowering::TypeScalarizeVector:
6143 case TargetLowering::TypeSplitVector:
6144 break;
6145 case TargetLowering::TypeWidenVector:
6146 // If the InOp is widened to the same size, convert it. Otherwise, fall
6147 // out of the switch and widen the widened input.
6148 InOp = GetWidenedVector(Op: InOp);
6149 InVT = InOp.getValueType();
6150 if (WidenVT.bitsEq(VT: InVT))
6151 // The input widens to the same size. Convert to the widen value.
6152 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6153 break;
6154 }
6155
6156 unsigned WidenSize = WidenVT.getSizeInBits();
6157 unsigned InSize = InVT.getSizeInBits();
6158 unsigned InScalarSize = InVT.getScalarSizeInBits();
6159 // x86mmx is not an acceptable vector element type, so don't try.
6160 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6161 // Determine new input vector type. The new input vector type will use
6162 // the same element type (if its a vector) or use the input type as a
6163 // vector. It is the same size as the type to widen to.
6164 EVT NewInVT;
6165 unsigned NewNumParts = WidenSize / InSize;
6166 if (InVT.isVector()) {
6167 EVT InEltVT = InVT.getVectorElementType();
6168 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6169 NumElements: WidenSize / InEltVT.getSizeInBits());
6170 } else {
6171 // For big endian systems, using the promoted input scalar type
6172 // to produce the scalar_to_vector would put the desired bits into
6173 // the least significant byte(s) of the wider element zero. This
6174 // will mean that the users of the result vector are using incorrect
6175 // bits. Use the original input type instead. Although either input
6176 // type can be used on little endian systems, for consistency we
6177 // use the original type there as well.
6178 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6179 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6180 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6181 }
6182
6183 if (TLI.isTypeLegal(VT: NewInVT)) {
6184 SDValue NewVec;
6185 if (InVT.isVector()) {
6186 // Because the result and the input are different vector types, widening
6187 // the result could create a legal type but widening the input might
6188 // make it an illegal type that might lead to repeatedly splitting the
6189 // input and then widening it. To avoid this, we widen the input only if
6190 // it results in a legal type.
6191 if (WidenSize % InSize == 0) {
6192 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6193 Ops[0] = InOp;
6194
6195 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6196 } else {
6197 SmallVector<SDValue, 16> Ops;
6198 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6199 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6200 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6201
6202 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6203 }
6204 } else {
6205 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6206 }
6207 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6208 }
6209 }
6210
6211 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6212}
6213
6214SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6215 return DAG.getNode(
6216 Opcode: N->getOpcode(), DL: SDLoc(N),
6217 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6218 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6219}
6220
6221SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6222 SDLoc dl(N);
6223 // Build a vector with poison for the new nodes.
6224 EVT VT = N->getValueType(ResNo: 0);
6225
6226 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6227 // type. The POISONs need to have the same type as the existing operands.
6228 EVT EltVT = N->getOperand(Num: 0).getValueType();
6229 unsigned NumElts = VT.getVectorNumElements();
6230
6231 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6232 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6233
6234 SmallVector<SDValue, 16> NewOps(N->ops());
6235 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6236 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6237
6238 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6239}
6240
6241SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6242 EVT InVT = N->getOperand(Num: 0).getValueType();
6243 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6244 SDLoc dl(N);
6245 unsigned NumOperands = N->getNumOperands();
6246
6247 bool InputWidened = false; // Indicates we need to widen the input.
6248 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6249 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6250 unsigned NumInElts = InVT.getVectorMinNumElements();
6251 if (WidenNumElts % NumInElts == 0) {
6252 // Add undef vectors to widen to correct length.
6253 unsigned NumConcat = WidenNumElts / NumInElts;
6254 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6255 SmallVector<SDValue, 16> Ops(NumConcat);
6256 for (unsigned i=0; i < NumOperands; ++i)
6257 Ops[i] = N->getOperand(Num: i);
6258 for (unsigned i = NumOperands; i != NumConcat; ++i)
6259 Ops[i] = UndefVal;
6260 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6261 }
6262 } else {
6263 InputWidened = true;
6264 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6265 // The inputs and the result are widen to the same value.
6266 unsigned i;
6267 for (i=1; i < NumOperands; ++i)
6268 if (!N->getOperand(Num: i).isUndef())
6269 break;
6270
6271 if (i == NumOperands)
6272 // Everything but the first operand is an UNDEF so just return the
6273 // widened first operand.
6274 return GetWidenedVector(Op: N->getOperand(Num: 0));
6275
6276 if (NumOperands == 2) {
6277 assert(!WidenVT.isScalableVector() &&
6278 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6279 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6280 unsigned NumInElts = InVT.getVectorNumElements();
6281
6282 // Replace concat of two operands with a shuffle.
6283 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6284 for (unsigned i = 0; i < NumInElts; ++i) {
6285 MaskOps[i] = i;
6286 MaskOps[i + NumInElts] = i + WidenNumElts;
6287 }
6288 return DAG.getVectorShuffle(VT: WidenVT, dl,
6289 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6290 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6291 Mask: MaskOps);
6292 }
6293 }
6294 }
6295
6296 assert(!WidenVT.isScalableVector() &&
6297 "Cannot use build vectors to widen CONCAT_VECTOR result");
6298 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6299 unsigned NumInElts = InVT.getVectorNumElements();
6300
6301 // Fall back to use extracts and build vector.
6302 EVT EltVT = WidenVT.getVectorElementType();
6303 SmallVector<SDValue, 16> Ops(WidenNumElts);
6304 unsigned Idx = 0;
6305 for (unsigned i=0; i < NumOperands; ++i) {
6306 SDValue InOp = N->getOperand(Num: i);
6307 if (InputWidened)
6308 InOp = GetWidenedVector(Op: InOp);
6309 for (unsigned j = 0; j < NumInElts; ++j)
6310 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6311 }
6312 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6313 for (; Idx < WidenNumElts; ++Idx)
6314 Ops[Idx] = UndefVal;
6315 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6316}
6317
6318SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6319 EVT VT = N->getValueType(ResNo: 0);
6320 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6321 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6322 SDValue InOp2 = N->getOperand(Num: 1);
6323 SDValue Idx = N->getOperand(Num: 2);
6324 SDLoc dl(N);
6325 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6326}
6327
6328SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6329 EVT VT = N->getValueType(ResNo: 0);
6330 EVT EltVT = VT.getVectorElementType();
6331 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6332 SDValue InOp = N->getOperand(Num: 0);
6333 SDValue Idx = N->getOperand(Num: 1);
6334 SDLoc dl(N);
6335
6336 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6337 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6338 InOp = GetWidenedVector(Op: InOp);
6339
6340 EVT InVT = InOp.getValueType();
6341
6342 // Check if we can just return the input vector after widening.
6343 uint64_t IdxVal = Idx->getAsZExtVal();
6344 if (IdxVal == 0 && InVT == WidenVT)
6345 return InOp;
6346
6347 // Check if we can extract from the vector.
6348 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6349 unsigned InNumElts = InVT.getVectorMinNumElements();
6350 unsigned VTNumElts = VT.getVectorMinNumElements();
6351 assert(IdxVal % VTNumElts == 0 &&
6352 "Expected Idx to be a multiple of subvector minimum vector length");
6353 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6354 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6355
6356 if (VT.isScalableVector()) {
6357 // Try to split the operation up into smaller extracts and concat the
6358 // results together, e.g.
6359 // nxv6i64 extract_subvector(nxv12i64, 6)
6360 // <->
6361 // nxv8i64 concat(
6362 // nxv2i64 extract_subvector(nxv16i64, 6)
6363 // nxv2i64 extract_subvector(nxv16i64, 8)
6364 // nxv2i64 extract_subvector(nxv16i64, 10)
6365 // undef)
6366 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6367 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6368 "down type's element count");
6369 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6370 EC: ElementCount::getScalable(MinVal: GCD));
6371 // Avoid recursion around e.g. nxv1i8.
6372 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6373 SmallVector<SDValue> Parts;
6374 unsigned I = 0;
6375 for (; I < VTNumElts / GCD; ++I)
6376 Parts.push_back(
6377 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6378 for (; I < WidenNumElts / GCD; ++I)
6379 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6380
6381 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6382 }
6383
6384 // Fallback to extracting through memory.
6385
6386 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6387 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6388 MachineFunction &MF = DAG.getMachineFunction();
6389 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6390 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6391
6392 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6393 PtrInfo, F: MachineMemOperand::MOStore,
6394 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6395 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6396 PtrInfo, F: MachineMemOperand::MOLoad,
6397 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6398
6399 // Write out the input vector.
6400 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6401
6402 // Build a mask to match the length of the non-widened result.
6403 SDValue Mask =
6404 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6405
6406 // Read back the sub-vector setting the remaining lanes to poison.
6407 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6408 return DAG.getMaskedLoad(
6409 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6410 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6411 }
6412
6413 // We could try widening the input to the right length but for now, extract
6414 // the original elements, fill the rest with undefs and build a vector.
6415 SmallVector<SDValue, 16> Ops(WidenNumElts);
6416 unsigned i;
6417 for (i = 0; i < VTNumElts; ++i)
6418 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6419
6420 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6421 for (; i < WidenNumElts; ++i)
6422 Ops[i] = UndefVal;
6423 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6424}
6425
6426SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6427 SDValue InOp = ModifyToType(
6428 InOp: N->getOperand(Num: 0),
6429 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6430 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6431 N2: N->getOperand(Num: 1));
6432}
6433
6434SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6435 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6436 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6437 VT: InOp.getValueType(), N1: InOp,
6438 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6439}
6440
6441/// Either return the same load or provide appropriate casts
6442/// from the load and return that.
6443static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6444 TypeSize LdWidth, TypeSize FirstVTWidth,
6445 SDLoc dl, SelectionDAG &DAG) {
6446 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6447 "Load width must be less than or equal to first value type width");
6448 TypeSize WidenWidth = WidenVT.getSizeInBits();
6449 if (!FirstVT.isVector()) {
6450 unsigned NumElts =
6451 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6452 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6453 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6454 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6455 }
6456 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6457 return LdOp;
6458}
6459
6460static std::optional<EVT> findMemType(SelectionDAG &DAG,
6461 const TargetLowering &TLI, unsigned Width,
6462 EVT WidenVT, unsigned Align,
6463 unsigned WidenEx);
6464
6465SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6466 EVT WidenVT =
6467 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6468 EVT LdVT = LD->getMemoryVT();
6469 SDLoc dl(LD);
6470 assert(LdVT.isVector() && WidenVT.isVector() && "Expected vectors");
6471 assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6472 "Must be scalable");
6473 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6474 "Expected equivalent element types");
6475
6476 // Load information
6477 SDValue Chain = LD->getChain();
6478 SDValue BasePtr = LD->getBasePtr();
6479
6480 TypeSize LdWidth = LdVT.getSizeInBits();
6481 TypeSize WidenWidth = WidenVT.getSizeInBits();
6482 TypeSize WidthDiff = WidenWidth - LdWidth;
6483
6484 // Find the vector type that can load from.
6485 std::optional<EVT> FirstVT =
6486 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6487 WidenEx: WidthDiff.getKnownMinValue());
6488
6489 if (!FirstVT)
6490 return SDValue();
6491
6492 SmallVector<EVT, 8> MemVTs;
6493 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6494
6495 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6496 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6497
6498 // Load the element with one instruction.
6499 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6500 FirstVTWidth, dl, DAG);
6501
6502 // Modified the chain - switch anything that used the old chain to use
6503 // the new one.
6504 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6505 return Result;
6506}
6507
6508SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6509 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6510 ISD::LoadExtType ExtType = LD->getExtensionType();
6511
6512 // A vector must always be stored in memory as-is, i.e. without any padding
6513 // between the elements, since various code depend on it, e.g. in the
6514 // handling of a bitcast of a vector type to int, which may be done with a
6515 // vector store followed by an integer load. A vector that does not have
6516 // elements that are byte-sized must therefore be stored as an integer
6517 // built out of the extracted vector elements.
6518 if (!LD->getMemoryVT().isByteSized()) {
6519 SDValue Value, NewChain;
6520 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6521 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6522 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6523 return SDValue();
6524 }
6525
6526 // Generate a vector-predicated load if it is custom/legal on the target. To
6527 // avoid possible recursion, only do this if the widened mask type is legal.
6528 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6529 // removed from the IR by the ExpandVectorPredication pass but we're
6530 // reintroducing them here.
6531 EVT VT = LD->getValueType(ResNo: 0);
6532 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6533 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6534
6535 if (ExtType == ISD::NON_EXTLOAD &&
6536 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6537 TLI.isTypeLegal(VT: WideMaskVT)) {
6538 SDLoc DL(N);
6539 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6540 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6541 EC: VT.getVectorElementCount());
6542 SDValue NewLoad =
6543 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6544 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6545 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6546
6547 // Modified the chain - switch anything that used the old chain to use
6548 // the new one.
6549 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6550
6551 return NewLoad;
6552 }
6553
6554 SDValue Result;
6555 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6556 if (ExtType != ISD::NON_EXTLOAD)
6557 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6558 else
6559 Result = GenWidenVectorLoads(LdChain, LD);
6560
6561 if (Result) {
6562 // If we generate a single load, we can use that for the chain. Otherwise,
6563 // build a factor node to remember the multiple loads are independent and
6564 // chain to that.
6565 SDValue NewChain;
6566 if (LdChain.size() == 1)
6567 NewChain = LdChain[0];
6568 else
6569 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6570
6571 // Modified the chain - switch anything that used the old chain to use
6572 // the new one.
6573 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6574
6575 return Result;
6576 }
6577
6578 if (VT.isVector()) {
6579 // If all else fails replace the load with a wide masked load.
6580 SDLoc DL(N);
6581 SDValue Mask =
6582 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6583
6584 SDValue NewLoad = DAG.getMaskedLoad(
6585 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6586 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6587 AM: LD->getAddressingMode(), LD->getExtensionType());
6588
6589 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6590 return NewLoad;
6591 }
6592
6593 report_fatal_error(reason: "Unable to widen vector load");
6594}
6595
6596SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6597 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6598 SDValue Mask = N->getMask();
6599 SDValue EVL = N->getVectorLength();
6600 ISD::LoadExtType ExtType = N->getExtensionType();
6601 SDLoc dl(N);
6602
6603 // The mask should be widened as well
6604 assert(getTypeAction(Mask.getValueType()) ==
6605 TargetLowering::TypeWidenVector &&
6606 "Unable to widen binary VP op");
6607 Mask = GetWidenedVector(Op: Mask);
6608 assert(Mask.getValueType().getVectorElementCount() ==
6609 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6610 .getVectorElementCount() &&
6611 "Unable to widen vector load");
6612
6613 SDValue Res =
6614 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6615 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6616 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6617 // Legalize the chain result - switch anything that used the old chain to
6618 // use the new one.
6619 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6620 return Res;
6621}
6622
6623SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6624 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6625 SDValue Mask = N->getMask();
6626 SDValue EVL = N->getVectorLength();
6627 SDLoc dl(N);
6628
6629 // The mask should be widened as well
6630 assert(getTypeAction(Mask.getValueType()) ==
6631 TargetLowering::TypeWidenVector &&
6632 "Unable to widen binary VP op");
6633 Mask = GetWidenedVector(Op: Mask);
6634 assert(Mask.getValueType().getVectorElementCount() ==
6635 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6636 .getVectorElementCount() &&
6637 "Unable to widen vector load");
6638
6639 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6640 Mask, EVL, MMO: N->getMemOperand());
6641 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6642 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6643 return Res;
6644}
6645
6646SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6647 SDLoc DL(N);
6648
6649 // The mask should be widened as well
6650 SDValue Mask = N->getMask();
6651 assert(getTypeAction(Mask.getValueType()) ==
6652 TargetLowering::TypeWidenVector &&
6653 "Unable to widen VP strided load");
6654 Mask = GetWidenedVector(Op: Mask);
6655
6656 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6657 assert(Mask.getValueType().getVectorElementCount() ==
6658 WidenVT.getVectorElementCount() &&
6659 "Data and mask vectors should have the same number of elements");
6660
6661 SDValue Res = DAG.getStridedLoadVP(
6662 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6663 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6664 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6665 IsExpanding: N->isExpandingLoad());
6666
6667 // Legalize the chain result - switch anything that used the old chain to
6668 // use the new one.
6669 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6670 return Res;
6671}
6672
6673SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6674 SDValue Vec = N->getOperand(Num: 0);
6675 SDValue Mask = N->getOperand(Num: 1);
6676 SDValue Passthru = N->getOperand(Num: 2);
6677 EVT WideVecVT =
6678 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6679 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6680 VT: Mask.getValueType().getVectorElementType(),
6681 EC: WideVecVT.getVectorElementCount());
6682
6683 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6684 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6685 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6686 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6687 N2: WideMask, N3: WidePassthru);
6688}
6689
6690SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6691 EVT VT = N->getValueType(ResNo: 0);
6692 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6693 SDValue Mask = N->getMask();
6694 EVT MaskVT = Mask.getValueType();
6695 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6696 ISD::LoadExtType ExtType = N->getExtensionType();
6697 SDLoc dl(N);
6698
6699 EVT WideMaskVT =
6700 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6701 EC: WidenVT.getVectorElementCount());
6702
6703 if (ExtType == ISD::NON_EXTLOAD &&
6704 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6705 TLI.isTypeLegal(VT: WideMaskVT) &&
6706 // If there is a passthru, we shouldn't use vp.load. However,
6707 // type legalizer will struggle on masked.load with
6708 // scalable vectors, so for scalable vectors, we still use vp.load
6709 // but manually merge the load result with the passthru using vp.select.
6710 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6711 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6712 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6713 EC: VT.getVectorElementCount());
6714 SDValue NewLoad =
6715 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6716 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6717 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6718 SDValue NewVal = NewLoad;
6719
6720 // Manually merge with vselect
6721 if (!N->getPassThru()->isUndef()) {
6722 assert(WidenVT.isScalableVector());
6723 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
6724 // The lanes past EVL are poison.
6725 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
6726 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
6727 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
6728 }
6729
6730 // Modified the chain - switch anything that used the old chain to use
6731 // the new one.
6732 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6733
6734 return NewVal;
6735 }
6736
6737 // The mask should be widened as well
6738 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6739
6740 SDValue Res = DAG.getMaskedLoad(
6741 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
6742 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
6743 ExtType, IsExpanding: N->isExpandingLoad());
6744 // Legalize the chain result - switch anything that used the old chain to
6745 // use the new one.
6746 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6747 return Res;
6748}
6749
6750SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
6751
6752 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6753 SDValue Mask = N->getMask();
6754 EVT MaskVT = Mask.getValueType();
6755 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6756 SDValue Scale = N->getScale();
6757 unsigned NumElts = WideVT.getVectorNumElements();
6758 SDLoc dl(N);
6759
6760 // The mask should be widened as well
6761 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6762 VT: MaskVT.getVectorElementType(),
6763 NumElements: WideVT.getVectorNumElements());
6764 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6765
6766 // Widen the Index operand
6767 SDValue Index = N->getIndex();
6768 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6769 VT: Index.getValueType().getScalarType(),
6770 NumElements: NumElts);
6771 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6772 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
6773 Scale };
6774
6775 // Widen the MemoryType
6776 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6777 VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
6778 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
6779 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
6780 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
6781
6782 // Legalize the chain result - switch anything that used the old chain to
6783 // use the new one.
6784 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6785 return Res;
6786}
6787
6788SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
6789 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6790 SDValue Mask = N->getMask();
6791 SDValue Scale = N->getScale();
6792 ElementCount WideEC = WideVT.getVectorElementCount();
6793 SDLoc dl(N);
6794
6795 SDValue Index = GetWidenedVector(Op: N->getIndex());
6796 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6797 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
6798 Mask = GetWidenedMask(Mask, EC: WideEC);
6799
6800 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
6801 Mask, N->getVectorLength()};
6802 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
6803 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
6804
6805 // Legalize the chain result - switch anything that used the old chain to
6806 // use the new one.
6807 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6808 return Res;
6809}
6810
6811SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
6812 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6813 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
6814}
6815
6816// Return true is this is a SETCC node or a strict version of it.
6817static inline bool isSETCCOp(unsigned Opcode) {
6818 switch (Opcode) {
6819 case ISD::SETCC:
6820 case ISD::STRICT_FSETCC:
6821 case ISD::STRICT_FSETCCS:
6822 return true;
6823 }
6824 return false;
6825}
6826
6827// Return true if this is a node that could have two SETCCs as operands.
6828static inline bool isLogicalMaskOp(unsigned Opcode) {
6829 switch (Opcode) {
6830 case ISD::AND:
6831 case ISD::OR:
6832 case ISD::XOR:
6833 return true;
6834 }
6835 return false;
6836}
6837
6838// If N is a SETCC or a strict variant of it, return the type
6839// of the compare operands.
6840static inline EVT getSETCCOperandType(SDValue N) {
6841 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
6842 return N->getOperand(Num: OpNo).getValueType();
6843}
6844
6845// This is used just for the assert in convertMask(). Check that this either
6846// a SETCC or a previously handled SETCC by convertMask().
6847#ifndef NDEBUG
6848static inline bool isSETCCorConvertedSETCC(SDValue N) {
6849 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
6850 N = N.getOperand(0);
6851 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
6852 for (unsigned i = 1; i < N->getNumOperands(); ++i)
6853 if (!N->getOperand(i)->isUndef())
6854 return false;
6855 N = N.getOperand(0);
6856 }
6857
6858 if (N.getOpcode() == ISD::TRUNCATE)
6859 N = N.getOperand(0);
6860 else if (N.getOpcode() == ISD::SIGN_EXTEND)
6861 N = N.getOperand(0);
6862
6863 if (isLogicalMaskOp(N.getOpcode()))
6864 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
6865 isSETCCorConvertedSETCC(N.getOperand(1));
6866
6867 return (isSETCCOp(N.getOpcode()) ||
6868 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
6869}
6870#endif
6871
6872// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
6873// to ToMaskVT if needed with vector extension or truncation.
6874SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
6875 EVT ToMaskVT) {
6876 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
6877 // FIXME: This code seems to be too restrictive, we might consider
6878 // generalizing it or dropping it.
6879 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
6880
6881 // Make a new Mask node, with a legal result VT.
6882 SDValue Mask;
6883 SmallVector<SDValue, 4> Ops;
6884 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
6885 Ops.push_back(Elt: InMask->getOperand(Num: i));
6886 if (InMask->isStrictFPOpcode()) {
6887 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
6888 ResultTys: { MaskVT, MVT::Other }, Ops);
6889 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
6890 }
6891 else
6892 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
6893 Flags: InMask->getFlags());
6894
6895 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
6896 // extend or truncate is needed.
6897 LLVMContext &Ctx = *DAG.getContext();
6898 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
6899 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
6900 if (MaskScalarBits < ToMaskScalBits) {
6901 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6902 NumElements: MaskVT.getVectorNumElements());
6903 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
6904 } else if (MaskScalarBits > ToMaskScalBits) {
6905 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6906 NumElements: MaskVT.getVectorNumElements());
6907 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
6908 }
6909
6910 assert(Mask->getValueType(0).getScalarSizeInBits() ==
6911 ToMaskVT.getScalarSizeInBits() &&
6912 "Mask should have the right element size by now.");
6913
6914 // Adjust Mask to the right number of elements.
6915 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
6916 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
6917 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
6918 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
6919 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
6920 EVT SubVT = Mask->getValueType(ResNo: 0);
6921 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
6922 SubOps[0] = Mask;
6923 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
6924 }
6925
6926 assert((Mask->getValueType(0) == ToMaskVT) &&
6927 "A mask of ToMaskVT should have been produced by now.");
6928
6929 return Mask;
6930}
6931
6932// This method tries to handle some special cases for the vselect mask
6933// and if needed adjusting the mask vector type to match that of the VSELECT.
6934// Without it, many cases end up with scalarization of the SETCC, with many
6935// unnecessary instructions.
6936SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
6937 LLVMContext &Ctx = *DAG.getContext();
6938 SDValue Cond = N->getOperand(Num: 0);
6939
6940 if (N->getOpcode() != ISD::VSELECT)
6941 return SDValue();
6942
6943 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
6944 return SDValue();
6945
6946 // If this is a splitted VSELECT that was previously already handled, do
6947 // nothing.
6948 EVT CondVT = Cond->getValueType(ResNo: 0);
6949 if (CondVT.getScalarSizeInBits() != 1)
6950 return SDValue();
6951
6952 EVT VSelVT = N->getValueType(ResNo: 0);
6953
6954 // This method can't handle scalable vector types.
6955 // FIXME: This support could be added in the future.
6956 if (VSelVT.isScalableVector())
6957 return SDValue();
6958
6959 // Only handle vector types which are a power of 2.
6960 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
6961 return SDValue();
6962
6963 // Don't touch if this will be scalarized.
6964 EVT FinalVT = VSelVT;
6965 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
6966 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
6967
6968 if (FinalVT.getVectorNumElements() == 1)
6969 return SDValue();
6970
6971 // If there is support for an i1 vector mask, don't touch.
6972 if (isSETCCOp(Opcode: Cond.getOpcode())) {
6973 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
6974 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
6975 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
6976 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
6977 if (SetCCResVT.getScalarSizeInBits() == 1)
6978 return SDValue();
6979 } else if (CondVT.getScalarType() == MVT::i1) {
6980 // If there is support for an i1 vector mask (or only scalar i1 conditions),
6981 // don't touch.
6982 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
6983 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
6984
6985 if (CondVT.getScalarType() == MVT::i1)
6986 return SDValue();
6987 }
6988
6989 // Widen the vselect result type if needed.
6990 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
6991 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
6992
6993 // The mask of the VSELECT should have integer elements.
6994 EVT ToMaskVT = VSelVT;
6995 if (!ToMaskVT.getScalarType().isInteger())
6996 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
6997
6998 SDValue Mask;
6999 if (isSETCCOp(Opcode: Cond->getOpcode())) {
7000 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
7001 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7002 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
7003 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
7004 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
7005 // Cond is (AND/OR/XOR (SETCC, SETCC))
7006 SDValue SETCC0 = Cond->getOperand(Num: 0);
7007 SDValue SETCC1 = Cond->getOperand(Num: 1);
7008 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
7009 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
7010 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
7011 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
7012 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
7013 EVT MaskVT;
7014 // If the two SETCCs have different VTs, either extend/truncate one of
7015 // them to the other "towards" ToMaskVT, or truncate one and extend the
7016 // other to ToMaskVT.
7017 if (ScalarBits0 != ScalarBits1) {
7018 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
7019 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
7020 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
7021 MaskVT = WideVT;
7022 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
7023 MaskVT = NarrowVT;
7024 else
7025 MaskVT = ToMaskVT;
7026 } else
7027 // If the two SETCCs have the same VT, don't change it.
7028 MaskVT = VT0;
7029
7030 // Make new SETCCs and logical nodes.
7031 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
7032 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
7033 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
7034
7035 // Convert the logical op for VSELECT if needed.
7036 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7037 } else
7038 return SDValue();
7039
7040 return Mask;
7041}
7042
7043SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7044 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7045 ElementCount WidenEC = WidenVT.getVectorElementCount();
7046
7047 SDValue Cond1 = N->getOperand(Num: 0);
7048 EVT CondVT = Cond1.getValueType();
7049 unsigned Opcode = N->getOpcode();
7050 if (CondVT.isVector()) {
7051 if (SDValue WideCond = WidenVSELECTMask(N)) {
7052 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7053 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7054 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7055 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7056 }
7057
7058 EVT CondEltVT = CondVT.getVectorElementType();
7059 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7060 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7061 Cond1 = GetWidenedVector(Op: Cond1);
7062
7063 // If we have to split the condition there is no point in widening the
7064 // select. This would result in an cycle of widening the select ->
7065 // widening the condition operand -> splitting the condition operand ->
7066 // splitting the select -> widening the select. Instead split this select
7067 // further and widen the resulting type.
7068 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7069 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7070 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7071 return Res;
7072 }
7073
7074 if (Cond1.getValueType() != CondWidenVT)
7075 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7076 }
7077
7078 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7079 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7080 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7081 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
7082 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7083 N4: N->getOperand(Num: 3));
7084 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7085}
7086
7087SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7088 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7089 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7090 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7091 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7092 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7093}
7094
7095SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7096 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7097 return DAG.getUNDEF(VT: WidenVT);
7098}
7099
7100SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7101 EVT VT = N->getValueType(ResNo: 0);
7102 SDLoc dl(N);
7103
7104 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7105 unsigned NumElts = VT.getVectorNumElements();
7106 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7107
7108 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7109 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7110
7111 // Adjust mask based on new input vector length.
7112 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7113 for (unsigned i = 0; i != NumElts; ++i) {
7114 int Idx = N->getMaskElt(Idx: i);
7115 if (Idx < (int)NumElts)
7116 NewMask[i] = Idx;
7117 else
7118 NewMask[i] = Idx - NumElts + WidenNumElts;
7119 }
7120 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7121}
7122
7123SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7124 EVT VT = N->getValueType(ResNo: 0);
7125 EVT EltVT = VT.getVectorElementType();
7126 SDLoc dl(N);
7127
7128 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7129 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7130 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7131
7132 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7133 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7134 unsigned VTNumElts = VT.getVectorMinNumElements();
7135 unsigned IdxVal = WidenNumElts - VTNumElts;
7136
7137 if (VT.isScalableVector()) {
7138 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7139 // results together, e.g.(nxv6i64 -> nxv8i64)
7140 // nxv8i64 vector_reverse
7141 // <->
7142 // nxv8i64 concat(
7143 // nxv2i64 extract_subvector(nxv8i64, 2)
7144 // nxv2i64 extract_subvector(nxv8i64, 4)
7145 // nxv2i64 extract_subvector(nxv8i64, 6)
7146 // nxv2i64 undef)
7147
7148 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7149 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7150 EC: ElementCount::getScalable(MinVal: GCD));
7151 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7152 "down type's element count");
7153 SmallVector<SDValue> Parts;
7154 unsigned i = 0;
7155 for (; i < VTNumElts / GCD; ++i)
7156 Parts.push_back(
7157 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7158 for (; i < WidenNumElts / GCD; ++i)
7159 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7160
7161 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7162 }
7163
7164 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7165 // fixed-vectors.
7166 SmallVector<int, 16> Mask(WidenNumElts, -1);
7167 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7168
7169 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7170 Mask);
7171}
7172
7173SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7174 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7175 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7176}
7177
7178SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7179 assert(N->getValueType(0).isVector() &&
7180 N->getOperand(0).getValueType().isVector() &&
7181 "Operands must be vectors");
7182 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7183 ElementCount WidenEC = WidenVT.getVectorElementCount();
7184
7185 SDValue InOp1 = N->getOperand(Num: 0);
7186 EVT InVT = InOp1.getValueType();
7187 assert(InVT.isVector() && "can not widen non-vector type");
7188 EVT WidenInVT =
7189 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7190
7191 // The input and output types often differ here, and it could be that while
7192 // we'd prefer to widen the result type, the input operands have been split.
7193 // In this case, we also need to split the result of this node as well.
7194 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7195 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7196 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7197 return Res;
7198 }
7199
7200 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7201 SDValue InOp2 = N->getOperand(Num: 1);
7202 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7203 InOp1 = GetWidenedVector(Op: InOp1);
7204 InOp2 = GetWidenedVector(Op: InOp2);
7205 } else {
7206 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7207 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7208 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7209 N2: InOp1, N3: ZeroIdx);
7210 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7211 N2: InOp2, N3: ZeroIdx);
7212 }
7213
7214 // Assume that the input and output will be widen appropriately. If not,
7215 // we will have to unroll it at some point.
7216 assert(InOp1.getValueType() == WidenInVT &&
7217 InOp2.getValueType() == WidenInVT &&
7218 "Input not widened to expected type!");
7219 (void)WidenInVT;
7220 if (N->getOpcode() == ISD::VP_SETCC) {
7221 SDValue Mask =
7222 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7223 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7224 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7225 }
7226 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7227 N3: N->getOperand(Num: 2));
7228}
7229
7230SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7231 assert(N->getValueType(0).isVector() &&
7232 N->getOperand(1).getValueType().isVector() &&
7233 "Operands must be vectors");
7234 EVT VT = N->getValueType(ResNo: 0);
7235 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7236 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7237 unsigned NumElts = VT.getVectorNumElements();
7238 EVT EltVT = VT.getVectorElementType();
7239
7240 SDLoc dl(N);
7241 SDValue Chain = N->getOperand(Num: 0);
7242 SDValue LHS = N->getOperand(Num: 1);
7243 SDValue RHS = N->getOperand(Num: 2);
7244 SDValue CC = N->getOperand(Num: 3);
7245 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7246
7247 // Fully unroll and reassemble.
7248 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7249 SmallVector<SDValue, 8> Chains(NumElts);
7250 for (unsigned i = 0; i != NumElts; ++i) {
7251 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7252 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7253
7254 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7255 Ops: {Chain, LHSElem, RHSElem, CC});
7256 Chains[i] = Scalars[i].getValue(R: 1);
7257 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7258 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7259 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7260 }
7261
7262 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7263 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7264
7265 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7266}
7267
7268//===----------------------------------------------------------------------===//
7269// Widen Vector Operand
7270//===----------------------------------------------------------------------===//
7271bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7272 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7273 SDValue Res = SDValue();
7274
7275 // See if the target wants to custom widen this node.
7276 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7277 return false;
7278
7279 switch (N->getOpcode()) {
7280 default:
7281#ifndef NDEBUG
7282 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7283 N->dump(&DAG);
7284 dbgs() << "\n";
7285#endif
7286 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7287
7288 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7289 case ISD::FAKE_USE:
7290 Res = WidenVecOp_FAKE_USE(N);
7291 break;
7292 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7293 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7294 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7295 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7296 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7297 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7298 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7299 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7300 break;
7301 case ISD::ANY_EXTEND_VECTOR_INREG:
7302 case ISD::SIGN_EXTEND_VECTOR_INREG:
7303 case ISD::ZERO_EXTEND_VECTOR_INREG:
7304 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7305 break;
7306 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7307 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7308 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7309 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7310 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7311 case ISD::STRICT_FSETCC:
7312 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7313 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7314 case ISD::FLDEXP:
7315 case ISD::FCOPYSIGN:
7316 case ISD::LROUND:
7317 case ISD::LLROUND:
7318 case ISD::LRINT:
7319 case ISD::LLRINT:
7320 Res = WidenVecOp_UnrollVectorOp(N);
7321 break;
7322 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7323
7324 case ISD::ANY_EXTEND:
7325 case ISD::SIGN_EXTEND:
7326 case ISD::ZERO_EXTEND:
7327 Res = WidenVecOp_EXTEND(N);
7328 break;
7329
7330 case ISD::SCMP:
7331 case ISD::UCMP:
7332 Res = WidenVecOp_CMP(N);
7333 break;
7334
7335 case ISD::FP_EXTEND:
7336 case ISD::STRICT_FP_EXTEND:
7337 case ISD::FP_ROUND:
7338 case ISD::STRICT_FP_ROUND:
7339 case ISD::FP_TO_SINT:
7340 case ISD::STRICT_FP_TO_SINT:
7341 case ISD::FP_TO_UINT:
7342 case ISD::STRICT_FP_TO_UINT:
7343 case ISD::SINT_TO_FP:
7344 case ISD::STRICT_SINT_TO_FP:
7345 case ISD::UINT_TO_FP:
7346 case ISD::STRICT_UINT_TO_FP:
7347 case ISD::TRUNCATE:
7348 case ISD::CONVERT_FROM_ARBITRARY_FP:
7349 Res = WidenVecOp_Convert(N);
7350 break;
7351
7352 case ISD::FP_TO_SINT_SAT:
7353 case ISD::FP_TO_UINT_SAT:
7354 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7355 break;
7356
7357 case ISD::VECREDUCE_FADD:
7358 case ISD::VECREDUCE_FMUL:
7359 case ISD::VECREDUCE_ADD:
7360 case ISD::VECREDUCE_MUL:
7361 case ISD::VECREDUCE_AND:
7362 case ISD::VECREDUCE_OR:
7363 case ISD::VECREDUCE_XOR:
7364 case ISD::VECREDUCE_SMAX:
7365 case ISD::VECREDUCE_SMIN:
7366 case ISD::VECREDUCE_UMAX:
7367 case ISD::VECREDUCE_UMIN:
7368 case ISD::VECREDUCE_FMAX:
7369 case ISD::VECREDUCE_FMIN:
7370 case ISD::VECREDUCE_FMAXIMUM:
7371 case ISD::VECREDUCE_FMINIMUM:
7372 Res = WidenVecOp_VECREDUCE(N);
7373 break;
7374 case ISD::VECREDUCE_SEQ_FADD:
7375 case ISD::VECREDUCE_SEQ_FMUL:
7376 Res = WidenVecOp_VECREDUCE_SEQ(N);
7377 break;
7378 case ISD::VP_REDUCE_FADD:
7379 case ISD::VP_REDUCE_SEQ_FADD:
7380 case ISD::VP_REDUCE_FMUL:
7381 case ISD::VP_REDUCE_SEQ_FMUL:
7382 case ISD::VP_REDUCE_ADD:
7383 case ISD::VP_REDUCE_MUL:
7384 case ISD::VP_REDUCE_AND:
7385 case ISD::VP_REDUCE_OR:
7386 case ISD::VP_REDUCE_XOR:
7387 case ISD::VP_REDUCE_SMAX:
7388 case ISD::VP_REDUCE_SMIN:
7389 case ISD::VP_REDUCE_UMAX:
7390 case ISD::VP_REDUCE_UMIN:
7391 case ISD::VP_REDUCE_FMAX:
7392 case ISD::VP_REDUCE_FMIN:
7393 case ISD::VP_REDUCE_FMAXIMUM:
7394 case ISD::VP_REDUCE_FMINIMUM:
7395 Res = WidenVecOp_VP_REDUCE(N);
7396 break;
7397 case ISD::VP_CTTZ_ELTS:
7398 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7399 Res = WidenVecOp_VP_CttzElements(N);
7400 break;
7401 case ISD::VECTOR_FIND_LAST_ACTIVE:
7402 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7403 break;
7404 }
7405
7406 // If Res is null, the sub-method took care of registering the result.
7407 if (!Res.getNode()) return false;
7408
7409 // If the result is N, the sub-method updated N in place. Tell the legalizer
7410 // core about this.
7411 if (Res.getNode() == N)
7412 return true;
7413
7414
7415 if (N->isStrictFPOpcode())
7416 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7417 "Invalid operand expansion");
7418 else
7419 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7420 "Invalid operand expansion");
7421
7422 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7423 return false;
7424}
7425
7426SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7427 SDLoc DL(N);
7428 EVT VT = N->getValueType(ResNo: 0);
7429
7430 SDValue InOp = N->getOperand(Num: 0);
7431 assert(getTypeAction(InOp.getValueType()) ==
7432 TargetLowering::TypeWidenVector &&
7433 "Unexpected type action");
7434 InOp = GetWidenedVector(Op: InOp);
7435 assert(VT.getVectorNumElements() <
7436 InOp.getValueType().getVectorNumElements() &&
7437 "Input wasn't widened!");
7438
7439 // We may need to further widen the operand until it has the same total
7440 // vector size as the result.
7441 EVT InVT = InOp.getValueType();
7442 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7443 EVT InEltVT = InVT.getVectorElementType();
7444 for (EVT FixedVT : MVT::vector_valuetypes()) {
7445 EVT FixedEltVT = FixedVT.getVectorElementType();
7446 if (TLI.isTypeLegal(VT: FixedVT) &&
7447 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7448 FixedEltVT == InEltVT) {
7449 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7450 "Not enough elements in the fixed type for the operand!");
7451 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7452 "We can't have the same type as we started with!");
7453 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7454 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7455 else
7456 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7457 break;
7458 }
7459 }
7460 InVT = InOp.getValueType();
7461 if (InVT.getSizeInBits() != VT.getSizeInBits())
7462 // We couldn't find a legal vector type that was a widening of the input
7463 // and could be extended in-register to the result type, so we have to
7464 // scalarize.
7465 return WidenVecOp_Convert(N);
7466 }
7467
7468 // Use special DAG nodes to represent the operation of extending the
7469 // low lanes.
7470 switch (N->getOpcode()) {
7471 default:
7472 llvm_unreachable("Extend legalization on extend operation!");
7473 case ISD::ANY_EXTEND:
7474 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7475 case ISD::SIGN_EXTEND:
7476 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7477 case ISD::ZERO_EXTEND:
7478 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7479 }
7480}
7481
7482SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7483 SDLoc dl(N);
7484
7485 EVT OpVT = N->getOperand(Num: 0).getValueType();
7486 EVT ResVT = N->getValueType(ResNo: 0);
7487 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7488 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7489
7490 // 1. EXTRACT_SUBVECTOR
7491 // 2. SIGN_EXTEND/ZERO_EXTEND
7492 // 3. CMP
7493 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7494 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7495
7496 // At this point the result type is guaranteed to be valid, so we can use it
7497 // as the operand type by extending it appropriately
7498 ISD::NodeType ExtendOpcode =
7499 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7500 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7501 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7502
7503 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7504}
7505
7506SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7507 // The result (and first input) is legal, but the second input is illegal.
7508 // We can't do much to fix that, so just unroll and let the extracts off of
7509 // the second input be widened as needed later.
7510 return DAG.UnrollVectorOp(N);
7511}
7512
7513SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7514 SDLoc DL(N);
7515 EVT ResultVT = N->getValueType(ResNo: 0);
7516 SDValue Test = N->getOperand(Num: 1);
7517 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7518
7519 // Process this node similarly to SETCC.
7520 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7521 if (ResultVT.getScalarType() == MVT::i1)
7522 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7523 NumElements: WideResultVT.getVectorNumElements());
7524
7525 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7526 Ops: {WideArg, Test}, Flags: N->getFlags());
7527
7528 // Extract the needed results from the result vector.
7529 EVT ResVT =
7530 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7531 NumElements: ResultVT.getVectorNumElements());
7532 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7533
7534 EVT OpVT = N->getOperand(Num: 0).getValueType();
7535 ISD::NodeType ExtendCode =
7536 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7537 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7538}
7539
7540SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7541 // Since the result is legal and the input is illegal.
7542 EVT VT = N->getValueType(ResNo: 0);
7543 EVT EltVT = VT.getVectorElementType();
7544 SDLoc dl(N);
7545 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7546 assert(getTypeAction(InOp.getValueType()) ==
7547 TargetLowering::TypeWidenVector &&
7548 "Unexpected type action");
7549 InOp = GetWidenedVector(Op: InOp);
7550 EVT InVT = InOp.getValueType();
7551 unsigned Opcode = N->getOpcode();
7552
7553 // See if a widened result type would be legal, if so widen the node.
7554 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7555 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7556 EC: InVT.getVectorElementCount());
7557 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7558 SDValue Res;
7559 if (N->isStrictFPOpcode()) {
7560 if (Opcode == ISD::STRICT_FP_ROUND)
7561 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7562 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7563 else
7564 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7565 Ops: { N->getOperand(Num: 0), InOp });
7566 // Legalize the chain result - switch anything that used the old chain to
7567 // use the new one.
7568 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7569 } else {
7570 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7571 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
7572 else
7573 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
7574 }
7575 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7576 }
7577
7578 EVT InEltVT = InVT.getVectorElementType();
7579
7580 // Unroll the convert into some scalar code and create a nasty build vector.
7581 unsigned NumElts = VT.getVectorNumElements();
7582 SmallVector<SDValue, 16> Ops(NumElts);
7583 if (N->isStrictFPOpcode()) {
7584 SmallVector<SDValue, 4> NewOps(N->ops());
7585 SmallVector<SDValue, 32> OpChains;
7586 for (unsigned i=0; i < NumElts; ++i) {
7587 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7588 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7589 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7590 }
7591 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7592 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7593 } else {
7594 for (unsigned i = 0; i < NumElts; ++i) {
7595 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7596 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7597 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT, N1: Elt, N2: N->getOperand(Num: 1));
7598 else
7599 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT, Operand: Elt);
7600 }
7601 }
7602
7603 return DAG.getBuildVector(VT, DL: dl, Ops);
7604}
7605
7606SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7607 EVT DstVT = N->getValueType(ResNo: 0);
7608 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7609 EVT SrcVT = Src.getValueType();
7610 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7611 SDLoc dl(N);
7612
7613 // See if a widened result type would be legal, if so widen the node.
7614 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7615 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7616 if (TLI.isTypeLegal(VT: WideDstVT)) {
7617 SDValue Res =
7618 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7619 return DAG.getNode(
7620 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7621 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7622 }
7623
7624 // Give up and unroll.
7625 return DAG.UnrollVectorOp(N);
7626}
7627
7628SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7629 EVT VT = N->getValueType(ResNo: 0);
7630 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7631 EVT InWidenVT = InOp.getValueType();
7632 SDLoc dl(N);
7633
7634 // Check if we can convert between two legal vector types and extract.
7635 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7636 TypeSize Size = VT.getSizeInBits();
7637 // x86mmx is not an acceptable vector element type, so don't try.
7638 if (!VT.isVector() && VT != MVT::x86mmx &&
7639 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7640 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7641 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7642 if (TLI.isTypeLegal(VT: NewVT)) {
7643 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7644 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7645 }
7646 }
7647
7648 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7649 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7650 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7651 // having to copy via memory.
7652 if (VT.isVector()) {
7653 EVT EltVT = VT.getVectorElementType();
7654 unsigned EltSize = EltVT.getFixedSizeInBits();
7655 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7656 ElementCount NewNumElts =
7657 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
7658 .divideCoefficientBy(RHS: EltSize);
7659 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
7660 if (TLI.isTypeLegal(VT: NewVT)) {
7661 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7662 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
7663 }
7664 }
7665 }
7666
7667 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
7668}
7669
7670// Vectors with sizes that are not powers of 2 need to be widened to the
7671// next largest power of 2. For example, we may get a vector of 3 32-bit
7672// integers or of 6 16-bit integers, both of which have to be widened to a
7673// 128-bit vector.
7674SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
7675 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
7676 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
7677 N2: WidenedOp);
7678}
7679
7680SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
7681 EVT VT = N->getValueType(ResNo: 0);
7682 EVT EltVT = VT.getVectorElementType();
7683 EVT InVT = N->getOperand(Num: 0).getValueType();
7684 SDLoc dl(N);
7685
7686 // If the widen width for this operand is the same as the width of the concat
7687 // and all but the first operand is undef, just use the widened operand.
7688 unsigned NumOperands = N->getNumOperands();
7689 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
7690 unsigned i;
7691 for (i = 1; i < NumOperands; ++i)
7692 if (!N->getOperand(Num: i).isUndef())
7693 break;
7694
7695 if (i == NumOperands)
7696 return GetWidenedVector(Op: N->getOperand(Num: 0));
7697 }
7698
7699 // Otherwise, fall back to a nasty build vector.
7700 unsigned NumElts = VT.getVectorNumElements();
7701 SmallVector<SDValue, 16> Ops(NumElts);
7702
7703 unsigned NumInElts = InVT.getVectorNumElements();
7704
7705 unsigned Idx = 0;
7706 for (unsigned i=0; i < NumOperands; ++i) {
7707 SDValue InOp = N->getOperand(Num: i);
7708 assert(getTypeAction(InOp.getValueType()) ==
7709 TargetLowering::TypeWidenVector &&
7710 "Unexpected type action");
7711 InOp = GetWidenedVector(Op: InOp);
7712 for (unsigned j = 0; j < NumInElts; ++j)
7713 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
7714 }
7715 return DAG.getBuildVector(VT, DL: dl, Ops);
7716}
7717
7718SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
7719 EVT VT = N->getValueType(ResNo: 0);
7720 SDValue SubVec = N->getOperand(Num: 1);
7721 SDValue InVec = N->getOperand(Num: 0);
7722
7723 EVT OrigVT = SubVec.getValueType();
7724 SubVec = GetWidenedVector(Op: SubVec);
7725 EVT SubVT = SubVec.getValueType();
7726
7727 // Whether or not all the elements of the widened SubVec will be inserted into
7728 // valid indices of VT.
7729 bool IndicesValid = false;
7730 // If we statically know that VT can fit SubVT, the indices are valid.
7731 if (VT.knownBitsGE(VT: SubVT))
7732 IndicesValid = true;
7733 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
7734 // Otherwise, if we're inserting a fixed vector into a scalable vector and
7735 // we know the minimum vscale we can work out if it's valid ourselves.
7736 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
7737 Kind: Attribute::VScaleRange);
7738 if (Attr.isValid()) {
7739 unsigned VScaleMin = Attr.getVScaleRangeMin();
7740 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
7741 SubVT.getFixedSizeInBits())
7742 IndicesValid = true;
7743 }
7744 }
7745
7746 if (!IndicesValid)
7747 report_fatal_error(
7748 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
7749
7750 SDLoc DL(N);
7751
7752 // We need to make sure that the indices are still valid, otherwise we might
7753 // widen what was previously well-defined to something undefined.
7754 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
7755 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
7756 N3: N->getOperand(Num: 2));
7757
7758 if (OrigVT.isScalableVector()) {
7759 // When the widened types match, overwriting the start of a vector is
7760 // effectively a merge operation that can be implement as a vselect.
7761 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
7762 SDValue Mask =
7763 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
7764 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
7765 }
7766
7767 // Fallback to inserting through memory.
7768 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
7769 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
7770 MachineFunction &MF = DAG.getMachineFunction();
7771 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7772 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
7773
7774 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
7775 PtrInfo, F: MachineMemOperand::MOStore,
7776 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7777 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
7778 PtrInfo, F: MachineMemOperand::MOLoad,
7779 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7780
7781 // Write out the vector being inserting into.
7782 SDValue Ch =
7783 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
7784
7785 // Build a mask to match the length of the sub-vector.
7786 SDValue Mask =
7787 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
7788
7789 // Overwrite the sub-vector at the required offset.
7790 SDValue SubVecPtr =
7791 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
7792 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
7793 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
7794 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
7795
7796 // Read back the result.
7797 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
7798 }
7799
7800 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
7801 // with a series of INSERT_VECTOR_ELT
7802 unsigned Idx = N->getConstantOperandVal(Num: 2);
7803
7804 SDValue InsertElt = InVec;
7805 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
7806 SDValue ExtractElt =
7807 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
7808 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
7809 }
7810
7811 return InsertElt;
7812}
7813
7814SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
7815 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7816 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
7817 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7818}
7819
7820SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
7821 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7822 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
7823 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7824}
7825
7826SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
7827 SDLoc DL(N);
7828 EVT ResVT = N->getValueType(ResNo: 0);
7829
7830 // Widen the input as requested by the legalizer.
7831 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7832 EVT WideInVT = WideInOp.getValueType();
7833
7834 // Simple case: if widened input is still smaller than or equal to result,
7835 // just use it directly.
7836 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
7837 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
7838
7839 // EXTEND_VECTOR_INREG requires input bits <= result bits.
7840 // If widening makes the input larger than the original result, widen the
7841 // result to match, then extract back down.
7842 EVT ResEltVT = ResVT.getVectorElementType();
7843 unsigned EltBits = ResEltVT.getSizeInBits();
7844 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
7845 "Widened input size must be a multiple of result element size");
7846
7847 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
7848 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
7849
7850 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
7851 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
7852}
7853
7854SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
7855 // We have to widen the value, but we want only to store the original
7856 // vector type.
7857 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
7858
7859 if (!ST->getMemoryVT().getScalarType().isByteSized())
7860 return TLI.scalarizeVectorStore(ST, DAG);
7861
7862 if (ST->isTruncatingStore())
7863 return TLI.scalarizeVectorStore(ST, DAG);
7864
7865 // Generate a vector-predicated store if it is custom/legal on the target.
7866 // To avoid possible recursion, only do this if the widened mask type is
7867 // legal.
7868 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
7869 // removed from the IR by the ExpandVectorPredication pass but we're
7870 // reintroducing them here.
7871 SDValue StVal = ST->getValue();
7872 EVT StVT = StVal.getValueType();
7873 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
7874 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
7875
7876 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7877 TLI.isTypeLegal(VT: WideMaskVT)) {
7878 // Widen the value.
7879 SDLoc DL(N);
7880 StVal = GetWidenedVector(Op: StVal);
7881 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
7882 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
7883 EC: StVT.getVectorElementCount());
7884 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
7885 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
7886 AM: ST->getAddressingMode());
7887 }
7888
7889 SmallVector<SDValue, 16> StChain;
7890 if (GenWidenVectorStores(StChain, ST)) {
7891 if (StChain.size() == 1)
7892 return StChain[0];
7893
7894 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
7895 }
7896
7897 if (StVT.isVector()) {
7898 // If all else fails replace the store with a wide masked store.
7899 SDLoc DL(N);
7900 SDValue WideStVal = GetWidenedVector(Op: StVal);
7901 SDValue Mask =
7902 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
7903
7904 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
7905 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
7906 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
7907 IsTruncating: ST->isTruncatingStore());
7908 }
7909
7910 report_fatal_error(reason: "Unable to widen vector store");
7911}
7912
7913SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
7914 assert((OpNo == 1 || OpNo == 3) &&
7915 "Can widen only data or mask operand of vp_store");
7916 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
7917 SDValue Mask = ST->getMask();
7918 SDValue StVal = ST->getValue();
7919 SDLoc dl(N);
7920
7921 if (OpNo == 1) {
7922 // Widen the value.
7923 StVal = GetWidenedVector(Op: StVal);
7924
7925 // We only handle the case where the mask needs widening to an
7926 // identically-sized type as the vector inputs.
7927 assert(getTypeAction(Mask.getValueType()) ==
7928 TargetLowering::TypeWidenVector &&
7929 "Unable to widen VP store");
7930 Mask = GetWidenedVector(Op: Mask);
7931 } else {
7932 Mask = GetWidenedVector(Op: Mask);
7933
7934 // We only handle the case where the stored value needs widening to an
7935 // identically-sized type as the mask.
7936 assert(getTypeAction(StVal.getValueType()) ==
7937 TargetLowering::TypeWidenVector &&
7938 "Unable to widen VP store");
7939 StVal = GetWidenedVector(Op: StVal);
7940 }
7941
7942 assert(Mask.getValueType().getVectorElementCount() ==
7943 StVal.getValueType().getVectorElementCount() &&
7944 "Mask and data vectors should have the same number of elements");
7945 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
7946 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
7947 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
7948 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
7949 IsCompressing: ST->isCompressingStore());
7950}
7951
7952SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
7953 unsigned OpNo) {
7954 assert((OpNo == 1 || OpNo == 4) &&
7955 "Can widen only data or mask operand of vp_strided_store");
7956 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
7957 SDValue Mask = SST->getMask();
7958 SDValue StVal = SST->getValue();
7959 SDLoc DL(N);
7960
7961 if (OpNo == 1)
7962 assert(getTypeAction(Mask.getValueType()) ==
7963 TargetLowering::TypeWidenVector &&
7964 "Unable to widen VP strided store");
7965 else
7966 assert(getTypeAction(StVal.getValueType()) ==
7967 TargetLowering::TypeWidenVector &&
7968 "Unable to widen VP strided store");
7969
7970 StVal = GetWidenedVector(Op: StVal);
7971 Mask = GetWidenedVector(Op: Mask);
7972
7973 assert(StVal.getValueType().getVectorElementCount() ==
7974 Mask.getValueType().getVectorElementCount() &&
7975 "Data and mask vectors should have the same number of elements");
7976
7977 return DAG.getStridedStoreVP(
7978 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
7979 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
7980 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
7981 IsCompressing: SST->isCompressingStore());
7982}
7983
7984SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
7985 assert((OpNo == 1 || OpNo == 4) &&
7986 "Can widen only data or mask operand of mstore");
7987 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
7988 SDValue Mask = MST->getMask();
7989 EVT MaskVT = Mask.getValueType();
7990 SDValue StVal = MST->getValue();
7991 EVT VT = StVal.getValueType();
7992 SDLoc dl(N);
7993
7994 EVT WideVT, WideMaskVT;
7995 if (OpNo == 1) {
7996 // Widen the value.
7997 StVal = GetWidenedVector(Op: StVal);
7998
7999 WideVT = StVal.getValueType();
8000 WideMaskVT =
8001 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8002 EC: WideVT.getVectorElementCount());
8003 } else {
8004 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8005
8006 EVT ValueVT = StVal.getValueType();
8007 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8008 EC: WideMaskVT.getVectorElementCount());
8009 }
8010
8011 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8012 TLI.isTypeLegal(VT: WideMaskVT)) {
8013 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8014 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8015 EC: VT.getVectorElementCount());
8016 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8017 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8018 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8019 }
8020
8021 if (OpNo == 1) {
8022 // The mask should be widened as well.
8023 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8024 } else {
8025 // Widen the mask.
8026 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8027
8028 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8029 }
8030
8031 assert(Mask.getValueType().getVectorElementCount() ==
8032 StVal.getValueType().getVectorElementCount() &&
8033 "Mask and data vectors should have the same number of elements");
8034 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8035 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8036 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8037 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8038}
8039
8040SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8041 assert(OpNo == 4 && "Can widen only the index of mgather");
8042 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8043 SDValue DataOp = MG->getPassThru();
8044 SDValue Mask = MG->getMask();
8045 SDValue Scale = MG->getScale();
8046
8047 // Just widen the index. It's allowed to have extra elements.
8048 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8049
8050 SDLoc dl(N);
8051 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8052 Scale};
8053 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8054 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8055 ExtTy: MG->getExtensionType());
8056 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8057 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8058 return SDValue();
8059}
8060
8061SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8062 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8063 SDValue DataOp = MSC->getValue();
8064 SDValue Mask = MSC->getMask();
8065 SDValue Index = MSC->getIndex();
8066 SDValue Scale = MSC->getScale();
8067 EVT WideMemVT = MSC->getMemoryVT();
8068
8069 if (OpNo == 1) {
8070 DataOp = GetWidenedVector(Op: DataOp);
8071 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
8072
8073 // Widen index.
8074 EVT IndexVT = Index.getValueType();
8075 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8076 VT: IndexVT.getVectorElementType(), NumElements: NumElts);
8077 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8078
8079 // The mask should be widened as well.
8080 EVT MaskVT = Mask.getValueType();
8081 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8082 VT: MaskVT.getVectorElementType(), NumElements: NumElts);
8083 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8084
8085 // Widen the MemoryType
8086 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8087 VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
8088 } else if (OpNo == 4) {
8089 // Just widen the index. It's allowed to have extra elements.
8090 Index = GetWidenedVector(Op: Index);
8091 } else
8092 llvm_unreachable("Can't widen this operand of mscatter");
8093
8094 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8095 Scale};
8096 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8097 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8098 IsTruncating: MSC->isTruncatingStore());
8099}
8100
8101SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8102 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8103 SDValue DataOp = VPSC->getValue();
8104 SDValue Mask = VPSC->getMask();
8105 SDValue Index = VPSC->getIndex();
8106 SDValue Scale = VPSC->getScale();
8107 EVT WideMemVT = VPSC->getMemoryVT();
8108
8109 if (OpNo == 1) {
8110 DataOp = GetWidenedVector(Op: DataOp);
8111 Index = GetWidenedVector(Op: Index);
8112 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8113 Mask = GetWidenedMask(Mask, EC: WideEC);
8114 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8115 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8116 } else if (OpNo == 3) {
8117 // Just widen the index. It's allowed to have extra elements.
8118 Index = GetWidenedVector(Op: Index);
8119 } else
8120 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8121
8122 SDValue Ops[] = {
8123 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8124 VPSC->getVectorLength()};
8125 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8126 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8127}
8128
8129SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8130 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8131 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8132 SDLoc dl(N);
8133 EVT VT = N->getValueType(ResNo: 0);
8134
8135 // WARNING: In this code we widen the compare instruction with garbage.
8136 // This garbage may contain denormal floats which may be slow. Is this a real
8137 // concern ? Should we zero the unused lanes if this is a float compare ?
8138
8139 // Get a new SETCC node to compare the newly widened operands.
8140 // Only some of the compared elements are legal.
8141 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8142 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8143 if (VT.getScalarType() == MVT::i1)
8144 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8145 EC: SVT.getVectorElementCount());
8146
8147 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8148 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8149
8150 // Extract the needed results from the result vector.
8151 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8152 VT: SVT.getVectorElementType(),
8153 EC: VT.getVectorElementCount());
8154 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8155
8156 EVT OpVT = N->getOperand(Num: 0).getValueType();
8157 ISD::NodeType ExtendCode =
8158 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8159 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8160}
8161
8162SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8163 SDValue Chain = N->getOperand(Num: 0);
8164 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8165 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8166 SDValue CC = N->getOperand(Num: 3);
8167 SDLoc dl(N);
8168
8169 EVT VT = N->getValueType(ResNo: 0);
8170 EVT EltVT = VT.getVectorElementType();
8171 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8172 unsigned NumElts = VT.getVectorNumElements();
8173
8174 // Unroll into a build vector.
8175 SmallVector<SDValue, 8> Scalars(NumElts);
8176 SmallVector<SDValue, 8> Chains(NumElts);
8177
8178 for (unsigned i = 0; i != NumElts; ++i) {
8179 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8180 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8181
8182 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8183 Ops: {Chain, LHSElem, RHSElem, CC});
8184 Chains[i] = Scalars[i].getValue(R: 1);
8185 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8186 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8187 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8188 }
8189
8190 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8191 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8192
8193 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8194}
8195
8196static unsigned getExtendForIntVecReduction(unsigned Opc) {
8197 switch (Opc) {
8198 default:
8199 llvm_unreachable("Expected integer vector reduction");
8200 case ISD::VECREDUCE_ADD:
8201 case ISD::VECREDUCE_MUL:
8202 case ISD::VECREDUCE_AND:
8203 case ISD::VECREDUCE_OR:
8204 case ISD::VECREDUCE_XOR:
8205 return ISD::ANY_EXTEND;
8206 case ISD::VECREDUCE_SMAX:
8207 case ISD::VECREDUCE_SMIN:
8208 return ISD::SIGN_EXTEND;
8209 case ISD::VECREDUCE_UMAX:
8210 case ISD::VECREDUCE_UMIN:
8211 return ISD::ZERO_EXTEND;
8212 }
8213}
8214
8215SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8216 SDLoc dl(N);
8217 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8218 EVT VT = N->getValueType(ResNo: 0);
8219 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8220 EVT WideVT = Op.getValueType();
8221 EVT ElemVT = OrigVT.getVectorElementType();
8222 SDNodeFlags Flags = N->getFlags();
8223
8224 unsigned Opc = N->getOpcode();
8225 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8226 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8227 assert(NeutralElem && "Neutral element must exist");
8228
8229 // Pad the vector with the neutral element.
8230 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8231 unsigned WideElts = WideVT.getVectorMinNumElements();
8232
8233 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8234 // needing to pad the source vector, because the inactive lanes can simply be
8235 // disabled and not contribute to the result.
8236 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8237 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8238 SDValue Start = NeutralElem;
8239 if (VT.isInteger())
8240 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8241 assert(Start.getValueType() == VT);
8242 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8243 EC: WideVT.getVectorElementCount());
8244 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8245 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8246 EC: OrigVT.getVectorElementCount());
8247 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8248 }
8249
8250 if (WideVT.isScalableVector()) {
8251 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8252 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8253 EC: ElementCount::getScalable(MinVal: GCD));
8254 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8255 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8256 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8257 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8258 }
8259
8260 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8261 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8262
8263 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8264}
8265
8266SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8267 SDLoc dl(N);
8268 SDValue AccOp = N->getOperand(Num: 0);
8269 SDValue VecOp = N->getOperand(Num: 1);
8270 SDValue Op = GetWidenedVector(Op: VecOp);
8271
8272 EVT VT = N->getValueType(ResNo: 0);
8273 EVT OrigVT = VecOp.getValueType();
8274 EVT WideVT = Op.getValueType();
8275 EVT ElemVT = OrigVT.getVectorElementType();
8276 SDNodeFlags Flags = N->getFlags();
8277
8278 unsigned Opc = N->getOpcode();
8279 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8280 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8281
8282 // Pad the vector with the neutral element.
8283 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8284 unsigned WideElts = WideVT.getVectorMinNumElements();
8285
8286 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8287 // needing to pad the source vector, because the inactive lanes can simply be
8288 // disabled and not contribute to the result.
8289 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8290 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8291 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8292 EC: WideVT.getVectorElementCount());
8293 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8294 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8295 EC: OrigVT.getVectorElementCount());
8296 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8297 }
8298
8299 if (WideVT.isScalableVector()) {
8300 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8301 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8302 EC: ElementCount::getScalable(MinVal: GCD));
8303 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8304 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8305 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8306 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8307 }
8308
8309 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8310 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8311
8312 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8313}
8314
8315SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8316 assert(N->isVPOpcode() && "Expected VP opcode");
8317
8318 SDLoc dl(N);
8319 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8320 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8321 EC: Op.getValueType().getVectorElementCount());
8322
8323 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8324 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8325 Flags: N->getFlags());
8326}
8327
8328SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8329 // This only gets called in the case that the left and right inputs and
8330 // result are of a legal odd vector type, and the condition is illegal i1 of
8331 // the same odd width that needs widening.
8332 EVT VT = N->getValueType(ResNo: 0);
8333 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8334
8335 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8336 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8337 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8338 SDLoc DL(N);
8339
8340 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8341 N2: LeftIn, N3: RightIn);
8342 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8343}
8344
8345SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8346 SDLoc DL(N);
8347 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8348 EVT SrcVT = Source.getValueType();
8349 SDValue Mask =
8350 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8351
8352 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8353 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8354}
8355
8356SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8357 SDLoc DL(N);
8358 SDValue Mask = N->getOperand(Num: 0);
8359 EVT OrigMaskVT = Mask.getValueType();
8360 SDValue WideMask = GetWidenedVector(Op: Mask);
8361 EVT WideMaskVT = WideMask.getValueType();
8362
8363 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8364 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8365 unsigned WideElts = WideMaskVT.getVectorNumElements();
8366 if (OrigElts != WideElts) {
8367 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8368 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8369 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8370 }
8371
8372 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8373 Operand: WideMask);
8374}
8375
8376//===----------------------------------------------------------------------===//
8377// Vector Widening Utilities
8378//===----------------------------------------------------------------------===//
8379
8380// Utility function to find the type to chop up a widen vector for load/store
8381// TLI: Target lowering used to determine legal types.
8382// Width: Width left need to load/store.
8383// WidenVT: The widen vector type to load to/store from
8384// Align: If 0, don't allow use of a wider type
8385// WidenEx: If Align is not 0, the amount additional we can load/store from.
8386
8387static std::optional<EVT> findMemType(SelectionDAG &DAG,
8388 const TargetLowering &TLI, unsigned Width,
8389 EVT WidenVT, unsigned Align = 0,
8390 unsigned WidenEx = 0) {
8391 EVT WidenEltVT = WidenVT.getVectorElementType();
8392 const bool Scalable = WidenVT.isScalableVector();
8393 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8394 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8395 unsigned AlignInBits = Align*8;
8396
8397 EVT RetVT = WidenEltVT;
8398 // Don't bother looking for an integer type if the vector is scalable, skip
8399 // to vector types.
8400 if (!Scalable) {
8401 // If we have one element to load/store, return it.
8402 if (Width == WidenEltWidth)
8403 return RetVT;
8404
8405 // See if there is larger legal integer than the element type to load/store.
8406 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8407 unsigned MemVTWidth = MemVT.getSizeInBits();
8408 if (MemVT.getSizeInBits() <= WidenEltWidth)
8409 break;
8410 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8411 if ((Action == TargetLowering::TypeLegal ||
8412 Action == TargetLowering::TypePromoteInteger) &&
8413 (WidenWidth % MemVTWidth) == 0 &&
8414 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8415 (MemVTWidth <= Width ||
8416 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8417 if (MemVTWidth == WidenWidth)
8418 return MemVT;
8419 RetVT = MemVT;
8420 break;
8421 }
8422 }
8423 }
8424
8425 // See if there is a larger vector type to load/store that has the same vector
8426 // element type and is evenly divisible with the WidenVT.
8427 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8428 // Skip vector MVTs which don't match the scalable property of WidenVT.
8429 if (Scalable != MemVT.isScalableVector())
8430 continue;
8431 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8432 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8433 if ((Action == TargetLowering::TypeLegal ||
8434 Action == TargetLowering::TypePromoteInteger) &&
8435 WidenEltVT == MemVT.getVectorElementType() &&
8436 (WidenWidth % MemVTWidth) == 0 &&
8437 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8438 (MemVTWidth <= Width ||
8439 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8440 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8441 return MemVT;
8442 }
8443 }
8444
8445 // Using element-wise loads and stores for widening operations is not
8446 // supported for scalable vectors
8447 if (Scalable)
8448 return std::nullopt;
8449
8450 return RetVT;
8451}
8452
8453// Builds a vector type from scalar loads
8454// VecTy: Resulting Vector type
8455// LDOps: Load operators to build a vector type
8456// [Start,End) the list of loads to use.
8457static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8458 SmallVectorImpl<SDValue> &LdOps,
8459 unsigned Start, unsigned End) {
8460 SDLoc dl(LdOps[Start]);
8461 EVT LdTy = LdOps[Start].getValueType();
8462 unsigned Width = VecTy.getSizeInBits();
8463 unsigned NumElts = Width / LdTy.getSizeInBits();
8464 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8465
8466 unsigned Idx = 1;
8467 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8468
8469 for (unsigned i = Start + 1; i != End; ++i) {
8470 EVT NewLdTy = LdOps[i].getValueType();
8471 if (NewLdTy != LdTy) {
8472 NumElts = Width / NewLdTy.getSizeInBits();
8473 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8474 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8475 // Readjust position and vector position based on new load type.
8476 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8477 LdTy = NewLdTy;
8478 }
8479 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8480 }
8481 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8482}
8483
8484SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8485 LoadSDNode *LD) {
8486 // The strategy assumes that we can efficiently load power-of-two widths.
8487 // The routine chops the vector into the largest vector loads with the same
8488 // element type or scalar loads and then recombines it to the widen vector
8489 // type.
8490 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8491 EVT LdVT = LD->getMemoryVT();
8492 SDLoc dl(LD);
8493 assert(LdVT.isVector() && WidenVT.isVector());
8494 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8495 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8496
8497 // Load information
8498 SDValue Chain = LD->getChain();
8499 SDValue BasePtr = LD->getBasePtr();
8500 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8501 AAMDNodes AAInfo = LD->getAAInfo();
8502
8503 TypeSize LdWidth = LdVT.getSizeInBits();
8504 TypeSize WidenWidth = WidenVT.getSizeInBits();
8505 TypeSize WidthDiff = WidenWidth - LdWidth;
8506 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8507 // and if the original load is simple.
8508 unsigned LdAlign =
8509 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8510
8511 // Find the vector type that can load from.
8512 std::optional<EVT> FirstVT =
8513 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8514 WidenEx: WidthDiff.getKnownMinValue());
8515
8516 if (!FirstVT)
8517 return SDValue();
8518
8519 SmallVector<EVT, 8> MemVTs;
8520 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8521
8522 // Unless we're able to load in one instruction we must work out how to load
8523 // the remainder.
8524 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8525 std::optional<EVT> NewVT = FirstVT;
8526 TypeSize RemainingWidth = LdWidth;
8527 TypeSize NewVTWidth = FirstVTWidth;
8528 do {
8529 RemainingWidth -= NewVTWidth;
8530 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8531 // The current type we are using is too large. Find a better size.
8532 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8533 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8534 if (!NewVT)
8535 return SDValue();
8536 NewVTWidth = NewVT->getSizeInBits();
8537 }
8538 MemVTs.push_back(Elt: *NewVT);
8539 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8540 }
8541
8542 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8543 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8544 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8545
8546 // Check if we can load the element with one instruction.
8547 if (MemVTs.empty())
8548 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8549 DAG);
8550
8551 // Load vector by using multiple loads from largest vector to scalar.
8552 SmallVector<SDValue, 16> LdOps;
8553 LdOps.push_back(Elt: LdOp);
8554
8555 uint64_t ScaledOffset = 0;
8556 MachinePointerInfo MPI = LD->getPointerInfo();
8557
8558 // First incremement past the first load.
8559 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8560 ScaledOffset: &ScaledOffset);
8561
8562 for (EVT MemVT : MemVTs) {
8563 Align NewAlign = ScaledOffset == 0
8564 ? LD->getBaseAlign()
8565 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8566 SDValue L =
8567 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8568
8569 LdOps.push_back(Elt: L);
8570 LdChain.push_back(Elt: L.getValue(R: 1));
8571 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8572 }
8573
8574 // Build the vector from the load operations.
8575 unsigned End = LdOps.size();
8576 if (!LdOps[0].getValueType().isVector())
8577 // All the loads are scalar loads.
8578 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8579
8580 // If the load contains vectors, build the vector using concat vector.
8581 // All of the vectors used to load are power-of-2, and the scalar loads can be
8582 // combined to make a power-of-2 vector.
8583 SmallVector<SDValue, 16> ConcatOps(End);
8584 int i = End - 1;
8585 int Idx = End;
8586 EVT LdTy = LdOps[i].getValueType();
8587 // First, combine the scalar loads to a vector.
8588 if (!LdTy.isVector()) {
8589 for (--i; i >= 0; --i) {
8590 LdTy = LdOps[i].getValueType();
8591 if (LdTy.isVector())
8592 break;
8593 }
8594 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8595 }
8596
8597 ConcatOps[--Idx] = LdOps[i];
8598 for (--i; i >= 0; --i) {
8599 EVT NewLdTy = LdOps[i].getValueType();
8600 if (NewLdTy != LdTy) {
8601 // Create a larger vector.
8602 TypeSize LdTySize = LdTy.getSizeInBits();
8603 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8604 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8605 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8606 unsigned NumOps =
8607 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8608 SmallVector<SDValue, 16> WidenOps(NumOps);
8609 unsigned j = 0;
8610 for (; j != End-Idx; ++j)
8611 WidenOps[j] = ConcatOps[Idx+j];
8612 for (; j != NumOps; ++j)
8613 WidenOps[j] = DAG.getPOISON(VT: LdTy);
8614
8615 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
8616 Ops: WidenOps);
8617 Idx = End - 1;
8618 LdTy = NewLdTy;
8619 }
8620 ConcatOps[--Idx] = LdOps[i];
8621 }
8622
8623 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
8624 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
8625 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
8626
8627 // We need to fill the rest with undefs to build the vector.
8628 unsigned NumOps =
8629 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
8630 SmallVector<SDValue, 16> WidenOps(NumOps);
8631 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
8632 {
8633 unsigned i = 0;
8634 for (; i != End-Idx; ++i)
8635 WidenOps[i] = ConcatOps[Idx+i];
8636 for (; i != NumOps; ++i)
8637 WidenOps[i] = UndefVal;
8638 }
8639 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
8640}
8641
8642SDValue
8643DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
8644 LoadSDNode *LD,
8645 ISD::LoadExtType ExtType) {
8646 // For extension loads, it may not be more efficient to chop up the vector
8647 // and then extend it. Instead, we unroll the load and build a new vector.
8648 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8649 EVT LdVT = LD->getMemoryVT();
8650 SDLoc dl(LD);
8651 assert(LdVT.isVector() && WidenVT.isVector());
8652 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8653
8654 // Load information
8655 SDValue Chain = LD->getChain();
8656 SDValue BasePtr = LD->getBasePtr();
8657 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8658 AAMDNodes AAInfo = LD->getAAInfo();
8659
8660 if (LdVT.isScalableVector())
8661 return SDValue();
8662
8663 EVT EltVT = WidenVT.getVectorElementType();
8664 EVT LdEltVT = LdVT.getVectorElementType();
8665 unsigned NumElts = LdVT.getVectorNumElements();
8666
8667 // Load each element and widen.
8668 unsigned WidenNumElts = WidenVT.getVectorNumElements();
8669 SmallVector<SDValue, 16> Ops(WidenNumElts);
8670 unsigned Increment = LdEltVT.getSizeInBits() / 8;
8671 Ops[0] =
8672 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8673 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8674 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
8675 unsigned i = 0, Offset = Increment;
8676 for (i=1; i < NumElts; ++i, Offset += Increment) {
8677 SDValue NewBasePtr =
8678 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
8679 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
8680 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
8681 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8682 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
8683 }
8684
8685 // Fill the rest with undefs.
8686 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8687 for (; i != WidenNumElts; ++i)
8688 Ops[i] = UndefVal;
8689
8690 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
8691}
8692
8693bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
8694 StoreSDNode *ST) {
8695 // The strategy assumes that we can efficiently store power-of-two widths.
8696 // The routine chops the vector into the largest vector stores with the same
8697 // element type or scalar stores.
8698 SDValue Chain = ST->getChain();
8699 SDValue BasePtr = ST->getBasePtr();
8700 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
8701 AAMDNodes AAInfo = ST->getAAInfo();
8702 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
8703 SDLoc dl(ST);
8704
8705 EVT StVT = ST->getMemoryVT();
8706 TypeSize StWidth = StVT.getSizeInBits();
8707 EVT ValVT = ValOp.getValueType();
8708 TypeSize ValWidth = ValVT.getSizeInBits();
8709 EVT ValEltVT = ValVT.getVectorElementType();
8710 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
8711 assert(StVT.getVectorElementType() == ValEltVT);
8712 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
8713 "Mismatch between store and value types");
8714
8715 int Idx = 0; // current index to store
8716
8717 MachinePointerInfo MPI = ST->getPointerInfo();
8718 uint64_t ScaledOffset = 0;
8719
8720 // A breakdown of how to widen this vector store. Each element of the vector
8721 // is a memory VT combined with the number of times it is to be stored to,
8722 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
8723 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
8724
8725 while (StWidth.isNonZero()) {
8726 // Find the largest vector type we can store with.
8727 std::optional<EVT> NewVT =
8728 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
8729 if (!NewVT)
8730 return false;
8731 MemVTs.push_back(Elt: {*NewVT, 0});
8732 TypeSize NewVTWidth = NewVT->getSizeInBits();
8733
8734 do {
8735 StWidth -= NewVTWidth;
8736 MemVTs.back().second++;
8737 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
8738 }
8739
8740 for (const auto &Pair : MemVTs) {
8741 EVT NewVT = Pair.first;
8742 unsigned Count = Pair.second;
8743 TypeSize NewVTWidth = NewVT.getSizeInBits();
8744
8745 if (NewVT.isVector()) {
8746 unsigned NumVTElts = NewVT.getVectorMinNumElements();
8747 do {
8748 Align NewAlign = ScaledOffset == 0
8749 ? ST->getBaseAlign()
8750 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
8751 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
8752 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
8753 MMOFlags, AAInfo);
8754 StChain.push_back(Elt: PartStore);
8755
8756 Idx += NumVTElts;
8757 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
8758 ScaledOffset: &ScaledOffset);
8759 } while (--Count);
8760 } else {
8761 // Cast the vector to the scalar type we can store.
8762 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
8763 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
8764 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
8765 // Readjust index position based on new vector type.
8766 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
8767 do {
8768 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
8769 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
8770 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
8771 StChain.push_back(Elt: PartStore);
8772
8773 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
8774 } while (--Count);
8775 // Restore index back to be relative to the original widen element type.
8776 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
8777 }
8778 }
8779
8780 return true;
8781}
8782
8783/// Modifies a vector input (widen or narrows) to a vector of NVT. The
8784/// input vector must have the same element type as NVT.
8785/// FillWithZeroes specifies that the vector should be widened with zeroes.
8786SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
8787 bool FillWithZeroes) {
8788 // Note that InOp might have been widened so it might already have
8789 // the right width or it might need be narrowed.
8790 EVT InVT = InOp.getValueType();
8791 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
8792 "input and widen element type must match");
8793 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
8794 "cannot modify scalable vectors in this way");
8795 SDLoc dl(InOp);
8796
8797 // Check if InOp already has the right width.
8798 if (InVT == NVT)
8799 return InOp;
8800
8801 ElementCount InEC = InVT.getVectorElementCount();
8802 ElementCount WidenEC = NVT.getVectorElementCount();
8803 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
8804 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
8805 SmallVector<SDValue, 16> Ops(NumConcat);
8806 SDValue FillVal =
8807 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
8808 Ops[0] = InOp;
8809 for (unsigned i = 1; i != NumConcat; ++i)
8810 Ops[i] = FillVal;
8811
8812 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
8813 }
8814
8815 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
8816 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
8817
8818 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
8819 "Scalable vectors should have been handled already.");
8820
8821 unsigned InNumElts = InEC.getFixedValue();
8822 unsigned WidenNumElts = WidenEC.getFixedValue();
8823
8824 // Fall back to extract and build (+ mask, if padding with zeros).
8825 SmallVector<SDValue, 16> Ops(WidenNumElts);
8826 EVT EltVT = NVT.getVectorElementType();
8827 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
8828 unsigned Idx;
8829 for (Idx = 0; Idx < MinNumElts; ++Idx)
8830 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
8831
8832 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8833 for (; Idx < WidenNumElts; ++Idx)
8834 Ops[Idx] = UndefVal;
8835
8836 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
8837 if (!FillWithZeroes)
8838 return Widened;
8839
8840 assert(NVT.isInteger() &&
8841 "We expect to never want to FillWithZeroes for non-integral types.");
8842
8843 SmallVector<SDValue, 16> MaskOps;
8844 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
8845 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
8846
8847 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
8848 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
8849}
8850