1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error(reason: "Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::LOOP_DEPENDENCE_WAR_MASK:
57 case ISD::LOOP_DEPENDENCE_RAW_MASK:
58 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
59 break;
60 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
61 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
62 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
63 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
64 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
65 case ISD::CONVERT_FROM_ARBITRARY_FP:
66 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
67 break;
68 case ISD::CONVERT_TO_ARBITRARY_FP:
69 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
70 break;
71 case ISD::AssertZext:
72 case ISD::AssertSext:
73 case ISD::FPOWI:
74 case ISD::AssertNoFPClass:
75 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
76 break;
77 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
78 case ISD::ATOMIC_LOAD:
79 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
80 break;
81 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
82 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
83 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
84 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
85 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
86 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
87 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
88 case ISD::POISON:
89 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
90 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
91 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
92 case ISD::ANY_EXTEND_VECTOR_INREG:
93 case ISD::SIGN_EXTEND_VECTOR_INREG:
94 case ISD::ZERO_EXTEND_VECTOR_INREG:
95 R = ScalarizeVecRes_VecInregOp(N);
96 break;
97 case ISD::ABS:
98 case ISD::ABS_MIN_POISON:
99 case ISD::ANY_EXTEND:
100 case ISD::BITREVERSE:
101 case ISD::BSWAP:
102 case ISD::CTLZ:
103 case ISD::CTLZ_ZERO_POISON:
104 case ISD::CTPOP:
105 case ISD::CTTZ:
106 case ISD::CTTZ_ZERO_POISON:
107 case ISD::FABS:
108 case ISD::FACOS:
109 case ISD::FASIN:
110 case ISD::FATAN:
111 case ISD::FCEIL:
112 case ISD::FCOS:
113 case ISD::FCOSH:
114 case ISD::FEXP:
115 case ISD::FEXP2:
116 case ISD::FEXP10:
117 case ISD::FFLOOR:
118 case ISD::FLOG:
119 case ISD::FLOG10:
120 case ISD::FLOG2:
121 case ISD::FNEARBYINT:
122 case ISD::FNEG:
123 case ISD::FREEZE:
124 case ISD::ARITH_FENCE:
125 case ISD::FP_EXTEND:
126 case ISD::FP_TO_SINT:
127 case ISD::FP_TO_UINT:
128 case ISD::FRINT:
129 case ISD::LRINT:
130 case ISD::LLRINT:
131 case ISD::FROUND:
132 case ISD::FROUNDEVEN:
133 case ISD::LROUND:
134 case ISD::LLROUND:
135 case ISD::FSIN:
136 case ISD::FSINH:
137 case ISD::FSQRT:
138 case ISD::FTAN:
139 case ISD::FTANH:
140 case ISD::FTRUNC:
141 case ISD::SIGN_EXTEND:
142 case ISD::SINT_TO_FP:
143 case ISD::TRUNCATE:
144 case ISD::UINT_TO_FP:
145 case ISD::ZERO_EXTEND:
146 case ISD::FCANONICALIZE:
147 R = ScalarizeVecRes_UnaryOp(N);
148 break;
149 case ISD::ADDRSPACECAST:
150 R = ScalarizeVecRes_ADDRSPACECAST(N);
151 break;
152 case ISD::FMODF:
153 case ISD::FFREXP:
154 case ISD::FSINCOS:
155 case ISD::FSINCOSPI:
156 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
157 break;
158 case ISD::ADD:
159 case ISD::AND:
160 case ISD::AVGCEILS:
161 case ISD::AVGCEILU:
162 case ISD::AVGFLOORS:
163 case ISD::AVGFLOORU:
164 case ISD::FADD:
165 case ISD::FCOPYSIGN:
166 case ISD::FDIV:
167 case ISD::FMUL:
168 case ISD::FMINNUM:
169 case ISD::FMAXNUM:
170 case ISD::FMINNUM_IEEE:
171 case ISD::FMAXNUM_IEEE:
172 case ISD::FMINIMUM:
173 case ISD::FMAXIMUM:
174 case ISD::FMINIMUMNUM:
175 case ISD::FMAXIMUMNUM:
176 case ISD::FLDEXP:
177 case ISD::ABDS:
178 case ISD::ABDU:
179 case ISD::SMIN:
180 case ISD::SMAX:
181 case ISD::UMIN:
182 case ISD::UMAX:
183
184 case ISD::SADDSAT:
185 case ISD::UADDSAT:
186 case ISD::SSUBSAT:
187 case ISD::USUBSAT:
188 case ISD::SSHLSAT:
189 case ISD::USHLSAT:
190
191 case ISD::FPOW:
192 case ISD::FATAN2:
193 case ISD::FREM:
194 case ISD::FSUB:
195 case ISD::MUL:
196 case ISD::MULHS:
197 case ISD::MULHU:
198 case ISD::OR:
199 case ISD::SDIV:
200 case ISD::SREM:
201 case ISD::SUB:
202 case ISD::UDIV:
203 case ISD::UREM:
204 case ISD::XOR:
205 case ISD::SHL:
206 case ISD::SRA:
207 case ISD::SRL:
208 case ISD::ROTL:
209 case ISD::ROTR:
210 case ISD::CLMUL:
211 case ISD::CLMULR:
212 case ISD::CLMULH:
213 case ISD::PEXT:
214 case ISD::PDEP:
215 R = ScalarizeVecRes_BinOp(N);
216 break;
217
218 case ISD::MASKED_UDIV:
219 case ISD::MASKED_SDIV:
220 case ISD::MASKED_UREM:
221 case ISD::MASKED_SREM:
222 R = ScalarizeVecRes_MaskedBinOp(N);
223 break;
224
225 case ISD::SCMP:
226 case ISD::UCMP:
227 R = ScalarizeVecRes_CMP(N);
228 break;
229
230 case ISD::FMA:
231 case ISD::FSHL:
232 case ISD::FSHR:
233 R = ScalarizeVecRes_TernaryOp(N);
234 break;
235
236#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
237 case ISD::STRICT_##DAGN:
238#include "llvm/IR/ConstrainedOps.def"
239 R = ScalarizeVecRes_StrictFPOp(N);
240 break;
241
242 case ISD::FP_TO_UINT_SAT:
243 case ISD::FP_TO_SINT_SAT:
244 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
245 break;
246
247 case ISD::UADDO:
248 case ISD::SADDO:
249 case ISD::USUBO:
250 case ISD::SSUBO:
251 case ISD::UMULO:
252 case ISD::SMULO:
253 R = ScalarizeVecRes_OverflowOp(N, ResNo);
254 break;
255 case ISD::SMULFIX:
256 case ISD::SMULFIXSAT:
257 case ISD::UMULFIX:
258 case ISD::UMULFIXSAT:
259 case ISD::SDIVFIX:
260 case ISD::SDIVFIXSAT:
261 case ISD::UDIVFIX:
262 case ISD::UDIVFIXSAT:
263 R = ScalarizeVecRes_FIX(N);
264 break;
265 }
266
267 // If R is null, the sub-method took care of registering the result.
268 if (R.getNode())
269 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
270}
271
272SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
273 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
274 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
275 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
276 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
277}
278
279SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
280 SDLoc DL(N);
281 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
282 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
283 SDValue Mask = N->getOperand(Num: 2);
284 EVT MaskVT = Mask.getValueType();
285 // The vselect result and input vectors need scalarizing, but it's
286 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
287 // See the similar logic in ScalarizeVecRes_SETCC.
288 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
289 Mask = GetScalarizedVector(Op: Mask);
290 else
291 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
292 // Vectors may have a different boolean contents to scalars, so truncate to i1
293 // and let type legalization promote appropriately.
294 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
295 // Masked binary ops don't have UB on disabled lanes but produce poison, so
296 // use 1 as the divisor to avoid division by zero and overflow.
297 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
298 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
299 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
300 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
301}
302
303SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
304 SDLoc DL(N);
305
306 SDValue LHS = N->getOperand(Num: 0);
307 SDValue RHS = N->getOperand(Num: 1);
308 if (getTypeAction(VT: LHS.getValueType()) ==
309 TargetLowering::TypeScalarizeVector) {
310 LHS = GetScalarizedVector(Op: LHS);
311 RHS = GetScalarizedVector(Op: RHS);
312 } else {
313 EVT VT = LHS.getValueType().getVectorElementType();
314 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
315 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
316 }
317
318 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
319 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
320}
321
322SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
323 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
324 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
325 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
326 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
327 N3: Op2, Flags: N->getFlags());
328}
329
330SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
331 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
332 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
333 SDValue Op2 = N->getOperand(Num: 2);
334 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
335 N3: Op2, Flags: N->getFlags());
336}
337
338SDValue
339DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
340 unsigned ResNo) {
341 assert(N->getValueType(0).getVectorNumElements() == 1 &&
342 "Unexpected vector type!");
343 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
344
345 EVT VT0 = N->getValueType(ResNo: 0);
346 EVT VT1 = N->getValueType(ResNo: 1);
347 SDLoc dl(N);
348
349 SDNode *ScalarNode =
350 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
351 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
352 .getNode();
353
354 // Replace the other vector result not being explicitly scalarized here.
355 unsigned OtherNo = 1 - ResNo;
356 EVT OtherVT = N->getValueType(ResNo: OtherNo);
357 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
358 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
359 } else {
360 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
361 Operand: SDValue(ScalarNode, OtherNo));
362 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
363 }
364
365 return SDValue(ScalarNode, ResNo);
366}
367
368SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
369 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
370 unsigned NumOpers = N->getNumOperands();
371 SDValue Chain = N->getOperand(Num: 0);
372 EVT ValueVTs[] = {VT, MVT::Other};
373 SDLoc dl(N);
374
375 SmallVector<SDValue, 4> Opers(NumOpers);
376
377 // The Chain is the first operand.
378 Opers[0] = Chain;
379
380 // Now process the remaining operands.
381 for (unsigned i = 1; i < NumOpers; ++i) {
382 SDValue Oper = N->getOperand(Num: i);
383 EVT OperVT = Oper.getValueType();
384
385 if (OperVT.isVector()) {
386 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
387 Oper = GetScalarizedVector(Op: Oper);
388 else
389 Oper =
390 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
391 }
392
393 Opers[i] = Oper;
394 }
395
396 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
397 Ops: Opers, Flags: N->getFlags());
398
399 // Legalize the chain result - switch anything that used the old chain to
400 // use the new one.
401 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
402 return Result;
403}
404
405SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
406 unsigned ResNo) {
407 SDLoc DL(N);
408 EVT ResVT = N->getValueType(ResNo: 0);
409 EVT OvVT = N->getValueType(ResNo: 1);
410
411 SDValue ScalarLHS, ScalarRHS;
412 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
413 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
414 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
415 } else {
416 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
417 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
418 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
419 ScalarLHS = ElemsLHS[0];
420 ScalarRHS = ElemsRHS[0];
421 }
422
423 SDVTList ScalarVTs = DAG.getVTList(
424 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
425 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
426 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
427 .getNode();
428
429 // Replace the other vector result not being explicitly scalarized here.
430 unsigned OtherNo = 1 - ResNo;
431 EVT OtherVT = N->getValueType(ResNo: OtherNo);
432 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
433 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
434 } else {
435 SDValue OtherVal = DAG.getNode(
436 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
437 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
438 }
439
440 return SDValue(ScalarNode, ResNo);
441}
442
443SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
444 unsigned ResNo) {
445 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
446 return GetScalarizedVector(Op);
447}
448
449SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
450 SDLoc DL(N);
451 SDValue SourceValue = N->getOperand(Num: 0);
452 SDValue SinkValue = N->getOperand(Num: 1);
453 SDValue EltSizeInBytes = N->getOperand(Num: 2);
454 SDValue LaneOffset = N->getOperand(Num: 3);
455
456 EVT PtrVT = SourceValue->getValueType(ResNo: 0);
457 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
458
459 // Take the difference between the pointers and divided by the element size,
460 // to see how many lanes separate them.
461 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: SinkValue, N2: SourceValue);
462 if (IsReadAfterWrite)
463 Diff = DAG.getNode(Opcode: ISD::ABS, DL, VT: PtrVT, Operand: Diff);
464 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: PtrVT, N1: Diff, N2: EltSizeInBytes);
465
466 // The pointers do not alias if:
467 // * Diff <= 0 || LaneOffset < Diff (WAR_MASK)
468 // * Diff == 0 || LaneOffset < abs(Diff) (RAW_MASK)
469 // Note: If LaneOffset is zero, both cases will fold to "true".
470 EVT CmpVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
471 VT: Diff.getValueType());
472 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
473 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Diff, RHS: Zero,
474 Cond: IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
475 return DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Cmp,
476 N2: DAG.getSetCC(DL, VT: CmpVT, LHS: LaneOffset, RHS: Diff, Cond: ISD::SETULT));
477}
478
479SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
480 SDValue Op = N->getOperand(Num: 0);
481 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
482 Op = GetScalarizedVector(Op);
483 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
484 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
485 VT: NewVT, Operand: Op);
486}
487
488SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
489 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
490 SDValue InOp = N->getOperand(Num: 0);
491 // The BUILD_VECTOR operands may be of wider element types and
492 // we may need to truncate them back to the requested return type.
493 if (EltVT.isInteger())
494 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
495 return InOp;
496}
497
498SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
499 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
500 VT: N->getValueType(ResNo: 0).getVectorElementType(),
501 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
502}
503
504SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
505 SDLoc DL(N);
506 SDValue Op = N->getOperand(Num: 0);
507 EVT OpVT = Op.getValueType();
508 // The result needs scalarizing, but it's not a given that the source does.
509 // See similar logic in ScalarizeVecRes_UnaryOp.
510 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
511 Op = GetScalarizedVector(Op);
512 } else {
513 EVT VT = OpVT.getVectorElementType();
514 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
515 }
516 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
517 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
518 N2: N->getOperand(Num: 1));
519}
520
521SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
522 SDLoc DL(N);
523 SDValue Op = N->getOperand(Num: 0);
524 EVT OpVT = Op.getValueType();
525 // The result needs scalarizing, but it's not a given that the source does.
526 // See similar logic in ScalarizeVecRes_UnaryOp.
527 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
528 Op = GetScalarizedVector(Op);
529 } else {
530 EVT VT = OpVT.getVectorElementType();
531 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
532 }
533 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
534 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
535 N2: N->getOperand(Num: 1));
536}
537
538SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
539 SDLoc DL(N);
540 SDValue Op = N->getOperand(Num: 0);
541 EVT OpVT = Op.getValueType();
542 // The result needs scalarizing, but it's not a given that the source does.
543 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
544 Op = GetScalarizedVector(Op);
545 } else {
546 EVT VT = OpVT.getVectorElementType();
547 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
548 }
549 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
550 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
551 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
552}
553
554SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
555 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
556 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
557 N2: N->getOperand(Num: 1));
558}
559
560SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
561 // The value to insert may have a wider type than the vector element type,
562 // so be sure to truncate it to the element type if necessary.
563 SDValue Op = N->getOperand(Num: 1);
564 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
565 if (Op.getValueType() != EltVT)
566 // FIXME: Can this happen for floating point types?
567 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
568 return Op;
569}
570
571SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
572 SDValue Result = DAG.getAtomicLoad(
573 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
574 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
575 MMO: N->getMemOperand());
576
577 // Legalize the chain result - switch anything that used the old chain to
578 // use the new one.
579 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
580 return Result;
581}
582
583SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
584 assert(N->isUnindexed() && "Indexed vector load?");
585
586 SDValue Result = DAG.getLoad(
587 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
588 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
589 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
590 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
591 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
592
593 // Legalize the chain result - switch anything that used the old chain to
594 // use the new one.
595 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
596 return Result;
597}
598
599SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
600 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
601 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
602 SDValue Op = N->getOperand(Num: 0);
603 EVT OpVT = Op.getValueType();
604 SDLoc DL(N);
605 // The result needs scalarizing, but it's not a given that the source does.
606 // This is a workaround for targets where it's impossible to scalarize the
607 // result of a conversion, because the source type is legal.
608 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
609 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
610 // legal and was not scalarized.
611 // See the similar logic in ScalarizeVecRes_SETCC
612 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
613 Op = GetScalarizedVector(Op);
614 } else {
615 EVT VT = OpVT.getVectorElementType();
616 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
617 }
618 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
619}
620
621SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
622 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
623 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
624 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
625 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
626 N1: LHS, N2: DAG.getValueType(ExtVT));
627}
628
629SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
630 SDLoc DL(N);
631 SDValue Op = N->getOperand(Num: 0);
632
633 EVT OpVT = Op.getValueType();
634 EVT OpEltVT = OpVT.getVectorElementType();
635 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
636
637 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
638 Op = GetScalarizedVector(Op);
639 } else {
640 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
641 }
642
643 switch (N->getOpcode()) {
644 case ISD::ANY_EXTEND_VECTOR_INREG:
645 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
646 case ISD::SIGN_EXTEND_VECTOR_INREG:
647 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
648 case ISD::ZERO_EXTEND_VECTOR_INREG:
649 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
650 }
651
652 llvm_unreachable("Illegal extend_vector_inreg opcode");
653}
654
655SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
656 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
657 SDValue Op = N->getOperand(Num: 0);
658 EVT OpVT = Op.getValueType();
659 SDLoc DL(N);
660 // The result needs scalarizing, but it's not a given that the source does.
661 // This is a workaround for targets where it's impossible to scalarize the
662 // result of a conversion, because the source type is legal.
663 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
664 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
665 // legal and was not scalarized.
666 // See the similar logic in ScalarizeVecRes_SETCC
667 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
668 Op = GetScalarizedVector(Op);
669 } else {
670 EVT VT = OpVT.getVectorElementType();
671 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
672 }
673 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
674 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
675 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
676 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
677}
678
679SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
680 // If the operand is wider than the vector element type then it is implicitly
681 // truncated. Make that explicit here.
682 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
683 SDValue InOp = N->getOperand(Num: 0);
684 if (InOp.getValueType() != EltVT)
685 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
686 return InOp;
687}
688
689SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
690 SDValue Cond = N->getOperand(Num: 0);
691 EVT OpVT = Cond.getValueType();
692 SDLoc DL(N);
693 // The vselect result and true/value operands needs scalarizing, but it's
694 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
695 // See the similar logic in ScalarizeVecRes_SETCC
696 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
697 Cond = GetScalarizedVector(Op: Cond);
698 } else {
699 EVT VT = OpVT.getVectorElementType();
700 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
701 }
702
703 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
704 TargetLowering::BooleanContent ScalarBool =
705 TLI.getBooleanContents(isVec: false, isFloat: false);
706 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
707
708 // If integer and float booleans have different contents then we can't
709 // reliably optimize in all cases. There is a full explanation for this in
710 // DAGCombiner::visitSELECT() where the same issue affects folding
711 // (select C, 0, 1) to (xor C, 1).
712 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
713 TLI.getBooleanContents(isVec: false, isFloat: true)) {
714 // At least try the common case where the boolean is generated by a
715 // comparison.
716 if (Cond->getOpcode() == ISD::SETCC) {
717 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
718 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
719 VecBool = TLI.getBooleanContents(Type: OpVT);
720 } else
721 ScalarBool = TargetLowering::UndefinedBooleanContent;
722 }
723
724 EVT CondVT = Cond.getValueType();
725 if (ScalarBool != VecBool) {
726 switch (ScalarBool) {
727 case TargetLowering::UndefinedBooleanContent:
728 break;
729 case TargetLowering::ZeroOrOneBooleanContent:
730 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
731 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
732 // Vector read from all ones, scalar expects a single 1 so mask.
733 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
734 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
735 break;
736 case TargetLowering::ZeroOrNegativeOneBooleanContent:
737 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
738 VecBool == TargetLowering::ZeroOrOneBooleanContent);
739 // Vector reads from a one, scalar from all ones so sign extend.
740 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
741 N1: Cond, N2: DAG.getValueType(MVT::i1));
742 break;
743 }
744 }
745
746 // Truncate the condition if needed
747 auto BoolVT = getSetCCResultType(VT: CondVT);
748 if (BoolVT.bitsLT(VT: CondVT))
749 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
750
751 return DAG.getSelect(DL: SDLoc(N),
752 VT: LHS.getValueType(), Cond, LHS,
753 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
754}
755
756SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
757 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
758 return DAG.getSelect(DL: SDLoc(N),
759 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
760 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
761}
762
763SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
764 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
765 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
766 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
767 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
768 N5: N->getOperand(Num: 4));
769}
770
771SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
772 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
773}
774
775SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
776 // Figure out if the scalar is the LHS or RHS and return it.
777 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
778 if (Arg.isUndef())
779 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
780 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
781 return GetScalarizedVector(Op: N->getOperand(Num: Op));
782}
783
784SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
785 SDValue Src = N->getOperand(Num: 0);
786 EVT SrcVT = Src.getValueType();
787 SDLoc dl(N);
788
789 // Handle case where result is scalarized but operand is not
790 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
791 Src = GetScalarizedVector(Op: Src);
792 else
793 Src = DAG.getNode(
794 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
795 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
796
797 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
798 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
799}
800
801SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
802 assert(N->getValueType(0).isVector() &&
803 N->getOperand(0).getValueType().isVector() &&
804 "Operand types must be vectors");
805 SDValue LHS = N->getOperand(Num: 0);
806 SDValue RHS = N->getOperand(Num: 1);
807 EVT OpVT = LHS.getValueType();
808 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
809 SDLoc DL(N);
810
811 // The result needs scalarizing, but it's not a given that the source does.
812 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
813 LHS = GetScalarizedVector(Op: LHS);
814 RHS = GetScalarizedVector(Op: RHS);
815 } else {
816 EVT VT = OpVT.getVectorElementType();
817 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
818 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
819 }
820
821 // Turn it into a scalar SETCC.
822 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
823 N3: N->getOperand(Num: 2));
824 // Vectors may have a different boolean contents to scalars. Promote the
825 // value appropriately.
826 ISD::NodeType ExtendCode =
827 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
828 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
829}
830
831SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
832 SDLoc DL(N);
833 SDValue Arg = N->getOperand(Num: 0);
834 SDValue Test = N->getOperand(Num: 1);
835 EVT ArgVT = Arg.getValueType();
836 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
837
838 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
839 Arg = GetScalarizedVector(Op: Arg);
840 } else {
841 EVT VT = ArgVT.getVectorElementType();
842 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
843 }
844
845 SDValue Res =
846 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
847 // Vectors may have a different boolean contents to scalars. Promote the
848 // value appropriately.
849 ISD::NodeType ExtendCode =
850 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
851 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
852}
853
854//===----------------------------------------------------------------------===//
855// Operand Vector Scalarization <1 x ty> -> ty.
856//===----------------------------------------------------------------------===//
857
858bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
859 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
860 N->dump(&DAG));
861 SDValue Res = SDValue();
862
863 switch (N->getOpcode()) {
864 default:
865#ifndef NDEBUG
866 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
867 N->dump(&DAG);
868 dbgs() << "\n";
869#endif
870 report_fatal_error(reason: "Do not know how to scalarize this operator's "
871 "operand!\n");
872 case ISD::BITCAST:
873 Res = ScalarizeVecOp_BITCAST(N);
874 break;
875 case ISD::FAKE_USE:
876 Res = ScalarizeVecOp_FAKE_USE(N);
877 break;
878 case ISD::ANY_EXTEND:
879 case ISD::ZERO_EXTEND:
880 case ISD::SIGN_EXTEND:
881 case ISD::TRUNCATE:
882 case ISD::FP_TO_SINT:
883 case ISD::FP_TO_UINT:
884 case ISD::SINT_TO_FP:
885 case ISD::UINT_TO_FP:
886 case ISD::LROUND:
887 case ISD::LLROUND:
888 case ISD::LRINT:
889 case ISD::LLRINT:
890 Res = ScalarizeVecOp_UnaryOp(N);
891 break;
892 case ISD::FP_TO_SINT_SAT:
893 case ISD::FP_TO_UINT_SAT:
894 case ISD::CONVERT_FROM_ARBITRARY_FP:
895 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
896 break;
897 case ISD::CONVERT_TO_ARBITRARY_FP: {
898 assert(N->getValueType(0).getVectorNumElements() == 1 &&
899 "Unexpected vector type!");
900 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
901 SDValue Op = DAG.getNode(
902 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
903 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
904 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
905 break;
906 }
907 case ISD::STRICT_SINT_TO_FP:
908 case ISD::STRICT_UINT_TO_FP:
909 case ISD::STRICT_FP_TO_SINT:
910 case ISD::STRICT_FP_TO_UINT:
911 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
912 break;
913 case ISD::CONCAT_VECTORS:
914 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
915 break;
916 case ISD::INSERT_SUBVECTOR:
917 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
918 break;
919 case ISD::EXTRACT_VECTOR_ELT:
920 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
921 break;
922 case ISD::VSELECT:
923 Res = ScalarizeVecOp_VSELECT(N);
924 break;
925 case ISD::SETCC:
926 Res = ScalarizeVecOp_VSETCC(N);
927 break;
928 case ISD::STRICT_FSETCC:
929 case ISD::STRICT_FSETCCS:
930 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
931 break;
932 case ISD::STORE:
933 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
934 break;
935 case ISD::ATOMIC_STORE:
936 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
937 break;
938 case ISD::STRICT_FP_ROUND:
939 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
940 break;
941 case ISD::FP_ROUND:
942 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
943 break;
944 case ISD::STRICT_FP_EXTEND:
945 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
946 break;
947 case ISD::FP_EXTEND:
948 Res = ScalarizeVecOp_FP_EXTEND(N);
949 break;
950 case ISD::VECREDUCE_FADD:
951 case ISD::VECREDUCE_FMUL:
952 case ISD::VECREDUCE_ADD:
953 case ISD::VECREDUCE_MUL:
954 case ISD::VECREDUCE_AND:
955 case ISD::VECREDUCE_OR:
956 case ISD::VECREDUCE_XOR:
957 case ISD::VECREDUCE_SMAX:
958 case ISD::VECREDUCE_SMIN:
959 case ISD::VECREDUCE_UMAX:
960 case ISD::VECREDUCE_UMIN:
961 case ISD::VECREDUCE_FMAX:
962 case ISD::VECREDUCE_FMIN:
963 case ISD::VECREDUCE_FMAXIMUM:
964 case ISD::VECREDUCE_FMINIMUM:
965 Res = ScalarizeVecOp_VECREDUCE(N);
966 break;
967 case ISD::VECREDUCE_SEQ_FADD:
968 case ISD::VECREDUCE_SEQ_FMUL:
969 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
970 break;
971 case ISD::SCMP:
972 case ISD::UCMP:
973 Res = ScalarizeVecOp_CMP(N);
974 break;
975 case ISD::VECTOR_FIND_LAST_ACTIVE:
976 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
977 break;
978 case ISD::CTTZ_ELTS:
979 case ISD::CTTZ_ELTS_ZERO_POISON:
980 Res = ScalarizeVecOp_CTTZ_ELTS(N);
981 break;
982 case ISD::MASKED_UDIV:
983 case ISD::MASKED_SDIV:
984 case ISD::MASKED_UREM:
985 case ISD::MASKED_SREM:
986 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
987 break;
988 }
989
990 // If the result is null, the sub-method took care of registering results etc.
991 if (!Res.getNode()) return false;
992
993 // If the result is N, the sub-method updated N in place. Tell the legalizer
994 // core about this.
995 if (Res.getNode() == N)
996 return true;
997
998 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
999 "Invalid operand expansion");
1000
1001 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1002 return false;
1003}
1004
1005/// If the value to convert is a vector that needs to be scalarized, it must be
1006/// <1 x ty>. Convert the element instead.
1007SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1008 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1009 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1010 VT: N->getValueType(ResNo: 0), Operand: Elt);
1011}
1012
1013// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1014SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1015 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1016 "Fake Use: Unexpected vector type!");
1017 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1018 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1019}
1020
1021/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1022/// Do the operation on the element instead.
1023SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1024 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1025 "Unexpected vector type!");
1026 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1027 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1028 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1029 // Revectorize the result so the types line up with what the uses of this
1030 // expression expect.
1031 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1032}
1033
1034/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1035/// typesize).
1036SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1037 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1038 "Unexpected vector type!");
1039 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1040 SDValue Op =
1041 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1042 N1: Elt, N2: N->getOperand(Num: 1));
1043 // Revectorize the result so the types line up with what the uses of this
1044 // expression expect.
1045 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1046}
1047
1048/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1049/// Do the strict FP operation on the element instead.
1050SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1051 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1052 "Unexpected vector type!");
1053 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1054 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1055 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1056 Ops: { N->getOperand(Num: 0), Elt });
1057 // Legalize the chain result - switch anything that used the old chain to
1058 // use the new one.
1059 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1060 // Revectorize the result so the types line up with what the uses of this
1061 // expression expect.
1062 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1063
1064 // Do our own replacement and return SDValue() to tell the caller that we
1065 // handled all replacements since caller can only handle a single result.
1066 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1067 return SDValue();
1068}
1069
1070/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1071SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1072 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1073 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1074 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1075 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1076}
1077
1078/// The inserted subvector is to be scalarized - use insert vector element
1079/// instead.
1080SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1081 unsigned OpNo) {
1082 // We should not be attempting to scalarize the containing vector
1083 assert(OpNo == 1);
1084 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1085 SDValue ContainingVec = N->getOperand(Num: 0);
1086 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1087 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1088 N3: N->getOperand(Num: 2));
1089}
1090
1091/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1092/// so just return the element, ignoring the index.
1093SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1094 EVT VT = N->getValueType(ResNo: 0);
1095 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1096 if (Res.getValueType() != VT)
1097 Res = VT.isFloatingPoint()
1098 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1099 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1100 return Res;
1101}
1102
1103/// If the input condition is a vector that needs to be scalarized, it must be
1104/// <1 x i1>, so just convert to a normal ISD::SELECT
1105/// (still with vector output type since that was acceptable if we got here).
1106SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1107 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1108 EVT VT = N->getValueType(ResNo: 0);
1109
1110 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1111 N3: N->getOperand(Num: 2));
1112}
1113
1114/// If the operand is a vector that needs to be scalarized then the
1115/// result must be v1i1, so just convert to a scalar SETCC and wrap
1116/// with a scalar_to_vector since the res type is legal if we got here
1117SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1118 assert(N->getValueType(0).isVector() &&
1119 N->getOperand(0).getValueType().isVector() &&
1120 "Operand types must be vectors");
1121 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1122
1123 EVT VT = N->getValueType(ResNo: 0);
1124 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1125 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1126
1127 EVT OpVT = N->getOperand(Num: 0).getValueType();
1128 EVT NVT = VT.getVectorElementType();
1129 SDLoc DL(N);
1130 // Turn it into a scalar SETCC.
1131 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1132 N3: N->getOperand(Num: 2));
1133
1134 // Vectors may have a different boolean contents to scalars. Promote the
1135 // value appropriately.
1136 ISD::NodeType ExtendCode =
1137 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1138
1139 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1140
1141 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1142}
1143
1144// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1145SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1146 unsigned OpNo) {
1147 assert(OpNo == 1 && "Wrong operand for scalarization!");
1148 assert(N->getValueType(0).isVector() &&
1149 N->getOperand(1).getValueType().isVector() &&
1150 "Operand types must be vectors");
1151 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1152
1153 EVT VT = N->getValueType(ResNo: 0);
1154 SDValue Ch = N->getOperand(Num: 0);
1155 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1156 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1157 SDValue CC = N->getOperand(Num: 3);
1158
1159 EVT OpVT = N->getOperand(Num: 1).getValueType();
1160 EVT NVT = VT.getVectorElementType();
1161 SDLoc DL(N);
1162 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1163 Ops: {Ch, LHS, RHS, CC});
1164
1165 // Legalize the chain result - switch anything that used the old chain to
1166 // use the new one.
1167 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1168
1169 ISD::NodeType ExtendCode =
1170 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1171
1172 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1173 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1174
1175 // Do our own replacement and return SDValue() to tell the caller that we
1176 // handled all replacements since caller can only handle a single result.
1177 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1178 return SDValue();
1179}
1180
1181/// If the value to store is a vector that needs to be scalarized, it must be
1182/// <1 x ty>. Just store the element.
1183SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1184 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1185 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1186 SDLoc dl(N);
1187
1188 if (N->isTruncatingStore())
1189 return DAG.getTruncStore(
1190 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1191 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1192 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1193 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1194
1195 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1196 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1197 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1198}
1199
1200/// If the value to store is a vector that needs to be scalarized, it must be
1201/// <1 x ty>. Just store the element.
1202SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1203 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1204 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1205 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1206 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1207}
1208
1209/// If the value to round is a vector that needs to be scalarized, it must be
1210/// <1 x ty>. Convert the element instead.
1211SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1212 assert(OpNo == 0 && "Wrong operand for scalarization!");
1213 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1214 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1215 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1216 N2: N->getOperand(Num: 1));
1217 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1218}
1219
1220SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1221 unsigned OpNo) {
1222 assert(OpNo == 1 && "Wrong operand for scalarization!");
1223 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1224 SDValue Res =
1225 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1226 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1227 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1228 // Legalize the chain result - switch anything that used the old chain to
1229 // use the new one.
1230 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1231
1232 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1233
1234 // Do our own replacement and return SDValue() to tell the caller that we
1235 // handled all replacements since caller can only handle a single result.
1236 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1237 return SDValue();
1238}
1239
1240/// If the value to extend is a vector that needs to be scalarized, it must be
1241/// <1 x ty>. Convert the element instead.
1242SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1243 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1244 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1245 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1246 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1247}
1248
1249/// If the value to extend is a vector that needs to be scalarized, it must be
1250/// <1 x ty>. Convert the element instead.
1251SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1252 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1253 SDValue Res =
1254 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1255 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1256 Ops: {N->getOperand(Num: 0), Elt});
1257 // Legalize the chain result - switch anything that used the old chain to
1258 // use the new one.
1259 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1260
1261 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1262
1263 // Do our own replacement and return SDValue() to tell the caller that we
1264 // handled all replacements since caller can only handle a single result.
1265 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1266 return SDValue();
1267}
1268
1269SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1270 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1271 // Result type may be wider than element type.
1272 if (Res.getValueType() != N->getValueType(ResNo: 0))
1273 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1274 return Res;
1275}
1276
1277SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1278 SDValue AccOp = N->getOperand(Num: 0);
1279 SDValue VecOp = N->getOperand(Num: 1);
1280
1281 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1282
1283 SDValue Op = GetScalarizedVector(Op: VecOp);
1284 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1285 N1: AccOp, N2: Op, Flags: N->getFlags());
1286}
1287
1288SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1289 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1290 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1291
1292 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1293 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1294 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1295}
1296
1297SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1298 // Since there is no "none-active" result, the only valid return for <1 x ty>
1299 // is 0. Note: Since we check the high mask during splitting this is safe.
1300 // As e.g., a <2 x ty> operation would split to:
1301 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1302 // : `last_active(%lo_mask)`
1303 // Which then scalarizes to:
1304 // %mask[1] ? 1 : 0
1305 EVT VT = N->getValueType(ResNo: 0);
1306 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1307}
1308
1309SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1310 // The number of trailing zero elements is 1 if the element is 0, and 0
1311 // otherwise.
1312 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1313 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1314 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1315 SDValue SetCC =
1316 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1317 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1318 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1319}
1320
1321SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1322 assert(OpNo == 2 && "Can only scalarize mask operand");
1323 SDLoc DL(N);
1324 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1325 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1326 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1327 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1328 // Vectors may have a different boolean contents to scalars, so truncate to i1
1329 // and let type legalization promote appropriately.
1330 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1331 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1332 // use 1 as the divisor to avoid division by zero and overflow.
1333 SDValue BinOp =
1334 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1335 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1336 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1337}
1338
1339//===----------------------------------------------------------------------===//
1340// Result Vector Splitting
1341//===----------------------------------------------------------------------===//
1342
1343/// This method is called when the specified result of the specified node is
1344/// found to need vector splitting. At this point, the node may also have
1345/// invalid operands or may have other results that need legalization, we just
1346/// know that (at least) one result needs vector splitting.
1347void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1348 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1349 SDValue Lo, Hi;
1350
1351 // See if the target wants to custom expand this node.
1352 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1353 return;
1354
1355 switch (N->getOpcode()) {
1356 default:
1357#ifndef NDEBUG
1358 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1359 N->dump(&DAG);
1360 dbgs() << "\n";
1361#endif
1362 report_fatal_error(reason: "Do not know how to split the result of this "
1363 "operator!\n");
1364
1365 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1366 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1367 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1368 break;
1369 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1370 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1371 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1372 case ISD::VSELECT:
1373 case ISD::SELECT:
1374 case ISD::VP_MERGE:
1375 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1376 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1377 case ISD::POISON:
1378 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1379 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1380 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1381 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1382 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1383 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1384 case ISD::FPOWI:
1385 case ISD::FLDEXP:
1386 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1387 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1388 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1389 case ISD::SPLAT_VECTOR:
1390 case ISD::SCALAR_TO_VECTOR:
1391 SplitVecRes_ScalarOp(N, Lo, Hi);
1392 break;
1393 case ISD::STEP_VECTOR:
1394 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1395 break;
1396 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1397 case ISD::ATOMIC_LOAD:
1398 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1399 break;
1400 case ISD::LOAD:
1401 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1402 break;
1403 case ISD::VP_LOAD:
1404 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1405 break;
1406 case ISD::VP_LOAD_FF:
1407 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1408 break;
1409 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1410 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1411 break;
1412 case ISD::MLOAD:
1413 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1414 break;
1415 case ISD::MGATHER:
1416 case ISD::VP_GATHER:
1417 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1418 break;
1419 case ISD::VECTOR_COMPRESS:
1420 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1421 break;
1422 case ISD::SETCC:
1423 case ISD::VP_SETCC:
1424 SplitVecRes_SETCC(N, Lo, Hi);
1425 break;
1426 case ISD::VECTOR_REVERSE:
1427 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1428 break;
1429 case ISD::VECTOR_SHUFFLE:
1430 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1431 break;
1432 case ISD::VECTOR_SPLICE_LEFT:
1433 case ISD::VECTOR_SPLICE_RIGHT:
1434 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1435 break;
1436 case ISD::VECTOR_DEINTERLEAVE:
1437 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1438 return;
1439 case ISD::VECTOR_INTERLEAVE:
1440 SplitVecRes_VECTOR_INTERLEAVE(N);
1441 return;
1442 case ISD::VAARG:
1443 SplitVecRes_VAARG(N, Lo, Hi);
1444 break;
1445
1446 case ISD::ANY_EXTEND_VECTOR_INREG:
1447 case ISD::SIGN_EXTEND_VECTOR_INREG:
1448 case ISD::ZERO_EXTEND_VECTOR_INREG:
1449 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1450 break;
1451
1452 case ISD::ABS:
1453 case ISD::ABS_MIN_POISON:
1454 case ISD::VP_ABS:
1455 case ISD::BITREVERSE:
1456 case ISD::VP_BITREVERSE:
1457 case ISD::BSWAP:
1458 case ISD::VP_BSWAP:
1459 case ISD::CTLZ:
1460 case ISD::VP_CTLZ:
1461 case ISD::CTTZ:
1462 case ISD::VP_CTTZ:
1463 case ISD::CTLZ_ZERO_POISON:
1464 case ISD::VP_CTLZ_ZERO_POISON:
1465 case ISD::CTTZ_ZERO_POISON:
1466 case ISD::VP_CTTZ_ZERO_POISON:
1467 case ISD::CTPOP:
1468 case ISD::VP_CTPOP:
1469 case ISD::FABS: case ISD::VP_FABS:
1470 case ISD::FACOS:
1471 case ISD::FASIN:
1472 case ISD::FATAN:
1473 case ISD::FCEIL:
1474 case ISD::VP_FCEIL:
1475 case ISD::FCOS:
1476 case ISD::FCOSH:
1477 case ISD::FEXP:
1478 case ISD::FEXP2:
1479 case ISD::FEXP10:
1480 case ISD::FFLOOR:
1481 case ISD::VP_FFLOOR:
1482 case ISD::FLOG:
1483 case ISD::FLOG10:
1484 case ISD::FLOG2:
1485 case ISD::FNEARBYINT:
1486 case ISD::VP_FNEARBYINT:
1487 case ISD::FNEG: case ISD::VP_FNEG:
1488 case ISD::FREEZE:
1489 case ISD::ARITH_FENCE:
1490 case ISD::FP_EXTEND:
1491 case ISD::VP_FP_EXTEND:
1492 case ISD::FP_ROUND:
1493 case ISD::VP_FP_ROUND:
1494 case ISD::FP_TO_SINT:
1495 case ISD::VP_FP_TO_SINT:
1496 case ISD::FP_TO_UINT:
1497 case ISD::VP_FP_TO_UINT:
1498 case ISD::FRINT:
1499 case ISD::VP_FRINT:
1500 case ISD::LRINT:
1501 case ISD::VP_LRINT:
1502 case ISD::LLRINT:
1503 case ISD::VP_LLRINT:
1504 case ISD::FROUND:
1505 case ISD::VP_FROUND:
1506 case ISD::FROUNDEVEN:
1507 case ISD::VP_FROUNDEVEN:
1508 case ISD::LROUND:
1509 case ISD::LLROUND:
1510 case ISD::FSIN:
1511 case ISD::FSINH:
1512 case ISD::FSQRT: case ISD::VP_SQRT:
1513 case ISD::FTAN:
1514 case ISD::FTANH:
1515 case ISD::FTRUNC:
1516 case ISD::VP_FROUNDTOZERO:
1517 case ISD::SINT_TO_FP:
1518 case ISD::VP_SINT_TO_FP:
1519 case ISD::TRUNCATE:
1520 case ISD::VP_TRUNCATE:
1521 case ISD::UINT_TO_FP:
1522 case ISD::VP_UINT_TO_FP:
1523 case ISD::FCANONICALIZE:
1524 case ISD::AssertNoFPClass:
1525 case ISD::CONVERT_FROM_ARBITRARY_FP:
1526 case ISD::CONVERT_TO_ARBITRARY_FP:
1527 SplitVecRes_UnaryOp(N, Lo, Hi);
1528 break;
1529 case ISD::ADDRSPACECAST:
1530 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1531 break;
1532 case ISD::FMODF:
1533 case ISD::FFREXP:
1534 case ISD::FSINCOS:
1535 case ISD::FSINCOSPI:
1536 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1537 break;
1538
1539 case ISD::ANY_EXTEND:
1540 case ISD::SIGN_EXTEND:
1541 case ISD::ZERO_EXTEND:
1542 case ISD::VP_SIGN_EXTEND:
1543 case ISD::VP_ZERO_EXTEND:
1544 SplitVecRes_ExtendOp(N, Lo, Hi);
1545 break;
1546
1547 case ISD::ADD: case ISD::VP_ADD:
1548 case ISD::SUB: case ISD::VP_SUB:
1549 case ISD::MUL: case ISD::VP_MUL:
1550 case ISD::CLMUL:
1551 case ISD::CLMULR:
1552 case ISD::CLMULH:
1553 case ISD::PEXT:
1554 case ISD::PDEP:
1555 case ISD::MULHS:
1556 case ISD::MULHU:
1557 case ISD::ABDS:
1558 case ISD::ABDU:
1559 case ISD::AVGCEILS:
1560 case ISD::AVGCEILU:
1561 case ISD::AVGFLOORS:
1562 case ISD::AVGFLOORU:
1563 case ISD::FADD: case ISD::VP_FADD:
1564 case ISD::FSUB: case ISD::VP_FSUB:
1565 case ISD::FMUL: case ISD::VP_FMUL:
1566 case ISD::FMINNUM:
1567 case ISD::FMINNUM_IEEE:
1568 case ISD::VP_FMINNUM:
1569 case ISD::FMAXNUM:
1570 case ISD::FMAXNUM_IEEE:
1571 case ISD::VP_FMAXNUM:
1572 case ISD::FMINIMUM:
1573 case ISD::VP_FMINIMUM:
1574 case ISD::FMAXIMUM:
1575 case ISD::VP_FMAXIMUM:
1576 case ISD::FMINIMUMNUM:
1577 case ISD::FMAXIMUMNUM:
1578 case ISD::SDIV: case ISD::VP_SDIV:
1579 case ISD::UDIV: case ISD::VP_UDIV:
1580 case ISD::FDIV: case ISD::VP_FDIV:
1581 case ISD::FPOW:
1582 case ISD::FATAN2:
1583 case ISD::AND: case ISD::VP_AND:
1584 case ISD::OR: case ISD::VP_OR:
1585 case ISD::XOR: case ISD::VP_XOR:
1586 case ISD::SHL: case ISD::VP_SHL:
1587 case ISD::SRA: case ISD::VP_SRA:
1588 case ISD::SRL: case ISD::VP_SRL:
1589 case ISD::UREM: case ISD::VP_UREM:
1590 case ISD::SREM: case ISD::VP_SREM:
1591 case ISD::FREM: case ISD::VP_FREM:
1592 case ISD::SMIN: case ISD::VP_SMIN:
1593 case ISD::SMAX: case ISD::VP_SMAX:
1594 case ISD::UMIN: case ISD::VP_UMIN:
1595 case ISD::UMAX: case ISD::VP_UMAX:
1596 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1597 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1598 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1599 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1600 case ISD::SSHLSAT:
1601 case ISD::USHLSAT:
1602 case ISD::ROTL:
1603 case ISD::ROTR:
1604 case ISD::VP_FCOPYSIGN:
1605 SplitVecRes_BinOp(N, Lo, Hi);
1606 break;
1607 case ISD::MASKED_UDIV:
1608 case ISD::MASKED_SDIV:
1609 case ISD::MASKED_UREM:
1610 case ISD::MASKED_SREM:
1611 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1612 break;
1613 case ISD::FMA: case ISD::VP_FMA:
1614 case ISD::FSHL:
1615 case ISD::VP_FSHL:
1616 case ISD::FSHR:
1617 case ISD::VP_FSHR:
1618 SplitVecRes_TernaryOp(N, Lo, Hi);
1619 break;
1620
1621 case ISD::SCMP: case ISD::UCMP:
1622 SplitVecRes_CMP(N, Lo, Hi);
1623 break;
1624
1625#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1626 case ISD::STRICT_##DAGN:
1627#include "llvm/IR/ConstrainedOps.def"
1628 SplitVecRes_StrictFPOp(N, Lo, Hi);
1629 break;
1630
1631 case ISD::FP_TO_UINT_SAT:
1632 case ISD::FP_TO_SINT_SAT:
1633 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1634 break;
1635
1636 case ISD::UADDO:
1637 case ISD::SADDO:
1638 case ISD::USUBO:
1639 case ISD::SSUBO:
1640 case ISD::UMULO:
1641 case ISD::SMULO:
1642 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1643 break;
1644 case ISD::SMULFIX:
1645 case ISD::SMULFIXSAT:
1646 case ISD::UMULFIX:
1647 case ISD::UMULFIXSAT:
1648 case ISD::SDIVFIX:
1649 case ISD::SDIVFIXSAT:
1650 case ISD::UDIVFIX:
1651 case ISD::UDIVFIXSAT:
1652 SplitVecRes_FIX(N, Lo, Hi);
1653 break;
1654 case ISD::EXPERIMENTAL_VP_SPLICE:
1655 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1656 break;
1657 case ISD::EXPERIMENTAL_VP_REVERSE:
1658 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1659 break;
1660 case ISD::PARTIAL_REDUCE_UMLA:
1661 case ISD::PARTIAL_REDUCE_SMLA:
1662 case ISD::PARTIAL_REDUCE_SUMLA:
1663 case ISD::PARTIAL_REDUCE_FMLA:
1664 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1665 break;
1666 case ISD::GET_ACTIVE_LANE_MASK:
1667 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1668 break;
1669 }
1670
1671 // If Lo/Hi is null, the sub-method took care of registering results etc.
1672 if (Lo.getNode())
1673 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1674}
1675
1676void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1677 MachinePointerInfo &MPI, SDValue &Ptr,
1678 uint64_t *ScaledOffset) {
1679 SDLoc DL(N);
1680 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1681
1682 if (MemVT.isScalableVector()) {
1683 SDValue BytesIncrement = DAG.getVScale(
1684 DL, VT: Ptr.getValueType(),
1685 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1686 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1687 if (ScaledOffset)
1688 *ScaledOffset += IncrementSize;
1689 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1690 Flags: SDNodeFlags::NoUnsignedWrap);
1691 } else {
1692 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1693 // Increment the pointer to the other half.
1694 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1695 }
1696}
1697
1698std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1699 return SplitMask(Mask, DL: SDLoc(Mask));
1700}
1701
1702std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1703 const SDLoc &DL) {
1704 SDValue MaskLo, MaskHi;
1705 EVT MaskVT = Mask.getValueType();
1706 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1707 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1708 else
1709 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1710 return std::make_pair(x&: MaskLo, y&: MaskHi);
1711}
1712
1713void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1714 SDValue LHSLo, LHSHi;
1715 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1716 SDValue RHSLo, RHSHi;
1717 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1718 SDLoc dl(N);
1719
1720 const SDNodeFlags Flags = N->getFlags();
1721 unsigned Opcode = N->getOpcode();
1722 if (N->getNumOperands() == 2) {
1723 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1724 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1725 return;
1726 }
1727
1728 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1729 assert(N->isVPOpcode() && "Expected VP opcode");
1730
1731 SDValue MaskLo, MaskHi;
1732 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1733
1734 SDValue EVLLo, EVLHi;
1735 std::tie(args&: EVLLo, args&: EVLHi) =
1736 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1737
1738 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1739 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1740 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1741 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1742}
1743
1744void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1745 SDValue &Hi) {
1746 SDValue LHSLo, LHSHi;
1747 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1748 SDValue RHSLo, RHSHi;
1749 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1750 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 2));
1751 SDLoc dl(N);
1752
1753 const SDNodeFlags Flags = N->getFlags();
1754 unsigned Opcode = N->getOpcode();
1755 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1756 Flags);
1757 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1758 Flags);
1759}
1760
1761void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1762 SDValue &Hi) {
1763 SDValue Op0Lo, Op0Hi;
1764 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1765 SDValue Op1Lo, Op1Hi;
1766 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1767 SDValue Op2Lo, Op2Hi;
1768 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1769 SDLoc dl(N);
1770
1771 const SDNodeFlags Flags = N->getFlags();
1772 unsigned Opcode = N->getOpcode();
1773 if (N->getNumOperands() == 3) {
1774 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1775 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1776 return;
1777 }
1778
1779 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1780 assert(N->isVPOpcode() && "Expected VP opcode");
1781
1782 SDValue MaskLo, MaskHi;
1783 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1784
1785 SDValue EVLLo, EVLHi;
1786 std::tie(args&: EVLLo, args&: EVLHi) =
1787 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1788
1789 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1790 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1791 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1792 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1793}
1794
1795void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1796 LLVMContext &Ctxt = *DAG.getContext();
1797 SDLoc dl(N);
1798
1799 SDValue LHS = N->getOperand(Num: 0);
1800 SDValue RHS = N->getOperand(Num: 1);
1801
1802 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1803 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1804 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1805 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1806 } else {
1807 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1808 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1809 }
1810
1811 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1812 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1813 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1814}
1815
1816void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1817 SDValue LHSLo, LHSHi;
1818 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1819 SDValue RHSLo, RHSHi;
1820 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1821 SDLoc dl(N);
1822 SDValue Op2 = N->getOperand(Num: 2);
1823
1824 unsigned Opcode = N->getOpcode();
1825 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1826 Flags: N->getFlags());
1827 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1828 Flags: N->getFlags());
1829}
1830
1831void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1832 SDValue &Hi) {
1833 // We know the result is a vector. The input may be either a vector or a
1834 // scalar value.
1835 EVT LoVT, HiVT;
1836 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1837 SDLoc dl(N);
1838
1839 SDValue InOp = N->getOperand(Num: 0);
1840 EVT InVT = InOp.getValueType();
1841
1842 // Handle some special cases efficiently.
1843 switch (getTypeAction(VT: InVT)) {
1844 case TargetLowering::TypeLegal:
1845 case TargetLowering::TypePromoteInteger:
1846 case TargetLowering::TypeSoftPromoteHalf:
1847 case TargetLowering::TypeSoftenFloat:
1848 case TargetLowering::TypeScalarizeVector:
1849 case TargetLowering::TypeWidenVector:
1850 break;
1851 case TargetLowering::TypeExpandInteger:
1852 case TargetLowering::TypeExpandFloat:
1853 // A scalar to vector conversion, where the scalar needs expansion.
1854 // If the vector is being split in two then we can just convert the
1855 // expanded pieces.
1856 if (LoVT == HiVT) {
1857 GetExpandedOp(Op: InOp, Lo, Hi);
1858 if (DAG.getDataLayout().isBigEndian())
1859 std::swap(a&: Lo, b&: Hi);
1860 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1861 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1862 return;
1863 }
1864 break;
1865 case TargetLowering::TypeSplitVector:
1866 // If the input is a vector that needs to be split, convert each split
1867 // piece of the input now.
1868 GetSplitVector(Op: InOp, Lo, Hi);
1869 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1870 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1871 return;
1872 case TargetLowering::TypeScalarizeScalableVector:
1873 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1874 }
1875
1876 if (LoVT.isScalableVector()) {
1877 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1878 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1879 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1880 return;
1881 }
1882
1883 // In the general case, convert the input to an integer and split it by hand.
1884 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1885 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1886 if (DAG.getDataLayout().isBigEndian())
1887 std::swap(a&: LoIntVT, b&: HiIntVT);
1888
1889 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1890
1891 if (DAG.getDataLayout().isBigEndian())
1892 std::swap(a&: Lo, b&: Hi);
1893 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1894 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1895}
1896
1897void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1898 SDValue &Hi) {
1899 SDLoc DL(N);
1900 EVT LoVT, HiVT;
1901 SDValue PtrA = N->getOperand(Num: 0);
1902 SDValue PtrB = N->getOperand(Num: 1);
1903 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1904
1905 // The lane offset for the "Lo" half of the mask is unchanged.
1906 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1907 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1908 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1909 // The lane offset for the "Hi" half of the mask is incremented by the number
1910 // of elements in the "Lo" half.
1911 unsigned LaneOffset =
1912 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1913 // Note: The lane offset is implicitly scalable for scalable masks.
1914 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1915 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1916 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1917}
1918
1919void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1920 SDValue &Hi) {
1921 EVT LoVT, HiVT;
1922 SDLoc dl(N);
1923 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1924 unsigned LoNumElts = LoVT.getVectorNumElements();
1925 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1926 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1927
1928 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1929 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1930}
1931
1932void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1933 SDValue &Hi) {
1934 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1935 SDLoc dl(N);
1936 unsigned NumSubvectors = N->getNumOperands() / 2;
1937 if (NumSubvectors == 1) {
1938 Lo = N->getOperand(Num: 0);
1939 Hi = N->getOperand(Num: 1);
1940 return;
1941 }
1942
1943 EVT LoVT, HiVT;
1944 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1945
1946 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1947 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1948
1949 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1950 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1951}
1952
1953void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1954 SDValue &Hi) {
1955 SDValue Vec = N->getOperand(Num: 0);
1956 SDValue Idx = N->getOperand(Num: 1);
1957 SDLoc dl(N);
1958
1959 EVT LoVT, HiVT;
1960 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1961
1962 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1963 uint64_t IdxVal = Idx->getAsZExtVal();
1964 Hi = DAG.getNode(
1965 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1966 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1967}
1968
1969void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1970 SDValue &Hi) {
1971 SDValue Vec = N->getOperand(Num: 0);
1972 SDValue SubVec = N->getOperand(Num: 1);
1973 SDValue Idx = N->getOperand(Num: 2);
1974 SDLoc dl(N);
1975 GetSplitVector(Op: Vec, Lo, Hi);
1976
1977 EVT VecVT = Vec.getValueType();
1978 EVT LoVT = Lo.getValueType();
1979 EVT SubVecVT = SubVec.getValueType();
1980 unsigned VecElems = VecVT.getVectorMinNumElements();
1981 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1982 unsigned LoElems = LoVT.getVectorMinNumElements();
1983
1984 // If we know the index is in the first half, and we know the subvector
1985 // doesn't cross the boundary between the halves, we can avoid spilling the
1986 // vector, and insert into the lower half of the split vector directly.
1987 unsigned IdxVal = Idx->getAsZExtVal();
1988 if (IdxVal + SubElems <= LoElems) {
1989 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1990 return;
1991 }
1992 // Similarly if the subvector is fully in the high half, but mind that we
1993 // can't tell whether a fixed-length subvector is fully within the high half
1994 // of a scalable vector.
1995 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1996 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1997 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1998 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1999 return;
2000 }
2001
2002 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
2003 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
2004 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
2005 if (WideSubVec.getValueType() == VecVT) {
2006 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
2007 return;
2008 }
2009 }
2010
2011 // Spill the vector to the stack.
2012 // In cases where the vector is illegal it will be broken down into parts
2013 // and stored in parts - we should use the alignment for the smallest part.
2014 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2015 SDValue StackPtr =
2016 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2017 auto &MF = DAG.getMachineFunction();
2018 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2019 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2020
2021 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2022 Alignment: SmallestAlign);
2023
2024 // Store the new subvector into the specified index.
2025 SDValue SubVecPtr =
2026 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2027 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2028 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2029
2030 // Load the Lo part from the stack slot.
2031 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2032 Alignment: SmallestAlign);
2033
2034 // Increment the pointer to the other part.
2035 auto *Load = cast<LoadSDNode>(Val&: Lo);
2036 MachinePointerInfo MPI = Load->getPointerInfo();
2037 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2038
2039 // Load the Hi part from the stack slot.
2040 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2041}
2042
2043// Handle splitting an FP where the second operand does not match the first
2044// type. The second operand may be a scalar, or a vector that has exactly as
2045// many elements as the first
2046void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2047 SDValue &Hi) {
2048 SDValue LHSLo, LHSHi;
2049 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2050 SDLoc DL(N);
2051
2052 SDValue RHSLo, RHSHi;
2053 SDValue RHS = N->getOperand(Num: 1);
2054 EVT RHSVT = RHS.getValueType();
2055 if (RHSVT.isVector()) {
2056 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2057 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2058 else
2059 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2060
2061 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2062 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2063 } else {
2064 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2065 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2066 }
2067}
2068
2069void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2070 SDValue &Hi) {
2071 SDLoc DL(N);
2072 SDValue ArgLo, ArgHi;
2073 SDValue Test = N->getOperand(Num: 1);
2074 SDValue FpValue = N->getOperand(Num: 0);
2075 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2076 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2077 else
2078 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2079 EVT LoVT, HiVT;
2080 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2081
2082 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2083 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2084}
2085
2086void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2087 SDValue &Hi) {
2088 SDValue LHSLo, LHSHi;
2089 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2090 SDLoc dl(N);
2091
2092 EVT LoVT, HiVT;
2093 std::tie(args&: LoVT, args&: HiVT) =
2094 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2095
2096 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2097 N2: DAG.getValueType(LoVT));
2098 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2099 N2: DAG.getValueType(HiVT));
2100}
2101
2102void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2103 SDValue &Hi) {
2104 unsigned Opcode = N->getOpcode();
2105 SDValue N0 = N->getOperand(Num: 0);
2106
2107 SDLoc dl(N);
2108 SDValue InLo, InHi;
2109
2110 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2111 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2112 else
2113 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2114
2115 EVT InLoVT = InLo.getValueType();
2116 unsigned InNumElements = InLoVT.getVectorNumElements();
2117
2118 EVT OutLoVT, OutHiVT;
2119 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2120 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2121 assert((2 * OutNumElements) <= InNumElements &&
2122 "Illegal extend vector in reg split");
2123
2124 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2125 // input vector (i.e. we only use InLo):
2126 // OutLo will extend the first OutNumElements from InLo.
2127 // OutHi will extend the next OutNumElements from InLo.
2128
2129 // Shuffle the elements from InLo for OutHi into the bottom elements to
2130 // create a 'fake' InHi.
2131 SmallVector<int, 8> SplitHi(InNumElements, -1);
2132 for (unsigned i = 0; i != OutNumElements; ++i)
2133 SplitHi[i] = i + OutNumElements;
2134 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2135
2136 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2137 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2138}
2139
2140void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2141 SDValue &Hi) {
2142 unsigned NumOps = N->getNumOperands();
2143 SDValue Chain = N->getOperand(Num: 0);
2144 EVT LoVT, HiVT;
2145 SDLoc dl(N);
2146 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2147
2148 SmallVector<SDValue, 4> OpsLo(NumOps);
2149 SmallVector<SDValue, 4> OpsHi(NumOps);
2150
2151 // The Chain is the first operand.
2152 OpsLo[0] = Chain;
2153 OpsHi[0] = Chain;
2154
2155 // Now process the remaining operands.
2156 for (unsigned i = 1; i < NumOps; ++i) {
2157 SDValue Op = N->getOperand(Num: i);
2158 SDValue OpLo = Op;
2159 SDValue OpHi = Op;
2160
2161 EVT InVT = Op.getValueType();
2162 if (InVT.isVector()) {
2163 // If the input also splits, handle it directly for a
2164 // compile time speedup. Otherwise split it by hand.
2165 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2166 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2167 else
2168 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2169 }
2170
2171 OpsLo[i] = OpLo;
2172 OpsHi[i] = OpHi;
2173 }
2174
2175 EVT LoValueVTs[] = {LoVT, MVT::Other};
2176 EVT HiValueVTs[] = {HiVT, MVT::Other};
2177 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2178 Flags: N->getFlags());
2179 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2180 Flags: N->getFlags());
2181
2182 // Build a factor node to remember that this Op is independent of the
2183 // other one.
2184 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2185 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2186
2187 // Legalize the chain result - switch anything that used the old chain to
2188 // use the new one.
2189 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2190}
2191
2192SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2193 SDValue Chain = N->getOperand(Num: 0);
2194 EVT VT = N->getValueType(ResNo: 0);
2195 unsigned NE = VT.getVectorNumElements();
2196 EVT EltVT = VT.getVectorElementType();
2197 SDLoc dl(N);
2198
2199 SmallVector<SDValue, 8> Scalars;
2200 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2201
2202 // If ResNE is 0, fully unroll the vector op.
2203 if (ResNE == 0)
2204 ResNE = NE;
2205 else if (NE > ResNE)
2206 NE = ResNE;
2207
2208 //The results of each unrolled operation, including the chain.
2209 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2210 SmallVector<SDValue, 8> Chains;
2211
2212 unsigned i;
2213 for (i = 0; i != NE; ++i) {
2214 Operands[0] = Chain;
2215 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2216 SDValue Operand = N->getOperand(Num: j);
2217 EVT OperandVT = Operand.getValueType();
2218 if (OperandVT.isVector()) {
2219 EVT OperandEltVT = OperandVT.getVectorElementType();
2220 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2221 } else {
2222 Operands[j] = Operand;
2223 }
2224 }
2225 SDValue Scalar =
2226 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2227
2228 //Add in the scalar as well as its chain value to the
2229 //result vectors.
2230 Scalars.push_back(Elt: Scalar);
2231 Chains.push_back(Elt: Scalar.getValue(R: 1));
2232 }
2233
2234 for (; i < ResNE; ++i)
2235 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2236
2237 // Build a new factor node to connect the chain back together.
2238 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2239 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2240
2241 // Create a new BUILD_VECTOR node
2242 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2243 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2244}
2245
2246void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2247 SDValue &Lo, SDValue &Hi) {
2248 SDLoc dl(N);
2249 EVT ResVT = N->getValueType(ResNo: 0);
2250 EVT OvVT = N->getValueType(ResNo: 1);
2251 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2252 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2253 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2254
2255 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2256 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2257 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2258 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2259 } else {
2260 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2261 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2262 }
2263
2264 unsigned Opcode = N->getOpcode();
2265 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2266 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2267 SDNode *LoNode =
2268 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2269 SDNode *HiNode =
2270 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2271
2272 Lo = SDValue(LoNode, ResNo);
2273 Hi = SDValue(HiNode, ResNo);
2274
2275 // Replace the other vector result not being explicitly split here.
2276 unsigned OtherNo = 1 - ResNo;
2277 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2278 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2279 SetSplitVector(Op: SDValue(N, OtherNo),
2280 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2281 } else {
2282 SDValue OtherVal = DAG.getNode(
2283 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2284 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2285 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2286 }
2287}
2288
2289void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2290 SDValue &Hi) {
2291 SDValue Vec = N->getOperand(Num: 0);
2292 SDValue Elt = N->getOperand(Num: 1);
2293 SDValue Idx = N->getOperand(Num: 2);
2294 SDLoc dl(N);
2295 GetSplitVector(Op: Vec, Lo, Hi);
2296
2297 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2298 unsigned IdxVal = CIdx->getZExtValue();
2299 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2300 if (IdxVal < LoNumElts) {
2301 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2302 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2303 return;
2304 } else if (!Vec.getValueType().isScalableVector()) {
2305 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2306 return;
2307 }
2308 }
2309
2310 // Make the vector elements byte-addressable if they aren't already.
2311 EVT VecVT = Vec.getValueType();
2312 EVT EltVT = VecVT.getVectorElementType();
2313 if (!EltVT.isByteSized()) {
2314 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2315 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2316 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2317 // Extend the element type to match if needed.
2318 if (EltVT.bitsGT(VT: Elt.getValueType()))
2319 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2320 }
2321
2322 // Spill the vector to the stack.
2323 // In cases where the vector is illegal it will be broken down into parts
2324 // and stored in parts - we should use the alignment for the smallest part.
2325 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2326 SDValue StackPtr =
2327 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2328 auto &MF = DAG.getMachineFunction();
2329 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2330 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2331
2332 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2333 Alignment: SmallestAlign);
2334
2335 // Store the new element. This may be larger than the vector element type,
2336 // so use a truncating store.
2337 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2338 Store = DAG.getTruncStore(
2339 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2340 Alignment: commonAlignment(A: SmallestAlign,
2341 Offset: EltVT.getFixedSizeInBits() / 8));
2342
2343 EVT LoVT, HiVT;
2344 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2345
2346 // Load the Lo part from the stack slot.
2347 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2348
2349 // Increment the pointer to the other part.
2350 auto Load = cast<LoadSDNode>(Val&: Lo);
2351 MachinePointerInfo MPI = Load->getPointerInfo();
2352 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2353
2354 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2355
2356 // If we adjusted the original type, we need to truncate the results.
2357 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2358 if (LoVT != Lo.getValueType())
2359 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2360 if (HiVT != Hi.getValueType())
2361 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2362}
2363
2364void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2365 SDValue &Hi) {
2366 EVT LoVT, HiVT;
2367 SDLoc dl(N);
2368 assert(N->getValueType(0).isScalableVector() &&
2369 "Only scalable vectors are supported for STEP_VECTOR");
2370 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2371 SDValue Step = N->getOperand(Num: 0);
2372
2373 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2374
2375 // Hi = Lo + (EltCnt * Step)
2376 EVT EltVT = Step.getValueType();
2377 APInt StepVal = Step->getAsAPIntVal();
2378 SDValue StartOfHi =
2379 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2380 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2381 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2382
2383 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2384 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2385}
2386
2387void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2388 SDValue &Hi) {
2389 EVT LoVT, HiVT;
2390 SDLoc dl(N);
2391 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2392 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2393 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2394 Hi = DAG.getPOISON(VT: HiVT);
2395 } else {
2396 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2397 Hi = Lo;
2398 }
2399}
2400
2401void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2402 SDValue &Hi) {
2403 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2404 "Extended load during type legalization!");
2405 SDLoc dl(LD);
2406 EVT VT = LD->getValueType(ResNo: 0);
2407 EVT LoVT, HiVT;
2408 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2409
2410 SDValue Ch = LD->getChain();
2411 SDValue Ptr = LD->getBasePtr();
2412
2413 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2414 EVT MemIntVT =
2415 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2416 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2417 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2418
2419 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2420 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2421 SDValue ExtractLo, ExtractHi;
2422 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2423
2424 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2425 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2426
2427 // Legalize the chain result - switch anything that used the old chain to
2428 // use the new one.
2429 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2430}
2431
2432void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2433 SDValue &Hi) {
2434 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2435 EVT LoVT, HiVT;
2436 SDLoc dl(LD);
2437 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2438
2439 ISD::LoadExtType ExtType = LD->getExtensionType();
2440 SDValue Ch = LD->getChain();
2441 SDValue Ptr = LD->getBasePtr();
2442 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2443 EVT MemoryVT = LD->getMemoryVT();
2444 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2445 AAMDNodes AAInfo = LD->getAAInfo();
2446
2447 EVT LoMemVT, HiMemVT;
2448 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2449
2450 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2451 SDValue Value, NewChain;
2452 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2453 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2454 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2455 return;
2456 }
2457
2458 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2459 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2460 AAInfo);
2461
2462 MachinePointerInfo MPI;
2463 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2464
2465 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2466 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2467
2468 // Build a factor node to remember that this load is independent of the
2469 // other one.
2470 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2471 N2: Hi.getValue(R: 1));
2472
2473 // Legalize the chain result - switch anything that used the old chain to
2474 // use the new one.
2475 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2476}
2477
2478void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2479 SDValue &Hi) {
2480 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2481 EVT LoVT, HiVT;
2482 SDLoc dl(LD);
2483 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2484
2485 ISD::LoadExtType ExtType = LD->getExtensionType();
2486 SDValue Ch = LD->getChain();
2487 SDValue Ptr = LD->getBasePtr();
2488 SDValue Offset = LD->getOffset();
2489 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2490 Align Alignment = LD->getBaseAlign();
2491 SDValue Mask = LD->getMask();
2492 SDValue EVL = LD->getVectorLength();
2493 EVT MemoryVT = LD->getMemoryVT();
2494
2495 EVT LoMemVT, HiMemVT;
2496 bool HiIsEmpty = false;
2497 std::tie(args&: LoMemVT, args&: HiMemVT) =
2498 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2499
2500 // Split Mask operand
2501 SDValue MaskLo, MaskHi;
2502 if (Mask.getOpcode() == ISD::SETCC) {
2503 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2504 } else {
2505 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2506 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2507 else
2508 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2509 }
2510
2511 // Split EVL operand
2512 SDValue EVLLo, EVLHi;
2513 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2514
2515 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2516 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2517 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2518 Ranges: LD->getRanges());
2519
2520 Lo =
2521 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2522 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2523
2524 if (HiIsEmpty) {
2525 // The hi vp_load has zero storage size. We therefore simply set it to
2526 // the low vp_load and rely on subsequent removal from the chain.
2527 Hi = Lo;
2528 } else {
2529 // Generate hi vp_load.
2530 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2531 IsCompressedMemory: LD->isExpandingLoad());
2532
2533 MachinePointerInfo MPI;
2534 if (LoMemVT.isScalableVector())
2535 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2536 else
2537 MPI = LD->getPointerInfo().getWithOffset(
2538 O: LoMemVT.getStoreSize().getFixedValue());
2539
2540 MMO = DAG.getMachineFunction().getMachineMemOperand(
2541 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2542 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2543
2544 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2545 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2546 IsExpanding: LD->isExpandingLoad());
2547 }
2548
2549 // Build a factor node to remember that this load is independent of the
2550 // other one.
2551 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2552 N2: Hi.getValue(R: 1));
2553
2554 // Legalize the chain result - switch anything that used the old chain to
2555 // use the new one.
2556 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2557}
2558
2559void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2560 SDValue &Hi) {
2561 SDLoc dl(LD);
2562 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2563
2564 SDValue Ch = LD->getChain();
2565 SDValue Ptr = LD->getBasePtr();
2566 Align Alignment = LD->getBaseAlign();
2567 SDValue Mask = LD->getMask();
2568 SDValue EVL = LD->getVectorLength();
2569
2570 // Split Mask operand
2571 SDValue MaskLo, MaskHi;
2572 if (Mask.getOpcode() == ISD::SETCC) {
2573 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2574 } else {
2575 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2576 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2577 else
2578 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2579 }
2580
2581 // Split EVL operand
2582 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2583
2584 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2585 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2586 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2587 Ranges: LD->getRanges());
2588
2589 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2590
2591 // Fill the upper half with poison.
2592 Hi = DAG.getPOISON(VT: HiVT);
2593
2594 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2595 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2596}
2597
2598void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2599 SDValue &Lo, SDValue &Hi) {
2600 assert(SLD->isUnindexed() &&
2601 "Indexed VP strided load during type legalization!");
2602 assert(SLD->getOffset().isUndef() &&
2603 "Unexpected indexed variable-length load offset");
2604
2605 SDLoc DL(SLD);
2606
2607 EVT LoVT, HiVT;
2608 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2609
2610 EVT LoMemVT, HiMemVT;
2611 bool HiIsEmpty = false;
2612 std::tie(args&: LoMemVT, args&: HiMemVT) =
2613 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2614
2615 SDValue Mask = SLD->getMask();
2616 SDValue LoMask, HiMask;
2617 if (Mask.getOpcode() == ISD::SETCC) {
2618 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2619 } else {
2620 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2621 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2622 else
2623 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2624 }
2625
2626 SDValue LoEVL, HiEVL;
2627 std::tie(args&: LoEVL, args&: HiEVL) =
2628 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2629
2630 // Generate the low vp_strided_load
2631 Lo = DAG.getStridedLoadVP(
2632 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2633 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2634 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2635
2636 if (HiIsEmpty) {
2637 // The high vp_strided_load has zero storage size. We therefore simply set
2638 // it to the low vp_strided_load and rely on subsequent removal from the
2639 // chain.
2640 Hi = Lo;
2641 } else {
2642 // Generate the high vp_strided_load.
2643 // To calculate the high base address, we need to sum to the low base
2644 // address stride number of bytes for each element already loaded by low,
2645 // that is: Ptr = Ptr + (LoEVL * Stride)
2646 EVT PtrVT = SLD->getBasePtr().getValueType();
2647 SDValue Increment =
2648 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2649 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2650 SDValue Ptr =
2651 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2652
2653 Align Alignment = SLD->getBaseAlign();
2654 if (LoMemVT.isScalableVector())
2655 Alignment = commonAlignment(
2656 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2657
2658 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2659 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2660 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2661 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2662
2663 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2664 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2665 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2666 IsExpanding: SLD->isExpandingLoad());
2667 }
2668
2669 // Build a factor node to remember that this load is independent of the
2670 // other one.
2671 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2672 N2: Hi.getValue(R: 1));
2673
2674 // Legalize the chain result - switch anything that used the old chain to
2675 // use the new one.
2676 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2677}
2678
2679void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2680 SDValue &Lo, SDValue &Hi) {
2681 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2682 EVT LoVT, HiVT;
2683 SDLoc dl(MLD);
2684 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2685
2686 SDValue Ch = MLD->getChain();
2687 SDValue Ptr = MLD->getBasePtr();
2688 SDValue Offset = MLD->getOffset();
2689 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2690 SDValue Mask = MLD->getMask();
2691 SDValue PassThru = MLD->getPassThru();
2692 Align Alignment = MLD->getBaseAlign();
2693 ISD::LoadExtType ExtType = MLD->getExtensionType();
2694 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2695
2696 // Split Mask operand
2697 SDValue MaskLo, MaskHi;
2698 if (Mask.getOpcode() == ISD::SETCC) {
2699 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2700 } else {
2701 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2702 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2703 else
2704 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2705 }
2706
2707 EVT MemoryVT = MLD->getMemoryVT();
2708 EVT LoMemVT, HiMemVT;
2709 bool HiIsEmpty = false;
2710 std::tie(args&: LoMemVT, args&: HiMemVT) =
2711 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2712
2713 SDValue PassThruLo, PassThruHi;
2714 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2715 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2716 else
2717 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2718
2719 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2720 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2721 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2722
2723 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2724 MMO, AM: MLD->getAddressingMode(), ExtType,
2725 IsExpanding: MLD->isExpandingLoad());
2726
2727 if (HiIsEmpty) {
2728 // The hi masked load has zero storage size. We therefore simply set it to
2729 // the low masked load and rely on subsequent removal from the chain.
2730 Hi = Lo;
2731 } else {
2732 // Generate hi masked load.
2733 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2734 IsCompressedMemory: MLD->isExpandingLoad());
2735
2736 MachinePointerInfo MPI;
2737 if (LoMemVT.isScalableVector())
2738 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2739 else
2740 MPI = MLD->getPointerInfo().getWithOffset(
2741 O: LoMemVT.getStoreSize().getFixedValue());
2742
2743 MMO = DAG.getMachineFunction().getMachineMemOperand(
2744 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2745 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2746
2747 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2748 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2749 IsExpanding: MLD->isExpandingLoad());
2750 }
2751
2752 // Build a factor node to remember that this load is independent of the
2753 // other one.
2754 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2755 N2: Hi.getValue(R: 1));
2756
2757 // Legalize the chain result - switch anything that used the old chain to
2758 // use the new one.
2759 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2760
2761}
2762
2763void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2764 SDValue &Hi, bool SplitSETCC) {
2765 EVT LoVT, HiVT;
2766 SDLoc dl(N);
2767 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2768
2769 SDValue Ch = N->getChain();
2770 SDValue Ptr = N->getBasePtr();
2771 struct Operands {
2772 SDValue Mask;
2773 SDValue Index;
2774 SDValue Scale;
2775 } Ops = [&]() -> Operands {
2776 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2777 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2778 }
2779 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2780 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2781 }();
2782
2783 EVT MemoryVT = N->getMemoryVT();
2784 Align Alignment = N->getBaseAlign();
2785
2786 // Split Mask operand
2787 SDValue MaskLo, MaskHi;
2788 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2789 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2790 } else {
2791 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2792 }
2793
2794 EVT LoMemVT, HiMemVT;
2795 // Split MemoryVT
2796 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2797
2798 SDValue IndexHi, IndexLo;
2799 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2800 TargetLowering::TypeSplitVector)
2801 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2802 else
2803 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2804
2805 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2806 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2807 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2808 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2809
2810 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2811 SDValue PassThru = MGT->getPassThru();
2812 SDValue PassThruLo, PassThruHi;
2813 if (getTypeAction(VT: PassThru.getValueType()) ==
2814 TargetLowering::TypeSplitVector)
2815 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2816 else
2817 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2818
2819 ISD::LoadExtType ExtType = MGT->getExtensionType();
2820 ISD::MemIndexType IndexTy = MGT->getIndexType();
2821
2822 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2823 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2824 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2825
2826 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2827 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2828 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2829 } else {
2830 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2831 SDValue EVLLo, EVLHi;
2832 std::tie(args&: EVLLo, args&: EVLHi) =
2833 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2834
2835 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2836 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2837 MMO, IndexType: VPGT->getIndexType());
2838
2839 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2840 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2841 MMO, IndexType: VPGT->getIndexType());
2842 }
2843
2844 // Build a factor node to remember that this load is independent of the
2845 // other one.
2846 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2847 N2: Hi.getValue(R: 1));
2848
2849 // Legalize the chain result - switch anything that used the old chain to
2850 // use the new one.
2851 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2852}
2853
2854void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2855 SDValue &Hi) {
2856 // This is not "trivial", as there is a dependency between the two subvectors.
2857 // Depending on the number of 1s in the mask, the elements from the Hi vector
2858 // need to be moved to the Lo vector. Passthru values make this even harder.
2859 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2860 // smaller types and passthru is undef, as it is most likely faster than the
2861 // fully expand path. Otherwise, just do the full expansion as one "big"
2862 // operation and then extract the Lo and Hi vectors from that. This gets
2863 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2864 SDLoc DL(N);
2865 EVT VecVT = N->getValueType(ResNo: 0);
2866
2867 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2868 bool HasCustomLowering = false;
2869 EVT CheckVT = LoVT;
2870 while (CheckVT.getVectorMinNumElements() > 1) {
2871 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2872 // custom lowering for illegal types. So we do the checks separately.
2873 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2874 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2875 HasCustomLowering = true;
2876 break;
2877 }
2878 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2879 }
2880
2881 SDValue Passthru = N->getOperand(Num: 2);
2882 if (!HasCustomLowering) {
2883 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2884 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2885 return;
2886 }
2887
2888 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2889 SDValue Mask = N->getOperand(Num: 1);
2890 SDValue LoMask, HiMask;
2891 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2892 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2893
2894 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2895 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2896 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2897
2898 SDValue StackPtr = DAG.CreateStackTemporary(
2899 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2900 MachineFunction &MF = DAG.getMachineFunction();
2901 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2902 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2903
2904 EVT MaskVT = LoMask.getValueType();
2905 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2906
2907 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2908 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2909 EC: MaskVT.getVectorElementCount());
2910 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2911 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2912 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2913
2914 SDValue Chain = DAG.getEntryNode();
2915 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2916 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2917 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2918
2919 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2920 if (!Passthru.isUndef()) {
2921 Compressed =
2922 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2923 }
2924 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2925}
2926
2927void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2928 assert(N->getValueType(0).isVector() &&
2929 N->getOperand(0).getValueType().isVector() &&
2930 "Operand types must be vectors");
2931
2932 EVT LoVT, HiVT;
2933 SDLoc DL(N);
2934 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2935
2936 // If the input also splits, handle it directly. Otherwise split it by hand.
2937 SDValue LL, LH, RL, RH;
2938 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2939 TargetLowering::TypeSplitVector)
2940 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2941 else
2942 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2943
2944 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2945 TargetLowering::TypeSplitVector)
2946 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2947 else
2948 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2949
2950 if (N->getOpcode() == ISD::SETCC) {
2951 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2952 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2953 } else {
2954 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2955 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2956 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2957 std::tie(args&: EVLLo, args&: EVLHi) =
2958 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2959 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2960 N5: EVLLo);
2961 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2962 N5: EVLHi);
2963 }
2964}
2965
2966void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2967 SDValue &Hi) {
2968 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2969 EVT LoVT, HiVT;
2970 SDLoc dl(N);
2971 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2972
2973 // If the input also splits, handle it directly for a compile time speedup.
2974 // Otherwise split it by hand.
2975 EVT InVT = N->getOperand(Num: 0).getValueType();
2976 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2977 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2978 else
2979 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2980
2981 const SDNodeFlags Flags = N->getFlags();
2982 unsigned Opcode = N->getOpcode();
2983 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
2984 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2985 N4: N->getOperand(Num: 3), Flags);
2986 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2987 N4: N->getOperand(Num: 3), Flags);
2988 return;
2989 }
2990 if (N->getNumOperands() <= 2) {
2991 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2992 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2993 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2994 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2995 } else {
2996 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2997 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2998 }
2999 return;
3000 }
3001
3002 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
3003 assert(N->isVPOpcode() && "Expected VP opcode");
3004
3005 SDValue MaskLo, MaskHi;
3006 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3007
3008 SDValue EVLLo, EVLHi;
3009 std::tie(args&: EVLLo, args&: EVLHi) =
3010 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3011
3012 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
3013 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
3014}
3015
3016void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3017 SDValue &Hi) {
3018 SDLoc dl(N);
3019 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3020
3021 // If the input also splits, handle it directly for a compile time speedup.
3022 // Otherwise split it by hand.
3023 EVT InVT = N->getOperand(Num: 0).getValueType();
3024 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3025 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3026 else
3027 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3028
3029 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3030 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3031 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3032 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
3033 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
3034}
3035
3036void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3037 unsigned ResNo,
3038 SDValue &Lo,
3039 SDValue &Hi) {
3040 SDLoc dl(N);
3041 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3042 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3043
3044 // If the input also splits, handle it directly for a compile time speedup.
3045 // Otherwise split it by hand.
3046 EVT InVT = N->getOperand(Num: 0).getValueType();
3047 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3048 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3049 else
3050 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3051
3052 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3053 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3054
3055 SDNode *HiNode = Hi.getNode();
3056 SDNode *LoNode = Lo.getNode();
3057
3058 // Replace the other vector result not being explicitly split here.
3059 unsigned OtherNo = 1 - ResNo;
3060 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3061 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3062 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3063 Hi: SDValue(HiNode, OtherNo));
3064 } else {
3065 SDValue OtherVal =
3066 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3067 N2: SDValue(HiNode, OtherNo));
3068 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3069 }
3070}
3071
3072void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3073 SDValue &Hi) {
3074 SDLoc dl(N);
3075 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3076 EVT DestVT = N->getValueType(ResNo: 0);
3077 EVT LoVT, HiVT;
3078 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3079
3080 // We can do better than a generic split operation if the extend is doing
3081 // more than just doubling the width of the elements and the following are
3082 // true:
3083 // - The number of vector elements is even,
3084 // - the source type is legal,
3085 // - the type of a split source is illegal,
3086 // - the type of an extended (by doubling element size) source is legal, and
3087 // - the type of that extended source when split is legal.
3088 //
3089 // This won't necessarily completely legalize the operation, but it will
3090 // more effectively move in the right direction and prevent falling down
3091 // to scalarization in many cases due to the input vector being split too
3092 // far.
3093 if (SrcVT.getVectorElementCount().isKnownEven() &&
3094 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3095 LLVMContext &Ctx = *DAG.getContext();
3096 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3097 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3098
3099 EVT SplitLoVT, SplitHiVT;
3100 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3101 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3102 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3103 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3104 N->dump(&DAG); dbgs() << "\n");
3105 if (!N->isVPOpcode()) {
3106 // Extend the source vector by one step.
3107 SDValue NewSrc =
3108 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3109 // Get the low and high halves of the new, extended one step, vector.
3110 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3111 // Extend those vector halves the rest of the way.
3112 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3113 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3114 return;
3115 }
3116
3117 // Extend the source vector by one step.
3118 SDValue NewSrc =
3119 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
3120 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
3121 // Get the low and high halves of the new, extended one step, vector.
3122 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3123
3124 SDValue MaskLo, MaskHi;
3125 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3126
3127 SDValue EVLLo, EVLHi;
3128 std::tie(args&: EVLLo, args&: EVLHi) =
3129 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3130 // Extend those vector halves the rest of the way.
3131 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
3132 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
3133 return;
3134 }
3135 }
3136 // Fall back to the generic unary operator splitting otherwise.
3137 SplitVecRes_UnaryOp(N, Lo, Hi);
3138}
3139
3140void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3141 SDValue &Lo, SDValue &Hi) {
3142 // The low and high parts of the original input give four input vectors.
3143 SDValue Inputs[4];
3144 SDLoc DL(N);
3145 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3146 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3147 EVT NewVT = Inputs[0].getValueType();
3148 unsigned NewElts = NewVT.getVectorNumElements();
3149
3150 auto &&IsConstant = [](const SDValue &N) {
3151 APInt SplatValue;
3152 return N.getResNo() == 0 &&
3153 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3154 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3155 };
3156 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3157 SDValue &Input2,
3158 ArrayRef<int> Mask) {
3159 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3160 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3161 "Expected build vector node.");
3162 EVT EltVT = NewVT.getVectorElementType();
3163 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3164 for (unsigned I = 0; I < NewElts; ++I) {
3165 if (Mask[I] == PoisonMaskElem)
3166 continue;
3167 unsigned Idx = Mask[I];
3168 if (Idx >= NewElts)
3169 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3170 else
3171 Ops[I] = Input1.getOperand(i: Idx);
3172 // Make the type of all elements the same as the element type.
3173 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3174 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3175 }
3176 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3177 };
3178
3179 // If Lo or Hi uses elements from at most two of the four input vectors, then
3180 // express it as a vector shuffle of those two inputs. Otherwise extract the
3181 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3182 SmallVector<int> OrigMask(N->getMask());
3183 // Try to pack incoming shuffles/inputs.
3184 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3185 &DL](SmallVectorImpl<int> &Mask) {
3186 // Check if all inputs are shuffles of the same operands or non-shuffles.
3187 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3188 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3189 SDValue Input = Inputs[Idx];
3190 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3191 if (!Shuffle ||
3192 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3193 continue;
3194 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3195 .push_back(Elt: Idx);
3196 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3197 .push_back(Elt: Idx);
3198 }
3199 for (auto &P : ShufflesIdxs) {
3200 if (P.second.size() < 2)
3201 continue;
3202 // Use shuffles operands instead of shuffles themselves.
3203 // 1. Adjust mask.
3204 for (int &Idx : Mask) {
3205 if (Idx == PoisonMaskElem)
3206 continue;
3207 unsigned SrcRegIdx = Idx / NewElts;
3208 if (Inputs[SrcRegIdx].isUndef()) {
3209 Idx = PoisonMaskElem;
3210 continue;
3211 }
3212 auto *Shuffle =
3213 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3214 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3215 continue;
3216 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3217 if (MaskElt == PoisonMaskElem) {
3218 Idx = PoisonMaskElem;
3219 continue;
3220 }
3221 Idx = MaskElt % NewElts +
3222 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3223 ? 0
3224 : 1] *
3225 NewElts;
3226 }
3227 // 2. Update inputs.
3228 Inputs[P.second[0]] = P.first.first;
3229 Inputs[P.second[1]] = P.first.second;
3230 // Clear the pair data.
3231 P.second.clear();
3232 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3233 }
3234 // Check if any concat_vectors can be simplified.
3235 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3236 for (int &Idx : Mask) {
3237 if (Idx == PoisonMaskElem)
3238 continue;
3239 unsigned SrcRegIdx = Idx / NewElts;
3240 if (Inputs[SrcRegIdx].isUndef()) {
3241 Idx = PoisonMaskElem;
3242 continue;
3243 }
3244 TargetLowering::LegalizeTypeAction TypeAction =
3245 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3246 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3247 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3248 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3249 (TypeAction == TargetLowering::TypeLegal ||
3250 TypeAction == TargetLowering::TypeWidenVector))
3251 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3252 }
3253 if (UsedSubVector.count() > 1) {
3254 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3255 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3256 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3257 continue;
3258 if (Pairs.empty() || Pairs.back().size() == 2)
3259 Pairs.emplace_back();
3260 if (UsedSubVector.test(Idx: 2 * I)) {
3261 Pairs.back().emplace_back(Args&: I, Args: 0);
3262 } else {
3263 assert(UsedSubVector.test(2 * I + 1) &&
3264 "Expected to be used one of the subvectors.");
3265 Pairs.back().emplace_back(Args&: I, Args: 1);
3266 }
3267 }
3268 if (!Pairs.empty() && Pairs.front().size() > 1) {
3269 // Adjust mask.
3270 for (int &Idx : Mask) {
3271 if (Idx == PoisonMaskElem)
3272 continue;
3273 unsigned SrcRegIdx = Idx / NewElts;
3274 auto *It = find_if(
3275 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3276 return Idxs.front().first == SrcRegIdx ||
3277 Idxs.back().first == SrcRegIdx;
3278 });
3279 if (It == Pairs.end())
3280 continue;
3281 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3282 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3283 }
3284 // Adjust inputs.
3285 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3286 Inputs[Idxs.front().first] = DAG.getNode(
3287 Opcode: ISD::CONCAT_VECTORS, DL,
3288 VT: Inputs[Idxs.front().first].getValueType(),
3289 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3290 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3291 }
3292 }
3293 }
3294 bool Changed;
3295 do {
3296 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3297 // reused operands.
3298 Changed = false;
3299 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3300 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3301 if (!Shuffle)
3302 continue;
3303 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3304 continue;
3305 int Op = -1;
3306 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3307 !Shuffle->isSplat()) {
3308 Op = 0;
3309 } else if (!Inputs[I].hasOneUse() &&
3310 !Shuffle->getOperand(Num: 1).isUndef()) {
3311 // Find the only used operand, if possible.
3312 for (int &Idx : Mask) {
3313 if (Idx == PoisonMaskElem)
3314 continue;
3315 unsigned SrcRegIdx = Idx / NewElts;
3316 if (SrcRegIdx != I)
3317 continue;
3318 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3319 if (MaskElt == PoisonMaskElem) {
3320 Idx = PoisonMaskElem;
3321 continue;
3322 }
3323 int OpIdx = MaskElt / NewElts;
3324 if (Op == -1) {
3325 Op = OpIdx;
3326 continue;
3327 }
3328 if (Op != OpIdx) {
3329 Op = -1;
3330 break;
3331 }
3332 }
3333 }
3334 if (Op < 0) {
3335 // Try to check if one of the shuffle operands is used already.
3336 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3337 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3338 continue;
3339 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3340 if (It == std::end(arr&: Inputs))
3341 continue;
3342 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3343 // Found that operand is used already.
3344 // 1. Fix the mask for the reused operand.
3345 for (int &Idx : Mask) {
3346 if (Idx == PoisonMaskElem)
3347 continue;
3348 unsigned SrcRegIdx = Idx / NewElts;
3349 if (SrcRegIdx != I)
3350 continue;
3351 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3352 if (MaskElt == PoisonMaskElem) {
3353 Idx = PoisonMaskElem;
3354 continue;
3355 }
3356 int MaskIdx = MaskElt / NewElts;
3357 if (OpIdx == MaskIdx)
3358 Idx = MaskElt % NewElts + FoundOp * NewElts;
3359 }
3360 // 2. Set Op to the unused OpIdx.
3361 Op = (OpIdx + 1) % 2;
3362 break;
3363 }
3364 }
3365 if (Op >= 0) {
3366 Changed = true;
3367 Inputs[I] = Shuffle->getOperand(Num: Op);
3368 // Adjust mask.
3369 for (int &Idx : Mask) {
3370 if (Idx == PoisonMaskElem)
3371 continue;
3372 unsigned SrcRegIdx = Idx / NewElts;
3373 if (SrcRegIdx != I)
3374 continue;
3375 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3376 int OpIdx = MaskElt / NewElts;
3377 if (OpIdx != Op)
3378 continue;
3379 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3380 }
3381 }
3382 }
3383 } while (Changed);
3384 };
3385 TryPeekThroughShufflesInputs(OrigMask);
3386 // Proces unique inputs.
3387 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3388 NewElts](SmallVectorImpl<int> &Mask) {
3389 SetVector<SDValue> UniqueInputs;
3390 SetVector<SDValue> UniqueConstantInputs;
3391 for (const auto &I : Inputs) {
3392 if (IsConstant(I))
3393 UniqueConstantInputs.insert(X: I);
3394 else if (!I.isUndef())
3395 UniqueInputs.insert(X: I);
3396 }
3397 // Adjust mask in case of reused inputs. Also, need to insert constant
3398 // inputs at first, otherwise it affects the final outcome.
3399 if (UniqueInputs.size() != std::size(Inputs)) {
3400 auto &&UniqueVec = UniqueInputs.takeVector();
3401 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3402 unsigned ConstNum = UniqueConstantVec.size();
3403 for (int &Idx : Mask) {
3404 if (Idx == PoisonMaskElem)
3405 continue;
3406 unsigned SrcRegIdx = Idx / NewElts;
3407 if (Inputs[SrcRegIdx].isUndef()) {
3408 Idx = PoisonMaskElem;
3409 continue;
3410 }
3411 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3412 if (It != UniqueConstantVec.end()) {
3413 Idx = (Idx % NewElts) +
3414 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3415 assert(Idx >= 0 && "Expected defined mask idx.");
3416 continue;
3417 }
3418 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3419 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3420 Idx = (Idx % NewElts) +
3421 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3422 assert(Idx >= 0 && "Expected defined mask idx.");
3423 }
3424 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3425 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3426 }
3427 };
3428 MakeUniqueInputs(OrigMask);
3429 SDValue OrigInputs[4];
3430 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3431 for (unsigned High = 0; High < 2; ++High) {
3432 SDValue &Output = High ? Hi : Lo;
3433
3434 // Build a shuffle mask for the output, discovering on the fly which
3435 // input vectors to use as shuffle operands.
3436 unsigned FirstMaskIdx = High * NewElts;
3437 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3438 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3439 assert(!Output && "Expected default initialized initial value.");
3440 TryPeekThroughShufflesInputs(Mask);
3441 MakeUniqueInputs(Mask);
3442 SDValue TmpInputs[4];
3443 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3444 // Track changes in the output registers.
3445 int UsedIdx = -1;
3446 bool SecondIteration = false;
3447 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3448 if (UsedIdx < 0) {
3449 UsedIdx = Idx;
3450 return false;
3451 }
3452 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3453 SecondIteration = true;
3454 return SecondIteration;
3455 };
3456 processShuffleMasks(
3457 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3458 /*NumOfUsedRegs=*/1,
3459 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3460 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3461 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3462 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3463 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3464 else
3465 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3466 N2: DAG.getPOISON(VT: NewVT), Mask);
3467 Inputs[Idx] = Output;
3468 },
3469 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3470 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3471 unsigned Idx2, bool /*Unused*/) {
3472 if (AccumulateResults(Idx1)) {
3473 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3474 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3475 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3476 else
3477 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3478 N2: Inputs[Idx2], Mask);
3479 } else {
3480 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3481 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3482 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3483 else
3484 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3485 N2: TmpInputs[Idx2], Mask);
3486 }
3487 Inputs[Idx1] = Output;
3488 });
3489 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3490 }
3491}
3492
3493void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3494 EVT OVT = N->getValueType(ResNo: 0);
3495 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3496 SDValue Chain = N->getOperand(Num: 0);
3497 SDValue Ptr = N->getOperand(Num: 1);
3498 SDValue SV = N->getOperand(Num: 2);
3499 SDLoc dl(N);
3500
3501 const Align Alignment =
3502 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3503
3504 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3505 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3506 Chain = Hi.getValue(R: 1);
3507
3508 // Modified the chain - switch anything that used the old chain to use
3509 // the new one.
3510 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3511}
3512
3513void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3514 SDValue &Hi) {
3515 EVT DstVTLo, DstVTHi;
3516 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3517 SDLoc dl(N);
3518
3519 SDValue SrcLo, SrcHi;
3520 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3521 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3522 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3523 else
3524 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3525
3526 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3527 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3528}
3529
3530void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3531 SDValue &Hi) {
3532 SDValue InLo, InHi;
3533 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3534 SDLoc DL(N);
3535
3536 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3537 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3538}
3539
3540void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3541 SDValue &Hi) {
3542 SDLoc DL(N);
3543
3544 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3545 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3546}
3547
3548void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3549 SDValue &Hi) {
3550 EVT VT = N->getValueType(ResNo: 0);
3551 SDValue Val = N->getOperand(Num: 0);
3552 SDValue Mask = N->getOperand(Num: 1);
3553 SDValue EVL = N->getOperand(Num: 2);
3554 SDLoc DL(N);
3555
3556 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3557 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3558 // then truncate back.
3559 EVT OrigVT = VT;
3560 if (!VT.getVectorElementType().isByteSized()) {
3561 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3562 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3563 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3564 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3565 }
3566
3567 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3568 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3569
3570 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3571 EC: VT.getVectorElementCount());
3572 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3573 EVT PtrVT = StackPtr.getValueType();
3574 auto &MF = DAG.getMachineFunction();
3575 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3576 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3577
3578 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3579 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3580 BaseAlignment: Alignment);
3581 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3582 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3583 BaseAlignment: Alignment);
3584
3585 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3586 SDValue NumElemMinus1 =
3587 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3588 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3589 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3590 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3591 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3592 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3593
3594 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3595 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3596 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3597 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3598
3599 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3600
3601 // Truncate back if we widened above.
3602 if (OrigVT != VT)
3603 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3604
3605 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3606}
3607
3608void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3609 SDValue &Hi) {
3610 EVT VT = N->getValueType(ResNo: 0);
3611 SDValue V1 = N->getOperand(Num: 0);
3612 SDValue V2 = N->getOperand(Num: 1);
3613 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3614 SDValue Mask = N->getOperand(Num: 3);
3615 SDValue EVL1 = N->getOperand(Num: 4);
3616 SDValue EVL2 = N->getOperand(Num: 5);
3617 SDLoc DL(N);
3618
3619 // Since EVL2 is considered the real VL it gets promoted during
3620 // SelectionDAGBuilder. Promote EVL1 here if needed.
3621 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3622 EVL1 = ZExtPromotedInteger(Op: EVL1);
3623
3624 // The stack splice addresses elements by byte offset/stride, which breaks for
3625 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3626 // stride is 0. Widen to a byte integer, splice, then truncate back.
3627 EVT OrigVT = VT;
3628 if (!VT.getVectorElementType().isByteSized()) {
3629 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3630 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3631 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3632 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3633 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3634 }
3635
3636 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3637
3638 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3639 EC: VT.getVectorElementCount() * 2);
3640 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3641 EVT PtrVT = StackPtr.getValueType();
3642 auto &MF = DAG.getMachineFunction();
3643 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3644 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3645
3646 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3647 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3648 BaseAlignment: Alignment);
3649 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3650 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3651 BaseAlignment: Alignment);
3652
3653 SDValue EltByteSize =
3654 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3655 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3656 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3657 // Clip EVL1Bytes to make sure we stay within the stack object.
3658 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3659 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3660 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3661 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3662
3663 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3664 SDValue StoreV1 =
3665 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3666 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3667
3668 SDValue StoreV2 =
3669 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3670 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3671
3672 SDValue Load;
3673 if (Imm >= 0) {
3674 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3675 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3676 } else {
3677 uint64_t TrailingElts = -Imm;
3678 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3679 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3680
3681 // Make sure TrailingBytes doesn't exceed the size of vec1.
3682 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3683 TrailingBytes =
3684 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3685
3686 // Calculate the start address of the spliced result.
3687 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3688 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3689 }
3690
3691 // Truncate back if we widened above.
3692 if (OrigVT != VT)
3693 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3694
3695 EVT LoVT, HiVT;
3696 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3697 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3698 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3699 Hi =
3700 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3701 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3702}
3703
3704void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3705 SDValue &Hi) {
3706 SDLoc DL(N);
3707 SDValue Acc = N->getOperand(Num: 0);
3708 SDValue Input1 = N->getOperand(Num: 1);
3709 SDValue Input2 = N->getOperand(Num: 2);
3710
3711 SDValue AccLo, AccHi;
3712 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3713 unsigned Opcode = N->getOpcode();
3714
3715 // If the input types don't need splitting, just accumulate into the
3716 // low part of the accumulator.
3717 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3718 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3719 Hi = AccHi;
3720 return;
3721 }
3722
3723 SDValue Input1Lo, Input1Hi;
3724 SDValue Input2Lo, Input2Hi;
3725 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3726 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3727 EVT ResultVT = AccLo.getValueType();
3728
3729 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3730 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3731}
3732
3733void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3734 SDValue &Hi) {
3735 SDLoc DL(N);
3736 SDValue Op0 = N->getOperand(Num: 0);
3737 SDValue Op1 = N->getOperand(Num: 1);
3738 EVT OpVT = Op0.getValueType();
3739
3740 EVT LoVT, HiVT;
3741 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3742
3743 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3744 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3745 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3746 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3747}
3748
3749void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3750 unsigned Factor = N->getNumOperands();
3751
3752 SmallVector<SDValue, 8> Ops(Factor * 2);
3753 for (unsigned i = 0; i != Factor; ++i) {
3754 SDValue OpLo, OpHi;
3755 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3756 Ops[i * 2] = OpLo;
3757 Ops[i * 2 + 1] = OpHi;
3758 }
3759
3760 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3761
3762 SDLoc DL(N);
3763 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3764 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3765 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3766 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3767
3768 for (unsigned i = 0; i != Factor; ++i)
3769 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3770}
3771
3772void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3773 unsigned Factor = N->getNumOperands();
3774
3775 SmallVector<SDValue, 8> Ops(Factor * 2);
3776 for (unsigned i = 0; i != Factor; ++i) {
3777 SDValue OpLo, OpHi;
3778 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3779 Ops[i] = OpLo;
3780 Ops[i + Factor] = OpHi;
3781 }
3782
3783 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3784
3785 SDLoc DL(N);
3786 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3787 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3788 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3789 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3790
3791 for (unsigned i = 0; i != Factor; ++i) {
3792 unsigned IdxLo = 2 * i;
3793 unsigned IdxHi = 2 * i + 1;
3794 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3795 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3796 }
3797}
3798
3799//===----------------------------------------------------------------------===//
3800// Operand Vector Splitting
3801//===----------------------------------------------------------------------===//
3802
3803/// This method is called when the specified operand of the specified node is
3804/// found to need vector splitting. At this point, all of the result types of
3805/// the node are known to be legal, but other operands of the node may need
3806/// legalization as well as the specified one.
3807bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3808 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3809 SDValue Res = SDValue();
3810
3811 // See if the target wants to custom split this node.
3812 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3813 return false;
3814
3815 switch (N->getOpcode()) {
3816 default:
3817#ifndef NDEBUG
3818 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3819 N->dump(&DAG);
3820 dbgs() << "\n";
3821#endif
3822 report_fatal_error(reason: "Do not know how to split this operator's "
3823 "operand!\n");
3824
3825 case ISD::VP_SETCC:
3826 case ISD::STRICT_FSETCC:
3827 case ISD::STRICT_FSETCCS:
3828 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3829 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3830 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3831 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3832 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3833 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3834 case ISD::VECTOR_FIND_LAST_ACTIVE:
3835 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3836 break;
3837 case ISD::VP_TRUNCATE:
3838 case ISD::TRUNCATE:
3839 Res = SplitVecOp_TruncateHelper(N);
3840 break;
3841 case ISD::STRICT_FP_ROUND:
3842 case ISD::VP_FP_ROUND:
3843 case ISD::FP_ROUND:
3844 case ISD::CONVERT_FROM_ARBITRARY_FP:
3845 case ISD::CONVERT_TO_ARBITRARY_FP:
3846 Res = SplitVecOp_FP_ROUND(N);
3847 break;
3848 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3849 case ISD::STORE:
3850 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3851 break;
3852 case ISD::ATOMIC_STORE:
3853 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3854 break;
3855 case ISD::VP_STORE:
3856 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3857 break;
3858 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3859 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3860 break;
3861 case ISD::MSTORE:
3862 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3863 break;
3864 case ISD::MSCATTER:
3865 case ISD::VP_SCATTER:
3866 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3867 break;
3868 case ISD::MGATHER:
3869 case ISD::VP_GATHER:
3870 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3871 break;
3872 case ISD::VSELECT:
3873 Res = SplitVecOp_VSELECT(N, OpNo);
3874 break;
3875 case ISD::VECTOR_COMPRESS:
3876 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3877 break;
3878 case ISD::STRICT_SINT_TO_FP:
3879 case ISD::STRICT_UINT_TO_FP:
3880 case ISD::SINT_TO_FP:
3881 case ISD::UINT_TO_FP:
3882 case ISD::VP_SINT_TO_FP:
3883 case ISD::VP_UINT_TO_FP:
3884 if (N->getValueType(ResNo: 0).bitsLT(
3885 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3886 Res = SplitVecOp_TruncateHelper(N);
3887 else
3888 Res = SplitVecOp_UnaryOp(N);
3889 break;
3890 case ISD::FP_TO_SINT_SAT:
3891 case ISD::FP_TO_UINT_SAT:
3892 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3893 break;
3894 case ISD::FP_TO_SINT:
3895 case ISD::FP_TO_UINT:
3896 case ISD::VP_FP_TO_SINT:
3897 case ISD::VP_FP_TO_UINT:
3898 case ISD::STRICT_FP_TO_SINT:
3899 case ISD::STRICT_FP_TO_UINT:
3900 case ISD::STRICT_FP_EXTEND:
3901 case ISD::FP_EXTEND:
3902 case ISD::SIGN_EXTEND:
3903 case ISD::ZERO_EXTEND:
3904 case ISD::ANY_EXTEND:
3905 case ISD::FTRUNC:
3906 case ISD::LROUND:
3907 case ISD::LLROUND:
3908 case ISD::LRINT:
3909 case ISD::LLRINT:
3910 Res = SplitVecOp_UnaryOp(N);
3911 break;
3912 case ISD::FLDEXP:
3913 Res = SplitVecOp_FPOpDifferentTypes(N);
3914 break;
3915
3916 case ISD::SCMP:
3917 case ISD::UCMP:
3918 Res = SplitVecOp_CMP(N);
3919 break;
3920
3921 case ISD::FAKE_USE:
3922 Res = SplitVecOp_FAKE_USE(N);
3923 break;
3924 case ISD::ANY_EXTEND_VECTOR_INREG:
3925 case ISD::SIGN_EXTEND_VECTOR_INREG:
3926 case ISD::ZERO_EXTEND_VECTOR_INREG:
3927 Res = SplitVecOp_ExtVecInRegOp(N);
3928 break;
3929
3930 case ISD::VECREDUCE_FADD:
3931 case ISD::VECREDUCE_FMUL:
3932 case ISD::VECREDUCE_ADD:
3933 case ISD::VECREDUCE_MUL:
3934 case ISD::VECREDUCE_AND:
3935 case ISD::VECREDUCE_OR:
3936 case ISD::VECREDUCE_XOR:
3937 case ISD::VECREDUCE_SMAX:
3938 case ISD::VECREDUCE_SMIN:
3939 case ISD::VECREDUCE_UMAX:
3940 case ISD::VECREDUCE_UMIN:
3941 case ISD::VECREDUCE_FMAX:
3942 case ISD::VECREDUCE_FMIN:
3943 case ISD::VECREDUCE_FMAXIMUM:
3944 case ISD::VECREDUCE_FMINIMUM:
3945 Res = SplitVecOp_VECREDUCE(N, OpNo);
3946 break;
3947 case ISD::VECREDUCE_SEQ_FADD:
3948 case ISD::VECREDUCE_SEQ_FMUL:
3949 Res = SplitVecOp_VECREDUCE_SEQ(N);
3950 break;
3951 case ISD::VP_REDUCE_FADD:
3952 case ISD::VP_REDUCE_SEQ_FADD:
3953 case ISD::VP_REDUCE_FMUL:
3954 case ISD::VP_REDUCE_SEQ_FMUL:
3955 case ISD::VP_REDUCE_ADD:
3956 case ISD::VP_REDUCE_MUL:
3957 case ISD::VP_REDUCE_AND:
3958 case ISD::VP_REDUCE_OR:
3959 case ISD::VP_REDUCE_XOR:
3960 case ISD::VP_REDUCE_SMAX:
3961 case ISD::VP_REDUCE_SMIN:
3962 case ISD::VP_REDUCE_UMAX:
3963 case ISD::VP_REDUCE_UMIN:
3964 case ISD::VP_REDUCE_FMAX:
3965 case ISD::VP_REDUCE_FMIN:
3966 case ISD::VP_REDUCE_FMAXIMUM:
3967 case ISD::VP_REDUCE_FMINIMUM:
3968 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3969 break;
3970 case ISD::CTTZ_ELTS:
3971 case ISD::CTTZ_ELTS_ZERO_POISON:
3972 Res = SplitVecOp_CttzElts(N);
3973 break;
3974 case ISD::VP_CTTZ_ELTS:
3975 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
3976 Res = SplitVecOp_VP_CttzElements(N);
3977 break;
3978 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3979 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3980 break;
3981 case ISD::PARTIAL_REDUCE_UMLA:
3982 case ISD::PARTIAL_REDUCE_SMLA:
3983 case ISD::PARTIAL_REDUCE_SUMLA:
3984 case ISD::PARTIAL_REDUCE_FMLA:
3985 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3986 break;
3987 }
3988
3989 // If the result is null, the sub-method took care of registering results etc.
3990 if (!Res.getNode()) return false;
3991
3992 // If the result is N, the sub-method updated N in place. Tell the legalizer
3993 // core about this.
3994 if (Res.getNode() == N)
3995 return true;
3996
3997 if (N->isStrictFPOpcode())
3998 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3999 "Invalid operand expansion");
4000 else
4001 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4002 "Invalid operand expansion");
4003
4004 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4005 return false;
4006}
4007
4008SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
4009 SDLoc DL(N);
4010
4011 SDValue LoMask, HiMask;
4012 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
4013
4014 EVT VT = N->getValueType(ResNo: 0);
4015 EVT SplitVT = LoMask.getValueType();
4016 ElementCount SplitEC = SplitVT.getVectorElementCount();
4017
4018 // Find the last active in both the low and the high masks.
4019 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
4020 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
4021
4022 // Check if any lane is active in the high mask.
4023 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4024 // sentinel value for "none active".
4025 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4026 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4027 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4028
4029 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4030 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4031 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4032 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4033 N3: LoFind);
4034}
4035
4036SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4037 // The only possibility for an illegal operand is the mask, since result type
4038 // legalization would have handled this node already otherwise.
4039 assert(OpNo == 0 && "Illegal operand must be mask");
4040
4041 SDValue Mask = N->getOperand(Num: 0);
4042 SDValue Src0 = N->getOperand(Num: 1);
4043 SDValue Src1 = N->getOperand(Num: 2);
4044 EVT Src0VT = Src0.getValueType();
4045 SDLoc DL(N);
4046 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4047
4048 SDValue Lo, Hi;
4049 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4050 assert(Lo.getValueType() == Hi.getValueType() &&
4051 "Lo and Hi have differing types");
4052
4053 EVT LoOpVT, HiOpVT;
4054 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4055 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4056
4057 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4058 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4059 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4060 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4061
4062 SDValue LoSelect =
4063 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4064 SDValue HiSelect =
4065 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4066
4067 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4068}
4069
4070SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4071 // The only possibility for an illegal operand is the mask, since result type
4072 // legalization would have handled this node already otherwise.
4073 assert(OpNo == 1 && "Illegal operand must be mask");
4074
4075 // To split the mask, we need to split the result type too, so we can just
4076 // reuse that logic here.
4077 SDValue Lo, Hi;
4078 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4079
4080 EVT VecVT = N->getValueType(ResNo: 0);
4081 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4082}
4083
4084SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4085 EVT ResVT = N->getValueType(ResNo: 0);
4086 SDValue Lo, Hi;
4087 SDLoc dl(N);
4088
4089 SDValue VecOp = N->getOperand(Num: OpNo);
4090 EVT VecVT = VecOp.getValueType();
4091 assert(VecVT.isVector() && "Can only split reduce vector operand");
4092 GetSplitVector(Op: VecOp, Lo, Hi);
4093 EVT LoOpVT, HiOpVT;
4094 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4095
4096 // Use the appropriate scalar instruction on the split subvectors before
4097 // reducing the now partially reduced smaller vector.
4098 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4099 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4100 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4101}
4102
4103SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4104 EVT ResVT = N->getValueType(ResNo: 0);
4105 SDValue Lo, Hi;
4106 SDLoc dl(N);
4107
4108 SDValue AccOp = N->getOperand(Num: 0);
4109 SDValue VecOp = N->getOperand(Num: 1);
4110 SDNodeFlags Flags = N->getFlags();
4111
4112 EVT VecVT = VecOp.getValueType();
4113 assert(VecVT.isVector() && "Can only split reduce vector operand");
4114 GetSplitVector(Op: VecOp, Lo, Hi);
4115 EVT LoOpVT, HiOpVT;
4116 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4117
4118 // Reduce low half.
4119 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4120
4121 // Reduce high half, using low half result as initial value.
4122 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4123}
4124
4125SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4126 assert(N->isVPOpcode() && "Expected VP opcode");
4127 assert(OpNo == 1 && "Can only split reduce vector operand");
4128
4129 unsigned Opc = N->getOpcode();
4130 EVT ResVT = N->getValueType(ResNo: 0);
4131 SDValue Lo, Hi;
4132 SDLoc dl(N);
4133
4134 SDValue VecOp = N->getOperand(Num: OpNo);
4135 EVT VecVT = VecOp.getValueType();
4136 assert(VecVT.isVector() && "Can only split reduce vector operand");
4137 GetSplitVector(Op: VecOp, Lo, Hi);
4138
4139 SDValue MaskLo, MaskHi;
4140 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4141
4142 SDValue EVLLo, EVLHi;
4143 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4144
4145 const SDNodeFlags Flags = N->getFlags();
4146
4147 SDValue ResLo =
4148 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4149 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4150}
4151
4152SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4153 // The result has a legal vector type, but the input needs splitting.
4154 EVT ResVT = N->getValueType(ResNo: 0);
4155 SDValue Lo, Hi;
4156 SDLoc dl(N);
4157 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4158 EVT InVT = Lo.getValueType();
4159
4160 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4161 EC: InVT.getVectorElementCount());
4162
4163 if (N->isStrictFPOpcode()) {
4164 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4165 Ops: {N->getOperand(Num: 0), Lo});
4166 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4167 Ops: {N->getOperand(Num: 0), Hi});
4168
4169 // Build a factor node to remember that this operation is independent
4170 // of the other one.
4171 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4172 N2: Hi.getValue(R: 1));
4173
4174 // Legalize the chain result - switch anything that used the old chain to
4175 // use the new one.
4176 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4177 } else if (N->getNumOperands() == 3) {
4178 assert(N->isVPOpcode() && "Expected VP opcode");
4179 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4180 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4181 std::tie(args&: EVLLo, args&: EVLHi) =
4182 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
4183 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4184 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4185 } else {
4186 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4187 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4188 }
4189
4190 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4191}
4192
4193// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4194SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4195 SDValue Lo, Hi;
4196 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4197 SDValue Chain =
4198 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4199 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4200}
4201
4202SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4203 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4204 // end up being split all the way down to individual components. Convert the
4205 // split pieces into integers and reassemble.
4206 EVT ResVT = N->getValueType(ResNo: 0);
4207 SDValue Lo, Hi;
4208 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4209 SDLoc dl(N);
4210
4211 if (ResVT.isScalableVector()) {
4212 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4213 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4214 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4215 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4216 }
4217
4218 Lo = BitConvertToInteger(Op: Lo);
4219 Hi = BitConvertToInteger(Op: Hi);
4220
4221 if (DAG.getDataLayout().isBigEndian())
4222 std::swap(a&: Lo, b&: Hi);
4223
4224 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4225}
4226
4227SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4228 unsigned OpNo) {
4229 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4230 // We know that the result type is legal.
4231 EVT ResVT = N->getValueType(ResNo: 0);
4232
4233 SDValue Vec = N->getOperand(Num: 0);
4234 SDValue SubVec = N->getOperand(Num: 1);
4235 SDValue Idx = N->getOperand(Num: 2);
4236 SDLoc dl(N);
4237
4238 SDValue Lo, Hi;
4239 GetSplitVector(Op: SubVec, Lo, Hi);
4240
4241 uint64_t IdxVal = Idx->getAsZExtVal();
4242 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4243
4244 SDValue FirstInsertion =
4245 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4246 SDValue SecondInsertion =
4247 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4248 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4249
4250 return SecondInsertion;
4251}
4252
4253SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4254 // We know that the extracted result type is legal.
4255 EVT SubVT = N->getValueType(ResNo: 0);
4256 SDValue Idx = N->getOperand(Num: 1);
4257 SDLoc dl(N);
4258 SDValue Lo, Hi;
4259
4260 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4261
4262 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4263 // Note: For scalable vectors, the index is scaled by vscale.
4264 ElementCount IdxVal =
4265 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4266 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4267
4268 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4269 ElementCount NumResultElts = SubVT.getVectorElementCount();
4270
4271 // If the extracted elements are all in the low half, do a simple extract.
4272 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4273 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4274
4275 unsigned LoEltsMin = LoElts.getKnownMinValue();
4276 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4277 SrcVT.isFixedLengthVector()) {
4278 // Extracted subvector crosses vector split, so we need to blend the two
4279 // halves.
4280 // TODO: May be able to emit partial extract_subvector.
4281 SmallVector<SDValue, 8> Elts;
4282 Elts.reserve(N: NumResultElts.getFixedValue());
4283
4284 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4285 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4286 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4287 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4288 /*Count=*/LoEltsMin - IdxValMin);
4289 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4290 /*Count=*/SubVT.getVectorNumElements() -
4291 Elts.size());
4292 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4293 }
4294
4295 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4296 ElementCount ExtractIdx = IdxVal - LoElts;
4297 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4298 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4299 Idx: ExtractIdx.getKnownMinValue());
4300
4301 EVT HiVT = Hi.getValueType();
4302 assert(HiVT.isFixedLengthVector() &&
4303 "Only fixed-vector extracts are supported in this case");
4304
4305 // We cannot create an extract_subvector that isn't a multiple of the
4306 // result size, which may go out of bounds for the last elements. Shuffle
4307 // the desired elements down to 0 and do a simple 0 extract.
4308 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4309 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4310 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4311
4312 SDValue Shuffle =
4313 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4314 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4315 }
4316
4317 // After this point the DAG node only permits extracting fixed-width
4318 // subvectors from scalable vectors.
4319 assert(SubVT.isFixedLengthVector() &&
4320 "Extracting scalable subvector from fixed-width unsupported");
4321
4322 // If the element type is i1 and we're not promoting the result, then we may
4323 // end up loading the wrong data since the bits are packed tightly into
4324 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4325 // type at index 4, then we will load a byte starting at index 0.
4326 if (SubVT.getScalarType() == MVT::i1)
4327 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4328 "subvector from a scalable predicate vector");
4329
4330 // Spill the vector to the stack. We should use the alignment for
4331 // the smallest part.
4332 SDValue Vec = N->getOperand(Num: 0);
4333 EVT VecVT = Vec.getValueType();
4334 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4335 SDValue StackPtr =
4336 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4337 auto &MF = DAG.getMachineFunction();
4338 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4339 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4340
4341 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4342 Alignment: SmallestAlign);
4343
4344 // Extract the subvector by loading the correct part.
4345 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4346
4347 return DAG.getLoad(
4348 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4349 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4350}
4351
4352SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4353 SDValue Vec = N->getOperand(Num: 0);
4354 SDValue Idx = N->getOperand(Num: 1);
4355 EVT VecVT = Vec.getValueType();
4356
4357 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4358 uint64_t IdxVal = Index->getZExtValue();
4359
4360 SDValue Lo, Hi;
4361 GetSplitVector(Op: Vec, Lo, Hi);
4362
4363 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4364
4365 if (IdxVal < LoElts)
4366 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4367 else if (!Vec.getValueType().isScalableVector())
4368 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4369 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4370 VT: Idx.getValueType())), 0);
4371 }
4372
4373 // See if the target wants to custom expand this node.
4374 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4375 return SDValue();
4376
4377 // Make the vector elements byte-addressable if they aren't already.
4378 SDLoc dl(N);
4379 EVT EltVT = VecVT.getVectorElementType();
4380 if (!EltVT.isByteSized()) {
4381 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4382 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4383 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4384 SDValue NewExtract =
4385 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4386 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4387 }
4388
4389 // Store the vector to the stack.
4390 // In cases where the vector is illegal it will be broken down into parts
4391 // and stored in parts - we should use the alignment for the smallest part.
4392 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4393 SDValue StackPtr =
4394 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4395 auto &MF = DAG.getMachineFunction();
4396 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4397 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4398 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4399 Alignment: SmallestAlign);
4400
4401 // Load back the required element.
4402 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4403
4404 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4405 // type, leaving the high bits undefined. But it can't truncate.
4406 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4407
4408 return DAG.getExtLoad(
4409 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4410 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4411 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4412}
4413
4414SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4415 SDValue Lo, Hi;
4416
4417 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4418 // splitting the result has the same effect as splitting the input operand.
4419 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4420
4421 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4422}
4423
4424SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4425 (void)OpNo;
4426 SDValue Lo, Hi;
4427 SplitVecRes_Gather(N, Lo, Hi);
4428
4429 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4430 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4431 return SDValue();
4432}
4433
4434SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4435 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4436 SDValue Ch = N->getChain();
4437 SDValue Ptr = N->getBasePtr();
4438 SDValue Offset = N->getOffset();
4439 assert(Offset.isUndef() && "Unexpected VP store offset");
4440 SDValue Mask = N->getMask();
4441 SDValue EVL = N->getVectorLength();
4442 SDValue Data = N->getValue();
4443 Align Alignment = N->getBaseAlign();
4444 SDLoc DL(N);
4445
4446 SDValue DataLo, DataHi;
4447 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4448 // Split Data operand
4449 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4450 else
4451 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4452
4453 // Split Mask operand
4454 SDValue MaskLo, MaskHi;
4455 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4456 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4457 } else {
4458 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4459 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4460 else
4461 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4462 }
4463
4464 EVT MemoryVT = N->getMemoryVT();
4465 EVT LoMemVT, HiMemVT;
4466 bool HiIsEmpty = false;
4467 std::tie(args&: LoMemVT, args&: HiMemVT) =
4468 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4469
4470 // Split EVL
4471 SDValue EVLLo, EVLHi;
4472 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4473
4474 SDValue Lo, Hi;
4475 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4476 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4477 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4478 Ranges: N->getRanges());
4479
4480 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4481 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4482 IsCompressing: N->isCompressingStore());
4483
4484 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4485 if (HiIsEmpty)
4486 return Lo;
4487
4488 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4489 IsCompressedMemory: N->isCompressingStore());
4490
4491 MachinePointerInfo MPI;
4492 if (LoMemVT.isScalableVector()) {
4493 Alignment = commonAlignment(A: Alignment,
4494 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4495 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4496 } else
4497 MPI = N->getPointerInfo().getWithOffset(
4498 O: LoMemVT.getStoreSize().getFixedValue());
4499
4500 MMO = DAG.getMachineFunction().getMachineMemOperand(
4501 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4502 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4503
4504 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4505 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4506 IsCompressing: N->isCompressingStore());
4507
4508 // Build a factor node to remember that this store is independent of the
4509 // other one.
4510 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4511}
4512
4513SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4514 unsigned OpNo) {
4515 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4516 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4517
4518 SDLoc DL(N);
4519
4520 SDValue Data = N->getValue();
4521 SDValue LoData, HiData;
4522 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4523 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4524 else
4525 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4526
4527 EVT LoMemVT, HiMemVT;
4528 bool HiIsEmpty = false;
4529 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4530 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4531
4532 SDValue Mask = N->getMask();
4533 SDValue LoMask, HiMask;
4534 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4535 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4536 else if (getTypeAction(VT: Mask.getValueType()) ==
4537 TargetLowering::TypeSplitVector)
4538 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4539 else
4540 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4541
4542 SDValue LoEVL, HiEVL;
4543 std::tie(args&: LoEVL, args&: HiEVL) =
4544 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4545
4546 // Generate the low vp_strided_store
4547 SDValue Lo = DAG.getStridedStoreVP(
4548 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4549 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4550 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4551
4552 // If the high vp_strided_store has zero storage size, only the low
4553 // vp_strided_store is needed.
4554 if (HiIsEmpty)
4555 return Lo;
4556
4557 // Generate the high vp_strided_store.
4558 // To calculate the high base address, we need to sum to the low base
4559 // address stride number of bytes for each element already stored by low,
4560 // that is: Ptr = Ptr + (LoEVL * Stride)
4561 EVT PtrVT = N->getBasePtr().getValueType();
4562 SDValue Increment =
4563 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4564 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4565 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4566
4567 Align Alignment = N->getBaseAlign();
4568 if (LoMemVT.isScalableVector())
4569 Alignment = commonAlignment(A: Alignment,
4570 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4571
4572 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4573 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4574 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4575 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4576
4577 SDValue Hi = DAG.getStridedStoreVP(
4578 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4579 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4580 IsCompressing: N->isCompressingStore());
4581
4582 // Build a factor node to remember that this store is independent of the
4583 // other one.
4584 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4585}
4586
4587SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4588 unsigned OpNo) {
4589 assert(N->isUnindexed() && "Indexed masked store of vector?");
4590 SDValue Ch = N->getChain();
4591 SDValue Ptr = N->getBasePtr();
4592 SDValue Offset = N->getOffset();
4593 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4594 SDValue Mask = N->getMask();
4595 SDValue Data = N->getValue();
4596 Align Alignment = N->getBaseAlign();
4597 SDLoc DL(N);
4598
4599 SDValue DataLo, DataHi;
4600 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4601 // Split Data operand
4602 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4603 else
4604 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4605
4606 // Split Mask operand
4607 SDValue MaskLo, MaskHi;
4608 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4609 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4610 } else {
4611 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4612 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4613 else
4614 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4615 }
4616
4617 EVT MemoryVT = N->getMemoryVT();
4618 EVT LoMemVT, HiMemVT;
4619 bool HiIsEmpty = false;
4620 std::tie(args&: LoMemVT, args&: HiMemVT) =
4621 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4622
4623 SDValue Lo, Hi, Res;
4624 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4625 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4626 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4627 Ranges: N->getRanges());
4628
4629 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4630 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4631 IsCompressing: N->isCompressingStore());
4632
4633 if (HiIsEmpty) {
4634 // The hi masked store has zero storage size.
4635 // Only the lo masked store is needed.
4636 Res = Lo;
4637 } else {
4638
4639 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4640 IsCompressedMemory: N->isCompressingStore());
4641
4642 MachinePointerInfo MPI;
4643 if (LoMemVT.isScalableVector()) {
4644 Alignment = commonAlignment(
4645 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4646 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4647 } else
4648 MPI = N->getPointerInfo().getWithOffset(
4649 O: LoMemVT.getStoreSize().getFixedValue());
4650
4651 MMO = DAG.getMachineFunction().getMachineMemOperand(
4652 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4653 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4654
4655 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4656 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4657 IsCompressing: N->isCompressingStore());
4658
4659 // Build a factor node to remember that this store is independent of the
4660 // other one.
4661 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4662 }
4663
4664 return Res;
4665}
4666
4667SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4668 SDValue Ch = N->getChain();
4669 SDValue Ptr = N->getBasePtr();
4670 EVT MemoryVT = N->getMemoryVT();
4671 Align Alignment = N->getBaseAlign();
4672 SDLoc DL(N);
4673 struct Operands {
4674 SDValue Mask;
4675 SDValue Index;
4676 SDValue Scale;
4677 SDValue Data;
4678 } Ops = [&]() -> Operands {
4679 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4680 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4681 .Data: MSC->getValue()};
4682 }
4683 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4684 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4685 .Data: VPSC->getValue()};
4686 }();
4687 // Split all operands
4688
4689 EVT LoMemVT, HiMemVT;
4690 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4691
4692 SDValue DataLo, DataHi;
4693 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4694 // Split Data operand
4695 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4696 else
4697 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4698
4699 // Split Mask operand
4700 SDValue MaskLo, MaskHi;
4701 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4702 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4703 } else {
4704 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4705 }
4706
4707 SDValue IndexHi, IndexLo;
4708 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4709 TargetLowering::TypeSplitVector)
4710 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4711 else
4712 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4713
4714 SDValue Lo;
4715 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4716 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4717 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4718 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4719
4720 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4721 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4722 Lo =
4723 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4724 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4725
4726 // The order of the Scatter operation after split is well defined. The "Hi"
4727 // part comes after the "Lo". So these two operations should be chained one
4728 // after another.
4729 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4730 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4731 MMO, IndexType: MSC->getIndexType(),
4732 IsTruncating: MSC->isTruncatingStore());
4733 }
4734 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4735 SDValue EVLLo, EVLHi;
4736 std::tie(args&: EVLLo, args&: EVLHi) =
4737 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4738
4739 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4740 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4741 IndexType: VPSC->getIndexType());
4742
4743 // The order of the Scatter operation after split is well defined. The "Hi"
4744 // part comes after the "Lo". So these two operations should be chained one
4745 // after another.
4746 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4747 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4748 IndexType: VPSC->getIndexType());
4749}
4750
4751SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4752 assert(N->isUnindexed() && "Indexed store of vector?");
4753 assert(OpNo == 1 && "Can only split the stored value");
4754 SDLoc DL(N);
4755
4756 bool isTruncating = N->isTruncatingStore();
4757 SDValue Ch = N->getChain();
4758 SDValue Ptr = N->getBasePtr();
4759 EVT MemoryVT = N->getMemoryVT();
4760 Align Alignment = N->getBaseAlign();
4761 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4762 AAMDNodes AAInfo = N->getAAInfo();
4763 SDValue Lo, Hi;
4764 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4765
4766 EVT LoMemVT, HiMemVT;
4767 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4768
4769 // Scalarize if the split halves are not byte-sized.
4770 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4771 return TLI.scalarizeVectorStore(ST: N, DAG);
4772
4773 if (isTruncating)
4774 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4775 Alignment, MMOFlags, AAInfo);
4776 else
4777 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4778 AAInfo);
4779
4780 MachinePointerInfo MPI;
4781 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4782
4783 if (isTruncating)
4784 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4785 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4786 else
4787 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4788
4789 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4790}
4791
4792SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4793 SDLoc DL(N);
4794 LLVMContext &Ctx = *DAG.getContext();
4795 SDValue StVal = N->getVal();
4796 EVT VT = StVal.getValueType();
4797 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4798
4799 // The store needs a single value spanning the full memory width. If the
4800 // value can be held in a legal vector register, keep it there and extract
4801 // the low integer element of the memory width. This lets the store be issued
4802 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4803 // bitcasting the split vector straight to a scalar integer, which would
4804 // reassemble the value element by element in GPRs.
4805 //
4806 // Reinterpret the value as a same-shaped integer vector first: an FP element
4807 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4808 // integer-of-element-size form does. Ask the target which legal vector type
4809 // it widens to.
4810 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4811 EVT IntEltVT = IntVecVT.getVectorElementType();
4812 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4813 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4814 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4815 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4816 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4817 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4818 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4819 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4820 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4821 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4822 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4823 Val: N->getBasePtr(), MMO: N->getMemOperand());
4824 }
4825
4826 // Otherwise issue a single atomic store of an integer that spans the full
4827 // memory width. Bitcasting the (illegal) vector value to that integer lets
4828 // the type legalizer further legalize the BITCAST input as needed, while the
4829 // ATOMIC_STORE itself uses only the legal integer type.
4830 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4831 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4832 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4833 Val: N->getBasePtr(), MMO: N->getMemOperand());
4834}
4835
4836SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4837 SDLoc DL(N);
4838
4839 // The input operands all must have the same type, and we know the result
4840 // type is valid. Convert this to a buildvector which extracts all the
4841 // input elements.
4842 // TODO: If the input elements are power-two vectors, we could convert this to
4843 // a new CONCAT_VECTORS node with elements that are half-wide.
4844 SmallVector<SDValue, 32> Elts;
4845 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4846 for (const SDValue &Op : N->op_values()) {
4847 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4848 i != e; ++i) {
4849 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4850 }
4851 }
4852
4853 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4854}
4855
4856SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4857 // The result type is legal, but the input type is illegal. If splitting
4858 // ends up with the result type of each half still being legal, just
4859 // do that. If, however, that would result in an illegal result type,
4860 // we can try to get more clever with power-two vectors. Specifically,
4861 // split the input type, but also widen the result element size, then
4862 // concatenate the halves and truncate again. For example, consider a target
4863 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4864 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4865 // %inlo = v4i32 extract_subvector %in, 0
4866 // %inhi = v4i32 extract_subvector %in, 4
4867 // %lo16 = v4i16 trunc v4i32 %inlo
4868 // %hi16 = v4i16 trunc v4i32 %inhi
4869 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4870 // %res = v8i8 trunc v8i16 %in16
4871 //
4872 // Without this transform, the original truncate would end up being
4873 // scalarized, which is pretty much always a last resort.
4874 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4875 SDValue InVec = N->getOperand(Num: OpNo);
4876 EVT InVT = InVec->getValueType(ResNo: 0);
4877 EVT OutVT = N->getValueType(ResNo: 0);
4878 ElementCount NumElements = OutVT.getVectorElementCount();
4879 bool IsFloat = OutVT.isFloatingPoint();
4880
4881 unsigned InElementSize = InVT.getScalarSizeInBits();
4882 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4883
4884 // Determine the split output VT. If its legal we can just split dirctly.
4885 EVT LoOutVT, HiOutVT;
4886 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4887 assert(LoOutVT == HiOutVT && "Unequal split?");
4888
4889 // If the input elements are only 1/2 the width of the result elements,
4890 // just use the normal splitting. Our trick only work if there's room
4891 // to split more than once.
4892 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4893 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4894 return SplitVecOp_UnaryOp(N);
4895 SDLoc DL(N);
4896
4897 // Don't touch if this will be scalarized.
4898 EVT FinalVT = InVT;
4899 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4900 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4901
4902 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4903 return SplitVecOp_UnaryOp(N);
4904
4905 // Get the split input vector.
4906 SDValue InLoVec, InHiVec;
4907 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4908
4909 // Truncate them to 1/2 the element size.
4910 //
4911 // This assumes the number of elements is a power of two; any vector that
4912 // isn't should be widened, not split.
4913 EVT HalfElementVT = IsFloat ?
4914 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4915 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4916 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4917 EC: NumElements.divideCoefficientBy(RHS: 2));
4918
4919 SDValue HalfLo;
4920 SDValue HalfHi;
4921 SDValue Chain;
4922 if (N->isStrictFPOpcode()) {
4923 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4924 Ops: {N->getOperand(Num: 0), InLoVec});
4925 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4926 Ops: {N->getOperand(Num: 0), InHiVec});
4927 // Legalize the chain result - switch anything that used the old chain to
4928 // use the new one.
4929 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4930 N2: HalfHi.getValue(R: 1));
4931 } else {
4932 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4933 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4934 }
4935
4936 // Concatenate them to get the full intermediate truncation result.
4937 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4938 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4939 N2: HalfHi);
4940 // Now finish up by truncating all the way down to the original result
4941 // type. This should normally be something that ends up being legal directly,
4942 // but in theory if a target has very wide vectors and an annoyingly
4943 // restricted set of legal types, this split can chain to build things up.
4944
4945 if (N->isStrictFPOpcode()) {
4946 SDValue Res = DAG.getNode(
4947 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4948 Ops: {Chain, InterVec,
4949 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4950 // Relink the chain
4951 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4952 return Res;
4953 }
4954
4955 return IsFloat
4956 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4957 N2: DAG.getTargetConstant(
4958 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4959 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4960}
4961
4962SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4963 unsigned Opc = N->getOpcode();
4964 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4965 assert(N->getValueType(0).isVector() &&
4966 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4967 "Operand types must be vectors");
4968 // The result has a legal vector type, but the input needs splitting.
4969 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4970 SDLoc DL(N);
4971 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4972 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4973
4974 EVT VT = N->getValueType(ResNo: 0);
4975 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
4976
4977 if (Opc == ISD::SETCC) {
4978 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4979 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4980 } else if (isStrict) {
4981 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4982 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4983 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4984 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4985 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4986 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4987 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4988 } else {
4989 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4990 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4991 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4992 std::tie(args&: EVLLo, args&: EVLHi) =
4993 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4994 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4995 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4996 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4997 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4998 }
4999
5000 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
5001 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
5002 if (VT == ConcatVT)
5003 return Con;
5004
5005 EVT OpVT = N->getOperand(Num: 0).getValueType();
5006 ISD::NodeType ExtendCode =
5007 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
5008 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
5009}
5010
5011
5012SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5013 // The result has a legal vector type, but the input needs splitting.
5014 EVT ResVT = N->getValueType(ResNo: 0);
5015 SDValue Lo, Hi;
5016 SDLoc DL(N);
5017 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5018 EVT InVT = Lo.getValueType();
5019
5020 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5021 EC: InVT.getVectorElementCount());
5022
5023 if (N->isStrictFPOpcode()) {
5024 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5025 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5026 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5027 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5028 // Legalize the chain result - switch anything that used the old chain to
5029 // use the new one.
5030 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5031 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5032 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5033 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
5034 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
5035 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
5036 std::tie(args&: EVLLo, args&: EVLHi) =
5037 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
5038 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5039 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5040 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5041 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5042 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5043 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5044 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5045 } else {
5046 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5047 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5048 }
5049
5050 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5051}
5052
5053// Split a vector type in an FP binary operation where the second operand has a
5054// different type from the first.
5055//
5056// The result (and the first input) has a legal vector type, but the second
5057// input needs splitting.
5058SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5059 SDLoc DL(N);
5060
5061 EVT LHSLoVT, LHSHiVT;
5062 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5063
5064 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5065 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5066
5067 SDValue LHSLo, LHSHi;
5068 std::tie(args&: LHSLo, args&: LHSHi) =
5069 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5070
5071 SDValue RHSLo, RHSHi;
5072 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5073
5074 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5075 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5076
5077 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5078}
5079
5080SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5081 LLVMContext &Ctxt = *DAG.getContext();
5082 SDLoc dl(N);
5083
5084 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5085 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5086 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5087
5088 EVT ResVT = N->getValueType(ResNo: 0);
5089 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5090 EVT NewResVT =
5091 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5092
5093 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5094 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5095
5096 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5097}
5098
5099SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5100 EVT ResVT = N->getValueType(ResNo: 0);
5101 SDValue Lo, Hi;
5102 SDLoc dl(N);
5103 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5104 EVT InVT = Lo.getValueType();
5105
5106 EVT NewResVT =
5107 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5108 EC: InVT.getVectorElementCount());
5109
5110 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5111 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5112
5113 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5114}
5115
5116SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5117 SDLoc DL(N);
5118 EVT ResVT = N->getValueType(ResNo: 0);
5119
5120 SDValue Lo, Hi;
5121 SDValue VecOp = N->getOperand(Num: 0);
5122 GetSplitVector(Op: VecOp, Lo, Hi);
5123
5124 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5125 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5126 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5127 SDValue VL =
5128 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5129 SDValue ResLoNotVL =
5130 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5131 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5132 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5133 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5134}
5135
5136SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5137 SDLoc DL(N);
5138 EVT ResVT = N->getValueType(ResNo: 0);
5139
5140 SDValue Lo, Hi;
5141 SDValue VecOp = N->getOperand(Num: 0);
5142 GetSplitVector(Op: VecOp, Lo, Hi);
5143
5144 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5145 auto [EVLLo, EVLHi] =
5146 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5147 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5148
5149 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5150 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5151 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5152 SDValue ResLoNotEVL =
5153 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5154 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5155 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5156 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5157}
5158
5159SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5160 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5161 SDLoc DL(HG);
5162 SDValue Inc = HG->getInc();
5163 SDValue Ptr = HG->getBasePtr();
5164 SDValue Scale = HG->getScale();
5165 SDValue IntID = HG->getIntID();
5166 EVT MemVT = HG->getMemoryVT();
5167 MachineMemOperand *MMO = HG->getMemOperand();
5168 ISD::MemIndexType IndexType = HG->getIndexType();
5169
5170 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5171 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5172 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5173 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5174 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5175 Ops: OpsLo, MMO, IndexType);
5176 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5177 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5178 MMO, IndexType);
5179}
5180
5181SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5182 SDValue Acc = N->getOperand(Num: 0);
5183 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5184 "Accumulator should already be a legal type, and shouldn't need "
5185 "further splitting");
5186
5187 SDLoc DL(N);
5188 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5189 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5190 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5191 unsigned Opcode = N->getOpcode();
5192 EVT ResultVT = Acc.getValueType();
5193
5194 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5195 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5196}
5197
5198//===----------------------------------------------------------------------===//
5199// Result Vector Widening
5200//===----------------------------------------------------------------------===//
5201
5202void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5203 unsigned WidenResNo) {
5204 unsigned NumResults = N->getNumValues();
5205 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5206 if (ResNo == WidenResNo)
5207 continue;
5208 EVT ResVT = N->getValueType(ResNo);
5209 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5210 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5211 } else {
5212 SDLoc DL(N);
5213 SDValue ResVal =
5214 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5215 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5216 }
5217 }
5218}
5219
5220void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5221 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5222
5223 // See if the target wants to custom widen this node.
5224 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5225 return;
5226
5227 SDValue Res = SDValue();
5228
5229 auto unrollExpandedOp = [&]() {
5230 // We're going to widen this vector op to a legal type by padding with undef
5231 // elements. If the wide vector op is eventually going to be expanded to
5232 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5233 // libcalls on the undef elements.
5234 EVT VT = N->getValueType(ResNo: 0);
5235 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5236 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5237 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
5238 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5239 if (N->getNumValues() > 1)
5240 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
5241 return true;
5242 }
5243 return false;
5244 };
5245
5246 switch (N->getOpcode()) {
5247 default:
5248#ifndef NDEBUG
5249 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5250 N->dump(&DAG);
5251 dbgs() << "\n";
5252#endif
5253 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
5254
5255 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5256 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5257 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5258 break;
5259 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5260 case ISD::ADDRSPACECAST:
5261 Res = WidenVecRes_ADDRSPACECAST(N);
5262 break;
5263 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5264 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5265 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5266 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5267 case ISD::INSERT_SUBVECTOR:
5268 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5269 break;
5270 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5271 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5272 case ISD::ATOMIC_LOAD:
5273 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5274 break;
5275 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5276 case ISD::STEP_VECTOR:
5277 case ISD::SPLAT_VECTOR:
5278 case ISD::SCALAR_TO_VECTOR:
5279 Res = WidenVecRes_ScalarOp(N);
5280 break;
5281 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5282 case ISD::VSELECT:
5283 case ISD::SELECT:
5284 case ISD::VP_SELECT:
5285 case ISD::VP_MERGE:
5286 Res = WidenVecRes_Select(N);
5287 break;
5288 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5289 case ISD::VP_SETCC:
5290 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5291 case ISD::POISON:
5292 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5293 case ISD::VECTOR_SHUFFLE:
5294 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5295 break;
5296 case ISD::VP_LOAD:
5297 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5298 break;
5299 case ISD::VP_LOAD_FF:
5300 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5301 break;
5302 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5303 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5304 break;
5305 case ISD::VECTOR_COMPRESS:
5306 Res = WidenVecRes_VECTOR_COMPRESS(N);
5307 break;
5308 case ISD::MLOAD:
5309 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5310 break;
5311 case ISD::MGATHER:
5312 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5313 break;
5314 case ISD::VP_GATHER:
5315 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5316 break;
5317 case ISD::VECTOR_REVERSE:
5318 Res = WidenVecRes_VECTOR_REVERSE(N);
5319 break;
5320 case ISD::GET_ACTIVE_LANE_MASK:
5321 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5322 break;
5323 case ISD::VECTOR_DEINTERLEAVE:
5324 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5325 break;
5326
5327 case ISD::ADD: case ISD::VP_ADD:
5328 case ISD::AND: case ISD::VP_AND:
5329 case ISD::MUL: case ISD::VP_MUL:
5330 case ISD::MULHS:
5331 case ISD::MULHU:
5332 case ISD::ABDS:
5333 case ISD::ABDU:
5334 case ISD::OR: case ISD::VP_OR:
5335 case ISD::SUB: case ISD::VP_SUB:
5336 case ISD::XOR: case ISD::VP_XOR:
5337 case ISD::SHL: case ISD::VP_SHL:
5338 case ISD::SRA: case ISD::VP_SRA:
5339 case ISD::SRL: case ISD::VP_SRL:
5340 case ISD::CLMUL:
5341 case ISD::CLMULR:
5342 case ISD::CLMULH:
5343 case ISD::PEXT:
5344 case ISD::PDEP:
5345 case ISD::FMINNUM:
5346 case ISD::FMINNUM_IEEE:
5347 case ISD::VP_FMINNUM:
5348 case ISD::FMAXNUM:
5349 case ISD::FMAXNUM_IEEE:
5350 case ISD::VP_FMAXNUM:
5351 case ISD::FMINIMUM:
5352 case ISD::VP_FMINIMUM:
5353 case ISD::FMAXIMUM:
5354 case ISD::VP_FMAXIMUM:
5355 case ISD::FMINIMUMNUM:
5356 case ISD::FMAXIMUMNUM:
5357 case ISD::SMIN: case ISD::VP_SMIN:
5358 case ISD::SMAX: case ISD::VP_SMAX:
5359 case ISD::UMIN: case ISD::VP_UMIN:
5360 case ISD::UMAX: case ISD::VP_UMAX:
5361 case ISD::UADDSAT: case ISD::VP_UADDSAT:
5362 case ISD::SADDSAT: case ISD::VP_SADDSAT:
5363 case ISD::USUBSAT: case ISD::VP_USUBSAT:
5364 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
5365 case ISD::SSHLSAT:
5366 case ISD::USHLSAT:
5367 case ISD::ROTL:
5368 case ISD::ROTR:
5369 case ISD::AVGFLOORS:
5370 case ISD::AVGFLOORU:
5371 case ISD::AVGCEILS:
5372 case ISD::AVGCEILU:
5373 // Vector-predicated binary op widening. Note that -- unlike the
5374 // unpredicated versions -- we don't have to worry about trapping on
5375 // operations like UDIV, FADD, etc., as we pass on the original vector
5376 // length parameter. This means the widened elements containing garbage
5377 // aren't active.
5378 case ISD::VP_SDIV:
5379 case ISD::VP_UDIV:
5380 case ISD::VP_SREM:
5381 case ISD::VP_UREM:
5382 case ISD::VP_FADD:
5383 case ISD::VP_FSUB:
5384 case ISD::VP_FMUL:
5385 case ISD::VP_FDIV:
5386 case ISD::VP_FREM:
5387 case ISD::VP_FCOPYSIGN:
5388 Res = WidenVecRes_Binary(N);
5389 break;
5390
5391 case ISD::MASKED_UDIV:
5392 case ISD::MASKED_SDIV:
5393 case ISD::MASKED_UREM:
5394 case ISD::MASKED_SREM:
5395 Res = WidenVecRes_MaskedBinary(N);
5396 break;
5397
5398 case ISD::SCMP:
5399 case ISD::UCMP:
5400 Res = WidenVecRes_CMP(N);
5401 break;
5402
5403 case ISD::FPOW:
5404 case ISD::FATAN2:
5405 case ISD::FREM:
5406 if (unrollExpandedOp())
5407 break;
5408 // If the target has custom/legal support for the scalar FP intrinsic ops
5409 // (they are probably not destined to become libcalls), then widen those
5410 // like any other binary ops.
5411 [[fallthrough]];
5412
5413 case ISD::FADD:
5414 case ISD::FMUL:
5415 case ISD::FSUB:
5416 case ISD::FDIV:
5417 case ISD::SDIV:
5418 case ISD::UDIV:
5419 case ISD::SREM:
5420 case ISD::UREM:
5421 Res = WidenVecRes_BinaryCanTrap(N);
5422 break;
5423
5424 case ISD::SMULFIX:
5425 case ISD::SMULFIXSAT:
5426 case ISD::UMULFIX:
5427 case ISD::UMULFIXSAT:
5428 // These are binary operations, but with an extra operand that shouldn't
5429 // be widened (the scale).
5430 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5431 break;
5432
5433#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5434 case ISD::STRICT_##DAGN:
5435#include "llvm/IR/ConstrainedOps.def"
5436 Res = WidenVecRes_StrictFP(N);
5437 break;
5438
5439 case ISD::UADDO:
5440 case ISD::SADDO:
5441 case ISD::USUBO:
5442 case ISD::SSUBO:
5443 case ISD::UMULO:
5444 case ISD::SMULO:
5445 Res = WidenVecRes_OverflowOp(N, ResNo);
5446 break;
5447
5448 case ISD::FCOPYSIGN:
5449 Res = WidenVecRes_FCOPYSIGN(N);
5450 break;
5451
5452 case ISD::IS_FPCLASS:
5453 case ISD::FPTRUNC_ROUND:
5454 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5455 break;
5456
5457 case ISD::FLDEXP:
5458 case ISD::FPOWI:
5459 if (!unrollExpandedOp())
5460 Res = WidenVecRes_ExpOp(N);
5461 break;
5462
5463 case ISD::ANY_EXTEND_VECTOR_INREG:
5464 case ISD::SIGN_EXTEND_VECTOR_INREG:
5465 case ISD::ZERO_EXTEND_VECTOR_INREG:
5466 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5467 break;
5468
5469 case ISD::ANY_EXTEND:
5470 case ISD::FP_EXTEND:
5471 case ISD::VP_FP_EXTEND:
5472 case ISD::FP_ROUND:
5473 case ISD::VP_FP_ROUND:
5474 case ISD::FP_TO_SINT:
5475 case ISD::VP_FP_TO_SINT:
5476 case ISD::FP_TO_UINT:
5477 case ISD::VP_FP_TO_UINT:
5478 case ISD::SIGN_EXTEND:
5479 case ISD::VP_SIGN_EXTEND:
5480 case ISD::SINT_TO_FP:
5481 case ISD::VP_SINT_TO_FP:
5482 case ISD::VP_TRUNCATE:
5483 case ISD::TRUNCATE:
5484 case ISD::UINT_TO_FP:
5485 case ISD::VP_UINT_TO_FP:
5486 case ISD::ZERO_EXTEND:
5487 case ISD::VP_ZERO_EXTEND:
5488 case ISD::CONVERT_FROM_ARBITRARY_FP:
5489 case ISD::CONVERT_TO_ARBITRARY_FP:
5490 Res = WidenVecRes_Convert(N);
5491 break;
5492
5493 case ISD::FP_TO_SINT_SAT:
5494 case ISD::FP_TO_UINT_SAT:
5495 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5496 break;
5497
5498 case ISD::LRINT:
5499 case ISD::LLRINT:
5500 case ISD::VP_LRINT:
5501 case ISD::VP_LLRINT:
5502 case ISD::LROUND:
5503 case ISD::LLROUND:
5504 Res = WidenVecRes_XROUND(N);
5505 break;
5506
5507 case ISD::FACOS:
5508 case ISD::FASIN:
5509 case ISD::FATAN:
5510 case ISD::FCEIL:
5511 case ISD::FCOS:
5512 case ISD::FCOSH:
5513 case ISD::FEXP:
5514 case ISD::FEXP2:
5515 case ISD::FEXP10:
5516 case ISD::FFLOOR:
5517 case ISD::FLOG:
5518 case ISD::FLOG10:
5519 case ISD::FLOG2:
5520 case ISD::FNEARBYINT:
5521 case ISD::FRINT:
5522 case ISD::FROUND:
5523 case ISD::FROUNDEVEN:
5524 case ISD::FSIN:
5525 case ISD::FSINH:
5526 case ISD::FSQRT:
5527 case ISD::FTAN:
5528 case ISD::FTANH:
5529 case ISD::FTRUNC:
5530 if (unrollExpandedOp())
5531 break;
5532 // If the target has custom/legal support for the scalar FP intrinsic ops
5533 // (they are probably not destined to become libcalls), then widen those
5534 // like any other unary ops.
5535 [[fallthrough]];
5536
5537 case ISD::ABS:
5538 case ISD::ABS_MIN_POISON:
5539 case ISD::VP_ABS:
5540 case ISD::BITREVERSE:
5541 case ISD::VP_BITREVERSE:
5542 case ISD::BSWAP:
5543 case ISD::VP_BSWAP:
5544 case ISD::CTLZ:
5545 case ISD::VP_CTLZ:
5546 case ISD::CTLZ_ZERO_POISON:
5547 case ISD::VP_CTLZ_ZERO_POISON:
5548 case ISD::CTPOP:
5549 case ISD::VP_CTPOP:
5550 case ISD::CTTZ:
5551 case ISD::VP_CTTZ:
5552 case ISD::CTTZ_ZERO_POISON:
5553 case ISD::VP_CTTZ_ZERO_POISON:
5554 case ISD::FNEG: case ISD::VP_FNEG:
5555 case ISD::FABS: case ISD::VP_FABS:
5556 case ISD::VP_SQRT:
5557 case ISD::VP_FCEIL:
5558 case ISD::VP_FFLOOR:
5559 case ISD::VP_FRINT:
5560 case ISD::VP_FNEARBYINT:
5561 case ISD::VP_FROUND:
5562 case ISD::VP_FROUNDEVEN:
5563 case ISD::VP_FROUNDTOZERO:
5564 case ISD::FREEZE:
5565 case ISD::ARITH_FENCE:
5566 case ISD::FCANONICALIZE:
5567 case ISD::AssertNoFPClass:
5568 Res = WidenVecRes_Unary(N);
5569 break;
5570 case ISD::FMA: case ISD::VP_FMA:
5571 case ISD::FSHL:
5572 case ISD::VP_FSHL:
5573 case ISD::FSHR:
5574 case ISD::VP_FSHR:
5575 Res = WidenVecRes_Ternary(N);
5576 break;
5577 case ISD::FMODF:
5578 case ISD::FFREXP:
5579 case ISD::FSINCOS:
5580 case ISD::FSINCOSPI: {
5581 if (!unrollExpandedOp())
5582 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5583 break;
5584 }
5585 }
5586
5587 // If Res is null, the sub-method took care of registering the result.
5588 if (Res.getNode())
5589 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5590}
5591
5592SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5593 // Ternary op widening.
5594 SDLoc dl(N);
5595 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5596 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5597 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5598 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5599 if (N->getNumOperands() == 3)
5600 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5601
5602 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5603 assert(N->isVPOpcode() && "Expected VP opcode");
5604
5605 SDValue Mask =
5606 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5607 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5608 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5609}
5610
5611SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5612 // Binary op widening.
5613 SDLoc dl(N);
5614 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5615 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5616 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5617 if (N->getNumOperands() == 2)
5618 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5619 Flags: N->getFlags());
5620
5621 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5622 assert(N->isVPOpcode() && "Expected VP opcode");
5623
5624 SDValue Mask =
5625 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5626 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5627 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5628}
5629
5630SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5631 SDLoc dl(N);
5632 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5633 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5634 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5635 SDValue Mask = N->getOperand(Num: 2);
5636 EVT WideMaskVT = WidenVT.changeVectorElementType(
5637 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5638 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5639 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5640 Flags: N->getFlags());
5641}
5642
5643SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5644 LLVMContext &Ctxt = *DAG.getContext();
5645 SDLoc dl(N);
5646
5647 SDValue LHS = N->getOperand(Num: 0);
5648 SDValue RHS = N->getOperand(Num: 1);
5649 EVT OpVT = LHS.getValueType();
5650 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5651 LHS = GetWidenedVector(Op: LHS);
5652 RHS = GetWidenedVector(Op: RHS);
5653 OpVT = LHS.getValueType();
5654 }
5655
5656 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5657 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5658 if (WidenResEC == OpVT.getVectorElementCount()) {
5659 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5660 }
5661
5662 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5663}
5664
5665SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5666 // Binary op widening, but with an extra operand that shouldn't be widened.
5667 SDLoc dl(N);
5668 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5669 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5670 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5671 SDValue InOp3 = N->getOperand(Num: 2);
5672 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5673 Flags: N->getFlags());
5674}
5675
5676// Given a vector of operations that have been broken up to widen, see
5677// if we can collect them together into the next widest legal VT. This
5678// implementation is trap-safe.
5679static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5680 SmallVectorImpl<SDValue> &ConcatOps,
5681 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5682 EVT WidenVT) {
5683 // Check to see if we have a single operation with the widen type.
5684 if (ConcatEnd == 1) {
5685 VT = ConcatOps[0].getValueType();
5686 if (VT == WidenVT)
5687 return ConcatOps[0];
5688 }
5689
5690 SDLoc dl(ConcatOps[0]);
5691 EVT WidenEltVT = WidenVT.getVectorElementType();
5692
5693 // while (Some element of ConcatOps is not of type MaxVT) {
5694 // From the end of ConcatOps, collect elements of the same type and put
5695 // them into an op of the next larger supported type
5696 // }
5697 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5698 int Idx = ConcatEnd - 1;
5699 VT = ConcatOps[Idx--].getValueType();
5700 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5701 Idx--;
5702
5703 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5704 EVT NextVT;
5705 do {
5706 NextSize *= 2;
5707 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5708 } while (!TLI.isTypeLegal(VT: NextVT));
5709
5710 if (!VT.isVector()) {
5711 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5712 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5713 unsigned NumToInsert = ConcatEnd - Idx - 1;
5714 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5715 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5716 ConcatOps[Idx+1] = VecOp;
5717 ConcatEnd = Idx + 2;
5718 } else {
5719 // Vector type, create a CONCAT_VECTORS of type NextVT
5720 SDValue undefVec = DAG.getPOISON(VT);
5721 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5722 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5723 unsigned RealVals = ConcatEnd - Idx - 1;
5724 unsigned SubConcatEnd = 0;
5725 unsigned SubConcatIdx = Idx + 1;
5726 while (SubConcatEnd < RealVals)
5727 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5728 while (SubConcatEnd < OpsToConcat)
5729 SubConcatOps[SubConcatEnd++] = undefVec;
5730 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5731 VT: NextVT, Ops: SubConcatOps);
5732 ConcatEnd = SubConcatIdx + 1;
5733 }
5734 }
5735
5736 // Check to see if we have a single operation with the widen type.
5737 if (ConcatEnd == 1) {
5738 VT = ConcatOps[0].getValueType();
5739 if (VT == WidenVT)
5740 return ConcatOps[0];
5741 }
5742
5743 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5744 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5745 if (NumOps != ConcatEnd ) {
5746 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5747 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5748 ConcatOps[j] = UndefVal;
5749 }
5750 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5751 Ops: ArrayRef(ConcatOps.data(), NumOps));
5752}
5753
5754SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5755 // Binary op widening for operations that can trap.
5756 unsigned Opcode = N->getOpcode();
5757 SDLoc dl(N);
5758 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5759 EVT WidenEltVT = WidenVT.getVectorElementType();
5760 EVT VT = WidenVT;
5761 unsigned NumElts = VT.getVectorMinNumElements();
5762 const SDNodeFlags Flags = N->getFlags();
5763 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5764 NumElts = NumElts / 2;
5765 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5766 }
5767
5768 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5769 // Operation doesn't trap so just widen as normal.
5770 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5771 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5772 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5773 }
5774
5775 // Generate a vp.op if it is custom/legal for the target. This avoids need
5776 // to split and tile the subvectors (below), because the inactive lanes can
5777 // simply be disabled. To avoid possible recursion, only do this if the
5778 // widened mask type is legal.
5779 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5780 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5781 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5782 EC: WidenVT.getVectorElementCount());
5783 TLI.isTypeLegal(VT: WideMaskVT)) {
5784 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5785 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5786 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5787 SDValue EVL =
5788 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5789 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5790 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5791 Flags);
5792 }
5793 }
5794
5795 // FIXME: Improve support for scalable vectors.
5796 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5797
5798 // No legal vector version so unroll the vector operation and then widen.
5799 if (NumElts == 1)
5800 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5801
5802 // Since the operation can trap, apply operation on the original vector.
5803 EVT MaxVT = VT;
5804 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5805 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5806 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5807
5808 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5809 unsigned ConcatEnd = 0; // Current ConcatOps index.
5810 int Idx = 0; // Current Idx into input vectors.
5811
5812 // NumElts := greatest legal vector size (at most WidenVT)
5813 // while (orig. vector has unhandled elements) {
5814 // take munches of size NumElts from the beginning and add to ConcatOps
5815 // NumElts := next smaller supported vector size or 1
5816 // }
5817 while (CurNumElts != 0) {
5818 while (CurNumElts >= NumElts) {
5819 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5820 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5821 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5822 Idx += NumElts;
5823 CurNumElts -= NumElts;
5824 }
5825 do {
5826 NumElts = NumElts / 2;
5827 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5828 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5829
5830 if (NumElts == 1) {
5831 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5832 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5833 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5834 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5835 N1: EOp1, N2: EOp2, Flags);
5836 }
5837 CurNumElts = 0;
5838 }
5839 }
5840
5841 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5842}
5843
5844SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5845 switch (N->getOpcode()) {
5846 case ISD::STRICT_FSETCC:
5847 case ISD::STRICT_FSETCCS:
5848 return WidenVecRes_STRICT_FSETCC(N);
5849 case ISD::STRICT_FP_EXTEND:
5850 case ISD::STRICT_FP_ROUND:
5851 case ISD::STRICT_FP_TO_SINT:
5852 case ISD::STRICT_FP_TO_UINT:
5853 case ISD::STRICT_SINT_TO_FP:
5854 case ISD::STRICT_UINT_TO_FP:
5855 return WidenVecRes_Convert_StrictFP(N);
5856 default:
5857 break;
5858 }
5859
5860 // StrictFP op widening for operations that can trap.
5861 unsigned NumOpers = N->getNumOperands();
5862 unsigned Opcode = N->getOpcode();
5863 SDLoc dl(N);
5864 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5865 EVT WidenEltVT = WidenVT.getVectorElementType();
5866 EVT VT = WidenVT;
5867 unsigned NumElts = VT.getVectorNumElements();
5868 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5869 NumElts = NumElts / 2;
5870 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5871 }
5872
5873 // No legal vector version so unroll the vector operation and then widen.
5874 if (NumElts == 1)
5875 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5876
5877 // Since the operation can trap, apply operation on the original vector.
5878 EVT MaxVT = VT;
5879 SmallVector<SDValue, 4> InOps;
5880 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5881
5882 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5883 SmallVector<SDValue, 16> Chains;
5884 unsigned ConcatEnd = 0; // Current ConcatOps index.
5885 int Idx = 0; // Current Idx into input vectors.
5886
5887 // The Chain is the first operand.
5888 InOps.push_back(Elt: N->getOperand(Num: 0));
5889
5890 // Now process the remaining operands.
5891 for (unsigned i = 1; i < NumOpers; ++i) {
5892 SDValue Oper = N->getOperand(Num: i);
5893
5894 EVT OpVT = Oper.getValueType();
5895 if (OpVT.isVector()) {
5896 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5897 Oper = GetWidenedVector(Op: Oper);
5898 else {
5899 EVT WideOpVT =
5900 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5901 EC: WidenVT.getVectorElementCount());
5902 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5903 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5904 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5905 }
5906 }
5907
5908 InOps.push_back(Elt: Oper);
5909 }
5910
5911 // NumElts := greatest legal vector size (at most WidenVT)
5912 // while (orig. vector has unhandled elements) {
5913 // take munches of size NumElts from the beginning and add to ConcatOps
5914 // NumElts := next smaller supported vector size or 1
5915 // }
5916 while (CurNumElts != 0) {
5917 while (CurNumElts >= NumElts) {
5918 SmallVector<SDValue, 4> EOps;
5919
5920 for (unsigned i = 0; i < NumOpers; ++i) {
5921 SDValue Op = InOps[i];
5922
5923 EVT OpVT = Op.getValueType();
5924 if (OpVT.isVector()) {
5925 EVT OpExtractVT =
5926 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5927 EC: VT.getVectorElementCount());
5928 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5929 }
5930
5931 EOps.push_back(Elt: Op);
5932 }
5933
5934 EVT OperVT[] = {VT, MVT::Other};
5935 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5936 ConcatOps[ConcatEnd++] = Oper;
5937 Chains.push_back(Elt: Oper.getValue(R: 1));
5938 Idx += NumElts;
5939 CurNumElts -= NumElts;
5940 }
5941 do {
5942 NumElts = NumElts / 2;
5943 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5944 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5945
5946 if (NumElts == 1) {
5947 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5948 SmallVector<SDValue, 4> EOps;
5949
5950 for (unsigned i = 0; i < NumOpers; ++i) {
5951 SDValue Op = InOps[i];
5952
5953 EVT OpVT = Op.getValueType();
5954 if (OpVT.isVector())
5955 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5956 Idx);
5957
5958 EOps.push_back(Elt: Op);
5959 }
5960
5961 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5962 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5963 ConcatOps[ConcatEnd++] = Oper;
5964 Chains.push_back(Elt: Oper.getValue(R: 1));
5965 }
5966 CurNumElts = 0;
5967 }
5968 }
5969
5970 // Build a factor node to remember all the Ops that have been created.
5971 SDValue NewChain;
5972 if (Chains.size() == 1)
5973 NewChain = Chains[0];
5974 else
5975 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5976 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5977
5978 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5979}
5980
5981SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5982 SDLoc DL(N);
5983 EVT ResVT = N->getValueType(ResNo: 0);
5984 EVT OvVT = N->getValueType(ResNo: 1);
5985 EVT WideResVT, WideOvVT;
5986 SDValue WideLHS, WideRHS;
5987
5988 // TODO: This might result in a widen/split loop.
5989 if (ResNo == 0) {
5990 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5991 WideOvVT = EVT::getVectorVT(
5992 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5993 NumElements: WideResVT.getVectorNumElements());
5994
5995 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5996 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5997 } else {
5998 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5999 WideResVT = EVT::getVectorVT(
6000 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
6001 NumElements: WideOvVT.getVectorNumElements());
6002
6003 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6004 SDValue Poison = DAG.getPOISON(VT: WideResVT);
6005
6006 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6007 N2: N->getOperand(Num: 0), N3: Zero);
6008 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6009 N2: N->getOperand(Num: 1), N3: Zero);
6010 }
6011
6012 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
6013 SDNode *WideNode = DAG.getNode(
6014 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
6015
6016 // Replace the other vector result not being explicitly widened here.
6017 unsigned OtherNo = 1 - ResNo;
6018 EVT OtherVT = N->getValueType(ResNo: OtherNo);
6019 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
6020 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
6021 } else {
6022 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6023 SDValue OtherVal = DAG.getNode(
6024 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
6025 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
6026 }
6027
6028 return SDValue(WideNode, ResNo);
6029}
6030
6031SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6032 LLVMContext &Ctx = *DAG.getContext();
6033 SDValue InOp = N->getOperand(Num: 0);
6034 SDLoc DL(N);
6035
6036 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6037 ElementCount WidenEC = WidenVT.getVectorElementCount();
6038
6039 EVT InVT = InOp.getValueType();
6040
6041 unsigned Opcode = N->getOpcode();
6042 const SDNodeFlags Flags = N->getFlags();
6043
6044 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6045 // not equal that of WidenVT.
6046 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6047 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6048 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6049 WidenVT.getScalarSizeInBits()) {
6050 InOp = ZExtPromotedInteger(Op: InOp);
6051 InVT = InOp.getValueType();
6052 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6053 Opcode = ISD::TRUNCATE;
6054 }
6055
6056 EVT InEltVT = InVT.getVectorElementType();
6057 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6058 ElementCount InVTEC = InVT.getVectorElementCount();
6059
6060 // Helper to build node with all scalar trailing operands.
6061 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6062 if (N->getNumOperands() == 1)
6063 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6064 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6065 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6066 N4: N->getOperand(Num: 3), Flags);
6067 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6068 };
6069
6070 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6071 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6072 InVT = InOp.getValueType();
6073 InVTEC = InVT.getVectorElementCount();
6074 if (InVTEC == WidenEC) {
6075 if (N->getNumOperands() == 3 && N->isVPOpcode()) {
6076 SDValue Mask =
6077 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6078 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
6079 }
6080 return MakeConvertNode(WidenVT, InOp);
6081 }
6082 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6083 // If both input and result vector types are of same width, extend
6084 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6085 // accepts fewer elements in the result than in the input.
6086 if (Opcode == ISD::ANY_EXTEND)
6087 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6088 if (Opcode == ISD::SIGN_EXTEND)
6089 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6090 if (Opcode == ISD::ZERO_EXTEND)
6091 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6092 }
6093
6094 // For TRUNCATE, try to widen using the legal EC of the input type instead
6095 // if the legalisation action for that intermediate type is not widening.
6096 // E.g. for trunc nxv1i64 -> nxv1i8 where
6097 // - nxv1i64 input gets widened to nxv2i64
6098 // - nxv1i8 output gets widened to nxv16i8
6099 // Then one can try widening the result to nxv2i8 (instead of going all the
6100 // way to nxv16i8) if this later allows type promotion.
6101 EVT MidResVT =
6102 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6103 if (N->getOpcode() == ISD::TRUNCATE &&
6104 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6105 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6106 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6107 }
6108 }
6109
6110 if (TLI.isTypeLegal(VT: InWidenVT)) {
6111 // Because the result and the input are different vector types, widening
6112 // the result could create a legal type but widening the input might make
6113 // it an illegal type that might lead to repeatedly splitting the input
6114 // and then widening it. To avoid this, we widen the input only if
6115 // it results in a legal type.
6116 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6117 // Widen the input and call convert on the widened input vector.
6118 unsigned NumConcat =
6119 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6120 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6121 Ops[0] = InOp;
6122 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6123 return MakeConvertNode(WidenVT, InVec);
6124 }
6125
6126 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6127 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6128 // Extract the input and convert the shorten input vector.
6129 return MakeConvertNode(WidenVT, InVal);
6130 }
6131 }
6132
6133 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6134 EVT EltVT = WidenVT.getVectorElementType();
6135 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6136 // Use the original element count so we don't do more scalar opts than
6137 // necessary.
6138 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6139 for (unsigned i=0; i < MinElts; ++i) {
6140 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6141 Ops[i] = MakeConvertNode(EltVT, Val);
6142 }
6143
6144 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6145}
6146
6147SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6148 SDLoc dl(N);
6149 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6150 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6151
6152 SDValue Src = N->getOperand(Num: 0);
6153 EVT SrcVT = Src.getValueType();
6154
6155 // Also widen the input.
6156 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6157 Src = GetWidenedVector(Op: Src);
6158 SrcVT = Src.getValueType();
6159 }
6160
6161 // Input and output not widened to the same size, give up.
6162 if (WidenNumElts != SrcVT.getVectorElementCount())
6163 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6164
6165 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6166}
6167
6168SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6169 SDLoc dl(N);
6170 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6171 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6172
6173 SDValue Src = N->getOperand(Num: 0);
6174 EVT SrcVT = Src.getValueType();
6175
6176 // Also widen the input.
6177 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6178 Src = GetWidenedVector(Op: Src);
6179 SrcVT = Src.getValueType();
6180 }
6181
6182 // Input and output not widened to the same size, give up.
6183 if (WidenNumElts != SrcVT.getVectorElementCount())
6184 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6185
6186 if (N->getNumOperands() == 1)
6187 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6188
6189 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6190 assert(N->isVPOpcode() && "Expected VP opcode");
6191
6192 SDValue Mask =
6193 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6194 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
6195}
6196
6197SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6198 SDValue InOp = N->getOperand(Num: 1);
6199 SDLoc DL(N);
6200 SmallVector<SDValue, 4> NewOps(N->ops());
6201
6202 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6203 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6204
6205 EVT InVT = InOp.getValueType();
6206 EVT InEltVT = InVT.getVectorElementType();
6207
6208 unsigned Opcode = N->getOpcode();
6209
6210 // FIXME: Optimizations need to be implemented here.
6211
6212 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6213 EVT EltVT = WidenVT.getVectorElementType();
6214 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6215 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6216 SmallVector<SDValue, 32> OpChains;
6217 // Use the original element count so we don't do more scalar opts than
6218 // necessary.
6219 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6220 for (unsigned i=0; i < MinElts; ++i) {
6221 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6222 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6223 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6224 }
6225 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6226 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6227
6228 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6229}
6230
6231SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6232 unsigned Opcode = N->getOpcode();
6233 SDValue InOp = N->getOperand(Num: 0);
6234 SDLoc DL(N);
6235
6236 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6237 EVT WidenSVT = WidenVT.getVectorElementType();
6238 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6239
6240 EVT InVT = InOp.getValueType();
6241 EVT InSVT = InVT.getVectorElementType();
6242 unsigned InVTNumElts = InVT.getVectorNumElements();
6243
6244 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6245 InOp = GetWidenedVector(Op: InOp);
6246 InVT = InOp.getValueType();
6247 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6248 switch (Opcode) {
6249 case ISD::ANY_EXTEND_VECTOR_INREG:
6250 case ISD::SIGN_EXTEND_VECTOR_INREG:
6251 case ISD::ZERO_EXTEND_VECTOR_INREG:
6252 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6253 }
6254 }
6255 }
6256
6257 // Unroll, extend the scalars and rebuild the vector.
6258 SmallVector<SDValue, 16> Ops;
6259 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6260 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6261 switch (Opcode) {
6262 case ISD::ANY_EXTEND_VECTOR_INREG:
6263 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6264 break;
6265 case ISD::SIGN_EXTEND_VECTOR_INREG:
6266 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6267 break;
6268 case ISD::ZERO_EXTEND_VECTOR_INREG:
6269 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6270 break;
6271 default:
6272 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6273 }
6274 Ops.push_back(Elt: Val);
6275 }
6276
6277 while (Ops.size() != WidenNumElts)
6278 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6279
6280 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6281}
6282
6283SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6284 // If this is an FCOPYSIGN with same input types, we can treat it as a
6285 // normal (can trap) binary op.
6286 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6287 return WidenVecRes_BinaryCanTrap(N);
6288
6289 // If the types are different, fall back to unrolling.
6290 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6291 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6292}
6293
6294/// Result and first source operand are different scalar types, but must have
6295/// the same number of elements. There is an additional control argument which
6296/// should be passed through unchanged.
6297SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6298 SDValue FpValue = N->getOperand(Num: 0);
6299 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6300 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6301 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6302 SDValue Arg = GetWidenedVector(Op: FpValue);
6303 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6304 Flags: N->getFlags());
6305}
6306
6307SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6308 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6309 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6310 SDValue RHS = N->getOperand(Num: 1);
6311 EVT ExpVT = RHS.getValueType();
6312 SDValue ExpOp = RHS;
6313 if (ExpVT.isVector()) {
6314 EVT WideExpVT = WidenVT.changeVectorElementType(
6315 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6316 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6317 }
6318
6319 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6320}
6321
6322SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6323 // Unary op widening.
6324 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6325 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6326 if (N->getNumOperands() == 1)
6327 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6328 if (N->getOpcode() == ISD::AssertNoFPClass)
6329 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
6330 N2: N->getOperand(Num: 1), Flags: N->getFlags());
6331
6332 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6333 assert(N->isVPOpcode() && "Expected VP opcode");
6334
6335 SDValue Mask =
6336 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6337 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
6338 Ops: {InOp, Mask, N->getOperand(Num: 2)});
6339}
6340
6341SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6342 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6343 EVT ExtVT = EVT::getVectorVT(
6344 Context&: *DAG.getContext(),
6345 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6346 EC: WidenVT.getVectorElementCount());
6347 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6348 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6349 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6350}
6351
6352SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6353 unsigned ResNo) {
6354 EVT VT0 = N->getValueType(ResNo: 0);
6355 EVT VT1 = N->getValueType(ResNo: 1);
6356
6357 assert(VT0.isVector() && VT1.isVector() &&
6358 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6359 "expected both results to be vectors of matching element count");
6360
6361 LLVMContext &Ctx = *DAG.getContext();
6362 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6363
6364 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6365 ElementCount WidenEC = WidenVT.getVectorElementCount();
6366
6367 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6368 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6369
6370 SDNode *WidenNode =
6371 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6372 .getNode();
6373
6374 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6375 return SDValue(WidenNode, ResNo);
6376}
6377
6378SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6379 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6380 return GetWidenedVector(Op: WidenVec);
6381}
6382
6383SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6384 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6385 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6386 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6387
6388 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
6389 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6390 DestAS: AddrSpaceCastN->getDestAddressSpace());
6391}
6392
6393SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6394 SDValue InOp = N->getOperand(Num: 0);
6395 EVT InVT = InOp.getValueType();
6396 EVT VT = N->getValueType(ResNo: 0);
6397 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6398 SDLoc dl(N);
6399
6400 switch (getTypeAction(VT: InVT)) {
6401 case TargetLowering::TypeLegal:
6402 break;
6403 case TargetLowering::TypeScalarizeScalableVector:
6404 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6405 case TargetLowering::TypePromoteInteger: {
6406 // If the incoming type is a vector that is being promoted, then
6407 // we know that the elements are arranged differently and that we
6408 // must perform the conversion using a stack slot.
6409 if (InVT.isVector())
6410 break;
6411
6412 // If the InOp is promoted to the same size, convert it. Otherwise,
6413 // fall out of the switch and widen the promoted input.
6414 SDValue NInOp = GetPromotedInteger(Op: InOp);
6415 EVT NInVT = NInOp.getValueType();
6416 if (WidenVT.bitsEq(VT: NInVT)) {
6417 // For big endian targets we need to shift the input integer or the
6418 // interesting bits will end up at the wrong place.
6419 if (DAG.getDataLayout().isBigEndian()) {
6420 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6421 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6422 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6423 }
6424 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6425 }
6426 InOp = NInOp;
6427 InVT = NInVT;
6428 break;
6429 }
6430 case TargetLowering::TypeSoftenFloat:
6431 case TargetLowering::TypeSoftPromoteHalf:
6432 case TargetLowering::TypeExpandInteger:
6433 case TargetLowering::TypeExpandFloat:
6434 case TargetLowering::TypeScalarizeVector:
6435 case TargetLowering::TypeSplitVector:
6436 break;
6437 case TargetLowering::TypeWidenVector:
6438 // If the InOp is widened to the same size, convert it. Otherwise, fall
6439 // out of the switch and widen the widened input.
6440 InOp = GetWidenedVector(Op: InOp);
6441 InVT = InOp.getValueType();
6442 if (WidenVT.bitsEq(VT: InVT))
6443 // The input widens to the same size. Convert to the widen value.
6444 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6445 break;
6446 }
6447
6448 unsigned WidenSize = WidenVT.getSizeInBits();
6449 unsigned InSize = InVT.getSizeInBits();
6450 unsigned InScalarSize = InVT.getScalarSizeInBits();
6451 // x86mmx is not an acceptable vector element type, so don't try.
6452 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6453 // Determine new input vector type. The new input vector type will use
6454 // the same element type (if its a vector) or use the input type as a
6455 // vector. It is the same size as the type to widen to.
6456 EVT NewInVT;
6457 unsigned NewNumParts = WidenSize / InSize;
6458 if (InVT.isVector()) {
6459 EVT InEltVT = InVT.getVectorElementType();
6460 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6461 NumElements: WidenSize / InEltVT.getSizeInBits());
6462 } else {
6463 // For big endian systems, using the promoted input scalar type
6464 // to produce the scalar_to_vector would put the desired bits into
6465 // the least significant byte(s) of the wider element zero. This
6466 // will mean that the users of the result vector are using incorrect
6467 // bits. Use the original input type instead. Although either input
6468 // type can be used on little endian systems, for consistency we
6469 // use the original type there as well.
6470 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6471 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6472 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6473 }
6474
6475 if (TLI.isTypeLegal(VT: NewInVT)) {
6476 SDValue NewVec;
6477 if (InVT.isVector()) {
6478 // Because the result and the input are different vector types, widening
6479 // the result could create a legal type but widening the input might
6480 // make it an illegal type that might lead to repeatedly splitting the
6481 // input and then widening it. To avoid this, we widen the input only if
6482 // it results in a legal type.
6483 if (WidenSize % InSize == 0) {
6484 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6485 Ops[0] = InOp;
6486
6487 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6488 } else {
6489 SmallVector<SDValue, 16> Ops;
6490 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6491 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6492 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6493
6494 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6495 }
6496 } else {
6497 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6498 }
6499 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6500 }
6501 }
6502
6503 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6504}
6505
6506SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6507 return DAG.getNode(
6508 Opcode: N->getOpcode(), DL: SDLoc(N),
6509 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6510 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6511}
6512
6513SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6514 SDLoc dl(N);
6515 // Build a vector with poison for the new nodes.
6516 EVT VT = N->getValueType(ResNo: 0);
6517
6518 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6519 // type. The POISONs need to have the same type as the existing operands.
6520 EVT EltVT = N->getOperand(Num: 0).getValueType();
6521 unsigned NumElts = VT.getVectorNumElements();
6522
6523 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6524 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6525
6526 SmallVector<SDValue, 16> NewOps(N->ops());
6527 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6528 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6529
6530 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6531}
6532
6533SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6534 EVT InVT = N->getOperand(Num: 0).getValueType();
6535 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6536 SDLoc dl(N);
6537 unsigned NumOperands = N->getNumOperands();
6538
6539 bool InputWidened = false; // Indicates we need to widen the input.
6540 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6541 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6542 unsigned NumInElts = InVT.getVectorMinNumElements();
6543 if (WidenNumElts % NumInElts == 0) {
6544 // Add undef vectors to widen to correct length.
6545 unsigned NumConcat = WidenNumElts / NumInElts;
6546 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6547 SmallVector<SDValue, 16> Ops(NumConcat);
6548 for (unsigned i=0; i < NumOperands; ++i)
6549 Ops[i] = N->getOperand(Num: i);
6550 for (unsigned i = NumOperands; i != NumConcat; ++i)
6551 Ops[i] = UndefVal;
6552 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6553 }
6554 } else {
6555 InputWidened = true;
6556 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6557 // The inputs and the result are widen to the same value.
6558 unsigned i;
6559 for (i=1; i < NumOperands; ++i)
6560 if (!N->getOperand(Num: i).isUndef())
6561 break;
6562
6563 if (i == NumOperands)
6564 // Everything but the first operand is an UNDEF so just return the
6565 // widened first operand.
6566 return GetWidenedVector(Op: N->getOperand(Num: 0));
6567
6568 if (NumOperands == 2) {
6569 assert(!WidenVT.isScalableVector() &&
6570 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6571 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6572 unsigned NumInElts = InVT.getVectorNumElements();
6573
6574 // Replace concat of two operands with a shuffle.
6575 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6576 for (unsigned i = 0; i < NumInElts; ++i) {
6577 MaskOps[i] = i;
6578 MaskOps[i + NumInElts] = i + WidenNumElts;
6579 }
6580 return DAG.getVectorShuffle(VT: WidenVT, dl,
6581 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6582 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6583 Mask: MaskOps);
6584 }
6585 }
6586 }
6587
6588 assert(!WidenVT.isScalableVector() &&
6589 "Cannot use build vectors to widen CONCAT_VECTOR result");
6590 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6591 unsigned NumInElts = InVT.getVectorNumElements();
6592
6593 // Fall back to use extracts and build vector.
6594 EVT EltVT = WidenVT.getVectorElementType();
6595 SmallVector<SDValue, 16> Ops(WidenNumElts);
6596 unsigned Idx = 0;
6597 for (unsigned i=0; i < NumOperands; ++i) {
6598 SDValue InOp = N->getOperand(Num: i);
6599 if (InputWidened)
6600 InOp = GetWidenedVector(Op: InOp);
6601 for (unsigned j = 0; j < NumInElts; ++j)
6602 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6603 }
6604 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6605 for (; Idx < WidenNumElts; ++Idx)
6606 Ops[Idx] = UndefVal;
6607 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6608}
6609
6610SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6611 EVT VT = N->getValueType(ResNo: 0);
6612 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6613 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6614 SDValue InOp2 = N->getOperand(Num: 1);
6615 SDValue Idx = N->getOperand(Num: 2);
6616 SDLoc dl(N);
6617 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6618}
6619
6620SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6621 EVT VT = N->getValueType(ResNo: 0);
6622 EVT EltVT = VT.getVectorElementType();
6623 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6624 SDValue InOp = N->getOperand(Num: 0);
6625 SDValue Idx = N->getOperand(Num: 1);
6626 SDLoc dl(N);
6627
6628 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6629 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6630 InOp = GetWidenedVector(Op: InOp);
6631
6632 EVT InVT = InOp.getValueType();
6633
6634 // Check if we can just return the input vector after widening.
6635 uint64_t IdxVal = Idx->getAsZExtVal();
6636 if (IdxVal == 0 && InVT == WidenVT)
6637 return InOp;
6638
6639 // Check if we can extract from the vector.
6640 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6641 unsigned InNumElts = InVT.getVectorMinNumElements();
6642 unsigned VTNumElts = VT.getVectorMinNumElements();
6643 assert(IdxVal % VTNumElts == 0 &&
6644 "Expected Idx to be a multiple of subvector minimum vector length");
6645 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6646 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6647
6648 if (VT.isScalableVector()) {
6649 // Try to split the operation up into smaller extracts and concat the
6650 // results together, e.g.
6651 // nxv6i64 extract_subvector(nxv12i64, 6)
6652 // <->
6653 // nxv8i64 concat(
6654 // nxv2i64 extract_subvector(nxv16i64, 6)
6655 // nxv2i64 extract_subvector(nxv16i64, 8)
6656 // nxv2i64 extract_subvector(nxv16i64, 10)
6657 // undef)
6658 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6659 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6660 "down type's element count");
6661 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6662 EC: ElementCount::getScalable(MinVal: GCD));
6663 // Avoid recursion around e.g. nxv1i8.
6664 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6665 SmallVector<SDValue> Parts;
6666 unsigned I = 0;
6667 for (; I < VTNumElts / GCD; ++I)
6668 Parts.push_back(
6669 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6670 for (; I < WidenNumElts / GCD; ++I)
6671 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6672
6673 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6674 }
6675
6676 // Fallback to extracting through memory.
6677
6678 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6679 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6680 MachineFunction &MF = DAG.getMachineFunction();
6681 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6682 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6683
6684 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6685 PtrInfo, F: MachineMemOperand::MOStore,
6686 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6687 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6688 PtrInfo, F: MachineMemOperand::MOLoad,
6689 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6690
6691 // Write out the input vector.
6692 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6693
6694 // Build a mask to match the length of the non-widened result.
6695 SDValue Mask =
6696 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6697
6698 // Read back the sub-vector setting the remaining lanes to poison.
6699 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6700 return DAG.getMaskedLoad(
6701 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6702 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6703 }
6704
6705 // We could try widening the input to the right length but for now, extract
6706 // the original elements, fill the rest with undefs and build a vector.
6707 SmallVector<SDValue, 16> Ops(WidenNumElts);
6708 unsigned i;
6709 for (i = 0; i < VTNumElts; ++i)
6710 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6711
6712 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6713 for (; i < WidenNumElts; ++i)
6714 Ops[i] = UndefVal;
6715 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6716}
6717
6718SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6719 SDValue InOp = ModifyToType(
6720 InOp: N->getOperand(Num: 0),
6721 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6722 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6723 N2: N->getOperand(Num: 1));
6724}
6725
6726SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6727 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6728 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6729 VT: InOp.getValueType(), N1: InOp,
6730 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6731}
6732
6733/// Either return the same load or provide appropriate casts
6734/// from the load and return that.
6735static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6736 TypeSize LdWidth, TypeSize FirstVTWidth,
6737 SDLoc dl, SelectionDAG &DAG) {
6738 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6739 "Load width must be less than or equal to first value type width");
6740 TypeSize WidenWidth = WidenVT.getSizeInBits();
6741 if (!FirstVT.isVector()) {
6742 unsigned NumElts =
6743 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6744 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6745 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6746 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6747 }
6748 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6749 return LdOp;
6750}
6751
6752/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6753/// widened value so it can be issued in a single atomic store.
6754static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6755 TypeSize FirstVTWidth, const SDLoc &dl,
6756 SelectionDAG &DAG) {
6757 TypeSize WidenWidth = WidenVT.getSizeInBits();
6758 if (!FirstVT.isVector()) {
6759 unsigned NumElts =
6760 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6761 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6762 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6763 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6764 }
6765 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6766 return StVal;
6767}
6768
6769static std::optional<EVT> findMemType(SelectionDAG &DAG,
6770 const TargetLowering &TLI, unsigned Width,
6771 EVT WidenVT, unsigned Align,
6772 unsigned WidenEx);
6773
6774SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6775 EVT WidenVT =
6776 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6777 EVT LdVT = LD->getMemoryVT();
6778 SDLoc dl(LD);
6779
6780 // Load information
6781 SDValue Chain = LD->getChain();
6782 SDValue BasePtr = LD->getBasePtr();
6783
6784 TypeSize LdWidth = LdVT.getSizeInBits();
6785 TypeSize WidenWidth = WidenVT.getSizeInBits();
6786 TypeSize WidthDiff = WidenWidth - LdWidth;
6787
6788 // Find the vector type that can load from.
6789 std::optional<EVT> FirstVT =
6790 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6791 WidenEx: WidthDiff.getKnownMinValue());
6792
6793 if (!FirstVT)
6794 return SDValue();
6795
6796 SmallVector<EVT, 8> MemVTs;
6797 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6798
6799 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6800 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6801
6802 // Load the element with one instruction.
6803 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6804 FirstVTWidth, dl, DAG);
6805
6806 // Modified the chain - switch anything that used the old chain to use
6807 // the new one.
6808 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6809 return Result;
6810}
6811
6812SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6813 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6814 ISD::LoadExtType ExtType = LD->getExtensionType();
6815
6816 // A vector must always be stored in memory as-is, i.e. without any padding
6817 // between the elements, since various code depend on it, e.g. in the
6818 // handling of a bitcast of a vector type to int, which may be done with a
6819 // vector store followed by an integer load. A vector that does not have
6820 // elements that are byte-sized must therefore be stored as an integer
6821 // built out of the extracted vector elements.
6822 if (!LD->getMemoryVT().isByteSized()) {
6823 SDValue Value, NewChain;
6824 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6825 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6826 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6827 return SDValue();
6828 }
6829
6830 // Generate a vector-predicated load if it is custom/legal on the target. To
6831 // avoid possible recursion, only do this if the widened mask type is legal.
6832 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6833 // removed from the IR by the ExpandVectorPredication pass but we're
6834 // reintroducing them here.
6835 EVT VT = LD->getValueType(ResNo: 0);
6836 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6837 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6838
6839 if (ExtType == ISD::NON_EXTLOAD &&
6840 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6841 TLI.isTypeLegal(VT: WideMaskVT)) {
6842 SDLoc DL(N);
6843 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6844 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6845 EC: VT.getVectorElementCount());
6846 SDValue NewLoad =
6847 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6848 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6849 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6850
6851 // Modified the chain - switch anything that used the old chain to use
6852 // the new one.
6853 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6854
6855 return NewLoad;
6856 }
6857
6858 SDValue Result;
6859 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6860 if (ExtType != ISD::NON_EXTLOAD)
6861 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6862 else
6863 Result = GenWidenVectorLoads(LdChain, LD);
6864
6865 if (Result) {
6866 // If we generate a single load, we can use that for the chain. Otherwise,
6867 // build a factor node to remember the multiple loads are independent and
6868 // chain to that.
6869 SDValue NewChain;
6870 if (LdChain.size() == 1)
6871 NewChain = LdChain[0];
6872 else
6873 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6874
6875 // Modified the chain - switch anything that used the old chain to use
6876 // the new one.
6877 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6878
6879 return Result;
6880 }
6881
6882 if (VT.isVector()) {
6883 // If all else fails replace the load with a wide masked load.
6884 SDLoc DL(N);
6885 SDValue Mask =
6886 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6887
6888 SDValue NewLoad = DAG.getMaskedLoad(
6889 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6890 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6891 AM: LD->getAddressingMode(), LD->getExtensionType());
6892
6893 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6894 return NewLoad;
6895 }
6896
6897 report_fatal_error(reason: "Unable to widen vector load");
6898}
6899
6900SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6901 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6902 SDValue Mask = N->getMask();
6903 SDValue EVL = N->getVectorLength();
6904 ISD::LoadExtType ExtType = N->getExtensionType();
6905 SDLoc dl(N);
6906
6907 // The mask should be widened as well
6908 assert(getTypeAction(Mask.getValueType()) ==
6909 TargetLowering::TypeWidenVector &&
6910 "Unable to widen binary VP op");
6911 Mask = GetWidenedVector(Op: Mask);
6912 assert(Mask.getValueType().getVectorElementCount() ==
6913 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6914 .getVectorElementCount() &&
6915 "Unable to widen vector load");
6916
6917 SDValue Res =
6918 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6919 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6920 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6921 // Legalize the chain result - switch anything that used the old chain to
6922 // use the new one.
6923 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6924 return Res;
6925}
6926
6927SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6928 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6929 SDValue Mask = N->getMask();
6930 SDValue EVL = N->getVectorLength();
6931 SDLoc dl(N);
6932
6933 // The mask should be widened as well
6934 assert(getTypeAction(Mask.getValueType()) ==
6935 TargetLowering::TypeWidenVector &&
6936 "Unable to widen binary VP op");
6937 Mask = GetWidenedVector(Op: Mask);
6938 assert(Mask.getValueType().getVectorElementCount() ==
6939 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6940 .getVectorElementCount() &&
6941 "Unable to widen vector load");
6942
6943 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6944 Mask, EVL, MMO: N->getMemOperand());
6945 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6946 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6947 return Res;
6948}
6949
6950SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6951 SDLoc DL(N);
6952
6953 // The mask should be widened as well
6954 SDValue Mask = N->getMask();
6955 assert(getTypeAction(Mask.getValueType()) ==
6956 TargetLowering::TypeWidenVector &&
6957 "Unable to widen VP strided load");
6958 Mask = GetWidenedVector(Op: Mask);
6959
6960 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6961 assert(Mask.getValueType().getVectorElementCount() ==
6962 WidenVT.getVectorElementCount() &&
6963 "Data and mask vectors should have the same number of elements");
6964
6965 SDValue Res = DAG.getStridedLoadVP(
6966 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6967 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6968 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6969 IsExpanding: N->isExpandingLoad());
6970
6971 // Legalize the chain result - switch anything that used the old chain to
6972 // use the new one.
6973 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6974 return Res;
6975}
6976
6977SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6978 SDValue Vec = N->getOperand(Num: 0);
6979 SDValue Mask = N->getOperand(Num: 1);
6980 SDValue Passthru = N->getOperand(Num: 2);
6981 EVT WideVecVT =
6982 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6983 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6984 VT: Mask.getValueType().getVectorElementType(),
6985 EC: WideVecVT.getVectorElementCount());
6986
6987 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6988 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6989 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6990 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6991 N2: WideMask, N3: WidePassthru);
6992}
6993
6994SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6995 EVT VT = N->getValueType(ResNo: 0);
6996 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6997 SDValue Mask = N->getMask();
6998 EVT MaskVT = Mask.getValueType();
6999 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7000 ISD::LoadExtType ExtType = N->getExtensionType();
7001 SDLoc dl(N);
7002
7003 EVT WideMaskVT =
7004 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7005 EC: WidenVT.getVectorElementCount());
7006
7007 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
7008 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
7009 TLI.isTypeLegal(VT: WideMaskVT) &&
7010 // If there is a passthru, we shouldn't use vp.load. However,
7011 // type legalizer will struggle on masked.load with
7012 // scalable vectors, so for scalable vectors, we still use vp.load
7013 // but manually merge the load result with the passthru using vp.select.
7014 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7015 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7016 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7017 EC: VT.getVectorElementCount());
7018 SDValue NewLoad =
7019 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
7020 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7021 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
7022 SDValue NewVal = NewLoad;
7023
7024 // Manually merge with vselect
7025 if (!N->getPassThru()->isUndef()) {
7026 assert(WidenVT.isScalableVector());
7027 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
7028 // The lanes past EVL are poison.
7029 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
7030 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
7031 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
7032 }
7033
7034 // Modified the chain - switch anything that used the old chain to use
7035 // the new one.
7036 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7037
7038 return NewVal;
7039 }
7040
7041 // The mask should be widened as well
7042 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7043
7044 SDValue Res = DAG.getMaskedLoad(
7045 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7046 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7047 ExtType, IsExpanding: N->isExpandingLoad());
7048 // Legalize the chain result - switch anything that used the old chain to
7049 // use the new one.
7050 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7051 return Res;
7052}
7053
7054SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7055
7056 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7057 SDValue Mask = N->getMask();
7058 EVT MaskVT = Mask.getValueType();
7059 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7060 SDValue Scale = N->getScale();
7061 ElementCount WideEC = WideVT.getVectorElementCount();
7062 SDLoc dl(N);
7063
7064 // The mask should be widened as well
7065 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7066 VT: MaskVT.getVectorElementType(), EC: WideEC);
7067 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7068
7069 // Widen the Index operand
7070 SDValue Index = N->getIndex();
7071 EVT WideIndexVT = EVT::getVectorVT(
7072 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7073 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7074 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7075 Scale };
7076
7077 // Widen the MemoryType
7078 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7079 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7080 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7081 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7082 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7083
7084 // Legalize the chain result - switch anything that used the old chain to
7085 // use the new one.
7086 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7087 return Res;
7088}
7089
7090SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7091 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7092 SDValue Mask = N->getMask();
7093 SDValue Scale = N->getScale();
7094 ElementCount WideEC = WideVT.getVectorElementCount();
7095 SDLoc dl(N);
7096
7097 SDValue Index = GetWidenedVector(Op: N->getIndex());
7098 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7099 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7100 Mask = GetWidenedMask(Mask, EC: WideEC);
7101
7102 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7103 Mask, N->getVectorLength()};
7104 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7105 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7106
7107 // Legalize the chain result - switch anything that used the old chain to
7108 // use the new one.
7109 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7110 return Res;
7111}
7112
7113SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7114 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7115 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7116}
7117
7118// Return true is this is a SETCC node or a strict version of it.
7119static inline bool isSETCCOp(unsigned Opcode) {
7120 switch (Opcode) {
7121 case ISD::SETCC:
7122 case ISD::STRICT_FSETCC:
7123 case ISD::STRICT_FSETCCS:
7124 return true;
7125 }
7126 return false;
7127}
7128
7129// Return true if this is a node that could have two SETCCs as operands.
7130static inline bool isLogicalMaskOp(unsigned Opcode) {
7131 switch (Opcode) {
7132 case ISD::AND:
7133 case ISD::OR:
7134 case ISD::XOR:
7135 return true;
7136 }
7137 return false;
7138}
7139
7140// If N is a SETCC or a strict variant of it, return the type
7141// of the compare operands.
7142static inline EVT getSETCCOperandType(SDValue N) {
7143 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7144 return N->getOperand(Num: OpNo).getValueType();
7145}
7146
7147// This is used just for the assert in convertMask(). Check that this either
7148// a SETCC or a previously handled SETCC by convertMask().
7149#ifndef NDEBUG
7150static inline bool isSETCCorConvertedSETCC(SDValue N) {
7151 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7152 N = N.getOperand(0);
7153 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7154 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7155 if (!N->getOperand(i)->isUndef())
7156 return false;
7157 N = N.getOperand(0);
7158 }
7159
7160 if (N.getOpcode() == ISD::TRUNCATE)
7161 N = N.getOperand(0);
7162 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7163 N = N.getOperand(0);
7164
7165 if (isLogicalMaskOp(N.getOpcode()))
7166 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7167 isSETCCorConvertedSETCC(N.getOperand(1));
7168
7169 return (isSETCCOp(N.getOpcode()) ||
7170 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7171}
7172#endif
7173
7174// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7175// to ToMaskVT if needed with vector extension or truncation.
7176SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7177 EVT ToMaskVT) {
7178 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
7179 // FIXME: This code seems to be too restrictive, we might consider
7180 // generalizing it or dropping it.
7181 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7182
7183 // Make a new Mask node, with a legal result VT.
7184 SDValue Mask;
7185 SmallVector<SDValue, 4> Ops;
7186 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7187 Ops.push_back(Elt: InMask->getOperand(Num: i));
7188 if (InMask->isStrictFPOpcode()) {
7189 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7190 ResultTys: { MaskVT, MVT::Other }, Ops);
7191 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7192 }
7193 else
7194 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7195 Flags: InMask->getFlags());
7196
7197 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
7198 // extend or truncate is needed.
7199 LLVMContext &Ctx = *DAG.getContext();
7200 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7201 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7202 if (MaskScalarBits < ToMaskScalBits) {
7203 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7204 NumElements: MaskVT.getVectorNumElements());
7205 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7206 } else if (MaskScalarBits > ToMaskScalBits) {
7207 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7208 NumElements: MaskVT.getVectorNumElements());
7209 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7210 }
7211
7212 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7213 ToMaskVT.getScalarSizeInBits() &&
7214 "Mask should have the right element size by now.");
7215
7216 // Adjust Mask to the right number of elements.
7217 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7218 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7219 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7220 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7221 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7222 EVT SubVT = Mask->getValueType(ResNo: 0);
7223 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7224 SubOps[0] = Mask;
7225 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7226 }
7227
7228 assert((Mask->getValueType(0) == ToMaskVT) &&
7229 "A mask of ToMaskVT should have been produced by now.");
7230
7231 return Mask;
7232}
7233
7234// This method tries to handle some special cases for the vselect mask
7235// and if needed adjusting the mask vector type to match that of the VSELECT.
7236// Without it, many cases end up with scalarization of the SETCC, with many
7237// unnecessary instructions.
7238SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7239 LLVMContext &Ctx = *DAG.getContext();
7240 SDValue Cond = N->getOperand(Num: 0);
7241
7242 if (N->getOpcode() != ISD::VSELECT)
7243 return SDValue();
7244
7245 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
7246 return SDValue();
7247
7248 // If this is a splitted VSELECT that was previously already handled, do
7249 // nothing.
7250 EVT CondVT = Cond->getValueType(ResNo: 0);
7251 if (CondVT.getScalarSizeInBits() != 1)
7252 return SDValue();
7253
7254 EVT VSelVT = N->getValueType(ResNo: 0);
7255
7256 // This method can't handle scalable vector types.
7257 // FIXME: This support could be added in the future.
7258 if (VSelVT.isScalableVector())
7259 return SDValue();
7260
7261 // Only handle vector types which are a power of 2.
7262 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7263 return SDValue();
7264
7265 // Don't touch if this will be scalarized.
7266 EVT FinalVT = VSelVT;
7267 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7268 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7269
7270 if (FinalVT.getVectorNumElements() == 1)
7271 return SDValue();
7272
7273 // If there is support for an i1 vector mask, don't touch.
7274 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7275 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7276 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7277 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7278 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7279 if (SetCCResVT.getScalarSizeInBits() == 1)
7280 return SDValue();
7281 } else if (CondVT.getScalarType() == MVT::i1) {
7282 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7283 // don't touch.
7284 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7285 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7286
7287 if (CondVT.getScalarType() == MVT::i1)
7288 return SDValue();
7289 }
7290
7291 // Widen the vselect result type if needed.
7292 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7293 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7294
7295 // The mask of the VSELECT should have integer elements.
7296 EVT ToMaskVT = VSelVT;
7297 if (!ToMaskVT.getScalarType().isInteger())
7298 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7299
7300 SDValue Mask;
7301 if (isSETCCOp(Opcode: Cond->getOpcode())) {
7302 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
7303 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7304 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
7305 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
7306 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
7307 // Cond is (AND/OR/XOR (SETCC, SETCC))
7308 SDValue SETCC0 = Cond->getOperand(Num: 0);
7309 SDValue SETCC1 = Cond->getOperand(Num: 1);
7310 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
7311 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
7312 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
7313 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
7314 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
7315 EVT MaskVT;
7316 // If the two SETCCs have different VTs, either extend/truncate one of
7317 // them to the other "towards" ToMaskVT, or truncate one and extend the
7318 // other to ToMaskVT.
7319 if (ScalarBits0 != ScalarBits1) {
7320 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
7321 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
7322 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
7323 MaskVT = WideVT;
7324 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
7325 MaskVT = NarrowVT;
7326 else
7327 MaskVT = ToMaskVT;
7328 } else
7329 // If the two SETCCs have the same VT, don't change it.
7330 MaskVT = VT0;
7331
7332 // Make new SETCCs and logical nodes.
7333 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
7334 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
7335 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
7336
7337 // Convert the logical op for VSELECT if needed.
7338 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7339 } else
7340 return SDValue();
7341
7342 return Mask;
7343}
7344
7345SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7346 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7347 ElementCount WidenEC = WidenVT.getVectorElementCount();
7348
7349 SDValue Cond1 = N->getOperand(Num: 0);
7350 EVT CondVT = Cond1.getValueType();
7351 unsigned Opcode = N->getOpcode();
7352 if (CondVT.isVector()) {
7353 if (SDValue WideCond = WidenVSELECTMask(N)) {
7354 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7355 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7356 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7357 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7358 }
7359
7360 EVT CondEltVT = CondVT.getVectorElementType();
7361 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7362 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7363 Cond1 = GetWidenedVector(Op: Cond1);
7364
7365 // If we have to split the condition there is no point in widening the
7366 // select. This would result in an cycle of widening the select ->
7367 // widening the condition operand -> splitting the condition operand ->
7368 // splitting the select -> widening the select. Instead split this select
7369 // further and widen the resulting type.
7370 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7371 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7372 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7373 return Res;
7374 }
7375
7376 if (Cond1.getValueType() != CondWidenVT)
7377 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7378 }
7379
7380 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7381 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7382 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7383 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
7384 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7385 N4: N->getOperand(Num: 3));
7386 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7387}
7388
7389SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7390 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7391 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7392 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7393 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7394 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7395}
7396
7397SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7398 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7399 return DAG.getUNDEF(VT: WidenVT);
7400}
7401
7402SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7403 EVT VT = N->getValueType(ResNo: 0);
7404 SDLoc dl(N);
7405
7406 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7407 unsigned NumElts = VT.getVectorNumElements();
7408 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7409
7410 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7411 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7412
7413 // Adjust mask based on new input vector length.
7414 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7415 for (unsigned i = 0; i != NumElts; ++i) {
7416 int Idx = N->getMaskElt(Idx: i);
7417 if (Idx < (int)NumElts)
7418 NewMask[i] = Idx;
7419 else
7420 NewMask[i] = Idx - NumElts + WidenNumElts;
7421 }
7422 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7423}
7424
7425SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7426 EVT VT = N->getValueType(ResNo: 0);
7427 EVT EltVT = VT.getVectorElementType();
7428 SDLoc dl(N);
7429
7430 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7431 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7432 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7433
7434 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7435 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7436 unsigned VTNumElts = VT.getVectorMinNumElements();
7437 unsigned IdxVal = WidenNumElts - VTNumElts;
7438
7439 if (VT.isScalableVector()) {
7440 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7441 // results together, e.g.(nxv6i64 -> nxv8i64)
7442 // nxv8i64 vector_reverse
7443 // <->
7444 // nxv8i64 concat(
7445 // nxv2i64 extract_subvector(nxv8i64, 2)
7446 // nxv2i64 extract_subvector(nxv8i64, 4)
7447 // nxv2i64 extract_subvector(nxv8i64, 6)
7448 // nxv2i64 undef)
7449
7450 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7451 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7452 EC: ElementCount::getScalable(MinVal: GCD));
7453 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7454 "down type's element count");
7455 SmallVector<SDValue> Parts;
7456 unsigned i = 0;
7457 for (; i < VTNumElts / GCD; ++i)
7458 Parts.push_back(
7459 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7460 for (; i < WidenNumElts / GCD; ++i)
7461 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7462
7463 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7464 }
7465
7466 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7467 // fixed-vectors.
7468 SmallVector<int, 16> Mask(WidenNumElts, -1);
7469 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7470
7471 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7472 Mask);
7473}
7474
7475SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7476 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7477 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7478}
7479
7480void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7481 EVT VT = N->getValueType(ResNo: 0);
7482 EVT EltVT = VT.getVectorElementType();
7483 ElementCount OrigEC = VT.getVectorElementCount();
7484 unsigned Factor = N->getNumOperands();
7485 SDLoc DL(N);
7486
7487 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7488 ElementCount WidenEC = WidenVT.getVectorElementCount();
7489 // We cannot just use the widened operands directly: since they might be
7490 // individually widened, using them directly will result in de-interleaving
7491 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7492 // not just concat but also "re-pack" these operands before extracting new
7493 // operand vectors with the widened type.
7494 EVT PackedWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7495 EC: WidenEC.multiplyCoefficientBy(RHS: Factor));
7496 SDValue PackedWidenVec = DAG.getUNDEF(VT: PackedWidenVT);
7497 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7498 const SDValue Op = N->getOperand(Num: Idx);
7499 // Note that we insert these widened operands with offsets derived from
7500 // the original vector length.
7501 PackedWidenVec = DAG.getInsertSubvector(
7502 DL, Vec: PackedWidenVec, SubVec: GetWidenedVector(Op),
7503 Idx: OrigEC.multiplyCoefficientBy(RHS: Idx).getKnownMinValue());
7504 }
7505
7506 // Extract the new widened operand vectors.
7507 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7508 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7509 NewOps[Idx] = DAG.getExtractSubvector(
7510 DL, VT: WidenVT, Vec: PackedWidenVec,
7511 Idx: WidenEC.multiplyCoefficientBy(RHS: Idx).getKnownMinValue());
7512 }
7513
7514 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7515 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7516 // Set the widened results manually.
7517 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7518 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7519}
7520
7521SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7522 assert(N->getValueType(0).isVector() &&
7523 N->getOperand(0).getValueType().isVector() &&
7524 "Operands must be vectors");
7525 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7526 ElementCount WidenEC = WidenVT.getVectorElementCount();
7527
7528 SDValue InOp1 = N->getOperand(Num: 0);
7529 EVT InVT = InOp1.getValueType();
7530 assert(InVT.isVector() && "can not widen non-vector type");
7531 EVT WidenInVT =
7532 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7533
7534 // The input and output types often differ here, and it could be that while
7535 // we'd prefer to widen the result type, the input operands have been split.
7536 // In this case, we also need to split the result of this node as well.
7537 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7538 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7539 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7540 return Res;
7541 }
7542
7543 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7544 SDValue InOp2 = N->getOperand(Num: 1);
7545 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7546 InOp1 = GetWidenedVector(Op: InOp1);
7547 InOp2 = GetWidenedVector(Op: InOp2);
7548 } else {
7549 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7550 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7551 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7552 N2: InOp1, N3: ZeroIdx);
7553 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7554 N2: InOp2, N3: ZeroIdx);
7555 }
7556
7557 // Assume that the input and output will be widen appropriately. If not,
7558 // we will have to unroll it at some point.
7559 assert(InOp1.getValueType() == WidenInVT &&
7560 InOp2.getValueType() == WidenInVT &&
7561 "Input not widened to expected type!");
7562 (void)WidenInVT;
7563 if (N->getOpcode() == ISD::VP_SETCC) {
7564 SDValue Mask =
7565 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7566 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7567 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7568 }
7569 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7570 N3: N->getOperand(Num: 2));
7571}
7572
7573SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7574 assert(N->getValueType(0).isVector() &&
7575 N->getOperand(1).getValueType().isVector() &&
7576 "Operands must be vectors");
7577 EVT VT = N->getValueType(ResNo: 0);
7578 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7579 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7580 unsigned NumElts = VT.getVectorNumElements();
7581 EVT EltVT = VT.getVectorElementType();
7582
7583 SDLoc dl(N);
7584 SDValue Chain = N->getOperand(Num: 0);
7585 SDValue LHS = N->getOperand(Num: 1);
7586 SDValue RHS = N->getOperand(Num: 2);
7587 SDValue CC = N->getOperand(Num: 3);
7588 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7589
7590 // Fully unroll and reassemble.
7591 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7592 SmallVector<SDValue, 8> Chains(NumElts);
7593 for (unsigned i = 0; i != NumElts; ++i) {
7594 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7595 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7596
7597 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7598 Ops: {Chain, LHSElem, RHSElem, CC});
7599 Chains[i] = Scalars[i].getValue(R: 1);
7600 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7601 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7602 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7603 }
7604
7605 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7606 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7607
7608 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7609}
7610
7611//===----------------------------------------------------------------------===//
7612// Widen Vector Operand
7613//===----------------------------------------------------------------------===//
7614bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7615 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7616 SDValue Res = SDValue();
7617
7618 // See if the target wants to custom widen this node.
7619 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7620 return false;
7621
7622 switch (N->getOpcode()) {
7623 default:
7624#ifndef NDEBUG
7625 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7626 N->dump(&DAG);
7627 dbgs() << "\n";
7628#endif
7629 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7630
7631 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7632 case ISD::FAKE_USE:
7633 Res = WidenVecOp_FAKE_USE(N);
7634 break;
7635 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7636 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7637 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7638 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7639 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7640 case ISD::ATOMIC_STORE:
7641 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7642 break;
7643 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7644 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7645 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7646 break;
7647 case ISD::ANY_EXTEND_VECTOR_INREG:
7648 case ISD::SIGN_EXTEND_VECTOR_INREG:
7649 case ISD::ZERO_EXTEND_VECTOR_INREG:
7650 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7651 break;
7652 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7653 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7654 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7655 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7656 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7657 case ISD::STRICT_FSETCC:
7658 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7659 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7660 case ISD::FLDEXP:
7661 case ISD::FCOPYSIGN:
7662 case ISD::LROUND:
7663 case ISD::LLROUND:
7664 case ISD::LRINT:
7665 case ISD::LLRINT:
7666 Res = WidenVecOp_UnrollVectorOp(N);
7667 break;
7668 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7669
7670 case ISD::ANY_EXTEND:
7671 case ISD::SIGN_EXTEND:
7672 case ISD::ZERO_EXTEND:
7673 Res = WidenVecOp_EXTEND(N);
7674 break;
7675
7676 case ISD::SCMP:
7677 case ISD::UCMP:
7678 Res = WidenVecOp_CMP(N);
7679 break;
7680
7681 case ISD::FP_EXTEND:
7682 case ISD::STRICT_FP_EXTEND:
7683 case ISD::FP_ROUND:
7684 case ISD::STRICT_FP_ROUND:
7685 case ISD::FP_TO_SINT:
7686 case ISD::STRICT_FP_TO_SINT:
7687 case ISD::FP_TO_UINT:
7688 case ISD::STRICT_FP_TO_UINT:
7689 case ISD::SINT_TO_FP:
7690 case ISD::STRICT_SINT_TO_FP:
7691 case ISD::UINT_TO_FP:
7692 case ISD::STRICT_UINT_TO_FP:
7693 case ISD::TRUNCATE:
7694 case ISD::CONVERT_FROM_ARBITRARY_FP:
7695 case ISD::CONVERT_TO_ARBITRARY_FP:
7696 Res = WidenVecOp_Convert(N);
7697 break;
7698
7699 case ISD::FP_TO_SINT_SAT:
7700 case ISD::FP_TO_UINT_SAT:
7701 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7702 break;
7703
7704 case ISD::VECREDUCE_FADD:
7705 case ISD::VECREDUCE_FMUL:
7706 case ISD::VECREDUCE_ADD:
7707 case ISD::VECREDUCE_MUL:
7708 case ISD::VECREDUCE_AND:
7709 case ISD::VECREDUCE_OR:
7710 case ISD::VECREDUCE_XOR:
7711 case ISD::VECREDUCE_SMAX:
7712 case ISD::VECREDUCE_SMIN:
7713 case ISD::VECREDUCE_UMAX:
7714 case ISD::VECREDUCE_UMIN:
7715 case ISD::VECREDUCE_FMAX:
7716 case ISD::VECREDUCE_FMIN:
7717 case ISD::VECREDUCE_FMAXIMUM:
7718 case ISD::VECREDUCE_FMINIMUM:
7719 Res = WidenVecOp_VECREDUCE(N);
7720 break;
7721 case ISD::VECREDUCE_SEQ_FADD:
7722 case ISD::VECREDUCE_SEQ_FMUL:
7723 Res = WidenVecOp_VECREDUCE_SEQ(N);
7724 break;
7725 case ISD::VP_REDUCE_FADD:
7726 case ISD::VP_REDUCE_SEQ_FADD:
7727 case ISD::VP_REDUCE_FMUL:
7728 case ISD::VP_REDUCE_SEQ_FMUL:
7729 case ISD::VP_REDUCE_ADD:
7730 case ISD::VP_REDUCE_MUL:
7731 case ISD::VP_REDUCE_AND:
7732 case ISD::VP_REDUCE_OR:
7733 case ISD::VP_REDUCE_XOR:
7734 case ISD::VP_REDUCE_SMAX:
7735 case ISD::VP_REDUCE_SMIN:
7736 case ISD::VP_REDUCE_UMAX:
7737 case ISD::VP_REDUCE_UMIN:
7738 case ISD::VP_REDUCE_FMAX:
7739 case ISD::VP_REDUCE_FMIN:
7740 case ISD::VP_REDUCE_FMAXIMUM:
7741 case ISD::VP_REDUCE_FMINIMUM:
7742 Res = WidenVecOp_VP_REDUCE(N);
7743 break;
7744 case ISD::VP_CTTZ_ELTS:
7745 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
7746 Res = WidenVecOp_VP_CttzElements(N);
7747 break;
7748 case ISD::VECTOR_FIND_LAST_ACTIVE:
7749 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7750 break;
7751 }
7752
7753 // If Res is null, the sub-method took care of registering the result.
7754 if (!Res.getNode()) return false;
7755
7756 // If the result is N, the sub-method updated N in place. Tell the legalizer
7757 // core about this.
7758 if (Res.getNode() == N)
7759 return true;
7760
7761
7762 if (N->isStrictFPOpcode())
7763 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7764 "Invalid operand expansion");
7765 else
7766 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7767 "Invalid operand expansion");
7768
7769 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7770 return false;
7771}
7772
7773SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7774 SDLoc DL(N);
7775 EVT VT = N->getValueType(ResNo: 0);
7776
7777 SDValue InOp = N->getOperand(Num: 0);
7778 assert(getTypeAction(InOp.getValueType()) ==
7779 TargetLowering::TypeWidenVector &&
7780 "Unexpected type action");
7781 InOp = GetWidenedVector(Op: InOp);
7782 assert(VT.getVectorNumElements() <
7783 InOp.getValueType().getVectorNumElements() &&
7784 "Input wasn't widened!");
7785
7786 // We may need to further widen the operand until it has the same total
7787 // vector size as the result.
7788 EVT InVT = InOp.getValueType();
7789 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7790 EVT InEltVT = InVT.getVectorElementType();
7791 for (EVT FixedVT : MVT::vector_valuetypes()) {
7792 EVT FixedEltVT = FixedVT.getVectorElementType();
7793 if (TLI.isTypeLegal(VT: FixedVT) &&
7794 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7795 FixedEltVT == InEltVT) {
7796 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7797 "Not enough elements in the fixed type for the operand!");
7798 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7799 "We can't have the same type as we started with!");
7800 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7801 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7802 else
7803 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7804 break;
7805 }
7806 }
7807 InVT = InOp.getValueType();
7808 if (InVT.getSizeInBits() != VT.getSizeInBits())
7809 // We couldn't find a legal vector type that was a widening of the input
7810 // and could be extended in-register to the result type, so we have to
7811 // scalarize.
7812 return WidenVecOp_Convert(N);
7813 }
7814
7815 // Use special DAG nodes to represent the operation of extending the
7816 // low lanes.
7817 switch (N->getOpcode()) {
7818 default:
7819 llvm_unreachable("Extend legalization on extend operation!");
7820 case ISD::ANY_EXTEND:
7821 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7822 case ISD::SIGN_EXTEND:
7823 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7824 case ISD::ZERO_EXTEND:
7825 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7826 }
7827}
7828
7829SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7830 SDLoc dl(N);
7831
7832 EVT OpVT = N->getOperand(Num: 0).getValueType();
7833 EVT ResVT = N->getValueType(ResNo: 0);
7834 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7835 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7836
7837 // 1. EXTRACT_SUBVECTOR
7838 // 2. SIGN_EXTEND/ZERO_EXTEND
7839 // 3. CMP
7840 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7841 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7842
7843 // At this point the result type is guaranteed to be valid, so we can use it
7844 // as the operand type by extending it appropriately
7845 ISD::NodeType ExtendOpcode =
7846 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7847 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7848 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7849
7850 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7851}
7852
7853SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7854 // The result (and first input) is legal, but the second input is illegal.
7855 // We can't do much to fix that, so just unroll and let the extracts off of
7856 // the second input be widened as needed later.
7857 return DAG.UnrollVectorOp(N);
7858}
7859
7860SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7861 SDLoc DL(N);
7862 EVT ResultVT = N->getValueType(ResNo: 0);
7863 SDValue Test = N->getOperand(Num: 1);
7864 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7865
7866 // Process this node similarly to SETCC.
7867 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7868 if (ResultVT.getScalarType() == MVT::i1)
7869 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7870 NumElements: WideResultVT.getVectorNumElements());
7871
7872 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7873 Ops: {WideArg, Test}, Flags: N->getFlags());
7874
7875 // Extract the needed results from the result vector.
7876 EVT ResVT =
7877 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7878 NumElements: ResultVT.getVectorNumElements());
7879 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7880
7881 EVT OpVT = N->getOperand(Num: 0).getValueType();
7882 ISD::NodeType ExtendCode =
7883 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7884 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7885}
7886
7887SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7888 // Since the result is legal and the input is illegal.
7889 EVT VT = N->getValueType(ResNo: 0);
7890 EVT EltVT = VT.getVectorElementType();
7891 SDLoc dl(N);
7892 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7893 assert(getTypeAction(InOp.getValueType()) ==
7894 TargetLowering::TypeWidenVector &&
7895 "Unexpected type action");
7896 InOp = GetWidenedVector(Op: InOp);
7897 EVT InVT = InOp.getValueType();
7898 unsigned Opcode = N->getOpcode();
7899
7900 // Helper to build a convert node with all scalar trailing operands.
7901 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
7902 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
7903 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
7904 N4: N->getOperand(Num: 3));
7905 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7906 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
7907 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
7908 };
7909
7910 // See if a widened result type would be legal, if so widen the node.
7911 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7912 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7913 EC: InVT.getVectorElementCount());
7914 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7915 SDValue Res;
7916 if (N->isStrictFPOpcode()) {
7917 if (Opcode == ISD::STRICT_FP_ROUND)
7918 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7919 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7920 else
7921 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7922 Ops: { N->getOperand(Num: 0), InOp });
7923 // Legalize the chain result - switch anything that used the old chain to
7924 // use the new one.
7925 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7926 } else {
7927 Res = MakeConvertNode(WideVT, InOp);
7928 }
7929 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7930 }
7931
7932 EVT InEltVT = InVT.getVectorElementType();
7933
7934 // Unroll the convert into some scalar code and create a nasty build vector.
7935 unsigned NumElts = VT.getVectorNumElements();
7936 SmallVector<SDValue, 16> Ops(NumElts);
7937 if (N->isStrictFPOpcode()) {
7938 SmallVector<SDValue, 4> NewOps(N->ops());
7939 SmallVector<SDValue, 32> OpChains;
7940 for (unsigned i=0; i < NumElts; ++i) {
7941 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7942 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7943 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7944 }
7945 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7946 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7947 } else {
7948 for (unsigned i = 0; i < NumElts; ++i) {
7949 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7950 Ops[i] = MakeConvertNode(EltVT, Elt);
7951 }
7952 }
7953
7954 return DAG.getBuildVector(VT, DL: dl, Ops);
7955}
7956
7957SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7958 EVT DstVT = N->getValueType(ResNo: 0);
7959 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7960 EVT SrcVT = Src.getValueType();
7961 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7962 SDLoc dl(N);
7963
7964 // See if a widened result type would be legal, if so widen the node.
7965 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7966 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7967 if (TLI.isTypeLegal(VT: WideDstVT)) {
7968 SDValue Res =
7969 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7970 return DAG.getNode(
7971 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7972 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7973 }
7974
7975 // Give up and unroll.
7976 return DAG.UnrollVectorOp(N);
7977}
7978
7979SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7980 EVT VT = N->getValueType(ResNo: 0);
7981 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7982 EVT InWidenVT = InOp.getValueType();
7983 SDLoc dl(N);
7984
7985 // Check if we can convert between two legal vector types and extract.
7986 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7987 TypeSize Size = VT.getSizeInBits();
7988 // x86mmx is not an acceptable vector element type, so don't try.
7989 if (!VT.isVector() && VT != MVT::x86mmx &&
7990 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7991 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7992 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7993 if (TLI.isTypeLegal(VT: NewVT)) {
7994 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7995 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7996 }
7997 }
7998
7999 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8000 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8001 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8002 // having to copy via memory.
8003 if (VT.isVector()) {
8004 EVT EltVT = VT.getVectorElementType();
8005 unsigned EltSize = EltVT.getFixedSizeInBits();
8006 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
8007 ElementCount NewNumElts =
8008 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8009 .divideCoefficientBy(RHS: EltSize);
8010 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8011 if (TLI.isTypeLegal(VT: NewVT)) {
8012 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8013 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8014 }
8015 }
8016 }
8017
8018 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8019}
8020
8021// Vectors with sizes that are not powers of 2 need to be widened to the
8022// next largest power of 2. For example, we may get a vector of 3 32-bit
8023// integers or of 6 16-bit integers, both of which have to be widened to a
8024// 128-bit vector.
8025SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8026 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8027 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8028 N2: WidenedOp);
8029}
8030
8031SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8032 EVT VT = N->getValueType(ResNo: 0);
8033 EVT EltVT = VT.getVectorElementType();
8034 EVT InVT = N->getOperand(Num: 0).getValueType();
8035 SDLoc dl(N);
8036
8037 // If the widen width for this operand is the same as the width of the concat
8038 // and all but the first operand is undef, just use the widened operand.
8039 unsigned NumOperands = N->getNumOperands();
8040 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8041 unsigned i;
8042 for (i = 1; i < NumOperands; ++i)
8043 if (!N->getOperand(Num: i).isUndef())
8044 break;
8045
8046 if (i == NumOperands)
8047 return GetWidenedVector(Op: N->getOperand(Num: 0));
8048 }
8049
8050 // Otherwise, fall back to a nasty build vector.
8051 unsigned NumElts = VT.getVectorNumElements();
8052 SmallVector<SDValue, 16> Ops(NumElts);
8053
8054 unsigned NumInElts = InVT.getVectorNumElements();
8055
8056 unsigned Idx = 0;
8057 for (unsigned i=0; i < NumOperands; ++i) {
8058 SDValue InOp = N->getOperand(Num: i);
8059 assert(getTypeAction(InOp.getValueType()) ==
8060 TargetLowering::TypeWidenVector &&
8061 "Unexpected type action");
8062 InOp = GetWidenedVector(Op: InOp);
8063 for (unsigned j = 0; j < NumInElts; ++j)
8064 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8065 }
8066 return DAG.getBuildVector(VT, DL: dl, Ops);
8067}
8068
8069SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8070 EVT VT = N->getValueType(ResNo: 0);
8071 SDValue SubVec = N->getOperand(Num: 1);
8072 SDValue InVec = N->getOperand(Num: 0);
8073
8074 EVT OrigVT = SubVec.getValueType();
8075 SubVec = GetWidenedVector(Op: SubVec);
8076 EVT SubVT = SubVec.getValueType();
8077
8078 // Whether or not all the elements of the widened SubVec will be inserted into
8079 // valid indices of VT.
8080 bool IndicesValid = false;
8081 // If we statically know that VT can fit SubVT, the indices are valid.
8082 if (VT.knownBitsGE(VT: SubVT))
8083 IndicesValid = true;
8084 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8085 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8086 // we know the minimum vscale we can work out if it's valid ourselves.
8087 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8088 Kind: Attribute::VScaleRange);
8089 if (Attr.isValid()) {
8090 unsigned VScaleMin = Attr.getVScaleRangeMin();
8091 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8092 SubVT.getFixedSizeInBits())
8093 IndicesValid = true;
8094 }
8095 }
8096
8097 if (!IndicesValid)
8098 report_fatal_error(
8099 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8100
8101 SDLoc DL(N);
8102
8103 // We need to make sure that the indices are still valid, otherwise we might
8104 // widen what was previously well-defined to something undefined.
8105 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8106 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8107 N3: N->getOperand(Num: 2));
8108
8109 if (OrigVT.isScalableVector()) {
8110 // When the widened types match, overwriting the start of a vector is
8111 // effectively a merge operation that can be implement as a vselect.
8112 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8113 SDValue Mask =
8114 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8115 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8116 }
8117
8118 // Fallback to inserting through memory.
8119 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8120 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8121 MachineFunction &MF = DAG.getMachineFunction();
8122 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8123 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8124
8125 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8126 PtrInfo, F: MachineMemOperand::MOStore,
8127 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8128 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8129 PtrInfo, F: MachineMemOperand::MOLoad,
8130 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8131
8132 // Write out the vector being inserting into.
8133 SDValue Ch =
8134 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8135
8136 // Build a mask to match the length of the sub-vector.
8137 SDValue Mask =
8138 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8139
8140 // Overwrite the sub-vector at the required offset.
8141 SDValue SubVecPtr =
8142 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8143 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8144 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8145 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8146
8147 // Read back the result.
8148 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8149 }
8150
8151 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8152 // with a series of INSERT_VECTOR_ELT
8153 unsigned Idx = N->getConstantOperandVal(Num: 2);
8154
8155 SDValue InsertElt = InVec;
8156 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8157 SDValue ExtractElt =
8158 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8159 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8160 }
8161
8162 return InsertElt;
8163}
8164
8165SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8166 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8167 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8168 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8169}
8170
8171SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8172 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8173 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8174 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8175}
8176
8177SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8178 SDLoc DL(N);
8179 EVT ResVT = N->getValueType(ResNo: 0);
8180
8181 // Widen the input as requested by the legalizer.
8182 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8183 EVT WideInVT = WideInOp.getValueType();
8184
8185 // Simple case: if widened input is still smaller than or equal to result,
8186 // just use it directly.
8187 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8188 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8189
8190 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8191 // If widening makes the input larger than the original result, widen the
8192 // result to match, then extract back down.
8193 EVT ResEltVT = ResVT.getVectorElementType();
8194 unsigned EltBits = ResEltVT.getSizeInBits();
8195 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8196 "Widened input size must be a multiple of result element size");
8197
8198 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8199 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8200
8201 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8202 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8203}
8204
8205SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8206 // We have to widen the value, but we want only to store the original
8207 // vector type.
8208 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8209
8210 if (!ST->getMemoryVT().getScalarType().isByteSized())
8211 return TLI.scalarizeVectorStore(ST, DAG);
8212
8213 if (ST->isTruncatingStore())
8214 return TLI.scalarizeVectorStore(ST, DAG);
8215
8216 // Generate a vector-predicated store if it is custom/legal on the target.
8217 // To avoid possible recursion, only do this if the widened mask type is
8218 // legal.
8219 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8220 // removed from the IR by the ExpandVectorPredication pass but we're
8221 // reintroducing them here.
8222 SDValue StVal = ST->getValue();
8223 EVT StVT = StVal.getValueType();
8224 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8225 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8226
8227 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8228 TLI.isTypeLegal(VT: WideMaskVT)) {
8229 // Widen the value.
8230 SDLoc DL(N);
8231 StVal = GetWidenedVector(Op: StVal);
8232 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8233 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8234 EC: StVT.getVectorElementCount());
8235 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8236 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8237 AM: ST->getAddressingMode());
8238 }
8239
8240 SmallVector<SDValue, 16> StChain;
8241 if (GenWidenVectorStores(StChain, ST)) {
8242 if (StChain.size() == 1)
8243 return StChain[0];
8244
8245 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8246 }
8247
8248 if (StVT.isVector()) {
8249 // If all else fails replace the store with a wide masked store.
8250 SDLoc DL(N);
8251 SDValue WideStVal = GetWidenedVector(Op: StVal);
8252 SDValue Mask =
8253 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8254
8255 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8256 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8257 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8258 IsTruncating: ST->isTruncatingStore());
8259 }
8260
8261 report_fatal_error(reason: "Unable to widen vector store");
8262}
8263
8264SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8265 EVT StVT = ST->getMemoryVT();
8266 SDLoc dl(ST);
8267
8268 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8269 EVT WidenVT = StVal.getValueType();
8270
8271 TypeSize StWidth = StVT.getSizeInBits();
8272 TypeSize WidenWidth = WidenVT.getSizeInBits();
8273 TypeSize WidthDiff = WidenWidth - StWidth;
8274
8275 // Find the vector type that can store the original memory width in one
8276 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8277 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8278 // implies a 4-byte movl, writing undef bytes past its object).
8279 std::optional<EVT> FirstVT =
8280 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8281 WidenEx: WidthDiff.getKnownMinValue());
8282 if (!FirstVT)
8283 return SDValue();
8284
8285 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8286
8287 SDValue StOp =
8288 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8289
8290 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8291 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8292}
8293
8294SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8295 assert((OpNo == 1 || OpNo == 3) &&
8296 "Can widen only data or mask operand of vp_store");
8297 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8298 SDValue Mask = ST->getMask();
8299 SDValue StVal = ST->getValue();
8300 SDLoc dl(N);
8301
8302 if (OpNo == 1) {
8303 // Widen the value.
8304 StVal = GetWidenedVector(Op: StVal);
8305
8306 // We only handle the case where the mask needs widening to an
8307 // identically-sized type as the vector inputs.
8308 assert(getTypeAction(Mask.getValueType()) ==
8309 TargetLowering::TypeWidenVector &&
8310 "Unable to widen VP store");
8311 Mask = GetWidenedVector(Op: Mask);
8312 } else {
8313 Mask = GetWidenedVector(Op: Mask);
8314
8315 // We only handle the case where the stored value needs widening to an
8316 // identically-sized type as the mask.
8317 assert(getTypeAction(StVal.getValueType()) ==
8318 TargetLowering::TypeWidenVector &&
8319 "Unable to widen VP store");
8320 StVal = GetWidenedVector(Op: StVal);
8321 }
8322
8323 assert(Mask.getValueType().getVectorElementCount() ==
8324 StVal.getValueType().getVectorElementCount() &&
8325 "Mask and data vectors should have the same number of elements");
8326 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8327 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8328 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8329 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8330 IsCompressing: ST->isCompressingStore());
8331}
8332
8333SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8334 unsigned OpNo) {
8335 assert((OpNo == 1 || OpNo == 4) &&
8336 "Can widen only data or mask operand of vp_strided_store");
8337 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8338 SDValue Mask = SST->getMask();
8339 SDValue StVal = SST->getValue();
8340 SDLoc DL(N);
8341
8342 if (OpNo == 1)
8343 assert(getTypeAction(Mask.getValueType()) ==
8344 TargetLowering::TypeWidenVector &&
8345 "Unable to widen VP strided store");
8346 else
8347 assert(getTypeAction(StVal.getValueType()) ==
8348 TargetLowering::TypeWidenVector &&
8349 "Unable to widen VP strided store");
8350
8351 StVal = GetWidenedVector(Op: StVal);
8352 Mask = GetWidenedVector(Op: Mask);
8353
8354 assert(StVal.getValueType().getVectorElementCount() ==
8355 Mask.getValueType().getVectorElementCount() &&
8356 "Data and mask vectors should have the same number of elements");
8357
8358 return DAG.getStridedStoreVP(
8359 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8360 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8361 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8362 IsCompressing: SST->isCompressingStore());
8363}
8364
8365SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8366 assert((OpNo == 1 || OpNo == 4) &&
8367 "Can widen only data or mask operand of mstore");
8368 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8369 SDValue Mask = MST->getMask();
8370 EVT MaskVT = Mask.getValueType();
8371 SDValue StVal = MST->getValue();
8372 EVT VT = StVal.getValueType();
8373 SDLoc dl(N);
8374
8375 EVT WideVT, WideMaskVT;
8376 if (OpNo == 1) {
8377 // Widen the value.
8378 StVal = GetWidenedVector(Op: StVal);
8379
8380 WideVT = StVal.getValueType();
8381 WideMaskVT =
8382 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8383 EC: WideVT.getVectorElementCount());
8384 } else {
8385 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8386
8387 EVT ValueVT = StVal.getValueType();
8388 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8389 EC: WideMaskVT.getVectorElementCount());
8390 }
8391
8392 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8393 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8394 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8395 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8396 EC: VT.getVectorElementCount());
8397 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8398 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8399 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8400 }
8401
8402 if (OpNo == 1) {
8403 // The mask should be widened as well.
8404 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8405 } else {
8406 // Widen the mask.
8407 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8408
8409 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8410 }
8411
8412 assert(Mask.getValueType().getVectorElementCount() ==
8413 StVal.getValueType().getVectorElementCount() &&
8414 "Mask and data vectors should have the same number of elements");
8415 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8416 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8417 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8418 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8419}
8420
8421SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8422 assert(OpNo == 4 && "Can widen only the index of mgather");
8423 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8424 SDValue DataOp = MG->getPassThru();
8425 SDValue Mask = MG->getMask();
8426 SDValue Scale = MG->getScale();
8427
8428 // Just widen the index. It's allowed to have extra elements.
8429 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8430
8431 SDLoc dl(N);
8432 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8433 Scale};
8434 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8435 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8436 ExtTy: MG->getExtensionType());
8437 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8438 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8439 return SDValue();
8440}
8441
8442SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8443 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8444 SDValue DataOp = MSC->getValue();
8445 SDValue Mask = MSC->getMask();
8446 SDValue Index = MSC->getIndex();
8447 SDValue Scale = MSC->getScale();
8448 EVT WideMemVT = MSC->getMemoryVT();
8449
8450 if (OpNo == 1) {
8451 DataOp = GetWidenedVector(Op: DataOp);
8452 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8453
8454 // Widen index.
8455 EVT IndexVT = Index.getValueType();
8456 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8457 VT: IndexVT.getVectorElementType(), EC: WideEC);
8458 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8459
8460 // The mask should be widened as well.
8461 EVT MaskVT = Mask.getValueType();
8462 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8463 VT: MaskVT.getVectorElementType(), EC: WideEC);
8464 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8465
8466 // Widen the MemoryType
8467 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8468 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8469 } else if (OpNo == 4) {
8470 // Just widen the index. It's allowed to have extra elements.
8471 Index = GetWidenedVector(Op: Index);
8472 } else
8473 llvm_unreachable("Can't widen this operand of mscatter");
8474
8475 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8476 Scale};
8477 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8478 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8479 IsTruncating: MSC->isTruncatingStore());
8480}
8481
8482SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8483 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8484 SDValue DataOp = VPSC->getValue();
8485 SDValue Mask = VPSC->getMask();
8486 SDValue Index = VPSC->getIndex();
8487 SDValue Scale = VPSC->getScale();
8488 EVT WideMemVT = VPSC->getMemoryVT();
8489
8490 if (OpNo == 1) {
8491 DataOp = GetWidenedVector(Op: DataOp);
8492 Index = GetWidenedVector(Op: Index);
8493 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8494 Mask = GetWidenedMask(Mask, EC: WideEC);
8495 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8496 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8497 } else if (OpNo == 3) {
8498 // Just widen the index. It's allowed to have extra elements.
8499 Index = GetWidenedVector(Op: Index);
8500 } else
8501 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8502
8503 SDValue Ops[] = {
8504 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8505 VPSC->getVectorLength()};
8506 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8507 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8508}
8509
8510SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8511 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8512 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8513 SDLoc dl(N);
8514 EVT VT = N->getValueType(ResNo: 0);
8515
8516 // WARNING: In this code we widen the compare instruction with garbage.
8517 // This garbage may contain denormal floats which may be slow. Is this a real
8518 // concern ? Should we zero the unused lanes if this is a float compare ?
8519
8520 // Get a new SETCC node to compare the newly widened operands.
8521 // Only some of the compared elements are legal.
8522 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8523 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8524 if (VT.getScalarType() == MVT::i1)
8525 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8526 EC: SVT.getVectorElementCount());
8527
8528 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8529 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8530
8531 // Extract the needed results from the result vector.
8532 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8533 VT: SVT.getVectorElementType(),
8534 EC: VT.getVectorElementCount());
8535 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8536
8537 EVT OpVT = N->getOperand(Num: 0).getValueType();
8538 ISD::NodeType ExtendCode =
8539 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8540 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8541}
8542
8543SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8544 SDValue Chain = N->getOperand(Num: 0);
8545 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8546 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8547 SDValue CC = N->getOperand(Num: 3);
8548 SDLoc dl(N);
8549
8550 EVT VT = N->getValueType(ResNo: 0);
8551 EVT EltVT = VT.getVectorElementType();
8552 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8553 unsigned NumElts = VT.getVectorNumElements();
8554
8555 // Unroll into a build vector.
8556 SmallVector<SDValue, 8> Scalars(NumElts);
8557 SmallVector<SDValue, 8> Chains(NumElts);
8558
8559 for (unsigned i = 0; i != NumElts; ++i) {
8560 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8561 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8562
8563 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8564 Ops: {Chain, LHSElem, RHSElem, CC});
8565 Chains[i] = Scalars[i].getValue(R: 1);
8566 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8567 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8568 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8569 }
8570
8571 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8572 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8573
8574 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8575}
8576
8577static unsigned getExtendForIntVecReduction(unsigned Opc) {
8578 switch (Opc) {
8579 default:
8580 llvm_unreachable("Expected integer vector reduction");
8581 case ISD::VECREDUCE_ADD:
8582 case ISD::VECREDUCE_MUL:
8583 case ISD::VECREDUCE_AND:
8584 case ISD::VECREDUCE_OR:
8585 case ISD::VECREDUCE_XOR:
8586 return ISD::ANY_EXTEND;
8587 case ISD::VECREDUCE_SMAX:
8588 case ISD::VECREDUCE_SMIN:
8589 return ISD::SIGN_EXTEND;
8590 case ISD::VECREDUCE_UMAX:
8591 case ISD::VECREDUCE_UMIN:
8592 return ISD::ZERO_EXTEND;
8593 }
8594}
8595
8596SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8597 SDLoc dl(N);
8598 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8599 EVT VT = N->getValueType(ResNo: 0);
8600 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8601 EVT WideVT = Op.getValueType();
8602 EVT ElemVT = OrigVT.getVectorElementType();
8603 SDNodeFlags Flags = N->getFlags();
8604
8605 unsigned Opc = N->getOpcode();
8606 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8607 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8608 assert(NeutralElem && "Neutral element must exist");
8609
8610 // Pad the vector with the neutral element.
8611 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8612 unsigned WideElts = WideVT.getVectorMinNumElements();
8613
8614 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8615 // needing to pad the source vector, because the inactive lanes can simply be
8616 // disabled and not contribute to the result.
8617 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8618 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8619 SDValue Start = NeutralElem;
8620 if (VT.isInteger())
8621 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8622 assert(Start.getValueType() == VT);
8623 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8624 EC: WideVT.getVectorElementCount());
8625 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8626 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8627 EC: OrigVT.getVectorElementCount());
8628 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8629 }
8630
8631 if (WideVT.isScalableVector()) {
8632 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8633 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8634 EC: ElementCount::getScalable(MinVal: GCD));
8635 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8636 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8637 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8638 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8639 }
8640
8641 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8642 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8643
8644 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8645}
8646
8647SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8648 SDLoc dl(N);
8649 SDValue AccOp = N->getOperand(Num: 0);
8650 SDValue VecOp = N->getOperand(Num: 1);
8651 SDValue Op = GetWidenedVector(Op: VecOp);
8652
8653 EVT VT = N->getValueType(ResNo: 0);
8654 EVT OrigVT = VecOp.getValueType();
8655 EVT WideVT = Op.getValueType();
8656 EVT ElemVT = OrigVT.getVectorElementType();
8657 SDNodeFlags Flags = N->getFlags();
8658
8659 unsigned Opc = N->getOpcode();
8660 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8661 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8662
8663 // Pad the vector with the neutral element.
8664 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8665 unsigned WideElts = WideVT.getVectorMinNumElements();
8666
8667 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8668 // needing to pad the source vector, because the inactive lanes can simply be
8669 // disabled and not contribute to the result.
8670 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8671 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8672 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8673 EC: WideVT.getVectorElementCount());
8674 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8675 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8676 EC: OrigVT.getVectorElementCount());
8677 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8678 }
8679
8680 if (WideVT.isScalableVector()) {
8681 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8682 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8683 EC: ElementCount::getScalable(MinVal: GCD));
8684 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8685 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8686 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8687 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8688 }
8689
8690 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8691 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8692
8693 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8694}
8695
8696SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8697 assert(N->isVPOpcode() && "Expected VP opcode");
8698
8699 SDLoc dl(N);
8700 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8701 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8702 EC: Op.getValueType().getVectorElementCount());
8703
8704 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8705 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8706 Flags: N->getFlags());
8707}
8708
8709SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8710 // This only gets called in the case that the left and right inputs and
8711 // result are of a legal odd vector type, and the condition is illegal i1 of
8712 // the same odd width that needs widening.
8713 EVT VT = N->getValueType(ResNo: 0);
8714 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8715
8716 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8717 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8718 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8719 SDLoc DL(N);
8720
8721 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8722 N2: LeftIn, N3: RightIn);
8723 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8724}
8725
8726SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8727 SDLoc DL(N);
8728 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8729 EVT SrcVT = Source.getValueType();
8730 SDValue Mask =
8731 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8732
8733 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8734 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8735}
8736
8737SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8738 SDLoc DL(N);
8739 SDValue Mask = N->getOperand(Num: 0);
8740 EVT OrigMaskVT = Mask.getValueType();
8741 SDValue WideMask = GetWidenedVector(Op: Mask);
8742 EVT WideMaskVT = WideMask.getValueType();
8743
8744 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8745 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8746 unsigned WideElts = WideMaskVT.getVectorNumElements();
8747 if (OrigElts != WideElts) {
8748 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8749 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8750 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8751 }
8752
8753 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8754 Operand: WideMask);
8755}
8756
8757//===----------------------------------------------------------------------===//
8758// Vector Widening Utilities
8759//===----------------------------------------------------------------------===//
8760
8761// Utility function to find the type to chop up a widen vector for load/store
8762// TLI: Target lowering used to determine legal types.
8763// Width: Width left need to load/store.
8764// WidenVT: The widen vector type to load to/store from
8765// Align: If 0, don't allow use of a wider type
8766// WidenEx: If Align is not 0, the amount additional we can load/store from.
8767
8768static std::optional<EVT> findMemType(SelectionDAG &DAG,
8769 const TargetLowering &TLI, unsigned Width,
8770 EVT WidenVT, unsigned Align = 0,
8771 unsigned WidenEx = 0) {
8772 EVT WidenEltVT = WidenVT.getVectorElementType();
8773 const bool Scalable = WidenVT.isScalableVector();
8774 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8775 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8776 unsigned AlignInBits = Align*8;
8777
8778 EVT RetVT = WidenEltVT;
8779 // Don't bother looking for an integer type if the vector is scalable, skip
8780 // to vector types.
8781 if (!Scalable) {
8782 // If we have one element to load/store, return it.
8783 if (Width == WidenEltWidth)
8784 return RetVT;
8785
8786 // See if there is larger legal integer than the element type to load/store.
8787 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8788 unsigned MemVTWidth = MemVT.getSizeInBits();
8789 if (MemVT.getSizeInBits() <= WidenEltWidth)
8790 break;
8791 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8792 if ((Action == TargetLowering::TypeLegal ||
8793 Action == TargetLowering::TypePromoteInteger) &&
8794 (WidenWidth % MemVTWidth) == 0 &&
8795 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8796 (MemVTWidth <= Width ||
8797 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8798 if (MemVTWidth == WidenWidth)
8799 return MemVT;
8800 RetVT = MemVT;
8801 break;
8802 }
8803 }
8804 }
8805
8806 // See if there is a larger vector type to load/store that has the same vector
8807 // element type and is evenly divisible with the WidenVT.
8808 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8809 // Skip vector MVTs which don't match the scalable property of WidenVT.
8810 if (Scalable != MemVT.isScalableVector())
8811 continue;
8812 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8813 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8814 if ((Action == TargetLowering::TypeLegal ||
8815 Action == TargetLowering::TypePromoteInteger) &&
8816 WidenEltVT == MemVT.getVectorElementType() &&
8817 (WidenWidth % MemVTWidth) == 0 &&
8818 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8819 (MemVTWidth <= Width ||
8820 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8821 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8822 return MemVT;
8823 }
8824 }
8825
8826 // Using element-wise loads and stores for widening operations is not
8827 // supported for scalable vectors
8828 if (Scalable)
8829 return std::nullopt;
8830
8831 return RetVT;
8832}
8833
8834// Builds a vector type from scalar loads
8835// VecTy: Resulting Vector type
8836// LDOps: Load operators to build a vector type
8837// [Start,End) the list of loads to use.
8838static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8839 SmallVectorImpl<SDValue> &LdOps,
8840 unsigned Start, unsigned End) {
8841 SDLoc dl(LdOps[Start]);
8842 EVT LdTy = LdOps[Start].getValueType();
8843 unsigned Width = VecTy.getSizeInBits();
8844 unsigned NumElts = Width / LdTy.getSizeInBits();
8845 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8846
8847 unsigned Idx = 1;
8848 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8849
8850 for (unsigned i = Start + 1; i != End; ++i) {
8851 EVT NewLdTy = LdOps[i].getValueType();
8852 if (NewLdTy != LdTy) {
8853 NumElts = Width / NewLdTy.getSizeInBits();
8854 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8855 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8856 // Readjust position and vector position based on new load type.
8857 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8858 LdTy = NewLdTy;
8859 }
8860 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8861 }
8862 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8863}
8864
8865SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8866 LoadSDNode *LD) {
8867 // The strategy assumes that we can efficiently load power-of-two widths.
8868 // The routine chops the vector into the largest vector loads with the same
8869 // element type or scalar loads and then recombines it to the widen vector
8870 // type.
8871 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8872 EVT LdVT = LD->getMemoryVT();
8873 SDLoc dl(LD);
8874 assert(LdVT.isVector() && WidenVT.isVector());
8875 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8876 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8877
8878 // Load information
8879 SDValue Chain = LD->getChain();
8880 SDValue BasePtr = LD->getBasePtr();
8881 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8882 AAMDNodes AAInfo = LD->getAAInfo();
8883
8884 TypeSize LdWidth = LdVT.getSizeInBits();
8885 TypeSize WidenWidth = WidenVT.getSizeInBits();
8886 TypeSize WidthDiff = WidenWidth - LdWidth;
8887 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8888 // and if the original load is simple.
8889 unsigned LdAlign =
8890 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8891
8892 // Find the vector type that can load from.
8893 std::optional<EVT> FirstVT =
8894 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8895 WidenEx: WidthDiff.getKnownMinValue());
8896
8897 if (!FirstVT)
8898 return SDValue();
8899
8900 SmallVector<EVT, 8> MemVTs;
8901 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8902
8903 // Unless we're able to load in one instruction we must work out how to load
8904 // the remainder.
8905 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8906 std::optional<EVT> NewVT = FirstVT;
8907 TypeSize RemainingWidth = LdWidth;
8908 TypeSize NewVTWidth = FirstVTWidth;
8909 do {
8910 RemainingWidth -= NewVTWidth;
8911 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8912 // The current type we are using is too large. Find a better size.
8913 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8914 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8915 if (!NewVT)
8916 return SDValue();
8917 NewVTWidth = NewVT->getSizeInBits();
8918 }
8919 MemVTs.push_back(Elt: *NewVT);
8920 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8921 }
8922
8923 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8924 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8925 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8926
8927 // Check if we can load the element with one instruction.
8928 if (MemVTs.empty())
8929 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8930 DAG);
8931
8932 // Load vector by using multiple loads from largest vector to scalar.
8933 SmallVector<SDValue, 16> LdOps;
8934 LdOps.push_back(Elt: LdOp);
8935
8936 uint64_t ScaledOffset = 0;
8937 MachinePointerInfo MPI = LD->getPointerInfo();
8938
8939 // First incremement past the first load.
8940 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8941 ScaledOffset: &ScaledOffset);
8942
8943 for (EVT MemVT : MemVTs) {
8944 Align NewAlign = ScaledOffset == 0
8945 ? LD->getBaseAlign()
8946 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8947 SDValue L =
8948 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8949
8950 LdOps.push_back(Elt: L);
8951 LdChain.push_back(Elt: L.getValue(R: 1));
8952 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8953 }
8954
8955 // Build the vector from the load operations.
8956 unsigned End = LdOps.size();
8957 if (!LdOps[0].getValueType().isVector())
8958 // All the loads are scalar loads.
8959 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8960
8961 // If the load contains vectors, build the vector using concat vector.
8962 // All of the vectors used to load are power-of-2, and the scalar loads can be
8963 // combined to make a power-of-2 vector.
8964 SmallVector<SDValue, 16> ConcatOps(End);
8965 int i = End - 1;
8966 int Idx = End;
8967 EVT LdTy = LdOps[i].getValueType();
8968 // First, combine the scalar loads to a vector.
8969 if (!LdTy.isVector()) {
8970 for (--i; i >= 0; --i) {
8971 LdTy = LdOps[i].getValueType();
8972 if (LdTy.isVector())
8973 break;
8974 }
8975 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8976 }
8977
8978 ConcatOps[--Idx] = LdOps[i];
8979 for (--i; i >= 0; --i) {
8980 EVT NewLdTy = LdOps[i].getValueType();
8981 if (NewLdTy != LdTy) {
8982 // Create a larger vector.
8983 TypeSize LdTySize = LdTy.getSizeInBits();
8984 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8985 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8986 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8987 unsigned NumOps =
8988 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8989 SmallVector<SDValue, 16> WidenOps(NumOps);
8990 unsigned j = 0;
8991 for (; j != End-Idx; ++j)
8992 WidenOps[j] = ConcatOps[Idx+j];
8993 for (; j != NumOps; ++j)
8994 WidenOps[j] = DAG.getPOISON(VT: LdTy);
8995
8996 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
8997 Ops: WidenOps);
8998 Idx = End - 1;
8999 LdTy = NewLdTy;
9000 }
9001 ConcatOps[--Idx] = LdOps[i];
9002 }
9003
9004 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9005 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9006 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9007
9008 // We need to fill the rest with undefs to build the vector.
9009 unsigned NumOps =
9010 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9011 SmallVector<SDValue, 16> WidenOps(NumOps);
9012 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9013 {
9014 unsigned i = 0;
9015 for (; i != End-Idx; ++i)
9016 WidenOps[i] = ConcatOps[Idx+i];
9017 for (; i != NumOps; ++i)
9018 WidenOps[i] = UndefVal;
9019 }
9020 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9021}
9022
9023SDValue
9024DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9025 LoadSDNode *LD,
9026 ISD::LoadExtType ExtType) {
9027 // For extension loads, it may not be more efficient to chop up the vector
9028 // and then extend it. Instead, we unroll the load and build a new vector.
9029 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9030 EVT LdVT = LD->getMemoryVT();
9031 SDLoc dl(LD);
9032 assert(LdVT.isVector() && WidenVT.isVector());
9033 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9034
9035 // Load information
9036 SDValue Chain = LD->getChain();
9037 SDValue BasePtr = LD->getBasePtr();
9038 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9039 AAMDNodes AAInfo = LD->getAAInfo();
9040
9041 if (LdVT.isScalableVector())
9042 return SDValue();
9043
9044 EVT EltVT = WidenVT.getVectorElementType();
9045 EVT LdEltVT = LdVT.getVectorElementType();
9046 unsigned NumElts = LdVT.getVectorNumElements();
9047
9048 // Load each element and widen.
9049 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9050 SmallVector<SDValue, 16> Ops(WidenNumElts);
9051 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9052 Ops[0] =
9053 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9054 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
9055 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9056 unsigned i = 0, Offset = Increment;
9057 for (i=1; i < NumElts; ++i, Offset += Increment) {
9058 SDValue NewBasePtr =
9059 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9060 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9061 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9062 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
9063 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9064 }
9065
9066 // Fill the rest with undefs.
9067 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9068 for (; i != WidenNumElts; ++i)
9069 Ops[i] = UndefVal;
9070
9071 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9072}
9073
9074bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9075 StoreSDNode *ST) {
9076 // The strategy assumes that we can efficiently store power-of-two widths.
9077 // The routine chops the vector into the largest vector stores with the same
9078 // element type or scalar stores.
9079 SDValue Chain = ST->getChain();
9080 SDValue BasePtr = ST->getBasePtr();
9081 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9082 AAMDNodes AAInfo = ST->getAAInfo();
9083 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9084 SDLoc dl(ST);
9085
9086 EVT StVT = ST->getMemoryVT();
9087 TypeSize StWidth = StVT.getSizeInBits();
9088 EVT ValVT = ValOp.getValueType();
9089 TypeSize ValWidth = ValVT.getSizeInBits();
9090 EVT ValEltVT = ValVT.getVectorElementType();
9091 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9092 assert(StVT.getVectorElementType() == ValEltVT);
9093 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9094 "Mismatch between store and value types");
9095
9096 int Idx = 0; // current index to store
9097
9098 MachinePointerInfo MPI = ST->getPointerInfo();
9099 uint64_t ScaledOffset = 0;
9100
9101 // A breakdown of how to widen this vector store. Each element of the vector
9102 // is a memory VT combined with the number of times it is to be stored to,
9103 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9104 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9105
9106 while (StWidth.isNonZero()) {
9107 // Find the largest vector type we can store with.
9108 std::optional<EVT> NewVT =
9109 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9110 if (!NewVT)
9111 return false;
9112 MemVTs.push_back(Elt: {*NewVT, 0});
9113 TypeSize NewVTWidth = NewVT->getSizeInBits();
9114
9115 do {
9116 StWidth -= NewVTWidth;
9117 MemVTs.back().second++;
9118 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9119 }
9120
9121 for (const auto &Pair : MemVTs) {
9122 EVT NewVT = Pair.first;
9123 unsigned Count = Pair.second;
9124 TypeSize NewVTWidth = NewVT.getSizeInBits();
9125
9126 if (NewVT.isVector()) {
9127 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9128 do {
9129 Align NewAlign = ScaledOffset == 0
9130 ? ST->getBaseAlign()
9131 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9132 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9133 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9134 MMOFlags, AAInfo);
9135 StChain.push_back(Elt: PartStore);
9136
9137 Idx += NumVTElts;
9138 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9139 ScaledOffset: &ScaledOffset);
9140 } while (--Count);
9141 } else {
9142 // Cast the vector to the scalar type we can store.
9143 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9144 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9145 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9146 // Readjust index position based on new vector type.
9147 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9148 do {
9149 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9150 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
9151 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
9152 StChain.push_back(Elt: PartStore);
9153
9154 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9155 } while (--Count);
9156 // Restore index back to be relative to the original widen element type.
9157 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9158 }
9159 }
9160
9161 return true;
9162}
9163
9164/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9165/// input vector must have the same element type as NVT.
9166/// FillWithZeroes specifies that the vector should be widened with zeroes.
9167SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9168 bool FillWithZeroes) {
9169 // Note that InOp might have been widened so it might already have
9170 // the right width or it might need be narrowed.
9171 EVT InVT = InOp.getValueType();
9172 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9173 "input and widen element type must match");
9174 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9175 "cannot modify scalable vectors in this way");
9176 SDLoc dl(InOp);
9177
9178 // Check if InOp already has the right width.
9179 if (InVT == NVT)
9180 return InOp;
9181
9182 ElementCount InEC = InVT.getVectorElementCount();
9183 ElementCount WidenEC = NVT.getVectorElementCount();
9184 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9185 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9186 SmallVector<SDValue, 16> Ops(NumConcat);
9187 SDValue FillVal =
9188 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9189 Ops[0] = InOp;
9190 for (unsigned i = 1; i != NumConcat; ++i)
9191 Ops[i] = FillVal;
9192
9193 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9194 }
9195
9196 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9197 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9198
9199 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9200 "Scalable vectors should have been handled already.");
9201
9202 unsigned InNumElts = InEC.getFixedValue();
9203 unsigned WidenNumElts = WidenEC.getFixedValue();
9204
9205 // Fall back to extract and build (+ mask, if padding with zeros).
9206 SmallVector<SDValue, 16> Ops(WidenNumElts);
9207 EVT EltVT = NVT.getVectorElementType();
9208 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9209 unsigned Idx;
9210 for (Idx = 0; Idx < MinNumElts; ++Idx)
9211 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9212
9213 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9214 for (; Idx < WidenNumElts; ++Idx)
9215 Ops[Idx] = UndefVal;
9216
9217 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9218 if (!FillWithZeroes)
9219 return Widened;
9220
9221 assert(NVT.isInteger() &&
9222 "We expect to never want to FillWithZeroes for non-integral types.");
9223
9224 SmallVector<SDValue, 16> MaskOps;
9225 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9226 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9227
9228 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9229 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9230}
9231