1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error(reason: "Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::LOOP_DEPENDENCE_WAR_MASK:
57 case ISD::LOOP_DEPENDENCE_RAW_MASK:
58 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
59 break;
60 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
61 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
62 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
63 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
64 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
65 case ISD::CONVERT_FROM_ARBITRARY_FP:
66 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
67 break;
68 case ISD::AssertZext:
69 case ISD::AssertSext:
70 case ISD::FPOWI:
71 case ISD::AssertNoFPClass:
72 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
73 break;
74 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
75 case ISD::ATOMIC_LOAD:
76 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
77 break;
78 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
79 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
80 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
81 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
82 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
83 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
84 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
85 case ISD::POISON:
86 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
87 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
88 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
89 case ISD::ANY_EXTEND_VECTOR_INREG:
90 case ISD::SIGN_EXTEND_VECTOR_INREG:
91 case ISD::ZERO_EXTEND_VECTOR_INREG:
92 R = ScalarizeVecRes_VecInregOp(N);
93 break;
94 case ISD::ABS:
95 case ISD::ANY_EXTEND:
96 case ISD::BITREVERSE:
97 case ISD::BSWAP:
98 case ISD::CTLZ:
99 case ISD::CTLZ_ZERO_UNDEF:
100 case ISD::CTPOP:
101 case ISD::CTTZ:
102 case ISD::CTTZ_ZERO_UNDEF:
103 case ISD::FABS:
104 case ISD::FACOS:
105 case ISD::FASIN:
106 case ISD::FATAN:
107 case ISD::FCEIL:
108 case ISD::FCOS:
109 case ISD::FCOSH:
110 case ISD::FEXP:
111 case ISD::FEXP2:
112 case ISD::FEXP10:
113 case ISD::FFLOOR:
114 case ISD::FLOG:
115 case ISD::FLOG10:
116 case ISD::FLOG2:
117 case ISD::FNEARBYINT:
118 case ISD::FNEG:
119 case ISD::FREEZE:
120 case ISD::ARITH_FENCE:
121 case ISD::FP_EXTEND:
122 case ISD::FP_TO_SINT:
123 case ISD::FP_TO_UINT:
124 case ISD::FRINT:
125 case ISD::LRINT:
126 case ISD::LLRINT:
127 case ISD::FROUND:
128 case ISD::FROUNDEVEN:
129 case ISD::LROUND:
130 case ISD::LLROUND:
131 case ISD::FSIN:
132 case ISD::FSINH:
133 case ISD::FSQRT:
134 case ISD::FTAN:
135 case ISD::FTANH:
136 case ISD::FTRUNC:
137 case ISD::SIGN_EXTEND:
138 case ISD::SINT_TO_FP:
139 case ISD::TRUNCATE:
140 case ISD::UINT_TO_FP:
141 case ISD::ZERO_EXTEND:
142 case ISD::FCANONICALIZE:
143 R = ScalarizeVecRes_UnaryOp(N);
144 break;
145 case ISD::ADDRSPACECAST:
146 R = ScalarizeVecRes_ADDRSPACECAST(N);
147 break;
148 case ISD::FMODF:
149 case ISD::FFREXP:
150 case ISD::FSINCOS:
151 case ISD::FSINCOSPI:
152 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
153 break;
154 case ISD::ADD:
155 case ISD::AND:
156 case ISD::AVGCEILS:
157 case ISD::AVGCEILU:
158 case ISD::AVGFLOORS:
159 case ISD::AVGFLOORU:
160 case ISD::FADD:
161 case ISD::FCOPYSIGN:
162 case ISD::FDIV:
163 case ISD::FMUL:
164 case ISD::FMINNUM:
165 case ISD::FMAXNUM:
166 case ISD::FMINNUM_IEEE:
167 case ISD::FMAXNUM_IEEE:
168 case ISD::FMINIMUM:
169 case ISD::FMAXIMUM:
170 case ISD::FMINIMUMNUM:
171 case ISD::FMAXIMUMNUM:
172 case ISD::FLDEXP:
173 case ISD::ABDS:
174 case ISD::ABDU:
175 case ISD::SMIN:
176 case ISD::SMAX:
177 case ISD::UMIN:
178 case ISD::UMAX:
179
180 case ISD::SADDSAT:
181 case ISD::UADDSAT:
182 case ISD::SSUBSAT:
183 case ISD::USUBSAT:
184 case ISD::SSHLSAT:
185 case ISD::USHLSAT:
186
187 case ISD::FPOW:
188 case ISD::FATAN2:
189 case ISD::FREM:
190 case ISD::FSUB:
191 case ISD::MUL:
192 case ISD::MULHS:
193 case ISD::MULHU:
194 case ISD::OR:
195 case ISD::SDIV:
196 case ISD::SREM:
197 case ISD::SUB:
198 case ISD::UDIV:
199 case ISD::UREM:
200 case ISD::XOR:
201 case ISD::SHL:
202 case ISD::SRA:
203 case ISD::SRL:
204 case ISD::ROTL:
205 case ISD::ROTR:
206 case ISD::CLMUL:
207 case ISD::CLMULR:
208 case ISD::CLMULH:
209 R = ScalarizeVecRes_BinOp(N);
210 break;
211
212 case ISD::SCMP:
213 case ISD::UCMP:
214 R = ScalarizeVecRes_CMP(N);
215 break;
216
217 case ISD::FMA:
218 case ISD::FSHL:
219 case ISD::FSHR:
220 R = ScalarizeVecRes_TernaryOp(N);
221 break;
222
223#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
224 case ISD::STRICT_##DAGN:
225#include "llvm/IR/ConstrainedOps.def"
226 R = ScalarizeVecRes_StrictFPOp(N);
227 break;
228
229 case ISD::FP_TO_UINT_SAT:
230 case ISD::FP_TO_SINT_SAT:
231 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
232 break;
233
234 case ISD::UADDO:
235 case ISD::SADDO:
236 case ISD::USUBO:
237 case ISD::SSUBO:
238 case ISD::UMULO:
239 case ISD::SMULO:
240 R = ScalarizeVecRes_OverflowOp(N, ResNo);
241 break;
242 case ISD::SMULFIX:
243 case ISD::SMULFIXSAT:
244 case ISD::UMULFIX:
245 case ISD::UMULFIXSAT:
246 case ISD::SDIVFIX:
247 case ISD::SDIVFIXSAT:
248 case ISD::UDIVFIX:
249 case ISD::UDIVFIXSAT:
250 R = ScalarizeVecRes_FIX(N);
251 break;
252 }
253
254 // If R is null, the sub-method took care of registering the result.
255 if (R.getNode())
256 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
257}
258
259SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
260 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
261 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
262 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
263 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
264}
265
266SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
267 SDLoc DL(N);
268
269 SDValue LHS = N->getOperand(Num: 0);
270 SDValue RHS = N->getOperand(Num: 1);
271 if (getTypeAction(VT: LHS.getValueType()) ==
272 TargetLowering::TypeScalarizeVector) {
273 LHS = GetScalarizedVector(Op: LHS);
274 RHS = GetScalarizedVector(Op: RHS);
275 } else {
276 EVT VT = LHS.getValueType().getVectorElementType();
277 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
278 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
279 }
280
281 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
282 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
283}
284
285SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
286 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
287 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
288 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
289 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
290 N3: Op2, Flags: N->getFlags());
291}
292
293SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
294 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
295 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
296 SDValue Op2 = N->getOperand(Num: 2);
297 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
298 N3: Op2, Flags: N->getFlags());
299}
300
301SDValue
302DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
303 unsigned ResNo) {
304 assert(N->getValueType(0).getVectorNumElements() == 1 &&
305 "Unexpected vector type!");
306 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
307
308 EVT VT0 = N->getValueType(ResNo: 0);
309 EVT VT1 = N->getValueType(ResNo: 1);
310 SDLoc dl(N);
311
312 SDNode *ScalarNode =
313 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
314 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
315 .getNode();
316
317 // Replace the other vector result not being explicitly scalarized here.
318 unsigned OtherNo = 1 - ResNo;
319 EVT OtherVT = N->getValueType(ResNo: OtherNo);
320 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
321 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
322 } else {
323 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
324 Operand: SDValue(ScalarNode, OtherNo));
325 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
326 }
327
328 return SDValue(ScalarNode, ResNo);
329}
330
331SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
332 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
333 unsigned NumOpers = N->getNumOperands();
334 SDValue Chain = N->getOperand(Num: 0);
335 EVT ValueVTs[] = {VT, MVT::Other};
336 SDLoc dl(N);
337
338 SmallVector<SDValue, 4> Opers(NumOpers);
339
340 // The Chain is the first operand.
341 Opers[0] = Chain;
342
343 // Now process the remaining operands.
344 for (unsigned i = 1; i < NumOpers; ++i) {
345 SDValue Oper = N->getOperand(Num: i);
346 EVT OperVT = Oper.getValueType();
347
348 if (OperVT.isVector()) {
349 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
350 Oper = GetScalarizedVector(Op: Oper);
351 else
352 Oper =
353 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
354 }
355
356 Opers[i] = Oper;
357 }
358
359 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
360 Ops: Opers, Flags: N->getFlags());
361
362 // Legalize the chain result - switch anything that used the old chain to
363 // use the new one.
364 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
365 return Result;
366}
367
368SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
369 unsigned ResNo) {
370 SDLoc DL(N);
371 EVT ResVT = N->getValueType(ResNo: 0);
372 EVT OvVT = N->getValueType(ResNo: 1);
373
374 SDValue ScalarLHS, ScalarRHS;
375 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
376 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
377 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
378 } else {
379 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
380 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
381 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
382 ScalarLHS = ElemsLHS[0];
383 ScalarRHS = ElemsRHS[0];
384 }
385
386 SDVTList ScalarVTs = DAG.getVTList(
387 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
388 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
389 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
390 .getNode();
391
392 // Replace the other vector result not being explicitly scalarized here.
393 unsigned OtherNo = 1 - ResNo;
394 EVT OtherVT = N->getValueType(ResNo: OtherNo);
395 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
396 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
397 } else {
398 SDValue OtherVal = DAG.getNode(
399 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
400 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
401 }
402
403 return SDValue(ScalarNode, ResNo);
404}
405
406SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
407 unsigned ResNo) {
408 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
409 return GetScalarizedVector(Op);
410}
411
412SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
413 SDLoc DL(N);
414 SDValue SourceValue = N->getOperand(Num: 0);
415 SDValue SinkValue = N->getOperand(Num: 1);
416 SDValue EltSizeInBytes = N->getOperand(Num: 2);
417 SDValue LaneOffset = N->getOperand(Num: 3);
418
419 EVT PtrVT = SourceValue->getValueType(ResNo: 0);
420 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
421
422 // Take the difference between the pointers and divided by the element size,
423 // to see how many lanes separate them.
424 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: SinkValue, N2: SourceValue);
425 if (IsReadAfterWrite)
426 Diff = DAG.getNode(Opcode: ISD::ABS, DL, VT: PtrVT, Operand: Diff);
427 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: PtrVT, N1: Diff, N2: EltSizeInBytes);
428
429 // The pointers do not alias if:
430 // * Diff <= 0 || LaneOffset < Diff (WAR_MASK)
431 // * Diff == 0 || LaneOffset < abs(Diff) (RAW_MASK)
432 // Note: If LaneOffset is zero, both cases will fold to "true".
433 EVT CmpVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
434 VT: Diff.getValueType());
435 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
436 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Diff, RHS: Zero,
437 Cond: IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
438 return DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Cmp,
439 N2: DAG.getSetCC(DL, VT: CmpVT, LHS: LaneOffset, RHS: Diff, Cond: ISD::SETULT));
440}
441
442SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
443 SDValue Op = N->getOperand(Num: 0);
444 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
445 Op = GetScalarizedVector(Op);
446 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
447 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
448 VT: NewVT, Operand: Op);
449}
450
451SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
452 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
453 SDValue InOp = N->getOperand(Num: 0);
454 // The BUILD_VECTOR operands may be of wider element types and
455 // we may need to truncate them back to the requested return type.
456 if (EltVT.isInteger())
457 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
458 return InOp;
459}
460
461SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
462 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
463 VT: N->getValueType(ResNo: 0).getVectorElementType(),
464 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
465}
466
467SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
468 SDLoc DL(N);
469 SDValue Op = N->getOperand(Num: 0);
470 EVT OpVT = Op.getValueType();
471 // The result needs scalarizing, but it's not a given that the source does.
472 // See similar logic in ScalarizeVecRes_UnaryOp.
473 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
474 Op = GetScalarizedVector(Op);
475 } else {
476 EVT VT = OpVT.getVectorElementType();
477 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
478 }
479 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
480 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
481 N2: N->getOperand(Num: 1));
482}
483
484SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
485 SDLoc DL(N);
486 SDValue Op = N->getOperand(Num: 0);
487 EVT OpVT = Op.getValueType();
488 // The result needs scalarizing, but it's not a given that the source does.
489 // See similar logic in ScalarizeVecRes_UnaryOp.
490 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
491 Op = GetScalarizedVector(Op);
492 } else {
493 EVT VT = OpVT.getVectorElementType();
494 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
495 }
496 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
497 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
498 N2: N->getOperand(Num: 1));
499}
500
501SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
502 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
503 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
504 N2: N->getOperand(Num: 1));
505}
506
507SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
508 // The value to insert may have a wider type than the vector element type,
509 // so be sure to truncate it to the element type if necessary.
510 SDValue Op = N->getOperand(Num: 1);
511 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
512 if (Op.getValueType() != EltVT)
513 // FIXME: Can this happen for floating point types?
514 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
515 return Op;
516}
517
518SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
519 SDValue Result = DAG.getAtomicLoad(
520 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
521 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
522 MMO: N->getMemOperand());
523
524 // Legalize the chain result - switch anything that used the old chain to
525 // use the new one.
526 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
527 return Result;
528}
529
530SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
531 assert(N->isUnindexed() && "Indexed vector load?");
532
533 SDValue Result = DAG.getLoad(
534 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
535 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
536 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
537 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
538 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
539
540 // Legalize the chain result - switch anything that used the old chain to
541 // use the new one.
542 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
543 return Result;
544}
545
546SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
547 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
548 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
549 SDValue Op = N->getOperand(Num: 0);
550 EVT OpVT = Op.getValueType();
551 SDLoc DL(N);
552 // The result needs scalarizing, but it's not a given that the source does.
553 // This is a workaround for targets where it's impossible to scalarize the
554 // result of a conversion, because the source type is legal.
555 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
556 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
557 // legal and was not scalarized.
558 // See the similar logic in ScalarizeVecRes_SETCC
559 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
560 Op = GetScalarizedVector(Op);
561 } else {
562 EVT VT = OpVT.getVectorElementType();
563 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
564 }
565 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
566}
567
568SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
569 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
570 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
571 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
572 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
573 N1: LHS, N2: DAG.getValueType(ExtVT));
574}
575
576SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
577 SDLoc DL(N);
578 SDValue Op = N->getOperand(Num: 0);
579
580 EVT OpVT = Op.getValueType();
581 EVT OpEltVT = OpVT.getVectorElementType();
582 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
583
584 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
585 Op = GetScalarizedVector(Op);
586 } else {
587 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
588 }
589
590 switch (N->getOpcode()) {
591 case ISD::ANY_EXTEND_VECTOR_INREG:
592 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
593 case ISD::SIGN_EXTEND_VECTOR_INREG:
594 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
595 case ISD::ZERO_EXTEND_VECTOR_INREG:
596 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
597 }
598
599 llvm_unreachable("Illegal extend_vector_inreg opcode");
600}
601
602SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
603 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
604 SDValue Op = N->getOperand(Num: 0);
605 EVT OpVT = Op.getValueType();
606 SDLoc DL(N);
607 // The result needs scalarizing, but it's not a given that the source does.
608 // This is a workaround for targets where it's impossible to scalarize the
609 // result of a conversion, because the source type is legal.
610 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
611 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
612 // legal and was not scalarized.
613 // See the similar logic in ScalarizeVecRes_SETCC
614 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
615 Op = GetScalarizedVector(Op);
616 } else {
617 EVT VT = OpVT.getVectorElementType();
618 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
619 }
620 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
621 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
622 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
623 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
624}
625
626SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
627 // If the operand is wider than the vector element type then it is implicitly
628 // truncated. Make that explicit here.
629 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
630 SDValue InOp = N->getOperand(Num: 0);
631 if (InOp.getValueType() != EltVT)
632 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
633 return InOp;
634}
635
636SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
637 SDValue Cond = N->getOperand(Num: 0);
638 EVT OpVT = Cond.getValueType();
639 SDLoc DL(N);
640 // The vselect result and true/value operands needs scalarizing, but it's
641 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
642 // See the similar logic in ScalarizeVecRes_SETCC
643 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
644 Cond = GetScalarizedVector(Op: Cond);
645 } else {
646 EVT VT = OpVT.getVectorElementType();
647 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
648 }
649
650 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
651 TargetLowering::BooleanContent ScalarBool =
652 TLI.getBooleanContents(isVec: false, isFloat: false);
653 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
654
655 // If integer and float booleans have different contents then we can't
656 // reliably optimize in all cases. There is a full explanation for this in
657 // DAGCombiner::visitSELECT() where the same issue affects folding
658 // (select C, 0, 1) to (xor C, 1).
659 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
660 TLI.getBooleanContents(isVec: false, isFloat: true)) {
661 // At least try the common case where the boolean is generated by a
662 // comparison.
663 if (Cond->getOpcode() == ISD::SETCC) {
664 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
665 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
666 VecBool = TLI.getBooleanContents(Type: OpVT);
667 } else
668 ScalarBool = TargetLowering::UndefinedBooleanContent;
669 }
670
671 EVT CondVT = Cond.getValueType();
672 if (ScalarBool != VecBool) {
673 switch (ScalarBool) {
674 case TargetLowering::UndefinedBooleanContent:
675 break;
676 case TargetLowering::ZeroOrOneBooleanContent:
677 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
678 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
679 // Vector read from all ones, scalar expects a single 1 so mask.
680 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
681 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
682 break;
683 case TargetLowering::ZeroOrNegativeOneBooleanContent:
684 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
685 VecBool == TargetLowering::ZeroOrOneBooleanContent);
686 // Vector reads from a one, scalar from all ones so sign extend.
687 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
688 N1: Cond, N2: DAG.getValueType(MVT::i1));
689 break;
690 }
691 }
692
693 // Truncate the condition if needed
694 auto BoolVT = getSetCCResultType(VT: CondVT);
695 if (BoolVT.bitsLT(VT: CondVT))
696 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
697
698 return DAG.getSelect(DL: SDLoc(N),
699 VT: LHS.getValueType(), Cond, LHS,
700 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
701}
702
703SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
704 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
705 return DAG.getSelect(DL: SDLoc(N),
706 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
707 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
708}
709
710SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
711 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
712 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
713 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
714 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
715 N5: N->getOperand(Num: 4));
716}
717
718SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
719 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
720}
721
722SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
723 // Figure out if the scalar is the LHS or RHS and return it.
724 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
725 if (Arg.isUndef())
726 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
727 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
728 return GetScalarizedVector(Op: N->getOperand(Num: Op));
729}
730
731SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
732 SDValue Src = N->getOperand(Num: 0);
733 EVT SrcVT = Src.getValueType();
734 SDLoc dl(N);
735
736 // Handle case where result is scalarized but operand is not
737 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
738 Src = GetScalarizedVector(Op: Src);
739 else
740 Src = DAG.getNode(
741 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
742 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
743
744 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
745 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
746}
747
748SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
749 assert(N->getValueType(0).isVector() &&
750 N->getOperand(0).getValueType().isVector() &&
751 "Operand types must be vectors");
752 SDValue LHS = N->getOperand(Num: 0);
753 SDValue RHS = N->getOperand(Num: 1);
754 EVT OpVT = LHS.getValueType();
755 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
756 SDLoc DL(N);
757
758 // The result needs scalarizing, but it's not a given that the source does.
759 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
760 LHS = GetScalarizedVector(Op: LHS);
761 RHS = GetScalarizedVector(Op: RHS);
762 } else {
763 EVT VT = OpVT.getVectorElementType();
764 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
765 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
766 }
767
768 // Turn it into a scalar SETCC.
769 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
770 N3: N->getOperand(Num: 2));
771 // Vectors may have a different boolean contents to scalars. Promote the
772 // value appropriately.
773 ISD::NodeType ExtendCode =
774 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
775 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
776}
777
778SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
779 SDLoc DL(N);
780 SDValue Arg = N->getOperand(Num: 0);
781 SDValue Test = N->getOperand(Num: 1);
782 EVT ArgVT = Arg.getValueType();
783 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
784
785 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
786 Arg = GetScalarizedVector(Op: Arg);
787 } else {
788 EVT VT = ArgVT.getVectorElementType();
789 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
790 }
791
792 SDValue Res =
793 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
794 // Vectors may have a different boolean contents to scalars. Promote the
795 // value appropriately.
796 ISD::NodeType ExtendCode =
797 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
798 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
799}
800
801//===----------------------------------------------------------------------===//
802// Operand Vector Scalarization <1 x ty> -> ty.
803//===----------------------------------------------------------------------===//
804
805bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
806 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
807 N->dump(&DAG));
808 SDValue Res = SDValue();
809
810 switch (N->getOpcode()) {
811 default:
812#ifndef NDEBUG
813 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
814 N->dump(&DAG);
815 dbgs() << "\n";
816#endif
817 report_fatal_error(reason: "Do not know how to scalarize this operator's "
818 "operand!\n");
819 case ISD::BITCAST:
820 Res = ScalarizeVecOp_BITCAST(N);
821 break;
822 case ISD::FAKE_USE:
823 Res = ScalarizeVecOp_FAKE_USE(N);
824 break;
825 case ISD::ANY_EXTEND:
826 case ISD::ZERO_EXTEND:
827 case ISD::SIGN_EXTEND:
828 case ISD::TRUNCATE:
829 case ISD::FP_TO_SINT:
830 case ISD::FP_TO_UINT:
831 case ISD::SINT_TO_FP:
832 case ISD::UINT_TO_FP:
833 case ISD::LROUND:
834 case ISD::LLROUND:
835 case ISD::LRINT:
836 case ISD::LLRINT:
837 Res = ScalarizeVecOp_UnaryOp(N);
838 break;
839 case ISD::FP_TO_SINT_SAT:
840 case ISD::FP_TO_UINT_SAT:
841 case ISD::CONVERT_FROM_ARBITRARY_FP:
842 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
843 break;
844 case ISD::STRICT_SINT_TO_FP:
845 case ISD::STRICT_UINT_TO_FP:
846 case ISD::STRICT_FP_TO_SINT:
847 case ISD::STRICT_FP_TO_UINT:
848 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
849 break;
850 case ISD::CONCAT_VECTORS:
851 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
852 break;
853 case ISD::INSERT_SUBVECTOR:
854 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
855 break;
856 case ISD::EXTRACT_VECTOR_ELT:
857 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
858 break;
859 case ISD::VSELECT:
860 Res = ScalarizeVecOp_VSELECT(N);
861 break;
862 case ISD::SETCC:
863 Res = ScalarizeVecOp_VSETCC(N);
864 break;
865 case ISD::STRICT_FSETCC:
866 case ISD::STRICT_FSETCCS:
867 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
868 break;
869 case ISD::STORE:
870 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
871 break;
872 case ISD::STRICT_FP_ROUND:
873 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
874 break;
875 case ISD::FP_ROUND:
876 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
877 break;
878 case ISD::STRICT_FP_EXTEND:
879 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
880 break;
881 case ISD::FP_EXTEND:
882 Res = ScalarizeVecOp_FP_EXTEND(N);
883 break;
884 case ISD::VECREDUCE_FADD:
885 case ISD::VECREDUCE_FMUL:
886 case ISD::VECREDUCE_ADD:
887 case ISD::VECREDUCE_MUL:
888 case ISD::VECREDUCE_AND:
889 case ISD::VECREDUCE_OR:
890 case ISD::VECREDUCE_XOR:
891 case ISD::VECREDUCE_SMAX:
892 case ISD::VECREDUCE_SMIN:
893 case ISD::VECREDUCE_UMAX:
894 case ISD::VECREDUCE_UMIN:
895 case ISD::VECREDUCE_FMAX:
896 case ISD::VECREDUCE_FMIN:
897 case ISD::VECREDUCE_FMAXIMUM:
898 case ISD::VECREDUCE_FMINIMUM:
899 Res = ScalarizeVecOp_VECREDUCE(N);
900 break;
901 case ISD::VECREDUCE_SEQ_FADD:
902 case ISD::VECREDUCE_SEQ_FMUL:
903 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
904 break;
905 case ISD::SCMP:
906 case ISD::UCMP:
907 Res = ScalarizeVecOp_CMP(N);
908 break;
909 case ISD::VECTOR_FIND_LAST_ACTIVE:
910 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
911 break;
912 }
913
914 // If the result is null, the sub-method took care of registering results etc.
915 if (!Res.getNode()) return false;
916
917 // If the result is N, the sub-method updated N in place. Tell the legalizer
918 // core about this.
919 if (Res.getNode() == N)
920 return true;
921
922 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
923 "Invalid operand expansion");
924
925 ReplaceValueWith(From: SDValue(N, 0), To: Res);
926 return false;
927}
928
929/// If the value to convert is a vector that needs to be scalarized, it must be
930/// <1 x ty>. Convert the element instead.
931SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
932 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
933 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
934 VT: N->getValueType(ResNo: 0), Operand: Elt);
935}
936
937// Need to legalize vector operands of fake uses. Must be <1 x ty>.
938SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
939 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
940 "Fake Use: Unexpected vector type!");
941 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
942 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
943}
944
945/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
946/// Do the operation on the element instead.
947SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
948 assert(N->getValueType(0).getVectorNumElements() == 1 &&
949 "Unexpected vector type!");
950 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
951 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
952 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
953 // Revectorize the result so the types line up with what the uses of this
954 // expression expect.
955 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
956}
957
958/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
959/// typesize).
960SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
961 assert(N->getValueType(0).getVectorNumElements() == 1 &&
962 "Unexpected vector type!");
963 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
964 SDValue Op =
965 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
966 N1: Elt, N2: N->getOperand(Num: 1));
967 // Revectorize the result so the types line up with what the uses of this
968 // expression expect.
969 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
970}
971
972/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
973/// Do the strict FP operation on the element instead.
974SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
975 assert(N->getValueType(0).getVectorNumElements() == 1 &&
976 "Unexpected vector type!");
977 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
978 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
979 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
980 Ops: { N->getOperand(Num: 0), Elt });
981 // Legalize the chain result - switch anything that used the old chain to
982 // use the new one.
983 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
984 // Revectorize the result so the types line up with what the uses of this
985 // expression expect.
986 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
987
988 // Do our own replacement and return SDValue() to tell the caller that we
989 // handled all replacements since caller can only handle a single result.
990 ReplaceValueWith(From: SDValue(N, 0), To: Res);
991 return SDValue();
992}
993
994/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
995SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
996 SmallVector<SDValue, 8> Ops(N->getNumOperands());
997 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
998 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
999 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1000}
1001
1002/// The inserted subvector is to be scalarized - use insert vector element
1003/// instead.
1004SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1005 unsigned OpNo) {
1006 // We should not be attempting to scalarize the containing vector
1007 assert(OpNo == 1);
1008 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1009 SDValue ContainingVec = N->getOperand(Num: 0);
1010 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1011 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1012 N3: N->getOperand(Num: 2));
1013}
1014
1015/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1016/// so just return the element, ignoring the index.
1017SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1018 EVT VT = N->getValueType(ResNo: 0);
1019 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1020 if (Res.getValueType() != VT)
1021 Res = VT.isFloatingPoint()
1022 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1023 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1024 return Res;
1025}
1026
1027/// If the input condition is a vector that needs to be scalarized, it must be
1028/// <1 x i1>, so just convert to a normal ISD::SELECT
1029/// (still with vector output type since that was acceptable if we got here).
1030SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1031 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1032 EVT VT = N->getValueType(ResNo: 0);
1033
1034 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1035 N3: N->getOperand(Num: 2));
1036}
1037
1038/// If the operand is a vector that needs to be scalarized then the
1039/// result must be v1i1, so just convert to a scalar SETCC and wrap
1040/// with a scalar_to_vector since the res type is legal if we got here
1041SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1042 assert(N->getValueType(0).isVector() &&
1043 N->getOperand(0).getValueType().isVector() &&
1044 "Operand types must be vectors");
1045 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1046
1047 EVT VT = N->getValueType(ResNo: 0);
1048 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1049 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1050
1051 EVT OpVT = N->getOperand(Num: 0).getValueType();
1052 EVT NVT = VT.getVectorElementType();
1053 SDLoc DL(N);
1054 // Turn it into a scalar SETCC.
1055 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1056 N3: N->getOperand(Num: 2));
1057
1058 // Vectors may have a different boolean contents to scalars. Promote the
1059 // value appropriately.
1060 ISD::NodeType ExtendCode =
1061 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1062
1063 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1064
1065 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1066}
1067
1068// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1069SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1070 unsigned OpNo) {
1071 assert(OpNo == 1 && "Wrong operand for scalarization!");
1072 assert(N->getValueType(0).isVector() &&
1073 N->getOperand(1).getValueType().isVector() &&
1074 "Operand types must be vectors");
1075 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1076
1077 EVT VT = N->getValueType(ResNo: 0);
1078 SDValue Ch = N->getOperand(Num: 0);
1079 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1080 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1081 SDValue CC = N->getOperand(Num: 3);
1082
1083 EVT OpVT = N->getOperand(Num: 1).getValueType();
1084 EVT NVT = VT.getVectorElementType();
1085 SDLoc DL(N);
1086 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1087 Ops: {Ch, LHS, RHS, CC});
1088
1089 // Legalize the chain result - switch anything that used the old chain to
1090 // use the new one.
1091 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1092
1093 ISD::NodeType ExtendCode =
1094 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1095
1096 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1097 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1098
1099 // Do our own replacement and return SDValue() to tell the caller that we
1100 // handled all replacements since caller can only handle a single result.
1101 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1102 return SDValue();
1103}
1104
1105/// If the value to store is a vector that needs to be scalarized, it must be
1106/// <1 x ty>. Just store the element.
1107SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1108 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1109 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1110 SDLoc dl(N);
1111
1112 if (N->isTruncatingStore())
1113 return DAG.getTruncStore(
1114 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1115 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1116 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1117 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1118
1119 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1120 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1121 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1122}
1123
1124/// If the value to round is a vector that needs to be scalarized, it must be
1125/// <1 x ty>. Convert the element instead.
1126SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1127 assert(OpNo == 0 && "Wrong operand for scalarization!");
1128 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1129 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1130 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1131 N2: N->getOperand(Num: 1));
1132 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1133}
1134
1135SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1136 unsigned OpNo) {
1137 assert(OpNo == 1 && "Wrong operand for scalarization!");
1138 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1139 SDValue Res =
1140 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1141 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1142 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1143 // Legalize the chain result - switch anything that used the old chain to
1144 // use the new one.
1145 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1146
1147 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1148
1149 // Do our own replacement and return SDValue() to tell the caller that we
1150 // handled all replacements since caller can only handle a single result.
1151 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1152 return SDValue();
1153}
1154
1155/// If the value to extend is a vector that needs to be scalarized, it must be
1156/// <1 x ty>. Convert the element instead.
1157SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1158 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1159 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1160 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1161 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1162}
1163
1164/// If the value to extend is a vector that needs to be scalarized, it must be
1165/// <1 x ty>. Convert the element instead.
1166SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1167 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1168 SDValue Res =
1169 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1170 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1171 Ops: {N->getOperand(Num: 0), Elt});
1172 // Legalize the chain result - switch anything that used the old chain to
1173 // use the new one.
1174 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1175
1176 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1177
1178 // Do our own replacement and return SDValue() to tell the caller that we
1179 // handled all replacements since caller can only handle a single result.
1180 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1181 return SDValue();
1182}
1183
1184SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1185 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1186 // Result type may be wider than element type.
1187 if (Res.getValueType() != N->getValueType(ResNo: 0))
1188 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1189 return Res;
1190}
1191
1192SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1193 SDValue AccOp = N->getOperand(Num: 0);
1194 SDValue VecOp = N->getOperand(Num: 1);
1195
1196 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1197
1198 SDValue Op = GetScalarizedVector(Op: VecOp);
1199 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1200 N1: AccOp, N2: Op, Flags: N->getFlags());
1201}
1202
1203SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1204 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1205 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1206
1207 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1208 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1209 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1210}
1211
1212SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1213 // Since there is no "none-active" result, the only valid return for <1 x ty>
1214 // is 0. Note: Since we check the high mask during splitting this is safe.
1215 // As e.g., a <2 x ty> operation would split to:
1216 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1217 // : `last_active(%lo_mask)`
1218 // Which then scalarizes to:
1219 // %mask[1] ? 1 : 0
1220 EVT VT = N->getValueType(ResNo: 0);
1221 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1222}
1223
1224//===----------------------------------------------------------------------===//
1225// Result Vector Splitting
1226//===----------------------------------------------------------------------===//
1227
1228/// This method is called when the specified result of the specified node is
1229/// found to need vector splitting. At this point, the node may also have
1230/// invalid operands or may have other results that need legalization, we just
1231/// know that (at least) one result needs vector splitting.
1232void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1233 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1234 SDValue Lo, Hi;
1235
1236 // See if the target wants to custom expand this node.
1237 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1238 return;
1239
1240 switch (N->getOpcode()) {
1241 default:
1242#ifndef NDEBUG
1243 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1244 N->dump(&DAG);
1245 dbgs() << "\n";
1246#endif
1247 report_fatal_error(reason: "Do not know how to split the result of this "
1248 "operator!\n");
1249
1250 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1251 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1252 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1253 break;
1254 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1255 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1256 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1257 case ISD::VSELECT:
1258 case ISD::SELECT:
1259 case ISD::VP_MERGE:
1260 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1261 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1262 case ISD::POISON:
1263 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1264 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1265 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1266 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1267 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1268 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1269 case ISD::FPOWI:
1270 case ISD::FLDEXP:
1271 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1272 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1273 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1274 case ISD::SPLAT_VECTOR:
1275 case ISD::SCALAR_TO_VECTOR:
1276 SplitVecRes_ScalarOp(N, Lo, Hi);
1277 break;
1278 case ISD::STEP_VECTOR:
1279 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1280 break;
1281 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1282 case ISD::LOAD:
1283 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1284 break;
1285 case ISD::VP_LOAD:
1286 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1287 break;
1288 case ISD::VP_LOAD_FF:
1289 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1290 break;
1291 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1292 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1293 break;
1294 case ISD::MLOAD:
1295 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1296 break;
1297 case ISD::MGATHER:
1298 case ISD::VP_GATHER:
1299 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1300 break;
1301 case ISD::VECTOR_COMPRESS:
1302 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1303 break;
1304 case ISD::SETCC:
1305 case ISD::VP_SETCC:
1306 SplitVecRes_SETCC(N, Lo, Hi);
1307 break;
1308 case ISD::VECTOR_REVERSE:
1309 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1310 break;
1311 case ISD::VECTOR_SHUFFLE:
1312 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1313 break;
1314 case ISD::VECTOR_SPLICE_LEFT:
1315 case ISD::VECTOR_SPLICE_RIGHT:
1316 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1317 break;
1318 case ISD::VECTOR_DEINTERLEAVE:
1319 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1320 return;
1321 case ISD::VECTOR_INTERLEAVE:
1322 SplitVecRes_VECTOR_INTERLEAVE(N);
1323 return;
1324 case ISD::VAARG:
1325 SplitVecRes_VAARG(N, Lo, Hi);
1326 break;
1327
1328 case ISD::ANY_EXTEND_VECTOR_INREG:
1329 case ISD::SIGN_EXTEND_VECTOR_INREG:
1330 case ISD::ZERO_EXTEND_VECTOR_INREG:
1331 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1332 break;
1333
1334 case ISD::ABS:
1335 case ISD::VP_ABS:
1336 case ISD::BITREVERSE:
1337 case ISD::VP_BITREVERSE:
1338 case ISD::BSWAP:
1339 case ISD::VP_BSWAP:
1340 case ISD::CTLZ:
1341 case ISD::VP_CTLZ:
1342 case ISD::CTTZ:
1343 case ISD::VP_CTTZ:
1344 case ISD::CTLZ_ZERO_UNDEF:
1345 case ISD::VP_CTLZ_ZERO_UNDEF:
1346 case ISD::CTTZ_ZERO_UNDEF:
1347 case ISD::VP_CTTZ_ZERO_UNDEF:
1348 case ISD::CTPOP:
1349 case ISD::VP_CTPOP:
1350 case ISD::FABS: case ISD::VP_FABS:
1351 case ISD::FACOS:
1352 case ISD::FASIN:
1353 case ISD::FATAN:
1354 case ISD::FCEIL:
1355 case ISD::VP_FCEIL:
1356 case ISD::FCOS:
1357 case ISD::FCOSH:
1358 case ISD::FEXP:
1359 case ISD::FEXP2:
1360 case ISD::FEXP10:
1361 case ISD::FFLOOR:
1362 case ISD::VP_FFLOOR:
1363 case ISD::FLOG:
1364 case ISD::FLOG10:
1365 case ISD::FLOG2:
1366 case ISD::FNEARBYINT:
1367 case ISD::VP_FNEARBYINT:
1368 case ISD::FNEG: case ISD::VP_FNEG:
1369 case ISD::FREEZE:
1370 case ISD::ARITH_FENCE:
1371 case ISD::FP_EXTEND:
1372 case ISD::VP_FP_EXTEND:
1373 case ISD::FP_ROUND:
1374 case ISD::VP_FP_ROUND:
1375 case ISD::FP_TO_SINT:
1376 case ISD::VP_FP_TO_SINT:
1377 case ISD::FP_TO_UINT:
1378 case ISD::VP_FP_TO_UINT:
1379 case ISD::FRINT:
1380 case ISD::VP_FRINT:
1381 case ISD::LRINT:
1382 case ISD::VP_LRINT:
1383 case ISD::LLRINT:
1384 case ISD::VP_LLRINT:
1385 case ISD::FROUND:
1386 case ISD::VP_FROUND:
1387 case ISD::FROUNDEVEN:
1388 case ISD::VP_FROUNDEVEN:
1389 case ISD::LROUND:
1390 case ISD::LLROUND:
1391 case ISD::FSIN:
1392 case ISD::FSINH:
1393 case ISD::FSQRT: case ISD::VP_SQRT:
1394 case ISD::FTAN:
1395 case ISD::FTANH:
1396 case ISD::FTRUNC:
1397 case ISD::VP_FROUNDTOZERO:
1398 case ISD::SINT_TO_FP:
1399 case ISD::VP_SINT_TO_FP:
1400 case ISD::TRUNCATE:
1401 case ISD::VP_TRUNCATE:
1402 case ISD::UINT_TO_FP:
1403 case ISD::VP_UINT_TO_FP:
1404 case ISD::FCANONICALIZE:
1405 case ISD::AssertNoFPClass:
1406 case ISD::CONVERT_FROM_ARBITRARY_FP:
1407 SplitVecRes_UnaryOp(N, Lo, Hi);
1408 break;
1409 case ISD::ADDRSPACECAST:
1410 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1411 break;
1412 case ISD::FMODF:
1413 case ISD::FFREXP:
1414 case ISD::FSINCOS:
1415 case ISD::FSINCOSPI:
1416 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1417 break;
1418
1419 case ISD::ANY_EXTEND:
1420 case ISD::SIGN_EXTEND:
1421 case ISD::ZERO_EXTEND:
1422 case ISD::VP_SIGN_EXTEND:
1423 case ISD::VP_ZERO_EXTEND:
1424 SplitVecRes_ExtendOp(N, Lo, Hi);
1425 break;
1426
1427 case ISD::ADD: case ISD::VP_ADD:
1428 case ISD::SUB: case ISD::VP_SUB:
1429 case ISD::MUL: case ISD::VP_MUL:
1430 case ISD::CLMUL:
1431 case ISD::CLMULR:
1432 case ISD::CLMULH:
1433 case ISD::MULHS:
1434 case ISD::MULHU:
1435 case ISD::ABDS:
1436 case ISD::ABDU:
1437 case ISD::AVGCEILS:
1438 case ISD::AVGCEILU:
1439 case ISD::AVGFLOORS:
1440 case ISD::AVGFLOORU:
1441 case ISD::FADD: case ISD::VP_FADD:
1442 case ISD::FSUB: case ISD::VP_FSUB:
1443 case ISD::FMUL: case ISD::VP_FMUL:
1444 case ISD::FMINNUM:
1445 case ISD::FMINNUM_IEEE:
1446 case ISD::VP_FMINNUM:
1447 case ISD::FMAXNUM:
1448 case ISD::FMAXNUM_IEEE:
1449 case ISD::VP_FMAXNUM:
1450 case ISD::FMINIMUM:
1451 case ISD::VP_FMINIMUM:
1452 case ISD::FMAXIMUM:
1453 case ISD::VP_FMAXIMUM:
1454 case ISD::FMINIMUMNUM:
1455 case ISD::FMAXIMUMNUM:
1456 case ISD::SDIV: case ISD::VP_SDIV:
1457 case ISD::UDIV: case ISD::VP_UDIV:
1458 case ISD::FDIV: case ISD::VP_FDIV:
1459 case ISD::FPOW:
1460 case ISD::FATAN2:
1461 case ISD::AND: case ISD::VP_AND:
1462 case ISD::OR: case ISD::VP_OR:
1463 case ISD::XOR: case ISD::VP_XOR:
1464 case ISD::SHL: case ISD::VP_SHL:
1465 case ISD::SRA: case ISD::VP_SRA:
1466 case ISD::SRL: case ISD::VP_SRL:
1467 case ISD::UREM: case ISD::VP_UREM:
1468 case ISD::SREM: case ISD::VP_SREM:
1469 case ISD::FREM: case ISD::VP_FREM:
1470 case ISD::SMIN: case ISD::VP_SMIN:
1471 case ISD::SMAX: case ISD::VP_SMAX:
1472 case ISD::UMIN: case ISD::VP_UMIN:
1473 case ISD::UMAX: case ISD::VP_UMAX:
1474 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1475 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1476 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1477 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1478 case ISD::SSHLSAT:
1479 case ISD::USHLSAT:
1480 case ISD::ROTL:
1481 case ISD::ROTR:
1482 case ISD::VP_FCOPYSIGN:
1483 SplitVecRes_BinOp(N, Lo, Hi);
1484 break;
1485 case ISD::FMA: case ISD::VP_FMA:
1486 case ISD::FSHL:
1487 case ISD::VP_FSHL:
1488 case ISD::FSHR:
1489 case ISD::VP_FSHR:
1490 SplitVecRes_TernaryOp(N, Lo, Hi);
1491 break;
1492
1493 case ISD::SCMP: case ISD::UCMP:
1494 SplitVecRes_CMP(N, Lo, Hi);
1495 break;
1496
1497#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1498 case ISD::STRICT_##DAGN:
1499#include "llvm/IR/ConstrainedOps.def"
1500 SplitVecRes_StrictFPOp(N, Lo, Hi);
1501 break;
1502
1503 case ISD::FP_TO_UINT_SAT:
1504 case ISD::FP_TO_SINT_SAT:
1505 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1506 break;
1507
1508 case ISD::UADDO:
1509 case ISD::SADDO:
1510 case ISD::USUBO:
1511 case ISD::SSUBO:
1512 case ISD::UMULO:
1513 case ISD::SMULO:
1514 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1515 break;
1516 case ISD::SMULFIX:
1517 case ISD::SMULFIXSAT:
1518 case ISD::UMULFIX:
1519 case ISD::UMULFIXSAT:
1520 case ISD::SDIVFIX:
1521 case ISD::SDIVFIXSAT:
1522 case ISD::UDIVFIX:
1523 case ISD::UDIVFIXSAT:
1524 SplitVecRes_FIX(N, Lo, Hi);
1525 break;
1526 case ISD::EXPERIMENTAL_VP_SPLICE:
1527 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1528 break;
1529 case ISD::EXPERIMENTAL_VP_REVERSE:
1530 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1531 break;
1532 case ISD::PARTIAL_REDUCE_UMLA:
1533 case ISD::PARTIAL_REDUCE_SMLA:
1534 case ISD::PARTIAL_REDUCE_SUMLA:
1535 case ISD::PARTIAL_REDUCE_FMLA:
1536 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1537 break;
1538 case ISD::GET_ACTIVE_LANE_MASK:
1539 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1540 break;
1541 }
1542
1543 // If Lo/Hi is null, the sub-method took care of registering results etc.
1544 if (Lo.getNode())
1545 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1546}
1547
1548void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1549 MachinePointerInfo &MPI, SDValue &Ptr,
1550 uint64_t *ScaledOffset) {
1551 SDLoc DL(N);
1552 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1553
1554 if (MemVT.isScalableVector()) {
1555 SDValue BytesIncrement = DAG.getVScale(
1556 DL, VT: Ptr.getValueType(),
1557 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1558 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1559 if (ScaledOffset)
1560 *ScaledOffset += IncrementSize;
1561 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1562 Flags: SDNodeFlags::NoUnsignedWrap);
1563 } else {
1564 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1565 // Increment the pointer to the other half.
1566 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1567 }
1568}
1569
1570std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1571 return SplitMask(Mask, DL: SDLoc(Mask));
1572}
1573
1574std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1575 const SDLoc &DL) {
1576 SDValue MaskLo, MaskHi;
1577 EVT MaskVT = Mask.getValueType();
1578 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1579 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1580 else
1581 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1582 return std::make_pair(x&: MaskLo, y&: MaskHi);
1583}
1584
1585void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1586 SDValue LHSLo, LHSHi;
1587 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1588 SDValue RHSLo, RHSHi;
1589 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1590 SDLoc dl(N);
1591
1592 const SDNodeFlags Flags = N->getFlags();
1593 unsigned Opcode = N->getOpcode();
1594 if (N->getNumOperands() == 2) {
1595 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1596 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1597 return;
1598 }
1599
1600 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1601 assert(N->isVPOpcode() && "Expected VP opcode");
1602
1603 SDValue MaskLo, MaskHi;
1604 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1605
1606 SDValue EVLLo, EVLHi;
1607 std::tie(args&: EVLLo, args&: EVLHi) =
1608 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1609
1610 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1611 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1612 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1613 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1614}
1615
1616void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1617 SDValue &Hi) {
1618 SDValue Op0Lo, Op0Hi;
1619 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1620 SDValue Op1Lo, Op1Hi;
1621 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1622 SDValue Op2Lo, Op2Hi;
1623 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1624 SDLoc dl(N);
1625
1626 const SDNodeFlags Flags = N->getFlags();
1627 unsigned Opcode = N->getOpcode();
1628 if (N->getNumOperands() == 3) {
1629 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1630 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1631 return;
1632 }
1633
1634 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1635 assert(N->isVPOpcode() && "Expected VP opcode");
1636
1637 SDValue MaskLo, MaskHi;
1638 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1639
1640 SDValue EVLLo, EVLHi;
1641 std::tie(args&: EVLLo, args&: EVLHi) =
1642 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1643
1644 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1645 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1646 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1647 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1648}
1649
1650void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1651 LLVMContext &Ctxt = *DAG.getContext();
1652 SDLoc dl(N);
1653
1654 SDValue LHS = N->getOperand(Num: 0);
1655 SDValue RHS = N->getOperand(Num: 1);
1656
1657 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1658 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1659 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1660 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1661 } else {
1662 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1663 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1664 }
1665
1666 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1667 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1668 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1669}
1670
1671void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1672 SDValue LHSLo, LHSHi;
1673 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1674 SDValue RHSLo, RHSHi;
1675 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1676 SDLoc dl(N);
1677 SDValue Op2 = N->getOperand(Num: 2);
1678
1679 unsigned Opcode = N->getOpcode();
1680 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1681 Flags: N->getFlags());
1682 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1683 Flags: N->getFlags());
1684}
1685
1686void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1687 SDValue &Hi) {
1688 // We know the result is a vector. The input may be either a vector or a
1689 // scalar value.
1690 EVT LoVT, HiVT;
1691 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1692 SDLoc dl(N);
1693
1694 SDValue InOp = N->getOperand(Num: 0);
1695 EVT InVT = InOp.getValueType();
1696
1697 // Handle some special cases efficiently.
1698 switch (getTypeAction(VT: InVT)) {
1699 case TargetLowering::TypeLegal:
1700 case TargetLowering::TypePromoteInteger:
1701 case TargetLowering::TypeSoftPromoteHalf:
1702 case TargetLowering::TypeSoftenFloat:
1703 case TargetLowering::TypeScalarizeVector:
1704 case TargetLowering::TypeWidenVector:
1705 break;
1706 case TargetLowering::TypeExpandInteger:
1707 case TargetLowering::TypeExpandFloat:
1708 // A scalar to vector conversion, where the scalar needs expansion.
1709 // If the vector is being split in two then we can just convert the
1710 // expanded pieces.
1711 if (LoVT == HiVT) {
1712 GetExpandedOp(Op: InOp, Lo, Hi);
1713 if (DAG.getDataLayout().isBigEndian())
1714 std::swap(a&: Lo, b&: Hi);
1715 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1716 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1717 return;
1718 }
1719 break;
1720 case TargetLowering::TypeSplitVector:
1721 // If the input is a vector that needs to be split, convert each split
1722 // piece of the input now.
1723 GetSplitVector(Op: InOp, Lo, Hi);
1724 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1725 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1726 return;
1727 case TargetLowering::TypeScalarizeScalableVector:
1728 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1729 }
1730
1731 if (LoVT.isScalableVector()) {
1732 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1733 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1734 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1735 return;
1736 }
1737
1738 // In the general case, convert the input to an integer and split it by hand.
1739 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1740 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1741 if (DAG.getDataLayout().isBigEndian())
1742 std::swap(a&: LoIntVT, b&: HiIntVT);
1743
1744 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1745
1746 if (DAG.getDataLayout().isBigEndian())
1747 std::swap(a&: Lo, b&: Hi);
1748 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1749 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1750}
1751
1752void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1753 SDValue &Hi) {
1754 SDLoc DL(N);
1755 EVT LoVT, HiVT;
1756 SDValue PtrA = N->getOperand(Num: 0);
1757 SDValue PtrB = N->getOperand(Num: 1);
1758 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1759
1760 // The lane offset for the "Lo" half of the mask is unchanged.
1761 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1762 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1763 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1764 // The lane offset for the "Hi" half of the mask is incremented by the number
1765 // of elements in the "Lo" half.
1766 unsigned LaneOffset =
1767 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1768 // Note: The lane offset is implicitly scalable for scalable masks.
1769 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1770 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1771 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1772}
1773
1774void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1775 SDValue &Hi) {
1776 EVT LoVT, HiVT;
1777 SDLoc dl(N);
1778 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1779 unsigned LoNumElts = LoVT.getVectorNumElements();
1780 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1781 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1782
1783 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1784 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1785}
1786
1787void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1788 SDValue &Hi) {
1789 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1790 SDLoc dl(N);
1791 unsigned NumSubvectors = N->getNumOperands() / 2;
1792 if (NumSubvectors == 1) {
1793 Lo = N->getOperand(Num: 0);
1794 Hi = N->getOperand(Num: 1);
1795 return;
1796 }
1797
1798 EVT LoVT, HiVT;
1799 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1800
1801 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1802 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1803
1804 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1805 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1806}
1807
1808void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1809 SDValue &Hi) {
1810 SDValue Vec = N->getOperand(Num: 0);
1811 SDValue Idx = N->getOperand(Num: 1);
1812 SDLoc dl(N);
1813
1814 EVT LoVT, HiVT;
1815 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1816
1817 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1818 uint64_t IdxVal = Idx->getAsZExtVal();
1819 Hi = DAG.getNode(
1820 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1821 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1822}
1823
1824void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1825 SDValue &Hi) {
1826 SDValue Vec = N->getOperand(Num: 0);
1827 SDValue SubVec = N->getOperand(Num: 1);
1828 SDValue Idx = N->getOperand(Num: 2);
1829 SDLoc dl(N);
1830 GetSplitVector(Op: Vec, Lo, Hi);
1831
1832 EVT VecVT = Vec.getValueType();
1833 EVT LoVT = Lo.getValueType();
1834 EVT SubVecVT = SubVec.getValueType();
1835 unsigned VecElems = VecVT.getVectorMinNumElements();
1836 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1837 unsigned LoElems = LoVT.getVectorMinNumElements();
1838
1839 // If we know the index is in the first half, and we know the subvector
1840 // doesn't cross the boundary between the halves, we can avoid spilling the
1841 // vector, and insert into the lower half of the split vector directly.
1842 unsigned IdxVal = Idx->getAsZExtVal();
1843 if (IdxVal + SubElems <= LoElems) {
1844 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1845 return;
1846 }
1847 // Similarly if the subvector is fully in the high half, but mind that we
1848 // can't tell whether a fixed-length subvector is fully within the high half
1849 // of a scalable vector.
1850 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1851 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1852 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1853 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1854 return;
1855 }
1856
1857 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1858 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1859 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1860 if (WideSubVec.getValueType() == VecVT) {
1861 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1862 return;
1863 }
1864 }
1865
1866 // Spill the vector to the stack.
1867 // In cases where the vector is illegal it will be broken down into parts
1868 // and stored in parts - we should use the alignment for the smallest part.
1869 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1870 SDValue StackPtr =
1871 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1872 auto &MF = DAG.getMachineFunction();
1873 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1874 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1875
1876 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1877 Alignment: SmallestAlign);
1878
1879 // Store the new subvector into the specified index.
1880 SDValue SubVecPtr =
1881 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1882 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1883 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1884
1885 // Load the Lo part from the stack slot.
1886 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1887 Alignment: SmallestAlign);
1888
1889 // Increment the pointer to the other part.
1890 auto *Load = cast<LoadSDNode>(Val&: Lo);
1891 MachinePointerInfo MPI = Load->getPointerInfo();
1892 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1893
1894 // Load the Hi part from the stack slot.
1895 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1896}
1897
1898// Handle splitting an FP where the second operand does not match the first
1899// type. The second operand may be a scalar, or a vector that has exactly as
1900// many elements as the first
1901void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1902 SDValue &Hi) {
1903 SDValue LHSLo, LHSHi;
1904 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1905 SDLoc DL(N);
1906
1907 SDValue RHSLo, RHSHi;
1908 SDValue RHS = N->getOperand(Num: 1);
1909 EVT RHSVT = RHS.getValueType();
1910 if (RHSVT.isVector()) {
1911 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1912 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1913 else
1914 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1915
1916 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1917 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1918 } else {
1919 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1920 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1921 }
1922}
1923
1924void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1925 SDValue &Hi) {
1926 SDLoc DL(N);
1927 SDValue ArgLo, ArgHi;
1928 SDValue Test = N->getOperand(Num: 1);
1929 SDValue FpValue = N->getOperand(Num: 0);
1930 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1931 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1932 else
1933 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1934 EVT LoVT, HiVT;
1935 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1936
1937 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1938 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1939}
1940
1941void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1942 SDValue &Hi) {
1943 SDValue LHSLo, LHSHi;
1944 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1945 SDLoc dl(N);
1946
1947 EVT LoVT, HiVT;
1948 std::tie(args&: LoVT, args&: HiVT) =
1949 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1950
1951 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1952 N2: DAG.getValueType(LoVT));
1953 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1954 N2: DAG.getValueType(HiVT));
1955}
1956
1957void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1958 SDValue &Hi) {
1959 unsigned Opcode = N->getOpcode();
1960 SDValue N0 = N->getOperand(Num: 0);
1961
1962 SDLoc dl(N);
1963 SDValue InLo, InHi;
1964
1965 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1966 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1967 else
1968 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1969
1970 EVT InLoVT = InLo.getValueType();
1971 unsigned InNumElements = InLoVT.getVectorNumElements();
1972
1973 EVT OutLoVT, OutHiVT;
1974 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1975 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1976 assert((2 * OutNumElements) <= InNumElements &&
1977 "Illegal extend vector in reg split");
1978
1979 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1980 // input vector (i.e. we only use InLo):
1981 // OutLo will extend the first OutNumElements from InLo.
1982 // OutHi will extend the next OutNumElements from InLo.
1983
1984 // Shuffle the elements from InLo for OutHi into the bottom elements to
1985 // create a 'fake' InHi.
1986 SmallVector<int, 8> SplitHi(InNumElements, -1);
1987 for (unsigned i = 0; i != OutNumElements; ++i)
1988 SplitHi[i] = i + OutNumElements;
1989 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
1990
1991 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
1992 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
1993}
1994
1995void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1996 SDValue &Hi) {
1997 unsigned NumOps = N->getNumOperands();
1998 SDValue Chain = N->getOperand(Num: 0);
1999 EVT LoVT, HiVT;
2000 SDLoc dl(N);
2001 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2002
2003 SmallVector<SDValue, 4> OpsLo(NumOps);
2004 SmallVector<SDValue, 4> OpsHi(NumOps);
2005
2006 // The Chain is the first operand.
2007 OpsLo[0] = Chain;
2008 OpsHi[0] = Chain;
2009
2010 // Now process the remaining operands.
2011 for (unsigned i = 1; i < NumOps; ++i) {
2012 SDValue Op = N->getOperand(Num: i);
2013 SDValue OpLo = Op;
2014 SDValue OpHi = Op;
2015
2016 EVT InVT = Op.getValueType();
2017 if (InVT.isVector()) {
2018 // If the input also splits, handle it directly for a
2019 // compile time speedup. Otherwise split it by hand.
2020 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2021 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2022 else
2023 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2024 }
2025
2026 OpsLo[i] = OpLo;
2027 OpsHi[i] = OpHi;
2028 }
2029
2030 EVT LoValueVTs[] = {LoVT, MVT::Other};
2031 EVT HiValueVTs[] = {HiVT, MVT::Other};
2032 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2033 Flags: N->getFlags());
2034 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2035 Flags: N->getFlags());
2036
2037 // Build a factor node to remember that this Op is independent of the
2038 // other one.
2039 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2040 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2041
2042 // Legalize the chain result - switch anything that used the old chain to
2043 // use the new one.
2044 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2045}
2046
2047SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2048 SDValue Chain = N->getOperand(Num: 0);
2049 EVT VT = N->getValueType(ResNo: 0);
2050 unsigned NE = VT.getVectorNumElements();
2051 EVT EltVT = VT.getVectorElementType();
2052 SDLoc dl(N);
2053
2054 SmallVector<SDValue, 8> Scalars;
2055 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2056
2057 // If ResNE is 0, fully unroll the vector op.
2058 if (ResNE == 0)
2059 ResNE = NE;
2060 else if (NE > ResNE)
2061 NE = ResNE;
2062
2063 //The results of each unrolled operation, including the chain.
2064 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2065 SmallVector<SDValue, 8> Chains;
2066
2067 unsigned i;
2068 for (i = 0; i != NE; ++i) {
2069 Operands[0] = Chain;
2070 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2071 SDValue Operand = N->getOperand(Num: j);
2072 EVT OperandVT = Operand.getValueType();
2073 if (OperandVT.isVector()) {
2074 EVT OperandEltVT = OperandVT.getVectorElementType();
2075 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2076 } else {
2077 Operands[j] = Operand;
2078 }
2079 }
2080 SDValue Scalar =
2081 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2082
2083 //Add in the scalar as well as its chain value to the
2084 //result vectors.
2085 Scalars.push_back(Elt: Scalar);
2086 Chains.push_back(Elt: Scalar.getValue(R: 1));
2087 }
2088
2089 for (; i < ResNE; ++i)
2090 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2091
2092 // Build a new factor node to connect the chain back together.
2093 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2094 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2095
2096 // Create a new BUILD_VECTOR node
2097 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2098 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2099}
2100
2101void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2102 SDValue &Lo, SDValue &Hi) {
2103 SDLoc dl(N);
2104 EVT ResVT = N->getValueType(ResNo: 0);
2105 EVT OvVT = N->getValueType(ResNo: 1);
2106 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2107 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2108 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2109
2110 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2111 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2112 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2113 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2114 } else {
2115 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2116 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2117 }
2118
2119 unsigned Opcode = N->getOpcode();
2120 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2121 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2122 SDNode *LoNode =
2123 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2124 SDNode *HiNode =
2125 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2126
2127 Lo = SDValue(LoNode, ResNo);
2128 Hi = SDValue(HiNode, ResNo);
2129
2130 // Replace the other vector result not being explicitly split here.
2131 unsigned OtherNo = 1 - ResNo;
2132 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2133 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2134 SetSplitVector(Op: SDValue(N, OtherNo),
2135 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2136 } else {
2137 SDValue OtherVal = DAG.getNode(
2138 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2139 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2140 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2141 }
2142}
2143
2144void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2145 SDValue &Hi) {
2146 SDValue Vec = N->getOperand(Num: 0);
2147 SDValue Elt = N->getOperand(Num: 1);
2148 SDValue Idx = N->getOperand(Num: 2);
2149 SDLoc dl(N);
2150 GetSplitVector(Op: Vec, Lo, Hi);
2151
2152 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2153 unsigned IdxVal = CIdx->getZExtValue();
2154 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2155 if (IdxVal < LoNumElts) {
2156 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2157 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2158 return;
2159 } else if (!Vec.getValueType().isScalableVector()) {
2160 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2161 return;
2162 }
2163 }
2164
2165 // Make the vector elements byte-addressable if they aren't already.
2166 EVT VecVT = Vec.getValueType();
2167 EVT EltVT = VecVT.getVectorElementType();
2168 if (!EltVT.isByteSized()) {
2169 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2170 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2171 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2172 // Extend the element type to match if needed.
2173 if (EltVT.bitsGT(VT: Elt.getValueType()))
2174 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2175 }
2176
2177 // Spill the vector to the stack.
2178 // In cases where the vector is illegal it will be broken down into parts
2179 // and stored in parts - we should use the alignment for the smallest part.
2180 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2181 SDValue StackPtr =
2182 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2183 auto &MF = DAG.getMachineFunction();
2184 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2185 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2186
2187 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2188 Alignment: SmallestAlign);
2189
2190 // Store the new element. This may be larger than the vector element type,
2191 // so use a truncating store.
2192 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2193 Store = DAG.getTruncStore(
2194 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2195 Alignment: commonAlignment(A: SmallestAlign,
2196 Offset: EltVT.getFixedSizeInBits() / 8));
2197
2198 EVT LoVT, HiVT;
2199 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2200
2201 // Load the Lo part from the stack slot.
2202 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2203
2204 // Increment the pointer to the other part.
2205 auto Load = cast<LoadSDNode>(Val&: Lo);
2206 MachinePointerInfo MPI = Load->getPointerInfo();
2207 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2208
2209 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2210
2211 // If we adjusted the original type, we need to truncate the results.
2212 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2213 if (LoVT != Lo.getValueType())
2214 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2215 if (HiVT != Hi.getValueType())
2216 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2217}
2218
2219void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2220 SDValue &Hi) {
2221 EVT LoVT, HiVT;
2222 SDLoc dl(N);
2223 assert(N->getValueType(0).isScalableVector() &&
2224 "Only scalable vectors are supported for STEP_VECTOR");
2225 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2226 SDValue Step = N->getOperand(Num: 0);
2227
2228 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2229
2230 // Hi = Lo + (EltCnt * Step)
2231 EVT EltVT = Step.getValueType();
2232 APInt StepVal = Step->getAsAPIntVal();
2233 SDValue StartOfHi =
2234 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2235 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2236 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2237
2238 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2239 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2240}
2241
2242void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2243 SDValue &Hi) {
2244 EVT LoVT, HiVT;
2245 SDLoc dl(N);
2246 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2247 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2248 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2249 Hi = DAG.getPOISON(VT: HiVT);
2250 } else {
2251 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2252 Hi = Lo;
2253 }
2254}
2255
2256void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2257 SDValue &Hi) {
2258 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2259 EVT LoVT, HiVT;
2260 SDLoc dl(LD);
2261 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2262
2263 ISD::LoadExtType ExtType = LD->getExtensionType();
2264 SDValue Ch = LD->getChain();
2265 SDValue Ptr = LD->getBasePtr();
2266 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2267 EVT MemoryVT = LD->getMemoryVT();
2268 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2269 AAMDNodes AAInfo = LD->getAAInfo();
2270
2271 EVT LoMemVT, HiMemVT;
2272 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2273
2274 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2275 SDValue Value, NewChain;
2276 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2277 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2278 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2279 return;
2280 }
2281
2282 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2283 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2284 AAInfo);
2285
2286 MachinePointerInfo MPI;
2287 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2288
2289 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2290 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2291
2292 // Build a factor node to remember that this load is independent of the
2293 // other one.
2294 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2295 N2: Hi.getValue(R: 1));
2296
2297 // Legalize the chain result - switch anything that used the old chain to
2298 // use the new one.
2299 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2300}
2301
2302void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2303 SDValue &Hi) {
2304 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2305 EVT LoVT, HiVT;
2306 SDLoc dl(LD);
2307 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2308
2309 ISD::LoadExtType ExtType = LD->getExtensionType();
2310 SDValue Ch = LD->getChain();
2311 SDValue Ptr = LD->getBasePtr();
2312 SDValue Offset = LD->getOffset();
2313 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2314 Align Alignment = LD->getBaseAlign();
2315 SDValue Mask = LD->getMask();
2316 SDValue EVL = LD->getVectorLength();
2317 EVT MemoryVT = LD->getMemoryVT();
2318
2319 EVT LoMemVT, HiMemVT;
2320 bool HiIsEmpty = false;
2321 std::tie(args&: LoMemVT, args&: HiMemVT) =
2322 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2323
2324 // Split Mask operand
2325 SDValue MaskLo, MaskHi;
2326 if (Mask.getOpcode() == ISD::SETCC) {
2327 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2328 } else {
2329 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2330 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2331 else
2332 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2333 }
2334
2335 // Split EVL operand
2336 SDValue EVLLo, EVLHi;
2337 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2338
2339 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2340 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2341 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2342 Ranges: LD->getRanges());
2343
2344 Lo =
2345 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2346 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2347
2348 if (HiIsEmpty) {
2349 // The hi vp_load has zero storage size. We therefore simply set it to
2350 // the low vp_load and rely on subsequent removal from the chain.
2351 Hi = Lo;
2352 } else {
2353 // Generate hi vp_load.
2354 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2355 IsCompressedMemory: LD->isExpandingLoad());
2356
2357 MachinePointerInfo MPI;
2358 if (LoMemVT.isScalableVector())
2359 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2360 else
2361 MPI = LD->getPointerInfo().getWithOffset(
2362 O: LoMemVT.getStoreSize().getFixedValue());
2363
2364 MMO = DAG.getMachineFunction().getMachineMemOperand(
2365 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2366 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2367
2368 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2369 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2370 IsExpanding: LD->isExpandingLoad());
2371 }
2372
2373 // Build a factor node to remember that this load is independent of the
2374 // other one.
2375 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2376 N2: Hi.getValue(R: 1));
2377
2378 // Legalize the chain result - switch anything that used the old chain to
2379 // use the new one.
2380 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2381}
2382
2383void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2384 SDValue &Hi) {
2385 SDLoc dl(LD);
2386 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2387
2388 SDValue Ch = LD->getChain();
2389 SDValue Ptr = LD->getBasePtr();
2390 Align Alignment = LD->getBaseAlign();
2391 SDValue Mask = LD->getMask();
2392 SDValue EVL = LD->getVectorLength();
2393
2394 // Split Mask operand
2395 SDValue MaskLo, MaskHi;
2396 if (Mask.getOpcode() == ISD::SETCC) {
2397 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2398 } else {
2399 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2400 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2401 else
2402 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2403 }
2404
2405 // Split EVL operand
2406 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2407
2408 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2409 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2410 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2411 Ranges: LD->getRanges());
2412
2413 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2414
2415 // Fill the upper half with poison.
2416 Hi = DAG.getPOISON(VT: HiVT);
2417
2418 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2419 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2420}
2421
2422void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2423 SDValue &Lo, SDValue &Hi) {
2424 assert(SLD->isUnindexed() &&
2425 "Indexed VP strided load during type legalization!");
2426 assert(SLD->getOffset().isUndef() &&
2427 "Unexpected indexed variable-length load offset");
2428
2429 SDLoc DL(SLD);
2430
2431 EVT LoVT, HiVT;
2432 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2433
2434 EVT LoMemVT, HiMemVT;
2435 bool HiIsEmpty = false;
2436 std::tie(args&: LoMemVT, args&: HiMemVT) =
2437 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2438
2439 SDValue Mask = SLD->getMask();
2440 SDValue LoMask, HiMask;
2441 if (Mask.getOpcode() == ISD::SETCC) {
2442 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2443 } else {
2444 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2445 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2446 else
2447 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2448 }
2449
2450 SDValue LoEVL, HiEVL;
2451 std::tie(args&: LoEVL, args&: HiEVL) =
2452 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2453
2454 // Generate the low vp_strided_load
2455 Lo = DAG.getStridedLoadVP(
2456 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2457 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2458 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2459
2460 if (HiIsEmpty) {
2461 // The high vp_strided_load has zero storage size. We therefore simply set
2462 // it to the low vp_strided_load and rely on subsequent removal from the
2463 // chain.
2464 Hi = Lo;
2465 } else {
2466 // Generate the high vp_strided_load.
2467 // To calculate the high base address, we need to sum to the low base
2468 // address stride number of bytes for each element already loaded by low,
2469 // that is: Ptr = Ptr + (LoEVL * Stride)
2470 EVT PtrVT = SLD->getBasePtr().getValueType();
2471 SDValue Increment =
2472 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2473 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2474 SDValue Ptr =
2475 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2476
2477 Align Alignment = SLD->getBaseAlign();
2478 if (LoMemVT.isScalableVector())
2479 Alignment = commonAlignment(
2480 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2481
2482 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2483 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2484 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2485 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2486
2487 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2488 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2489 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2490 IsExpanding: SLD->isExpandingLoad());
2491 }
2492
2493 // Build a factor node to remember that this load is independent of the
2494 // other one.
2495 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2496 N2: Hi.getValue(R: 1));
2497
2498 // Legalize the chain result - switch anything that used the old chain to
2499 // use the new one.
2500 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2501}
2502
2503void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2504 SDValue &Lo, SDValue &Hi) {
2505 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2506 EVT LoVT, HiVT;
2507 SDLoc dl(MLD);
2508 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2509
2510 SDValue Ch = MLD->getChain();
2511 SDValue Ptr = MLD->getBasePtr();
2512 SDValue Offset = MLD->getOffset();
2513 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2514 SDValue Mask = MLD->getMask();
2515 SDValue PassThru = MLD->getPassThru();
2516 Align Alignment = MLD->getBaseAlign();
2517 ISD::LoadExtType ExtType = MLD->getExtensionType();
2518 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2519
2520 // Split Mask operand
2521 SDValue MaskLo, MaskHi;
2522 if (Mask.getOpcode() == ISD::SETCC) {
2523 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2524 } else {
2525 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2526 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2527 else
2528 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2529 }
2530
2531 EVT MemoryVT = MLD->getMemoryVT();
2532 EVT LoMemVT, HiMemVT;
2533 bool HiIsEmpty = false;
2534 std::tie(args&: LoMemVT, args&: HiMemVT) =
2535 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2536
2537 SDValue PassThruLo, PassThruHi;
2538 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2539 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2540 else
2541 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2542
2543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2544 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2545 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2546
2547 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2548 MMO, AM: MLD->getAddressingMode(), ExtType,
2549 IsExpanding: MLD->isExpandingLoad());
2550
2551 if (HiIsEmpty) {
2552 // The hi masked load has zero storage size. We therefore simply set it to
2553 // the low masked load and rely on subsequent removal from the chain.
2554 Hi = Lo;
2555 } else {
2556 // Generate hi masked load.
2557 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2558 IsCompressedMemory: MLD->isExpandingLoad());
2559
2560 MachinePointerInfo MPI;
2561 if (LoMemVT.isScalableVector())
2562 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2563 else
2564 MPI = MLD->getPointerInfo().getWithOffset(
2565 O: LoMemVT.getStoreSize().getFixedValue());
2566
2567 MMO = DAG.getMachineFunction().getMachineMemOperand(
2568 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2569 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2570
2571 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2572 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2573 IsExpanding: MLD->isExpandingLoad());
2574 }
2575
2576 // Build a factor node to remember that this load is independent of the
2577 // other one.
2578 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2579 N2: Hi.getValue(R: 1));
2580
2581 // Legalize the chain result - switch anything that used the old chain to
2582 // use the new one.
2583 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2584
2585}
2586
2587void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2588 SDValue &Hi, bool SplitSETCC) {
2589 EVT LoVT, HiVT;
2590 SDLoc dl(N);
2591 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2592
2593 SDValue Ch = N->getChain();
2594 SDValue Ptr = N->getBasePtr();
2595 struct Operands {
2596 SDValue Mask;
2597 SDValue Index;
2598 SDValue Scale;
2599 } Ops = [&]() -> Operands {
2600 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2601 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2602 }
2603 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2604 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2605 }();
2606
2607 EVT MemoryVT = N->getMemoryVT();
2608 Align Alignment = N->getBaseAlign();
2609
2610 // Split Mask operand
2611 SDValue MaskLo, MaskHi;
2612 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2613 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2614 } else {
2615 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2616 }
2617
2618 EVT LoMemVT, HiMemVT;
2619 // Split MemoryVT
2620 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2621
2622 SDValue IndexHi, IndexLo;
2623 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2624 TargetLowering::TypeSplitVector)
2625 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2626 else
2627 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2628
2629 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2630 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2631 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2632 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2633
2634 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2635 SDValue PassThru = MGT->getPassThru();
2636 SDValue PassThruLo, PassThruHi;
2637 if (getTypeAction(VT: PassThru.getValueType()) ==
2638 TargetLowering::TypeSplitVector)
2639 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2640 else
2641 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2642
2643 ISD::LoadExtType ExtType = MGT->getExtensionType();
2644 ISD::MemIndexType IndexTy = MGT->getIndexType();
2645
2646 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2647 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2648 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2649
2650 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2651 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2652 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2653 } else {
2654 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2655 SDValue EVLLo, EVLHi;
2656 std::tie(args&: EVLLo, args&: EVLHi) =
2657 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2658
2659 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2660 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2661 MMO, IndexType: VPGT->getIndexType());
2662
2663 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2664 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2665 MMO, IndexType: VPGT->getIndexType());
2666 }
2667
2668 // Build a factor node to remember that this load is independent of the
2669 // other one.
2670 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2671 N2: Hi.getValue(R: 1));
2672
2673 // Legalize the chain result - switch anything that used the old chain to
2674 // use the new one.
2675 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2676}
2677
2678void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2679 SDValue &Hi) {
2680 // This is not "trivial", as there is a dependency between the two subvectors.
2681 // Depending on the number of 1s in the mask, the elements from the Hi vector
2682 // need to be moved to the Lo vector. Passthru values make this even harder.
2683 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2684 // smaller types and passthru is undef, as it is most likely faster than the
2685 // fully expand path. Otherwise, just do the full expansion as one "big"
2686 // operation and then extract the Lo and Hi vectors from that. This gets
2687 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2688 SDLoc DL(N);
2689 EVT VecVT = N->getValueType(ResNo: 0);
2690
2691 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2692 bool HasCustomLowering = false;
2693 EVT CheckVT = LoVT;
2694 while (CheckVT.getVectorMinNumElements() > 1) {
2695 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2696 // custom lowering for illegal types. So we do the checks separately.
2697 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2698 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2699 HasCustomLowering = true;
2700 break;
2701 }
2702 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2703 }
2704
2705 SDValue Passthru = N->getOperand(Num: 2);
2706 if (!HasCustomLowering) {
2707 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2708 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2709 return;
2710 }
2711
2712 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2713 SDValue Mask = N->getOperand(Num: 1);
2714 SDValue LoMask, HiMask;
2715 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2716 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2717
2718 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2719 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2720 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2721
2722 SDValue StackPtr = DAG.CreateStackTemporary(
2723 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2724 MachineFunction &MF = DAG.getMachineFunction();
2725 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2726 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2727
2728 EVT MaskVT = LoMask.getValueType();
2729 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2730
2731 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2732 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2733 EC: MaskVT.getVectorElementCount());
2734 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2735 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2736 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2737
2738 SDValue Chain = DAG.getEntryNode();
2739 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2740 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2741 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2742
2743 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2744 if (!Passthru.isUndef()) {
2745 Compressed =
2746 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2747 }
2748 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2749}
2750
2751void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2752 assert(N->getValueType(0).isVector() &&
2753 N->getOperand(0).getValueType().isVector() &&
2754 "Operand types must be vectors");
2755
2756 EVT LoVT, HiVT;
2757 SDLoc DL(N);
2758 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2759
2760 // If the input also splits, handle it directly. Otherwise split it by hand.
2761 SDValue LL, LH, RL, RH;
2762 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2763 TargetLowering::TypeSplitVector)
2764 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2765 else
2766 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2767
2768 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2769 TargetLowering::TypeSplitVector)
2770 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2771 else
2772 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2773
2774 if (N->getOpcode() == ISD::SETCC) {
2775 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2776 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2777 } else {
2778 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2779 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2780 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2781 std::tie(args&: EVLLo, args&: EVLHi) =
2782 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2783 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2784 N5: EVLLo);
2785 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2786 N5: EVLHi);
2787 }
2788}
2789
2790void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2791 SDValue &Hi) {
2792 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2793 EVT LoVT, HiVT;
2794 SDLoc dl(N);
2795 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2796
2797 // If the input also splits, handle it directly for a compile time speedup.
2798 // Otherwise split it by hand.
2799 EVT InVT = N->getOperand(Num: 0).getValueType();
2800 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2801 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2802 else
2803 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2804
2805 const SDNodeFlags Flags = N->getFlags();
2806 unsigned Opcode = N->getOpcode();
2807 if (N->getNumOperands() <= 2) {
2808 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2809 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2810 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2811 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2812 } else {
2813 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2814 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2815 }
2816 return;
2817 }
2818
2819 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2820 assert(N->isVPOpcode() && "Expected VP opcode");
2821
2822 SDValue MaskLo, MaskHi;
2823 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2824
2825 SDValue EVLLo, EVLHi;
2826 std::tie(args&: EVLLo, args&: EVLHi) =
2827 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2828
2829 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2830 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2831}
2832
2833void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2834 SDValue &Hi) {
2835 SDLoc dl(N);
2836 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2837
2838 // If the input also splits, handle it directly for a compile time speedup.
2839 // Otherwise split it by hand.
2840 EVT InVT = N->getOperand(Num: 0).getValueType();
2841 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2842 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2843 else
2844 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2845
2846 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
2847 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2848 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2849 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
2850 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
2851}
2852
2853void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
2854 unsigned ResNo,
2855 SDValue &Lo,
2856 SDValue &Hi) {
2857 SDLoc dl(N);
2858 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2859 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2860
2861 // If the input also splits, handle it directly for a compile time speedup.
2862 // Otherwise split it by hand.
2863 EVT InVT = N->getOperand(Num: 0).getValueType();
2864 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2865 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2866 else
2867 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2868
2869 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
2870 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
2871
2872 SDNode *HiNode = Hi.getNode();
2873 SDNode *LoNode = Lo.getNode();
2874
2875 // Replace the other vector result not being explicitly split here.
2876 unsigned OtherNo = 1 - ResNo;
2877 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2878 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2879 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2880 Hi: SDValue(HiNode, OtherNo));
2881 } else {
2882 SDValue OtherVal =
2883 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2884 N2: SDValue(HiNode, OtherNo));
2885 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2886 }
2887}
2888
2889void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2890 SDValue &Hi) {
2891 SDLoc dl(N);
2892 EVT SrcVT = N->getOperand(Num: 0).getValueType();
2893 EVT DestVT = N->getValueType(ResNo: 0);
2894 EVT LoVT, HiVT;
2895 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2896
2897 // We can do better than a generic split operation if the extend is doing
2898 // more than just doubling the width of the elements and the following are
2899 // true:
2900 // - The number of vector elements is even,
2901 // - the source type is legal,
2902 // - the type of a split source is illegal,
2903 // - the type of an extended (by doubling element size) source is legal, and
2904 // - the type of that extended source when split is legal.
2905 //
2906 // This won't necessarily completely legalize the operation, but it will
2907 // more effectively move in the right direction and prevent falling down
2908 // to scalarization in many cases due to the input vector being split too
2909 // far.
2910 if (SrcVT.getVectorElementCount().isKnownEven() &&
2911 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2912 LLVMContext &Ctx = *DAG.getContext();
2913 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2914 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2915
2916 EVT SplitLoVT, SplitHiVT;
2917 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2918 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2919 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2920 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2921 N->dump(&DAG); dbgs() << "\n");
2922 if (!N->isVPOpcode()) {
2923 // Extend the source vector by one step.
2924 SDValue NewSrc =
2925 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2926 // Get the low and high halves of the new, extended one step, vector.
2927 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2928 // Extend those vector halves the rest of the way.
2929 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2930 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2931 return;
2932 }
2933
2934 // Extend the source vector by one step.
2935 SDValue NewSrc =
2936 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2937 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2938 // Get the low and high halves of the new, extended one step, vector.
2939 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2940
2941 SDValue MaskLo, MaskHi;
2942 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2943
2944 SDValue EVLLo, EVLHi;
2945 std::tie(args&: EVLLo, args&: EVLHi) =
2946 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2947 // Extend those vector halves the rest of the way.
2948 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2949 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2950 return;
2951 }
2952 }
2953 // Fall back to the generic unary operator splitting otherwise.
2954 SplitVecRes_UnaryOp(N, Lo, Hi);
2955}
2956
2957void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2958 SDValue &Lo, SDValue &Hi) {
2959 // The low and high parts of the original input give four input vectors.
2960 SDValue Inputs[4];
2961 SDLoc DL(N);
2962 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2963 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2964 EVT NewVT = Inputs[0].getValueType();
2965 unsigned NewElts = NewVT.getVectorNumElements();
2966
2967 auto &&IsConstant = [](const SDValue &N) {
2968 APInt SplatValue;
2969 return N.getResNo() == 0 &&
2970 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2971 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2972 };
2973 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2974 SDValue &Input2,
2975 ArrayRef<int> Mask) {
2976 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2977 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2978 "Expected build vector node.");
2979 EVT EltVT = NewVT.getVectorElementType();
2980 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
2981 for (unsigned I = 0; I < NewElts; ++I) {
2982 if (Mask[I] == PoisonMaskElem)
2983 continue;
2984 unsigned Idx = Mask[I];
2985 if (Idx >= NewElts)
2986 Ops[I] = Input2.getOperand(i: Idx - NewElts);
2987 else
2988 Ops[I] = Input1.getOperand(i: Idx);
2989 // Make the type of all elements the same as the element type.
2990 if (Ops[I].getValueType().bitsGT(VT: EltVT))
2991 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
2992 }
2993 return DAG.getBuildVector(VT: NewVT, DL, Ops);
2994 };
2995
2996 // If Lo or Hi uses elements from at most two of the four input vectors, then
2997 // express it as a vector shuffle of those two inputs. Otherwise extract the
2998 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2999 SmallVector<int> OrigMask(N->getMask());
3000 // Try to pack incoming shuffles/inputs.
3001 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3002 &DL](SmallVectorImpl<int> &Mask) {
3003 // Check if all inputs are shuffles of the same operands or non-shuffles.
3004 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3005 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3006 SDValue Input = Inputs[Idx];
3007 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3008 if (!Shuffle ||
3009 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3010 continue;
3011 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3012 .push_back(Elt: Idx);
3013 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3014 .push_back(Elt: Idx);
3015 }
3016 for (auto &P : ShufflesIdxs) {
3017 if (P.second.size() < 2)
3018 continue;
3019 // Use shuffles operands instead of shuffles themselves.
3020 // 1. Adjust mask.
3021 for (int &Idx : Mask) {
3022 if (Idx == PoisonMaskElem)
3023 continue;
3024 unsigned SrcRegIdx = Idx / NewElts;
3025 if (Inputs[SrcRegIdx].isUndef()) {
3026 Idx = PoisonMaskElem;
3027 continue;
3028 }
3029 auto *Shuffle =
3030 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3031 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3032 continue;
3033 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3034 if (MaskElt == PoisonMaskElem) {
3035 Idx = PoisonMaskElem;
3036 continue;
3037 }
3038 Idx = MaskElt % NewElts +
3039 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3040 ? 0
3041 : 1] *
3042 NewElts;
3043 }
3044 // 2. Update inputs.
3045 Inputs[P.second[0]] = P.first.first;
3046 Inputs[P.second[1]] = P.first.second;
3047 // Clear the pair data.
3048 P.second.clear();
3049 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3050 }
3051 // Check if any concat_vectors can be simplified.
3052 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3053 for (int &Idx : Mask) {
3054 if (Idx == PoisonMaskElem)
3055 continue;
3056 unsigned SrcRegIdx = Idx / NewElts;
3057 if (Inputs[SrcRegIdx].isUndef()) {
3058 Idx = PoisonMaskElem;
3059 continue;
3060 }
3061 TargetLowering::LegalizeTypeAction TypeAction =
3062 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3063 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3064 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3065 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3066 (TypeAction == TargetLowering::TypeLegal ||
3067 TypeAction == TargetLowering::TypeWidenVector))
3068 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3069 }
3070 if (UsedSubVector.count() > 1) {
3071 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3072 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3073 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3074 continue;
3075 if (Pairs.empty() || Pairs.back().size() == 2)
3076 Pairs.emplace_back();
3077 if (UsedSubVector.test(Idx: 2 * I)) {
3078 Pairs.back().emplace_back(Args&: I, Args: 0);
3079 } else {
3080 assert(UsedSubVector.test(2 * I + 1) &&
3081 "Expected to be used one of the subvectors.");
3082 Pairs.back().emplace_back(Args&: I, Args: 1);
3083 }
3084 }
3085 if (!Pairs.empty() && Pairs.front().size() > 1) {
3086 // Adjust mask.
3087 for (int &Idx : Mask) {
3088 if (Idx == PoisonMaskElem)
3089 continue;
3090 unsigned SrcRegIdx = Idx / NewElts;
3091 auto *It = find_if(
3092 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3093 return Idxs.front().first == SrcRegIdx ||
3094 Idxs.back().first == SrcRegIdx;
3095 });
3096 if (It == Pairs.end())
3097 continue;
3098 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3099 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3100 }
3101 // Adjust inputs.
3102 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3103 Inputs[Idxs.front().first] = DAG.getNode(
3104 Opcode: ISD::CONCAT_VECTORS, DL,
3105 VT: Inputs[Idxs.front().first].getValueType(),
3106 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3107 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3108 }
3109 }
3110 }
3111 bool Changed;
3112 do {
3113 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3114 // reused operands.
3115 Changed = false;
3116 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3117 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3118 if (!Shuffle)
3119 continue;
3120 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3121 continue;
3122 int Op = -1;
3123 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3124 !Shuffle->isSplat()) {
3125 Op = 0;
3126 } else if (!Inputs[I].hasOneUse() &&
3127 !Shuffle->getOperand(Num: 1).isUndef()) {
3128 // Find the only used operand, if possible.
3129 for (int &Idx : Mask) {
3130 if (Idx == PoisonMaskElem)
3131 continue;
3132 unsigned SrcRegIdx = Idx / NewElts;
3133 if (SrcRegIdx != I)
3134 continue;
3135 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3136 if (MaskElt == PoisonMaskElem) {
3137 Idx = PoisonMaskElem;
3138 continue;
3139 }
3140 int OpIdx = MaskElt / NewElts;
3141 if (Op == -1) {
3142 Op = OpIdx;
3143 continue;
3144 }
3145 if (Op != OpIdx) {
3146 Op = -1;
3147 break;
3148 }
3149 }
3150 }
3151 if (Op < 0) {
3152 // Try to check if one of the shuffle operands is used already.
3153 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3154 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3155 continue;
3156 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3157 if (It == std::end(arr&: Inputs))
3158 continue;
3159 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3160 // Found that operand is used already.
3161 // 1. Fix the mask for the reused operand.
3162 for (int &Idx : Mask) {
3163 if (Idx == PoisonMaskElem)
3164 continue;
3165 unsigned SrcRegIdx = Idx / NewElts;
3166 if (SrcRegIdx != I)
3167 continue;
3168 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3169 if (MaskElt == PoisonMaskElem) {
3170 Idx = PoisonMaskElem;
3171 continue;
3172 }
3173 int MaskIdx = MaskElt / NewElts;
3174 if (OpIdx == MaskIdx)
3175 Idx = MaskElt % NewElts + FoundOp * NewElts;
3176 }
3177 // 2. Set Op to the unused OpIdx.
3178 Op = (OpIdx + 1) % 2;
3179 break;
3180 }
3181 }
3182 if (Op >= 0) {
3183 Changed = true;
3184 Inputs[I] = Shuffle->getOperand(Num: Op);
3185 // Adjust mask.
3186 for (int &Idx : Mask) {
3187 if (Idx == PoisonMaskElem)
3188 continue;
3189 unsigned SrcRegIdx = Idx / NewElts;
3190 if (SrcRegIdx != I)
3191 continue;
3192 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3193 int OpIdx = MaskElt / NewElts;
3194 if (OpIdx != Op)
3195 continue;
3196 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3197 }
3198 }
3199 }
3200 } while (Changed);
3201 };
3202 TryPeekThroughShufflesInputs(OrigMask);
3203 // Proces unique inputs.
3204 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3205 NewElts](SmallVectorImpl<int> &Mask) {
3206 SetVector<SDValue> UniqueInputs;
3207 SetVector<SDValue> UniqueConstantInputs;
3208 for (const auto &I : Inputs) {
3209 if (IsConstant(I))
3210 UniqueConstantInputs.insert(X: I);
3211 else if (!I.isUndef())
3212 UniqueInputs.insert(X: I);
3213 }
3214 // Adjust mask in case of reused inputs. Also, need to insert constant
3215 // inputs at first, otherwise it affects the final outcome.
3216 if (UniqueInputs.size() != std::size(Inputs)) {
3217 auto &&UniqueVec = UniqueInputs.takeVector();
3218 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3219 unsigned ConstNum = UniqueConstantVec.size();
3220 for (int &Idx : Mask) {
3221 if (Idx == PoisonMaskElem)
3222 continue;
3223 unsigned SrcRegIdx = Idx / NewElts;
3224 if (Inputs[SrcRegIdx].isUndef()) {
3225 Idx = PoisonMaskElem;
3226 continue;
3227 }
3228 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3229 if (It != UniqueConstantVec.end()) {
3230 Idx = (Idx % NewElts) +
3231 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3232 assert(Idx >= 0 && "Expected defined mask idx.");
3233 continue;
3234 }
3235 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3236 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3237 Idx = (Idx % NewElts) +
3238 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3239 assert(Idx >= 0 && "Expected defined mask idx.");
3240 }
3241 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3242 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3243 }
3244 };
3245 MakeUniqueInputs(OrigMask);
3246 SDValue OrigInputs[4];
3247 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3248 for (unsigned High = 0; High < 2; ++High) {
3249 SDValue &Output = High ? Hi : Lo;
3250
3251 // Build a shuffle mask for the output, discovering on the fly which
3252 // input vectors to use as shuffle operands.
3253 unsigned FirstMaskIdx = High * NewElts;
3254 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3255 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3256 assert(!Output && "Expected default initialized initial value.");
3257 TryPeekThroughShufflesInputs(Mask);
3258 MakeUniqueInputs(Mask);
3259 SDValue TmpInputs[4];
3260 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3261 // Track changes in the output registers.
3262 int UsedIdx = -1;
3263 bool SecondIteration = false;
3264 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3265 if (UsedIdx < 0) {
3266 UsedIdx = Idx;
3267 return false;
3268 }
3269 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3270 SecondIteration = true;
3271 return SecondIteration;
3272 };
3273 processShuffleMasks(
3274 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3275 /*NumOfUsedRegs=*/1,
3276 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3277 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3278 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3279 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3280 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3281 else
3282 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3283 N2: DAG.getPOISON(VT: NewVT), Mask);
3284 Inputs[Idx] = Output;
3285 },
3286 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3287 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3288 unsigned Idx2, bool /*Unused*/) {
3289 if (AccumulateResults(Idx1)) {
3290 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3291 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3292 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3293 else
3294 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3295 N2: Inputs[Idx2], Mask);
3296 } else {
3297 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3298 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3299 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3300 else
3301 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3302 N2: TmpInputs[Idx2], Mask);
3303 }
3304 Inputs[Idx1] = Output;
3305 });
3306 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3307 }
3308}
3309
3310void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3311 EVT OVT = N->getValueType(ResNo: 0);
3312 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3313 SDValue Chain = N->getOperand(Num: 0);
3314 SDValue Ptr = N->getOperand(Num: 1);
3315 SDValue SV = N->getOperand(Num: 2);
3316 SDLoc dl(N);
3317
3318 const Align Alignment =
3319 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3320
3321 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3322 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3323 Chain = Hi.getValue(R: 1);
3324
3325 // Modified the chain - switch anything that used the old chain to use
3326 // the new one.
3327 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3328}
3329
3330void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3331 SDValue &Hi) {
3332 EVT DstVTLo, DstVTHi;
3333 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3334 SDLoc dl(N);
3335
3336 SDValue SrcLo, SrcHi;
3337 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3338 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3339 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3340 else
3341 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3342
3343 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3344 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3345}
3346
3347void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3348 SDValue &Hi) {
3349 SDValue InLo, InHi;
3350 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3351 SDLoc DL(N);
3352
3353 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3354 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3355}
3356
3357void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3358 SDValue &Hi) {
3359 SDLoc DL(N);
3360
3361 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3362 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3363}
3364
3365void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3366 SDValue &Hi) {
3367 EVT VT = N->getValueType(ResNo: 0);
3368 SDValue Val = N->getOperand(Num: 0);
3369 SDValue Mask = N->getOperand(Num: 1);
3370 SDValue EVL = N->getOperand(Num: 2);
3371 SDLoc DL(N);
3372
3373 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3374 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3375
3376 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3377 EC: VT.getVectorElementCount());
3378 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3379 EVT PtrVT = StackPtr.getValueType();
3380 auto &MF = DAG.getMachineFunction();
3381 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3382 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3383
3384 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3385 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3386 BaseAlignment: Alignment);
3387 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3388 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3389 BaseAlignment: Alignment);
3390
3391 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3392 SDValue NumElemMinus1 =
3393 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3394 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3395 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3396 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3397 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3398 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3399
3400 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3401 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3402 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3403 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3404
3405 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3406
3407 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3408}
3409
3410void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3411 SDValue &Hi) {
3412 EVT VT = N->getValueType(ResNo: 0);
3413 SDValue V1 = N->getOperand(Num: 0);
3414 SDValue V2 = N->getOperand(Num: 1);
3415 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3416 SDValue Mask = N->getOperand(Num: 3);
3417 SDValue EVL1 = N->getOperand(Num: 4);
3418 SDValue EVL2 = N->getOperand(Num: 5);
3419 SDLoc DL(N);
3420
3421 // Since EVL2 is considered the real VL it gets promoted during
3422 // SelectionDAGBuilder. Promote EVL1 here if needed.
3423 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3424 EVL1 = ZExtPromotedInteger(Op: EVL1);
3425
3426 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3427
3428 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3429 EC: VT.getVectorElementCount() * 2);
3430 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3431 EVT PtrVT = StackPtr.getValueType();
3432 auto &MF = DAG.getMachineFunction();
3433 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3434 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3435
3436 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3437 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3438 BaseAlignment: Alignment);
3439 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3440 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3441 BaseAlignment: Alignment);
3442
3443 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: EVL1);
3444 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3445
3446 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3447 SDValue StoreV1 =
3448 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3449 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3450
3451 SDValue StoreV2 =
3452 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3453 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3454
3455 SDValue Load;
3456 if (Imm >= 0) {
3457 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3458 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3459 } else {
3460 uint64_t TrailingElts = -Imm;
3461 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3462 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3463
3464 // Make sure TrailingBytes doesn't exceed the size of vec1.
3465 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3466 TrailingBytes =
3467 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3468
3469 // Calculate the start address of the spliced result.
3470 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3471 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3472 }
3473
3474 EVT LoVT, HiVT;
3475 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
3476 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3477 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3478 Hi =
3479 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3480 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3481}
3482
3483void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3484 SDValue &Hi) {
3485 SDLoc DL(N);
3486 SDValue Acc = N->getOperand(Num: 0);
3487 SDValue Input1 = N->getOperand(Num: 1);
3488 SDValue Input2 = N->getOperand(Num: 2);
3489
3490 SDValue AccLo, AccHi;
3491 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3492 unsigned Opcode = N->getOpcode();
3493
3494 // If the input types don't need splitting, just accumulate into the
3495 // low part of the accumulator.
3496 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3497 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3498 Hi = AccHi;
3499 return;
3500 }
3501
3502 SDValue Input1Lo, Input1Hi;
3503 SDValue Input2Lo, Input2Hi;
3504 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3505 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3506 EVT ResultVT = AccLo.getValueType();
3507
3508 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3509 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3510}
3511
3512void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3513 SDValue &Hi) {
3514 SDLoc DL(N);
3515 SDValue Op0 = N->getOperand(Num: 0);
3516 SDValue Op1 = N->getOperand(Num: 1);
3517 EVT OpVT = Op0.getValueType();
3518
3519 EVT LoVT, HiVT;
3520 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3521
3522 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3523 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3524 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3525 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3526}
3527
3528void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3529 unsigned Factor = N->getNumOperands();
3530
3531 SmallVector<SDValue, 8> Ops(Factor * 2);
3532 for (unsigned i = 0; i != Factor; ++i) {
3533 SDValue OpLo, OpHi;
3534 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3535 Ops[i * 2] = OpLo;
3536 Ops[i * 2 + 1] = OpHi;
3537 }
3538
3539 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3540
3541 SDLoc DL(N);
3542 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3543 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3544 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3545 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3546
3547 for (unsigned i = 0; i != Factor; ++i)
3548 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3549}
3550
3551void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3552 unsigned Factor = N->getNumOperands();
3553
3554 SmallVector<SDValue, 8> Ops(Factor * 2);
3555 for (unsigned i = 0; i != Factor; ++i) {
3556 SDValue OpLo, OpHi;
3557 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3558 Ops[i] = OpLo;
3559 Ops[i + Factor] = OpHi;
3560 }
3561
3562 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3563
3564 SDLoc DL(N);
3565 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3566 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3567 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3568 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3569
3570 for (unsigned i = 0; i != Factor; ++i) {
3571 unsigned IdxLo = 2 * i;
3572 unsigned IdxHi = 2 * i + 1;
3573 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3574 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3575 }
3576}
3577
3578//===----------------------------------------------------------------------===//
3579// Operand Vector Splitting
3580//===----------------------------------------------------------------------===//
3581
3582/// This method is called when the specified operand of the specified node is
3583/// found to need vector splitting. At this point, all of the result types of
3584/// the node are known to be legal, but other operands of the node may need
3585/// legalization as well as the specified one.
3586bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3587 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3588 SDValue Res = SDValue();
3589
3590 // See if the target wants to custom split this node.
3591 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3592 return false;
3593
3594 switch (N->getOpcode()) {
3595 default:
3596#ifndef NDEBUG
3597 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3598 N->dump(&DAG);
3599 dbgs() << "\n";
3600#endif
3601 report_fatal_error(reason: "Do not know how to split this operator's "
3602 "operand!\n");
3603
3604 case ISD::VP_SETCC:
3605 case ISD::STRICT_FSETCC:
3606 case ISD::STRICT_FSETCCS:
3607 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3608 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3609 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3610 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3611 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3612 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3613 case ISD::VECTOR_FIND_LAST_ACTIVE:
3614 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3615 break;
3616 case ISD::VP_TRUNCATE:
3617 case ISD::TRUNCATE:
3618 Res = SplitVecOp_TruncateHelper(N);
3619 break;
3620 case ISD::STRICT_FP_ROUND:
3621 case ISD::VP_FP_ROUND:
3622 case ISD::FP_ROUND:
3623 case ISD::CONVERT_FROM_ARBITRARY_FP:
3624 Res = SplitVecOp_FP_ROUND(N);
3625 break;
3626 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3627 case ISD::STORE:
3628 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3629 break;
3630 case ISD::VP_STORE:
3631 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3632 break;
3633 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3634 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3635 break;
3636 case ISD::MSTORE:
3637 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3638 break;
3639 case ISD::MSCATTER:
3640 case ISD::VP_SCATTER:
3641 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3642 break;
3643 case ISD::MGATHER:
3644 case ISD::VP_GATHER:
3645 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3646 break;
3647 case ISD::VSELECT:
3648 Res = SplitVecOp_VSELECT(N, OpNo);
3649 break;
3650 case ISD::VECTOR_COMPRESS:
3651 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3652 break;
3653 case ISD::STRICT_SINT_TO_FP:
3654 case ISD::STRICT_UINT_TO_FP:
3655 case ISD::SINT_TO_FP:
3656 case ISD::UINT_TO_FP:
3657 case ISD::VP_SINT_TO_FP:
3658 case ISD::VP_UINT_TO_FP:
3659 if (N->getValueType(ResNo: 0).bitsLT(
3660 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3661 Res = SplitVecOp_TruncateHelper(N);
3662 else
3663 Res = SplitVecOp_UnaryOp(N);
3664 break;
3665 case ISD::FP_TO_SINT_SAT:
3666 case ISD::FP_TO_UINT_SAT:
3667 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3668 break;
3669 case ISD::FP_TO_SINT:
3670 case ISD::FP_TO_UINT:
3671 case ISD::VP_FP_TO_SINT:
3672 case ISD::VP_FP_TO_UINT:
3673 case ISD::STRICT_FP_TO_SINT:
3674 case ISD::STRICT_FP_TO_UINT:
3675 case ISD::STRICT_FP_EXTEND:
3676 case ISD::FP_EXTEND:
3677 case ISD::SIGN_EXTEND:
3678 case ISD::ZERO_EXTEND:
3679 case ISD::ANY_EXTEND:
3680 case ISD::FTRUNC:
3681 case ISD::LROUND:
3682 case ISD::LLROUND:
3683 case ISD::LRINT:
3684 case ISD::LLRINT:
3685 Res = SplitVecOp_UnaryOp(N);
3686 break;
3687 case ISD::FLDEXP:
3688 Res = SplitVecOp_FPOpDifferentTypes(N);
3689 break;
3690
3691 case ISD::SCMP:
3692 case ISD::UCMP:
3693 Res = SplitVecOp_CMP(N);
3694 break;
3695
3696 case ISD::FAKE_USE:
3697 Res = SplitVecOp_FAKE_USE(N);
3698 break;
3699 case ISD::ANY_EXTEND_VECTOR_INREG:
3700 case ISD::SIGN_EXTEND_VECTOR_INREG:
3701 case ISD::ZERO_EXTEND_VECTOR_INREG:
3702 Res = SplitVecOp_ExtVecInRegOp(N);
3703 break;
3704
3705 case ISD::VECREDUCE_FADD:
3706 case ISD::VECREDUCE_FMUL:
3707 case ISD::VECREDUCE_ADD:
3708 case ISD::VECREDUCE_MUL:
3709 case ISD::VECREDUCE_AND:
3710 case ISD::VECREDUCE_OR:
3711 case ISD::VECREDUCE_XOR:
3712 case ISD::VECREDUCE_SMAX:
3713 case ISD::VECREDUCE_SMIN:
3714 case ISD::VECREDUCE_UMAX:
3715 case ISD::VECREDUCE_UMIN:
3716 case ISD::VECREDUCE_FMAX:
3717 case ISD::VECREDUCE_FMIN:
3718 case ISD::VECREDUCE_FMAXIMUM:
3719 case ISD::VECREDUCE_FMINIMUM:
3720 Res = SplitVecOp_VECREDUCE(N, OpNo);
3721 break;
3722 case ISD::VECREDUCE_SEQ_FADD:
3723 case ISD::VECREDUCE_SEQ_FMUL:
3724 Res = SplitVecOp_VECREDUCE_SEQ(N);
3725 break;
3726 case ISD::VP_REDUCE_FADD:
3727 case ISD::VP_REDUCE_SEQ_FADD:
3728 case ISD::VP_REDUCE_FMUL:
3729 case ISD::VP_REDUCE_SEQ_FMUL:
3730 case ISD::VP_REDUCE_ADD:
3731 case ISD::VP_REDUCE_MUL:
3732 case ISD::VP_REDUCE_AND:
3733 case ISD::VP_REDUCE_OR:
3734 case ISD::VP_REDUCE_XOR:
3735 case ISD::VP_REDUCE_SMAX:
3736 case ISD::VP_REDUCE_SMIN:
3737 case ISD::VP_REDUCE_UMAX:
3738 case ISD::VP_REDUCE_UMIN:
3739 case ISD::VP_REDUCE_FMAX:
3740 case ISD::VP_REDUCE_FMIN:
3741 case ISD::VP_REDUCE_FMAXIMUM:
3742 case ISD::VP_REDUCE_FMINIMUM:
3743 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3744 break;
3745 case ISD::VP_CTTZ_ELTS:
3746 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
3747 Res = SplitVecOp_VP_CttzElements(N);
3748 break;
3749 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3750 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3751 break;
3752 case ISD::PARTIAL_REDUCE_UMLA:
3753 case ISD::PARTIAL_REDUCE_SMLA:
3754 case ISD::PARTIAL_REDUCE_SUMLA:
3755 case ISD::PARTIAL_REDUCE_FMLA:
3756 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3757 break;
3758 }
3759
3760 // If the result is null, the sub-method took care of registering results etc.
3761 if (!Res.getNode()) return false;
3762
3763 // If the result is N, the sub-method updated N in place. Tell the legalizer
3764 // core about this.
3765 if (Res.getNode() == N)
3766 return true;
3767
3768 if (N->isStrictFPOpcode())
3769 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3770 "Invalid operand expansion");
3771 else
3772 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3773 "Invalid operand expansion");
3774
3775 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3776 return false;
3777}
3778
3779SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3780 SDLoc DL(N);
3781
3782 SDValue LoMask, HiMask;
3783 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
3784
3785 EVT VT = N->getValueType(ResNo: 0);
3786 EVT SplitVT = LoMask.getValueType();
3787 ElementCount SplitEC = SplitVT.getVectorElementCount();
3788
3789 // Find the last active in both the low and the high masks.
3790 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
3791 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
3792
3793 // Check if any lane is active in the high mask.
3794 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
3795 // sentinel value for "none active".
3796 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
3797 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
3798 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
3799
3800 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
3801 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
3802 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
3803 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
3804 N3: LoFind);
3805}
3806
3807SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3808 // The only possibility for an illegal operand is the mask, since result type
3809 // legalization would have handled this node already otherwise.
3810 assert(OpNo == 0 && "Illegal operand must be mask");
3811
3812 SDValue Mask = N->getOperand(Num: 0);
3813 SDValue Src0 = N->getOperand(Num: 1);
3814 SDValue Src1 = N->getOperand(Num: 2);
3815 EVT Src0VT = Src0.getValueType();
3816 SDLoc DL(N);
3817 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3818
3819 SDValue Lo, Hi;
3820 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3821 assert(Lo.getValueType() == Hi.getValueType() &&
3822 "Lo and Hi have differing types");
3823
3824 EVT LoOpVT, HiOpVT;
3825 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3826 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3827
3828 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3829 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3830 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3831 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3832
3833 SDValue LoSelect =
3834 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3835 SDValue HiSelect =
3836 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3837
3838 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3839}
3840
3841SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
3842 // The only possibility for an illegal operand is the mask, since result type
3843 // legalization would have handled this node already otherwise.
3844 assert(OpNo == 1 && "Illegal operand must be mask");
3845
3846 // To split the mask, we need to split the result type too, so we can just
3847 // reuse that logic here.
3848 SDValue Lo, Hi;
3849 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
3850
3851 EVT VecVT = N->getValueType(ResNo: 0);
3852 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
3853}
3854
3855SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3856 EVT ResVT = N->getValueType(ResNo: 0);
3857 SDValue Lo, Hi;
3858 SDLoc dl(N);
3859
3860 SDValue VecOp = N->getOperand(Num: OpNo);
3861 EVT VecVT = VecOp.getValueType();
3862 assert(VecVT.isVector() && "Can only split reduce vector operand");
3863 GetSplitVector(Op: VecOp, Lo, Hi);
3864 EVT LoOpVT, HiOpVT;
3865 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3866
3867 // Use the appropriate scalar instruction on the split subvectors before
3868 // reducing the now partially reduced smaller vector.
3869 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3870 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3871 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3872}
3873
3874SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3875 EVT ResVT = N->getValueType(ResNo: 0);
3876 SDValue Lo, Hi;
3877 SDLoc dl(N);
3878
3879 SDValue AccOp = N->getOperand(Num: 0);
3880 SDValue VecOp = N->getOperand(Num: 1);
3881 SDNodeFlags Flags = N->getFlags();
3882
3883 EVT VecVT = VecOp.getValueType();
3884 assert(VecVT.isVector() && "Can only split reduce vector operand");
3885 GetSplitVector(Op: VecOp, Lo, Hi);
3886 EVT LoOpVT, HiOpVT;
3887 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3888
3889 // Reduce low half.
3890 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3891
3892 // Reduce high half, using low half result as initial value.
3893 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3894}
3895
3896SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3897 assert(N->isVPOpcode() && "Expected VP opcode");
3898 assert(OpNo == 1 && "Can only split reduce vector operand");
3899
3900 unsigned Opc = N->getOpcode();
3901 EVT ResVT = N->getValueType(ResNo: 0);
3902 SDValue Lo, Hi;
3903 SDLoc dl(N);
3904
3905 SDValue VecOp = N->getOperand(Num: OpNo);
3906 EVT VecVT = VecOp.getValueType();
3907 assert(VecVT.isVector() && "Can only split reduce vector operand");
3908 GetSplitVector(Op: VecOp, Lo, Hi);
3909
3910 SDValue MaskLo, MaskHi;
3911 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3912
3913 SDValue EVLLo, EVLHi;
3914 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3915
3916 const SDNodeFlags Flags = N->getFlags();
3917
3918 SDValue ResLo =
3919 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3920 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3921}
3922
3923SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3924 // The result has a legal vector type, but the input needs splitting.
3925 EVT ResVT = N->getValueType(ResNo: 0);
3926 SDValue Lo, Hi;
3927 SDLoc dl(N);
3928 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3929 EVT InVT = Lo.getValueType();
3930
3931 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3932 EC: InVT.getVectorElementCount());
3933
3934 if (N->isStrictFPOpcode()) {
3935 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3936 Ops: {N->getOperand(Num: 0), Lo});
3937 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3938 Ops: {N->getOperand(Num: 0), Hi});
3939
3940 // Build a factor node to remember that this operation is independent
3941 // of the other one.
3942 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
3943 N2: Hi.getValue(R: 1));
3944
3945 // Legalize the chain result - switch anything that used the old chain to
3946 // use the new one.
3947 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3948 } else if (N->getNumOperands() == 3) {
3949 assert(N->isVPOpcode() && "Expected VP opcode");
3950 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3951 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3952 std::tie(args&: EVLLo, args&: EVLHi) =
3953 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3954 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3955 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3956 } else {
3957 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3958 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3959 }
3960
3961 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3962}
3963
3964// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
3965SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
3966 SDValue Lo, Hi;
3967 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3968 SDValue Chain =
3969 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
3970 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
3971}
3972
3973SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3974 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3975 // end up being split all the way down to individual components. Convert the
3976 // split pieces into integers and reassemble.
3977 EVT ResVT = N->getValueType(ResNo: 0);
3978 SDValue Lo, Hi;
3979 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3980 SDLoc dl(N);
3981
3982 if (ResVT.isScalableVector()) {
3983 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
3984 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
3985 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
3986 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3987 }
3988
3989 Lo = BitConvertToInteger(Op: Lo);
3990 Hi = BitConvertToInteger(Op: Hi);
3991
3992 if (DAG.getDataLayout().isBigEndian())
3993 std::swap(a&: Lo, b&: Hi);
3994
3995 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
3996}
3997
3998SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3999 unsigned OpNo) {
4000 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4001 // We know that the result type is legal.
4002 EVT ResVT = N->getValueType(ResNo: 0);
4003
4004 SDValue Vec = N->getOperand(Num: 0);
4005 SDValue SubVec = N->getOperand(Num: 1);
4006 SDValue Idx = N->getOperand(Num: 2);
4007 SDLoc dl(N);
4008
4009 SDValue Lo, Hi;
4010 GetSplitVector(Op: SubVec, Lo, Hi);
4011
4012 uint64_t IdxVal = Idx->getAsZExtVal();
4013 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4014
4015 SDValue FirstInsertion =
4016 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4017 SDValue SecondInsertion =
4018 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4019 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4020
4021 return SecondInsertion;
4022}
4023
4024SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4025 // We know that the extracted result type is legal.
4026 EVT SubVT = N->getValueType(ResNo: 0);
4027 SDValue Idx = N->getOperand(Num: 1);
4028 SDLoc dl(N);
4029 SDValue Lo, Hi;
4030
4031 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4032
4033 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4034 // Note: For scalable vectors, the index is scaled by vscale.
4035 ElementCount IdxVal =
4036 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4037 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4038
4039 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4040 ElementCount NumResultElts = SubVT.getVectorElementCount();
4041
4042 // If the extracted elements are all in the low half, do a simple extract.
4043 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4044 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4045
4046 unsigned LoEltsMin = LoElts.getKnownMinValue();
4047 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4048 SrcVT.isFixedLengthVector()) {
4049 // Extracted subvector crosses vector split, so we need to blend the two
4050 // halves.
4051 // TODO: May be able to emit partial extract_subvector.
4052 SmallVector<SDValue, 8> Elts;
4053 Elts.reserve(N: NumResultElts.getFixedValue());
4054
4055 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4056 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4057 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4058 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4059 /*Count=*/LoEltsMin - IdxValMin);
4060 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4061 /*Count=*/SubVT.getVectorNumElements() -
4062 Elts.size());
4063 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4064 }
4065
4066 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4067 ElementCount ExtractIdx = IdxVal - LoElts;
4068 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4069 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4070 Idx: ExtractIdx.getKnownMinValue());
4071
4072 EVT HiVT = Hi.getValueType();
4073 assert(HiVT.isFixedLengthVector() &&
4074 "Only fixed-vector extracts are supported in this case");
4075
4076 // We cannot create an extract_subvector that isn't a multiple of the
4077 // result size, which may go out of bounds for the last elements. Shuffle
4078 // the desired elements down to 0 and do a simple 0 extract.
4079 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4080 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4081 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4082
4083 SDValue Shuffle =
4084 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4085 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4086 }
4087
4088 // After this point the DAG node only permits extracting fixed-width
4089 // subvectors from scalable vectors.
4090 assert(SubVT.isFixedLengthVector() &&
4091 "Extracting scalable subvector from fixed-width unsupported");
4092
4093 // If the element type is i1 and we're not promoting the result, then we may
4094 // end up loading the wrong data since the bits are packed tightly into
4095 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4096 // type at index 4, then we will load a byte starting at index 0.
4097 if (SubVT.getScalarType() == MVT::i1)
4098 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4099 "subvector from a scalable predicate vector");
4100
4101 // Spill the vector to the stack. We should use the alignment for
4102 // the smallest part.
4103 SDValue Vec = N->getOperand(Num: 0);
4104 EVT VecVT = Vec.getValueType();
4105 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4106 SDValue StackPtr =
4107 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4108 auto &MF = DAG.getMachineFunction();
4109 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4110 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4111
4112 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4113 Alignment: SmallestAlign);
4114
4115 // Extract the subvector by loading the correct part.
4116 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4117
4118 return DAG.getLoad(
4119 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4120 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4121}
4122
4123SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4124 SDValue Vec = N->getOperand(Num: 0);
4125 SDValue Idx = N->getOperand(Num: 1);
4126 EVT VecVT = Vec.getValueType();
4127
4128 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4129 uint64_t IdxVal = Index->getZExtValue();
4130
4131 SDValue Lo, Hi;
4132 GetSplitVector(Op: Vec, Lo, Hi);
4133
4134 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4135
4136 if (IdxVal < LoElts)
4137 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4138 else if (!Vec.getValueType().isScalableVector())
4139 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4140 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4141 VT: Idx.getValueType())), 0);
4142 }
4143
4144 // See if the target wants to custom expand this node.
4145 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4146 return SDValue();
4147
4148 // Make the vector elements byte-addressable if they aren't already.
4149 SDLoc dl(N);
4150 EVT EltVT = VecVT.getVectorElementType();
4151 if (!EltVT.isByteSized()) {
4152 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4153 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4154 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4155 SDValue NewExtract =
4156 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4157 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4158 }
4159
4160 // Store the vector to the stack.
4161 // In cases where the vector is illegal it will be broken down into parts
4162 // and stored in parts - we should use the alignment for the smallest part.
4163 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4164 SDValue StackPtr =
4165 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4166 auto &MF = DAG.getMachineFunction();
4167 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4168 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4169 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4170 Alignment: SmallestAlign);
4171
4172 // Load back the required element.
4173 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4174
4175 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4176 // type, leaving the high bits undefined. But it can't truncate.
4177 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4178
4179 return DAG.getExtLoad(
4180 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4181 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4182 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4183}
4184
4185SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4186 SDValue Lo, Hi;
4187
4188 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4189 // splitting the result has the same effect as splitting the input operand.
4190 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4191
4192 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4193}
4194
4195SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4196 (void)OpNo;
4197 SDValue Lo, Hi;
4198 SplitVecRes_Gather(N, Lo, Hi);
4199
4200 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4201 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4202 return SDValue();
4203}
4204
4205SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4206 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4207 SDValue Ch = N->getChain();
4208 SDValue Ptr = N->getBasePtr();
4209 SDValue Offset = N->getOffset();
4210 assert(Offset.isUndef() && "Unexpected VP store offset");
4211 SDValue Mask = N->getMask();
4212 SDValue EVL = N->getVectorLength();
4213 SDValue Data = N->getValue();
4214 Align Alignment = N->getBaseAlign();
4215 SDLoc DL(N);
4216
4217 SDValue DataLo, DataHi;
4218 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4219 // Split Data operand
4220 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4221 else
4222 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4223
4224 // Split Mask operand
4225 SDValue MaskLo, MaskHi;
4226 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4227 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4228 } else {
4229 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4230 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4231 else
4232 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4233 }
4234
4235 EVT MemoryVT = N->getMemoryVT();
4236 EVT LoMemVT, HiMemVT;
4237 bool HiIsEmpty = false;
4238 std::tie(args&: LoMemVT, args&: HiMemVT) =
4239 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4240
4241 // Split EVL
4242 SDValue EVLLo, EVLHi;
4243 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4244
4245 SDValue Lo, Hi;
4246 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4247 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4248 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4249 Ranges: N->getRanges());
4250
4251 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4252 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4253 IsCompressing: N->isCompressingStore());
4254
4255 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4256 if (HiIsEmpty)
4257 return Lo;
4258
4259 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4260 IsCompressedMemory: N->isCompressingStore());
4261
4262 MachinePointerInfo MPI;
4263 if (LoMemVT.isScalableVector()) {
4264 Alignment = commonAlignment(A: Alignment,
4265 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4266 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4267 } else
4268 MPI = N->getPointerInfo().getWithOffset(
4269 O: LoMemVT.getStoreSize().getFixedValue());
4270
4271 MMO = DAG.getMachineFunction().getMachineMemOperand(
4272 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4273 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4274
4275 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4276 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4277 IsCompressing: N->isCompressingStore());
4278
4279 // Build a factor node to remember that this store is independent of the
4280 // other one.
4281 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4282}
4283
4284SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4285 unsigned OpNo) {
4286 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4287 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4288
4289 SDLoc DL(N);
4290
4291 SDValue Data = N->getValue();
4292 SDValue LoData, HiData;
4293 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4294 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4295 else
4296 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4297
4298 EVT LoMemVT, HiMemVT;
4299 bool HiIsEmpty = false;
4300 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4301 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4302
4303 SDValue Mask = N->getMask();
4304 SDValue LoMask, HiMask;
4305 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4306 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4307 else if (getTypeAction(VT: Mask.getValueType()) ==
4308 TargetLowering::TypeSplitVector)
4309 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4310 else
4311 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4312
4313 SDValue LoEVL, HiEVL;
4314 std::tie(args&: LoEVL, args&: HiEVL) =
4315 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4316
4317 // Generate the low vp_strided_store
4318 SDValue Lo = DAG.getStridedStoreVP(
4319 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4320 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4321 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4322
4323 // If the high vp_strided_store has zero storage size, only the low
4324 // vp_strided_store is needed.
4325 if (HiIsEmpty)
4326 return Lo;
4327
4328 // Generate the high vp_strided_store.
4329 // To calculate the high base address, we need to sum to the low base
4330 // address stride number of bytes for each element already stored by low,
4331 // that is: Ptr = Ptr + (LoEVL * Stride)
4332 EVT PtrVT = N->getBasePtr().getValueType();
4333 SDValue Increment =
4334 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4335 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4336 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4337
4338 Align Alignment = N->getBaseAlign();
4339 if (LoMemVT.isScalableVector())
4340 Alignment = commonAlignment(A: Alignment,
4341 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4342
4343 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4344 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4345 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4346 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4347
4348 SDValue Hi = DAG.getStridedStoreVP(
4349 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4350 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4351 IsCompressing: N->isCompressingStore());
4352
4353 // Build a factor node to remember that this store is independent of the
4354 // other one.
4355 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4356}
4357
4358SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4359 unsigned OpNo) {
4360 assert(N->isUnindexed() && "Indexed masked store of vector?");
4361 SDValue Ch = N->getChain();
4362 SDValue Ptr = N->getBasePtr();
4363 SDValue Offset = N->getOffset();
4364 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4365 SDValue Mask = N->getMask();
4366 SDValue Data = N->getValue();
4367 Align Alignment = N->getBaseAlign();
4368 SDLoc DL(N);
4369
4370 SDValue DataLo, DataHi;
4371 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4372 // Split Data operand
4373 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4374 else
4375 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4376
4377 // Split Mask operand
4378 SDValue MaskLo, MaskHi;
4379 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4380 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4381 } else {
4382 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4383 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4384 else
4385 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4386 }
4387
4388 EVT MemoryVT = N->getMemoryVT();
4389 EVT LoMemVT, HiMemVT;
4390 bool HiIsEmpty = false;
4391 std::tie(args&: LoMemVT, args&: HiMemVT) =
4392 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4393
4394 SDValue Lo, Hi, Res;
4395 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4396 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4397 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4398 Ranges: N->getRanges());
4399
4400 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4401 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4402 IsCompressing: N->isCompressingStore());
4403
4404 if (HiIsEmpty) {
4405 // The hi masked store has zero storage size.
4406 // Only the lo masked store is needed.
4407 Res = Lo;
4408 } else {
4409
4410 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4411 IsCompressedMemory: N->isCompressingStore());
4412
4413 MachinePointerInfo MPI;
4414 if (LoMemVT.isScalableVector()) {
4415 Alignment = commonAlignment(
4416 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4417 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4418 } else
4419 MPI = N->getPointerInfo().getWithOffset(
4420 O: LoMemVT.getStoreSize().getFixedValue());
4421
4422 MMO = DAG.getMachineFunction().getMachineMemOperand(
4423 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4424 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4425
4426 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4427 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4428 IsCompressing: N->isCompressingStore());
4429
4430 // Build a factor node to remember that this store is independent of the
4431 // other one.
4432 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4433 }
4434
4435 return Res;
4436}
4437
4438SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4439 SDValue Ch = N->getChain();
4440 SDValue Ptr = N->getBasePtr();
4441 EVT MemoryVT = N->getMemoryVT();
4442 Align Alignment = N->getBaseAlign();
4443 SDLoc DL(N);
4444 struct Operands {
4445 SDValue Mask;
4446 SDValue Index;
4447 SDValue Scale;
4448 SDValue Data;
4449 } Ops = [&]() -> Operands {
4450 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4451 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4452 .Data: MSC->getValue()};
4453 }
4454 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4455 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4456 .Data: VPSC->getValue()};
4457 }();
4458 // Split all operands
4459
4460 EVT LoMemVT, HiMemVT;
4461 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4462
4463 SDValue DataLo, DataHi;
4464 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4465 // Split Data operand
4466 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4467 else
4468 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4469
4470 // Split Mask operand
4471 SDValue MaskLo, MaskHi;
4472 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4473 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4474 } else {
4475 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4476 }
4477
4478 SDValue IndexHi, IndexLo;
4479 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4480 TargetLowering::TypeSplitVector)
4481 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4482 else
4483 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4484
4485 SDValue Lo;
4486 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4487 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4488 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4489 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4490
4491 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4492 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4493 Lo =
4494 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4495 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4496
4497 // The order of the Scatter operation after split is well defined. The "Hi"
4498 // part comes after the "Lo". So these two operations should be chained one
4499 // after another.
4500 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4501 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4502 MMO, IndexType: MSC->getIndexType(),
4503 IsTruncating: MSC->isTruncatingStore());
4504 }
4505 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4506 SDValue EVLLo, EVLHi;
4507 std::tie(args&: EVLLo, args&: EVLHi) =
4508 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4509
4510 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4511 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4512 IndexType: VPSC->getIndexType());
4513
4514 // The order of the Scatter operation after split is well defined. The "Hi"
4515 // part comes after the "Lo". So these two operations should be chained one
4516 // after another.
4517 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4518 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4519 IndexType: VPSC->getIndexType());
4520}
4521
4522SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4523 assert(N->isUnindexed() && "Indexed store of vector?");
4524 assert(OpNo == 1 && "Can only split the stored value");
4525 SDLoc DL(N);
4526
4527 bool isTruncating = N->isTruncatingStore();
4528 SDValue Ch = N->getChain();
4529 SDValue Ptr = N->getBasePtr();
4530 EVT MemoryVT = N->getMemoryVT();
4531 Align Alignment = N->getBaseAlign();
4532 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4533 AAMDNodes AAInfo = N->getAAInfo();
4534 SDValue Lo, Hi;
4535 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4536
4537 EVT LoMemVT, HiMemVT;
4538 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4539
4540 // Scalarize if the split halves are not byte-sized.
4541 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4542 return TLI.scalarizeVectorStore(ST: N, DAG);
4543
4544 if (isTruncating)
4545 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4546 Alignment, MMOFlags, AAInfo);
4547 else
4548 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4549 AAInfo);
4550
4551 MachinePointerInfo MPI;
4552 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4553
4554 if (isTruncating)
4555 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4556 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4557 else
4558 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4559
4560 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4561}
4562
4563SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4564 SDLoc DL(N);
4565
4566 // The input operands all must have the same type, and we know the result
4567 // type is valid. Convert this to a buildvector which extracts all the
4568 // input elements.
4569 // TODO: If the input elements are power-two vectors, we could convert this to
4570 // a new CONCAT_VECTORS node with elements that are half-wide.
4571 SmallVector<SDValue, 32> Elts;
4572 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4573 for (const SDValue &Op : N->op_values()) {
4574 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4575 i != e; ++i) {
4576 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4577 }
4578 }
4579
4580 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4581}
4582
4583SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4584 // The result type is legal, but the input type is illegal. If splitting
4585 // ends up with the result type of each half still being legal, just
4586 // do that. If, however, that would result in an illegal result type,
4587 // we can try to get more clever with power-two vectors. Specifically,
4588 // split the input type, but also widen the result element size, then
4589 // concatenate the halves and truncate again. For example, consider a target
4590 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4591 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4592 // %inlo = v4i32 extract_subvector %in, 0
4593 // %inhi = v4i32 extract_subvector %in, 4
4594 // %lo16 = v4i16 trunc v4i32 %inlo
4595 // %hi16 = v4i16 trunc v4i32 %inhi
4596 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4597 // %res = v8i8 trunc v8i16 %in16
4598 //
4599 // Without this transform, the original truncate would end up being
4600 // scalarized, which is pretty much always a last resort.
4601 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4602 SDValue InVec = N->getOperand(Num: OpNo);
4603 EVT InVT = InVec->getValueType(ResNo: 0);
4604 EVT OutVT = N->getValueType(ResNo: 0);
4605 ElementCount NumElements = OutVT.getVectorElementCount();
4606 bool IsFloat = OutVT.isFloatingPoint();
4607
4608 unsigned InElementSize = InVT.getScalarSizeInBits();
4609 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4610
4611 // Determine the split output VT. If its legal we can just split dirctly.
4612 EVT LoOutVT, HiOutVT;
4613 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4614 assert(LoOutVT == HiOutVT && "Unequal split?");
4615
4616 // If the input elements are only 1/2 the width of the result elements,
4617 // just use the normal splitting. Our trick only work if there's room
4618 // to split more than once.
4619 if (isTypeLegal(VT: LoOutVT) ||
4620 InElementSize <= OutElementSize * 2)
4621 return SplitVecOp_UnaryOp(N);
4622 SDLoc DL(N);
4623
4624 // Don't touch if this will be scalarized.
4625 EVT FinalVT = InVT;
4626 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4627 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4628
4629 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4630 return SplitVecOp_UnaryOp(N);
4631
4632 // Get the split input vector.
4633 SDValue InLoVec, InHiVec;
4634 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4635
4636 // Truncate them to 1/2 the element size.
4637 //
4638 // This assumes the number of elements is a power of two; any vector that
4639 // isn't should be widened, not split.
4640 EVT HalfElementVT = IsFloat ?
4641 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4642 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4643 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4644 EC: NumElements.divideCoefficientBy(RHS: 2));
4645
4646 SDValue HalfLo;
4647 SDValue HalfHi;
4648 SDValue Chain;
4649 if (N->isStrictFPOpcode()) {
4650 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4651 Ops: {N->getOperand(Num: 0), InLoVec});
4652 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4653 Ops: {N->getOperand(Num: 0), InHiVec});
4654 // Legalize the chain result - switch anything that used the old chain to
4655 // use the new one.
4656 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4657 N2: HalfHi.getValue(R: 1));
4658 } else {
4659 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4660 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4661 }
4662
4663 // Concatenate them to get the full intermediate truncation result.
4664 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4665 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4666 N2: HalfHi);
4667 // Now finish up by truncating all the way down to the original result
4668 // type. This should normally be something that ends up being legal directly,
4669 // but in theory if a target has very wide vectors and an annoyingly
4670 // restricted set of legal types, this split can chain to build things up.
4671
4672 if (N->isStrictFPOpcode()) {
4673 SDValue Res = DAG.getNode(
4674 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4675 Ops: {Chain, InterVec,
4676 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4677 // Relink the chain
4678 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4679 return Res;
4680 }
4681
4682 return IsFloat
4683 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4684 N2: DAG.getTargetConstant(
4685 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4686 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4687}
4688
4689SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4690 unsigned Opc = N->getOpcode();
4691 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4692 assert(N->getValueType(0).isVector() &&
4693 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4694 "Operand types must be vectors");
4695 // The result has a legal vector type, but the input needs splitting.
4696 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4697 SDLoc DL(N);
4698 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4699 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4700
4701 EVT VT = N->getValueType(ResNo: 0);
4702 EVT PartResVT = Lo0.getValueType().changeElementType(Context&: *DAG.getContext(),
4703 EltVT: VT.getScalarType());
4704
4705 if (Opc == ISD::SETCC) {
4706 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4707 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4708 } else if (isStrict) {
4709 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4710 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4711 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4712 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4713 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4714 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4715 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4716 } else {
4717 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4718 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4719 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4720 std::tie(args&: EVLLo, args&: EVLHi) =
4721 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4722 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4723 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4724 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4725 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4726 }
4727
4728 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: LoRes, N2: HiRes);
4729}
4730
4731
4732SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4733 // The result has a legal vector type, but the input needs splitting.
4734 EVT ResVT = N->getValueType(ResNo: 0);
4735 SDValue Lo, Hi;
4736 SDLoc DL(N);
4737 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4738 EVT InVT = Lo.getValueType();
4739
4740 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4741 EC: InVT.getVectorElementCount());
4742
4743 if (N->isStrictFPOpcode()) {
4744 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4745 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
4746 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4747 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
4748 // Legalize the chain result - switch anything that used the old chain to
4749 // use the new one.
4750 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4751 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
4752 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4753 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4754 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4755 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4756 std::tie(args&: EVLLo, args&: EVLHi) =
4757 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
4758 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4759 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4760 } else {
4761 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4762 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4763 }
4764
4765 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4766}
4767
4768// Split a vector type in an FP binary operation where the second operand has a
4769// different type from the first.
4770//
4771// The result (and the first input) has a legal vector type, but the second
4772// input needs splitting.
4773SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4774 SDLoc DL(N);
4775
4776 EVT LHSLoVT, LHSHiVT;
4777 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4778
4779 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4780 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4781
4782 SDValue LHSLo, LHSHi;
4783 std::tie(args&: LHSLo, args&: LHSHi) =
4784 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4785
4786 SDValue RHSLo, RHSHi;
4787 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4788
4789 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4790 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4791
4792 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4793}
4794
4795SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
4796 LLVMContext &Ctxt = *DAG.getContext();
4797 SDLoc dl(N);
4798
4799 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4800 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
4801 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
4802
4803 EVT ResVT = N->getValueType(ResNo: 0);
4804 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
4805 EVT NewResVT =
4806 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
4807
4808 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
4809 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
4810
4811 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4812}
4813
4814SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4815 EVT ResVT = N->getValueType(ResNo: 0);
4816 SDValue Lo, Hi;
4817 SDLoc dl(N);
4818 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4819 EVT InVT = Lo.getValueType();
4820
4821 EVT NewResVT =
4822 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4823 EC: InVT.getVectorElementCount());
4824
4825 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4826 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4827
4828 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4829}
4830
4831SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
4832 SDLoc DL(N);
4833 EVT ResVT = N->getValueType(ResNo: 0);
4834
4835 SDValue Lo, Hi;
4836 SDValue VecOp = N->getOperand(Num: 0);
4837 GetSplitVector(Op: VecOp, Lo, Hi);
4838
4839 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
4840 auto [EVLLo, EVLHi] =
4841 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
4842 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
4843
4844 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
4845 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_UNDEF(Hi)).
4846 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4847 SDValue ResLoNotEVL =
4848 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
4849 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4850 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
4851 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
4852}
4853
4854SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
4855 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
4856 SDLoc DL(HG);
4857 SDValue Inc = HG->getInc();
4858 SDValue Ptr = HG->getBasePtr();
4859 SDValue Scale = HG->getScale();
4860 SDValue IntID = HG->getIntID();
4861 EVT MemVT = HG->getMemoryVT();
4862 MachineMemOperand *MMO = HG->getMemOperand();
4863 ISD::MemIndexType IndexType = HG->getIndexType();
4864
4865 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
4866 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
4867 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
4868 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
4869 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
4870 Ops: OpsLo, MMO, IndexType);
4871 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
4872 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
4873 MMO, IndexType);
4874}
4875
4876SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
4877 SDValue Acc = N->getOperand(Num: 0);
4878 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
4879 "Accumulator should already be a legal type, and shouldn't need "
4880 "further splitting");
4881
4882 SDLoc DL(N);
4883 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
4884 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
4885 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
4886 unsigned Opcode = N->getOpcode();
4887 EVT ResultVT = Acc.getValueType();
4888
4889 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
4890 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
4891}
4892
4893//===----------------------------------------------------------------------===//
4894// Result Vector Widening
4895//===----------------------------------------------------------------------===//
4896
4897void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
4898 unsigned WidenResNo) {
4899 unsigned NumResults = N->getNumValues();
4900 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
4901 if (ResNo == WidenResNo)
4902 continue;
4903 EVT ResVT = N->getValueType(ResNo);
4904 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
4905 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
4906 } else {
4907 SDLoc DL(N);
4908 SDValue ResVal =
4909 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
4910 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
4911 }
4912 }
4913}
4914
4915void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4916 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4917
4918 // See if the target wants to custom widen this node.
4919 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4920 return;
4921
4922 SDValue Res = SDValue();
4923
4924 auto unrollExpandedOp = [&]() {
4925 // We're going to widen this vector op to a legal type by padding with undef
4926 // elements. If the wide vector op is eventually going to be expanded to
4927 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4928 // libcalls on the undef elements.
4929 EVT VT = N->getValueType(ResNo: 0);
4930 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4931 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
4932 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
4933 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4934 if (N->getNumValues() > 1)
4935 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
4936 return true;
4937 }
4938 return false;
4939 };
4940
4941 switch (N->getOpcode()) {
4942 default:
4943#ifndef NDEBUG
4944 dbgs() << "WidenVectorResult #" << ResNo << ": ";
4945 N->dump(&DAG);
4946 dbgs() << "\n";
4947#endif
4948 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4949
4950 case ISD::LOOP_DEPENDENCE_RAW_MASK:
4951 case ISD::LOOP_DEPENDENCE_WAR_MASK:
4952 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
4953 break;
4954 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4955 case ISD::ADDRSPACECAST:
4956 Res = WidenVecRes_ADDRSPACECAST(N);
4957 break;
4958 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4959 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4960 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4961 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4962 case ISD::INSERT_SUBVECTOR:
4963 Res = WidenVecRes_INSERT_SUBVECTOR(N);
4964 break;
4965 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4966 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4967 case ISD::ATOMIC_LOAD:
4968 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
4969 break;
4970 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4971 case ISD::STEP_VECTOR:
4972 case ISD::SPLAT_VECTOR:
4973 case ISD::SCALAR_TO_VECTOR:
4974 Res = WidenVecRes_ScalarOp(N);
4975 break;
4976 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4977 case ISD::VSELECT:
4978 case ISD::SELECT:
4979 case ISD::VP_SELECT:
4980 case ISD::VP_MERGE:
4981 Res = WidenVecRes_Select(N);
4982 break;
4983 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4984 case ISD::VP_SETCC:
4985 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4986 case ISD::POISON:
4987 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4988 case ISD::VECTOR_SHUFFLE:
4989 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
4990 break;
4991 case ISD::VP_LOAD:
4992 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
4993 break;
4994 case ISD::VP_LOAD_FF:
4995 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
4996 break;
4997 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4998 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
4999 break;
5000 case ISD::VECTOR_COMPRESS:
5001 Res = WidenVecRes_VECTOR_COMPRESS(N);
5002 break;
5003 case ISD::MLOAD:
5004 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5005 break;
5006 case ISD::MGATHER:
5007 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5008 break;
5009 case ISD::VP_GATHER:
5010 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5011 break;
5012 case ISD::VECTOR_REVERSE:
5013 Res = WidenVecRes_VECTOR_REVERSE(N);
5014 break;
5015 case ISD::GET_ACTIVE_LANE_MASK:
5016 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5017 break;
5018
5019 case ISD::ADD: case ISD::VP_ADD:
5020 case ISD::AND: case ISD::VP_AND:
5021 case ISD::MUL: case ISD::VP_MUL:
5022 case ISD::MULHS:
5023 case ISD::MULHU:
5024 case ISD::ABDS:
5025 case ISD::ABDU:
5026 case ISD::OR: case ISD::VP_OR:
5027 case ISD::SUB: case ISD::VP_SUB:
5028 case ISD::XOR: case ISD::VP_XOR:
5029 case ISD::SHL: case ISD::VP_SHL:
5030 case ISD::SRA: case ISD::VP_SRA:
5031 case ISD::SRL: case ISD::VP_SRL:
5032 case ISD::CLMUL:
5033 case ISD::CLMULR:
5034 case ISD::CLMULH:
5035 case ISD::FMINNUM:
5036 case ISD::FMINNUM_IEEE:
5037 case ISD::VP_FMINNUM:
5038 case ISD::FMAXNUM:
5039 case ISD::FMAXNUM_IEEE:
5040 case ISD::VP_FMAXNUM:
5041 case ISD::FMINIMUM:
5042 case ISD::VP_FMINIMUM:
5043 case ISD::FMAXIMUM:
5044 case ISD::VP_FMAXIMUM:
5045 case ISD::FMINIMUMNUM:
5046 case ISD::FMAXIMUMNUM:
5047 case ISD::SMIN: case ISD::VP_SMIN:
5048 case ISD::SMAX: case ISD::VP_SMAX:
5049 case ISD::UMIN: case ISD::VP_UMIN:
5050 case ISD::UMAX: case ISD::VP_UMAX:
5051 case ISD::UADDSAT: case ISD::VP_UADDSAT:
5052 case ISD::SADDSAT: case ISD::VP_SADDSAT:
5053 case ISD::USUBSAT: case ISD::VP_USUBSAT:
5054 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
5055 case ISD::SSHLSAT:
5056 case ISD::USHLSAT:
5057 case ISD::ROTL:
5058 case ISD::ROTR:
5059 case ISD::AVGFLOORS:
5060 case ISD::AVGFLOORU:
5061 case ISD::AVGCEILS:
5062 case ISD::AVGCEILU:
5063 // Vector-predicated binary op widening. Note that -- unlike the
5064 // unpredicated versions -- we don't have to worry about trapping on
5065 // operations like UDIV, FADD, etc., as we pass on the original vector
5066 // length parameter. This means the widened elements containing garbage
5067 // aren't active.
5068 case ISD::VP_SDIV:
5069 case ISD::VP_UDIV:
5070 case ISD::VP_SREM:
5071 case ISD::VP_UREM:
5072 case ISD::VP_FADD:
5073 case ISD::VP_FSUB:
5074 case ISD::VP_FMUL:
5075 case ISD::VP_FDIV:
5076 case ISD::VP_FREM:
5077 case ISD::VP_FCOPYSIGN:
5078 Res = WidenVecRes_Binary(N);
5079 break;
5080
5081 case ISD::SCMP:
5082 case ISD::UCMP:
5083 Res = WidenVecRes_CMP(N);
5084 break;
5085
5086 case ISD::FPOW:
5087 case ISD::FATAN2:
5088 case ISD::FREM:
5089 if (unrollExpandedOp())
5090 break;
5091 // If the target has custom/legal support for the scalar FP intrinsic ops
5092 // (they are probably not destined to become libcalls), then widen those
5093 // like any other binary ops.
5094 [[fallthrough]];
5095
5096 case ISD::FADD:
5097 case ISD::FMUL:
5098 case ISD::FSUB:
5099 case ISD::FDIV:
5100 case ISD::SDIV:
5101 case ISD::UDIV:
5102 case ISD::SREM:
5103 case ISD::UREM:
5104 Res = WidenVecRes_BinaryCanTrap(N);
5105 break;
5106
5107 case ISD::SMULFIX:
5108 case ISD::SMULFIXSAT:
5109 case ISD::UMULFIX:
5110 case ISD::UMULFIXSAT:
5111 // These are binary operations, but with an extra operand that shouldn't
5112 // be widened (the scale).
5113 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5114 break;
5115
5116#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5117 case ISD::STRICT_##DAGN:
5118#include "llvm/IR/ConstrainedOps.def"
5119 Res = WidenVecRes_StrictFP(N);
5120 break;
5121
5122 case ISD::UADDO:
5123 case ISD::SADDO:
5124 case ISD::USUBO:
5125 case ISD::SSUBO:
5126 case ISD::UMULO:
5127 case ISD::SMULO:
5128 Res = WidenVecRes_OverflowOp(N, ResNo);
5129 break;
5130
5131 case ISD::FCOPYSIGN:
5132 Res = WidenVecRes_FCOPYSIGN(N);
5133 break;
5134
5135 case ISD::IS_FPCLASS:
5136 case ISD::FPTRUNC_ROUND:
5137 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5138 break;
5139
5140 case ISD::FLDEXP:
5141 case ISD::FPOWI:
5142 if (!unrollExpandedOp())
5143 Res = WidenVecRes_ExpOp(N);
5144 break;
5145
5146 case ISD::ANY_EXTEND_VECTOR_INREG:
5147 case ISD::SIGN_EXTEND_VECTOR_INREG:
5148 case ISD::ZERO_EXTEND_VECTOR_INREG:
5149 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5150 break;
5151
5152 case ISD::ANY_EXTEND:
5153 case ISD::FP_EXTEND:
5154 case ISD::VP_FP_EXTEND:
5155 case ISD::FP_ROUND:
5156 case ISD::VP_FP_ROUND:
5157 case ISD::FP_TO_SINT:
5158 case ISD::VP_FP_TO_SINT:
5159 case ISD::FP_TO_UINT:
5160 case ISD::VP_FP_TO_UINT:
5161 case ISD::SIGN_EXTEND:
5162 case ISD::VP_SIGN_EXTEND:
5163 case ISD::SINT_TO_FP:
5164 case ISD::VP_SINT_TO_FP:
5165 case ISD::VP_TRUNCATE:
5166 case ISD::TRUNCATE:
5167 case ISD::UINT_TO_FP:
5168 case ISD::VP_UINT_TO_FP:
5169 case ISD::ZERO_EXTEND:
5170 case ISD::VP_ZERO_EXTEND:
5171 case ISD::CONVERT_FROM_ARBITRARY_FP:
5172 Res = WidenVecRes_Convert(N);
5173 break;
5174
5175 case ISD::FP_TO_SINT_SAT:
5176 case ISD::FP_TO_UINT_SAT:
5177 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5178 break;
5179
5180 case ISD::LRINT:
5181 case ISD::LLRINT:
5182 case ISD::VP_LRINT:
5183 case ISD::VP_LLRINT:
5184 case ISD::LROUND:
5185 case ISD::LLROUND:
5186 Res = WidenVecRes_XROUND(N);
5187 break;
5188
5189 case ISD::FACOS:
5190 case ISD::FASIN:
5191 case ISD::FATAN:
5192 case ISD::FCEIL:
5193 case ISD::FCOS:
5194 case ISD::FCOSH:
5195 case ISD::FEXP:
5196 case ISD::FEXP2:
5197 case ISD::FEXP10:
5198 case ISD::FFLOOR:
5199 case ISD::FLOG:
5200 case ISD::FLOG10:
5201 case ISD::FLOG2:
5202 case ISD::FNEARBYINT:
5203 case ISD::FRINT:
5204 case ISD::FROUND:
5205 case ISD::FROUNDEVEN:
5206 case ISD::FSIN:
5207 case ISD::FSINH:
5208 case ISD::FSQRT:
5209 case ISD::FTAN:
5210 case ISD::FTANH:
5211 case ISD::FTRUNC:
5212 if (unrollExpandedOp())
5213 break;
5214 // If the target has custom/legal support for the scalar FP intrinsic ops
5215 // (they are probably not destined to become libcalls), then widen those
5216 // like any other unary ops.
5217 [[fallthrough]];
5218
5219 case ISD::ABS:
5220 case ISD::VP_ABS:
5221 case ISD::BITREVERSE:
5222 case ISD::VP_BITREVERSE:
5223 case ISD::BSWAP:
5224 case ISD::VP_BSWAP:
5225 case ISD::CTLZ:
5226 case ISD::VP_CTLZ:
5227 case ISD::CTLZ_ZERO_UNDEF:
5228 case ISD::VP_CTLZ_ZERO_UNDEF:
5229 case ISD::CTPOP:
5230 case ISD::VP_CTPOP:
5231 case ISD::CTTZ:
5232 case ISD::VP_CTTZ:
5233 case ISD::CTTZ_ZERO_UNDEF:
5234 case ISD::VP_CTTZ_ZERO_UNDEF:
5235 case ISD::FNEG: case ISD::VP_FNEG:
5236 case ISD::FABS: case ISD::VP_FABS:
5237 case ISD::VP_SQRT:
5238 case ISD::VP_FCEIL:
5239 case ISD::VP_FFLOOR:
5240 case ISD::VP_FRINT:
5241 case ISD::VP_FNEARBYINT:
5242 case ISD::VP_FROUND:
5243 case ISD::VP_FROUNDEVEN:
5244 case ISD::VP_FROUNDTOZERO:
5245 case ISD::FREEZE:
5246 case ISD::ARITH_FENCE:
5247 case ISD::FCANONICALIZE:
5248 case ISD::AssertNoFPClass:
5249 Res = WidenVecRes_Unary(N);
5250 break;
5251 case ISD::FMA: case ISD::VP_FMA:
5252 case ISD::FSHL:
5253 case ISD::VP_FSHL:
5254 case ISD::FSHR:
5255 case ISD::VP_FSHR:
5256 Res = WidenVecRes_Ternary(N);
5257 break;
5258 case ISD::FMODF:
5259 case ISD::FFREXP:
5260 case ISD::FSINCOS:
5261 case ISD::FSINCOSPI: {
5262 if (!unrollExpandedOp())
5263 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5264 break;
5265 }
5266 }
5267
5268 // If Res is null, the sub-method took care of registering the result.
5269 if (Res.getNode())
5270 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5271}
5272
5273SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5274 // Ternary op widening.
5275 SDLoc dl(N);
5276 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5277 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5278 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5279 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5280 if (N->getNumOperands() == 3)
5281 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5282
5283 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5284 assert(N->isVPOpcode() && "Expected VP opcode");
5285
5286 SDValue Mask =
5287 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5288 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5289 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5290}
5291
5292SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5293 // Binary op widening.
5294 SDLoc dl(N);
5295 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5296 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5297 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5298 if (N->getNumOperands() == 2)
5299 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5300 Flags: N->getFlags());
5301
5302 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5303 assert(N->isVPOpcode() && "Expected VP opcode");
5304
5305 SDValue Mask =
5306 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5307 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5308 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5309}
5310
5311SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5312 LLVMContext &Ctxt = *DAG.getContext();
5313 SDLoc dl(N);
5314
5315 SDValue LHS = N->getOperand(Num: 0);
5316 SDValue RHS = N->getOperand(Num: 1);
5317 EVT OpVT = LHS.getValueType();
5318 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5319 LHS = GetWidenedVector(Op: LHS);
5320 RHS = GetWidenedVector(Op: RHS);
5321 OpVT = LHS.getValueType();
5322 }
5323
5324 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5325 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5326 if (WidenResEC == OpVT.getVectorElementCount()) {
5327 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5328 }
5329
5330 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5331}
5332
5333SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5334 // Binary op widening, but with an extra operand that shouldn't be widened.
5335 SDLoc dl(N);
5336 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5337 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5338 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5339 SDValue InOp3 = N->getOperand(Num: 2);
5340 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5341 Flags: N->getFlags());
5342}
5343
5344// Given a vector of operations that have been broken up to widen, see
5345// if we can collect them together into the next widest legal VT. This
5346// implementation is trap-safe.
5347static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5348 SmallVectorImpl<SDValue> &ConcatOps,
5349 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5350 EVT WidenVT) {
5351 // Check to see if we have a single operation with the widen type.
5352 if (ConcatEnd == 1) {
5353 VT = ConcatOps[0].getValueType();
5354 if (VT == WidenVT)
5355 return ConcatOps[0];
5356 }
5357
5358 SDLoc dl(ConcatOps[0]);
5359 EVT WidenEltVT = WidenVT.getVectorElementType();
5360
5361 // while (Some element of ConcatOps is not of type MaxVT) {
5362 // From the end of ConcatOps, collect elements of the same type and put
5363 // them into an op of the next larger supported type
5364 // }
5365 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5366 int Idx = ConcatEnd - 1;
5367 VT = ConcatOps[Idx--].getValueType();
5368 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5369 Idx--;
5370
5371 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5372 EVT NextVT;
5373 do {
5374 NextSize *= 2;
5375 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5376 } while (!TLI.isTypeLegal(VT: NextVT));
5377
5378 if (!VT.isVector()) {
5379 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5380 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5381 unsigned NumToInsert = ConcatEnd - Idx - 1;
5382 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5383 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5384 ConcatOps[Idx+1] = VecOp;
5385 ConcatEnd = Idx + 2;
5386 } else {
5387 // Vector type, create a CONCAT_VECTORS of type NextVT
5388 SDValue undefVec = DAG.getPOISON(VT);
5389 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5390 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5391 unsigned RealVals = ConcatEnd - Idx - 1;
5392 unsigned SubConcatEnd = 0;
5393 unsigned SubConcatIdx = Idx + 1;
5394 while (SubConcatEnd < RealVals)
5395 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5396 while (SubConcatEnd < OpsToConcat)
5397 SubConcatOps[SubConcatEnd++] = undefVec;
5398 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5399 VT: NextVT, Ops: SubConcatOps);
5400 ConcatEnd = SubConcatIdx + 1;
5401 }
5402 }
5403
5404 // Check to see if we have a single operation with the widen type.
5405 if (ConcatEnd == 1) {
5406 VT = ConcatOps[0].getValueType();
5407 if (VT == WidenVT)
5408 return ConcatOps[0];
5409 }
5410
5411 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5412 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5413 if (NumOps != ConcatEnd ) {
5414 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5415 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5416 ConcatOps[j] = UndefVal;
5417 }
5418 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5419 Ops: ArrayRef(ConcatOps.data(), NumOps));
5420}
5421
5422SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5423 // Binary op widening for operations that can trap.
5424 unsigned Opcode = N->getOpcode();
5425 SDLoc dl(N);
5426 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5427 EVT WidenEltVT = WidenVT.getVectorElementType();
5428 EVT VT = WidenVT;
5429 unsigned NumElts = VT.getVectorMinNumElements();
5430 const SDNodeFlags Flags = N->getFlags();
5431 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5432 NumElts = NumElts / 2;
5433 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5434 }
5435
5436 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5437 // Operation doesn't trap so just widen as normal.
5438 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5439 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5440 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5441 }
5442
5443 // Generate a vp.op if it is custom/legal for the target. This avoids need
5444 // to split and tile the subvectors (below), because the inactive lanes can
5445 // simply be disabled. To avoid possible recursion, only do this if the
5446 // widened mask type is legal.
5447 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5448 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5449 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5450 EC: WidenVT.getVectorElementCount());
5451 TLI.isTypeLegal(VT: WideMaskVT)) {
5452 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5453 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5454 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5455 SDValue EVL =
5456 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5457 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5458 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5459 Flags);
5460 }
5461 }
5462
5463 // FIXME: Improve support for scalable vectors.
5464 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5465
5466 // No legal vector version so unroll the vector operation and then widen.
5467 if (NumElts == 1)
5468 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5469
5470 // Since the operation can trap, apply operation on the original vector.
5471 EVT MaxVT = VT;
5472 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5473 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5474 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5475
5476 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5477 unsigned ConcatEnd = 0; // Current ConcatOps index.
5478 int Idx = 0; // Current Idx into input vectors.
5479
5480 // NumElts := greatest legal vector size (at most WidenVT)
5481 // while (orig. vector has unhandled elements) {
5482 // take munches of size NumElts from the beginning and add to ConcatOps
5483 // NumElts := next smaller supported vector size or 1
5484 // }
5485 while (CurNumElts != 0) {
5486 while (CurNumElts >= NumElts) {
5487 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5488 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5489 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5490 Idx += NumElts;
5491 CurNumElts -= NumElts;
5492 }
5493 do {
5494 NumElts = NumElts / 2;
5495 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5496 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5497
5498 if (NumElts == 1) {
5499 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5500 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5501 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5502 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5503 N1: EOp1, N2: EOp2, Flags);
5504 }
5505 CurNumElts = 0;
5506 }
5507 }
5508
5509 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5510}
5511
5512SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5513 switch (N->getOpcode()) {
5514 case ISD::STRICT_FSETCC:
5515 case ISD::STRICT_FSETCCS:
5516 return WidenVecRes_STRICT_FSETCC(N);
5517 case ISD::STRICT_FP_EXTEND:
5518 case ISD::STRICT_FP_ROUND:
5519 case ISD::STRICT_FP_TO_SINT:
5520 case ISD::STRICT_FP_TO_UINT:
5521 case ISD::STRICT_SINT_TO_FP:
5522 case ISD::STRICT_UINT_TO_FP:
5523 return WidenVecRes_Convert_StrictFP(N);
5524 default:
5525 break;
5526 }
5527
5528 // StrictFP op widening for operations that can trap.
5529 unsigned NumOpers = N->getNumOperands();
5530 unsigned Opcode = N->getOpcode();
5531 SDLoc dl(N);
5532 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5533 EVT WidenEltVT = WidenVT.getVectorElementType();
5534 EVT VT = WidenVT;
5535 unsigned NumElts = VT.getVectorNumElements();
5536 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5537 NumElts = NumElts / 2;
5538 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5539 }
5540
5541 // No legal vector version so unroll the vector operation and then widen.
5542 if (NumElts == 1)
5543 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5544
5545 // Since the operation can trap, apply operation on the original vector.
5546 EVT MaxVT = VT;
5547 SmallVector<SDValue, 4> InOps;
5548 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5549
5550 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5551 SmallVector<SDValue, 16> Chains;
5552 unsigned ConcatEnd = 0; // Current ConcatOps index.
5553 int Idx = 0; // Current Idx into input vectors.
5554
5555 // The Chain is the first operand.
5556 InOps.push_back(Elt: N->getOperand(Num: 0));
5557
5558 // Now process the remaining operands.
5559 for (unsigned i = 1; i < NumOpers; ++i) {
5560 SDValue Oper = N->getOperand(Num: i);
5561
5562 EVT OpVT = Oper.getValueType();
5563 if (OpVT.isVector()) {
5564 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5565 Oper = GetWidenedVector(Op: Oper);
5566 else {
5567 EVT WideOpVT =
5568 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5569 EC: WidenVT.getVectorElementCount());
5570 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5571 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5572 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5573 }
5574 }
5575
5576 InOps.push_back(Elt: Oper);
5577 }
5578
5579 // NumElts := greatest legal vector size (at most WidenVT)
5580 // while (orig. vector has unhandled elements) {
5581 // take munches of size NumElts from the beginning and add to ConcatOps
5582 // NumElts := next smaller supported vector size or 1
5583 // }
5584 while (CurNumElts != 0) {
5585 while (CurNumElts >= NumElts) {
5586 SmallVector<SDValue, 4> EOps;
5587
5588 for (unsigned i = 0; i < NumOpers; ++i) {
5589 SDValue Op = InOps[i];
5590
5591 EVT OpVT = Op.getValueType();
5592 if (OpVT.isVector()) {
5593 EVT OpExtractVT =
5594 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5595 EC: VT.getVectorElementCount());
5596 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5597 }
5598
5599 EOps.push_back(Elt: Op);
5600 }
5601
5602 EVT OperVT[] = {VT, MVT::Other};
5603 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5604 ConcatOps[ConcatEnd++] = Oper;
5605 Chains.push_back(Elt: Oper.getValue(R: 1));
5606 Idx += NumElts;
5607 CurNumElts -= NumElts;
5608 }
5609 do {
5610 NumElts = NumElts / 2;
5611 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5612 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5613
5614 if (NumElts == 1) {
5615 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5616 SmallVector<SDValue, 4> EOps;
5617
5618 for (unsigned i = 0; i < NumOpers; ++i) {
5619 SDValue Op = InOps[i];
5620
5621 EVT OpVT = Op.getValueType();
5622 if (OpVT.isVector())
5623 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5624 Idx);
5625
5626 EOps.push_back(Elt: Op);
5627 }
5628
5629 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5630 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5631 ConcatOps[ConcatEnd++] = Oper;
5632 Chains.push_back(Elt: Oper.getValue(R: 1));
5633 }
5634 CurNumElts = 0;
5635 }
5636 }
5637
5638 // Build a factor node to remember all the Ops that have been created.
5639 SDValue NewChain;
5640 if (Chains.size() == 1)
5641 NewChain = Chains[0];
5642 else
5643 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5644 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5645
5646 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5647}
5648
5649SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5650 SDLoc DL(N);
5651 EVT ResVT = N->getValueType(ResNo: 0);
5652 EVT OvVT = N->getValueType(ResNo: 1);
5653 EVT WideResVT, WideOvVT;
5654 SDValue WideLHS, WideRHS;
5655
5656 // TODO: This might result in a widen/split loop.
5657 if (ResNo == 0) {
5658 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5659 WideOvVT = EVT::getVectorVT(
5660 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5661 NumElements: WideResVT.getVectorNumElements());
5662
5663 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5664 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5665 } else {
5666 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5667 WideResVT = EVT::getVectorVT(
5668 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5669 NumElements: WideOvVT.getVectorNumElements());
5670
5671 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5672 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5673
5674 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5675 N2: N->getOperand(Num: 0), N3: Zero);
5676 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5677 N2: N->getOperand(Num: 1), N3: Zero);
5678 }
5679
5680 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5681 SDNode *WideNode = DAG.getNode(
5682 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5683
5684 // Replace the other vector result not being explicitly widened here.
5685 unsigned OtherNo = 1 - ResNo;
5686 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5687 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5688 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5689 } else {
5690 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5691 SDValue OtherVal = DAG.getNode(
5692 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5693 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5694 }
5695
5696 return SDValue(WideNode, ResNo);
5697}
5698
5699SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5700 LLVMContext &Ctx = *DAG.getContext();
5701 SDValue InOp = N->getOperand(Num: 0);
5702 SDLoc DL(N);
5703
5704 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
5705 ElementCount WidenEC = WidenVT.getVectorElementCount();
5706
5707 EVT InVT = InOp.getValueType();
5708
5709 unsigned Opcode = N->getOpcode();
5710 const SDNodeFlags Flags = N->getFlags();
5711
5712 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
5713 // not equal that of WidenVT.
5714 if (N->getOpcode() == ISD::ZERO_EXTEND &&
5715 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
5716 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
5717 WidenVT.getScalarSizeInBits()) {
5718 InOp = ZExtPromotedInteger(Op: InOp);
5719 InVT = InOp.getValueType();
5720 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
5721 Opcode = ISD::TRUNCATE;
5722 }
5723
5724 EVT InEltVT = InVT.getVectorElementType();
5725 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
5726 ElementCount InVTEC = InVT.getVectorElementCount();
5727
5728 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5729 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5730 InVT = InOp.getValueType();
5731 InVTEC = InVT.getVectorElementCount();
5732 if (InVTEC == WidenEC) {
5733 if (N->getNumOperands() == 1)
5734 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp, Flags);
5735 if (N->getNumOperands() == 3) {
5736 assert(N->isVPOpcode() && "Expected VP opcode");
5737 SDValue Mask =
5738 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5739 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
5740 }
5741 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
5742 }
5743 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
5744 // If both input and result vector types are of same width, extend
5745 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
5746 // accepts fewer elements in the result than in the input.
5747 if (Opcode == ISD::ANY_EXTEND)
5748 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5749 if (Opcode == ISD::SIGN_EXTEND)
5750 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5751 if (Opcode == ISD::ZERO_EXTEND)
5752 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5753 }
5754
5755 // For TRUNCATE, try to widen using the legal EC of the input type instead
5756 // if the legalisation action for that intermediate type is not widening.
5757 // E.g. for trunc nxv1i64 -> nxv1i8 where
5758 // - nxv1i64 input gets widened to nxv2i64
5759 // - nxv1i8 output gets widened to nxv16i8
5760 // Then one can try widening the result to nxv2i8 (instead of going all the
5761 // way to nxv16i8) if this later allows type promotion.
5762 EVT MidResVT =
5763 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
5764 if (N->getOpcode() == ISD::TRUNCATE &&
5765 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
5766 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
5767 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
5768 }
5769 }
5770
5771 if (TLI.isTypeLegal(VT: InWidenVT)) {
5772 // Because the result and the input are different vector types, widening
5773 // the result could create a legal type but widening the input might make
5774 // it an illegal type that might lead to repeatedly splitting the input
5775 // and then widening it. To avoid this, we widen the input only if
5776 // it results in a legal type.
5777 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
5778 // Widen the input and call convert on the widened input vector.
5779 unsigned NumConcat =
5780 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
5781 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
5782 Ops[0] = InOp;
5783 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
5784 if (N->getNumOperands() == 1)
5785 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec, Flags);
5786 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
5787 }
5788
5789 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
5790 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
5791 // Extract the input and convert the shorten input vector.
5792 if (N->getNumOperands() == 1)
5793 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal, Flags);
5794 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
5795 }
5796 }
5797
5798 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5799 EVT EltVT = WidenVT.getVectorElementType();
5800 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
5801 // Use the original element count so we don't do more scalar opts than
5802 // necessary.
5803 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5804 for (unsigned i=0; i < MinElts; ++i) {
5805 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5806 if (N->getNumOperands() == 1)
5807 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val, Flags);
5808 else
5809 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
5810 }
5811
5812 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5813}
5814
5815SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
5816 SDLoc dl(N);
5817 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5818 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5819
5820 SDValue Src = N->getOperand(Num: 0);
5821 EVT SrcVT = Src.getValueType();
5822
5823 // Also widen the input.
5824 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5825 Src = GetWidenedVector(Op: Src);
5826 SrcVT = Src.getValueType();
5827 }
5828
5829 // Input and output not widened to the same size, give up.
5830 if (WidenNumElts != SrcVT.getVectorElementCount())
5831 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5832
5833 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
5834}
5835
5836SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
5837 SDLoc dl(N);
5838 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5839 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5840
5841 SDValue Src = N->getOperand(Num: 0);
5842 EVT SrcVT = Src.getValueType();
5843
5844 // Also widen the input.
5845 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5846 Src = GetWidenedVector(Op: Src);
5847 SrcVT = Src.getValueType();
5848 }
5849
5850 // Input and output not widened to the same size, give up.
5851 if (WidenNumElts != SrcVT.getVectorElementCount())
5852 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5853
5854 if (N->getNumOperands() == 1)
5855 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
5856
5857 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5858 assert(N->isVPOpcode() && "Expected VP opcode");
5859
5860 SDValue Mask =
5861 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5862 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
5863}
5864
5865SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
5866 SDValue InOp = N->getOperand(Num: 1);
5867 SDLoc DL(N);
5868 SmallVector<SDValue, 4> NewOps(N->ops());
5869
5870 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5871 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5872
5873 EVT InVT = InOp.getValueType();
5874 EVT InEltVT = InVT.getVectorElementType();
5875
5876 unsigned Opcode = N->getOpcode();
5877
5878 // FIXME: Optimizations need to be implemented here.
5879
5880 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5881 EVT EltVT = WidenVT.getVectorElementType();
5882 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
5883 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
5884 SmallVector<SDValue, 32> OpChains;
5885 // Use the original element count so we don't do more scalar opts than
5886 // necessary.
5887 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5888 for (unsigned i=0; i < MinElts; ++i) {
5889 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5890 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
5891 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
5892 }
5893 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
5894 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5895
5896 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5897}
5898
5899SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
5900 unsigned Opcode = N->getOpcode();
5901 SDValue InOp = N->getOperand(Num: 0);
5902 SDLoc DL(N);
5903
5904 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5905 EVT WidenSVT = WidenVT.getVectorElementType();
5906 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5907
5908 EVT InVT = InOp.getValueType();
5909 EVT InSVT = InVT.getVectorElementType();
5910 unsigned InVTNumElts = InVT.getVectorNumElements();
5911
5912 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5913 InOp = GetWidenedVector(Op: InOp);
5914 InVT = InOp.getValueType();
5915 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
5916 switch (Opcode) {
5917 case ISD::ANY_EXTEND_VECTOR_INREG:
5918 case ISD::SIGN_EXTEND_VECTOR_INREG:
5919 case ISD::ZERO_EXTEND_VECTOR_INREG:
5920 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
5921 }
5922 }
5923 }
5924
5925 // Unroll, extend the scalars and rebuild the vector.
5926 SmallVector<SDValue, 16> Ops;
5927 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
5928 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
5929 switch (Opcode) {
5930 case ISD::ANY_EXTEND_VECTOR_INREG:
5931 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
5932 break;
5933 case ISD::SIGN_EXTEND_VECTOR_INREG:
5934 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
5935 break;
5936 case ISD::ZERO_EXTEND_VECTOR_INREG:
5937 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
5938 break;
5939 default:
5940 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
5941 }
5942 Ops.push_back(Elt: Val);
5943 }
5944
5945 while (Ops.size() != WidenNumElts)
5946 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
5947
5948 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5949}
5950
5951SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
5952 // If this is an FCOPYSIGN with same input types, we can treat it as a
5953 // normal (can trap) binary op.
5954 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
5955 return WidenVecRes_BinaryCanTrap(N);
5956
5957 // If the types are different, fall back to unrolling.
5958 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5959 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5960}
5961
5962/// Result and first source operand are different scalar types, but must have
5963/// the same number of elements. There is an additional control argument which
5964/// should be passed through unchanged.
5965SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
5966 SDValue FpValue = N->getOperand(Num: 0);
5967 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5968 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
5969 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5970 SDValue Arg = GetWidenedVector(Op: FpValue);
5971 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
5972 Flags: N->getFlags());
5973}
5974
5975SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
5976 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5977 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5978 SDValue RHS = N->getOperand(Num: 1);
5979 EVT ExpVT = RHS.getValueType();
5980 SDValue ExpOp = RHS;
5981 if (ExpVT.isVector()) {
5982 EVT WideExpVT = WidenVT.changeVectorElementType(
5983 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
5984 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
5985 }
5986
5987 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
5988}
5989
5990SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
5991 // Unary op widening.
5992 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5993 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5994 if (N->getNumOperands() == 1)
5995 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
5996 if (N->getOpcode() == ISD::AssertNoFPClass)
5997 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
5998 N2: N->getOperand(Num: 1), Flags: N->getFlags());
5999
6000 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
6001 assert(N->isVPOpcode() && "Expected VP opcode");
6002
6003 SDValue Mask =
6004 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
6005 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
6006 Ops: {InOp, Mask, N->getOperand(Num: 2)});
6007}
6008
6009SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6010 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6011 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6012 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
6013 .getVectorElementType(),
6014 NumElements: WidenVT.getVectorNumElements());
6015 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6016 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6017 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6018}
6019
6020SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6021 unsigned ResNo) {
6022 EVT VT0 = N->getValueType(ResNo: 0);
6023 EVT VT1 = N->getValueType(ResNo: 1);
6024
6025 assert(VT0.isVector() && VT1.isVector() &&
6026 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6027 "expected both results to be vectors of matching element count");
6028
6029 LLVMContext &Ctx = *DAG.getContext();
6030 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6031
6032 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6033 ElementCount WidenEC = WidenVT.getVectorElementCount();
6034
6035 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6036 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6037
6038 SDNode *WidenNode =
6039 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6040 .getNode();
6041
6042 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6043 return SDValue(WidenNode, ResNo);
6044}
6045
6046SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6047 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6048 return GetWidenedVector(Op: WidenVec);
6049}
6050
6051SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6052 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6053 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6054 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6055
6056 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
6057 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6058 DestAS: AddrSpaceCastN->getDestAddressSpace());
6059}
6060
6061SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6062 SDValue InOp = N->getOperand(Num: 0);
6063 EVT InVT = InOp.getValueType();
6064 EVT VT = N->getValueType(ResNo: 0);
6065 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6066 SDLoc dl(N);
6067
6068 switch (getTypeAction(VT: InVT)) {
6069 case TargetLowering::TypeLegal:
6070 break;
6071 case TargetLowering::TypeScalarizeScalableVector:
6072 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6073 case TargetLowering::TypePromoteInteger: {
6074 // If the incoming type is a vector that is being promoted, then
6075 // we know that the elements are arranged differently and that we
6076 // must perform the conversion using a stack slot.
6077 if (InVT.isVector())
6078 break;
6079
6080 // If the InOp is promoted to the same size, convert it. Otherwise,
6081 // fall out of the switch and widen the promoted input.
6082 SDValue NInOp = GetPromotedInteger(Op: InOp);
6083 EVT NInVT = NInOp.getValueType();
6084 if (WidenVT.bitsEq(VT: NInVT)) {
6085 // For big endian targets we need to shift the input integer or the
6086 // interesting bits will end up at the wrong place.
6087 if (DAG.getDataLayout().isBigEndian()) {
6088 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6089 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6090 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6091 }
6092 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6093 }
6094 InOp = NInOp;
6095 InVT = NInVT;
6096 break;
6097 }
6098 case TargetLowering::TypeSoftenFloat:
6099 case TargetLowering::TypeSoftPromoteHalf:
6100 case TargetLowering::TypeExpandInteger:
6101 case TargetLowering::TypeExpandFloat:
6102 case TargetLowering::TypeScalarizeVector:
6103 case TargetLowering::TypeSplitVector:
6104 break;
6105 case TargetLowering::TypeWidenVector:
6106 // If the InOp is widened to the same size, convert it. Otherwise, fall
6107 // out of the switch and widen the widened input.
6108 InOp = GetWidenedVector(Op: InOp);
6109 InVT = InOp.getValueType();
6110 if (WidenVT.bitsEq(VT: InVT))
6111 // The input widens to the same size. Convert to the widen value.
6112 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6113 break;
6114 }
6115
6116 unsigned WidenSize = WidenVT.getSizeInBits();
6117 unsigned InSize = InVT.getSizeInBits();
6118 unsigned InScalarSize = InVT.getScalarSizeInBits();
6119 // x86mmx is not an acceptable vector element type, so don't try.
6120 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6121 // Determine new input vector type. The new input vector type will use
6122 // the same element type (if its a vector) or use the input type as a
6123 // vector. It is the same size as the type to widen to.
6124 EVT NewInVT;
6125 unsigned NewNumParts = WidenSize / InSize;
6126 if (InVT.isVector()) {
6127 EVT InEltVT = InVT.getVectorElementType();
6128 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6129 NumElements: WidenSize / InEltVT.getSizeInBits());
6130 } else {
6131 // For big endian systems, using the promoted input scalar type
6132 // to produce the scalar_to_vector would put the desired bits into
6133 // the least significant byte(s) of the wider element zero. This
6134 // will mean that the users of the result vector are using incorrect
6135 // bits. Use the original input type instead. Although either input
6136 // type can be used on little endian systems, for consistency we
6137 // use the original type there as well.
6138 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6139 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6140 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6141 }
6142
6143 if (TLI.isTypeLegal(VT: NewInVT)) {
6144 SDValue NewVec;
6145 if (InVT.isVector()) {
6146 // Because the result and the input are different vector types, widening
6147 // the result could create a legal type but widening the input might
6148 // make it an illegal type that might lead to repeatedly splitting the
6149 // input and then widening it. To avoid this, we widen the input only if
6150 // it results in a legal type.
6151 if (WidenSize % InSize == 0) {
6152 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6153 Ops[0] = InOp;
6154
6155 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6156 } else {
6157 SmallVector<SDValue, 16> Ops;
6158 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6159 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6160 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6161
6162 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6163 }
6164 } else {
6165 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6166 }
6167 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6168 }
6169 }
6170
6171 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6172}
6173
6174SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6175 return DAG.getNode(
6176 Opcode: N->getOpcode(), DL: SDLoc(N),
6177 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6178 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6179}
6180
6181SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6182 SDLoc dl(N);
6183 // Build a vector with poison for the new nodes.
6184 EVT VT = N->getValueType(ResNo: 0);
6185
6186 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6187 // type. The POISONs need to have the same type as the existing operands.
6188 EVT EltVT = N->getOperand(Num: 0).getValueType();
6189 unsigned NumElts = VT.getVectorNumElements();
6190
6191 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6192 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6193
6194 SmallVector<SDValue, 16> NewOps(N->ops());
6195 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6196 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6197
6198 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6199}
6200
6201SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6202 EVT InVT = N->getOperand(Num: 0).getValueType();
6203 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6204 SDLoc dl(N);
6205 unsigned NumOperands = N->getNumOperands();
6206
6207 bool InputWidened = false; // Indicates we need to widen the input.
6208 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6209 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6210 unsigned NumInElts = InVT.getVectorMinNumElements();
6211 if (WidenNumElts % NumInElts == 0) {
6212 // Add undef vectors to widen to correct length.
6213 unsigned NumConcat = WidenNumElts / NumInElts;
6214 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6215 SmallVector<SDValue, 16> Ops(NumConcat);
6216 for (unsigned i=0; i < NumOperands; ++i)
6217 Ops[i] = N->getOperand(Num: i);
6218 for (unsigned i = NumOperands; i != NumConcat; ++i)
6219 Ops[i] = UndefVal;
6220 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6221 }
6222 } else {
6223 InputWidened = true;
6224 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6225 // The inputs and the result are widen to the same value.
6226 unsigned i;
6227 for (i=1; i < NumOperands; ++i)
6228 if (!N->getOperand(Num: i).isUndef())
6229 break;
6230
6231 if (i == NumOperands)
6232 // Everything but the first operand is an UNDEF so just return the
6233 // widened first operand.
6234 return GetWidenedVector(Op: N->getOperand(Num: 0));
6235
6236 if (NumOperands == 2) {
6237 assert(!WidenVT.isScalableVector() &&
6238 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6239 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6240 unsigned NumInElts = InVT.getVectorNumElements();
6241
6242 // Replace concat of two operands with a shuffle.
6243 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6244 for (unsigned i = 0; i < NumInElts; ++i) {
6245 MaskOps[i] = i;
6246 MaskOps[i + NumInElts] = i + WidenNumElts;
6247 }
6248 return DAG.getVectorShuffle(VT: WidenVT, dl,
6249 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6250 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6251 Mask: MaskOps);
6252 }
6253 }
6254 }
6255
6256 assert(!WidenVT.isScalableVector() &&
6257 "Cannot use build vectors to widen CONCAT_VECTOR result");
6258 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6259 unsigned NumInElts = InVT.getVectorNumElements();
6260
6261 // Fall back to use extracts and build vector.
6262 EVT EltVT = WidenVT.getVectorElementType();
6263 SmallVector<SDValue, 16> Ops(WidenNumElts);
6264 unsigned Idx = 0;
6265 for (unsigned i=0; i < NumOperands; ++i) {
6266 SDValue InOp = N->getOperand(Num: i);
6267 if (InputWidened)
6268 InOp = GetWidenedVector(Op: InOp);
6269 for (unsigned j = 0; j < NumInElts; ++j)
6270 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6271 }
6272 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6273 for (; Idx < WidenNumElts; ++Idx)
6274 Ops[Idx] = UndefVal;
6275 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6276}
6277
6278SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6279 EVT VT = N->getValueType(ResNo: 0);
6280 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6281 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6282 SDValue InOp2 = N->getOperand(Num: 1);
6283 SDValue Idx = N->getOperand(Num: 2);
6284 SDLoc dl(N);
6285 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6286}
6287
6288SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6289 EVT VT = N->getValueType(ResNo: 0);
6290 EVT EltVT = VT.getVectorElementType();
6291 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6292 SDValue InOp = N->getOperand(Num: 0);
6293 SDValue Idx = N->getOperand(Num: 1);
6294 SDLoc dl(N);
6295
6296 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6297 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6298 InOp = GetWidenedVector(Op: InOp);
6299
6300 EVT InVT = InOp.getValueType();
6301
6302 // Check if we can just return the input vector after widening.
6303 uint64_t IdxVal = Idx->getAsZExtVal();
6304 if (IdxVal == 0 && InVT == WidenVT)
6305 return InOp;
6306
6307 // Check if we can extract from the vector.
6308 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6309 unsigned InNumElts = InVT.getVectorMinNumElements();
6310 unsigned VTNumElts = VT.getVectorMinNumElements();
6311 assert(IdxVal % VTNumElts == 0 &&
6312 "Expected Idx to be a multiple of subvector minimum vector length");
6313 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6314 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6315
6316 if (VT.isScalableVector()) {
6317 // Try to split the operation up into smaller extracts and concat the
6318 // results together, e.g.
6319 // nxv6i64 extract_subvector(nxv12i64, 6)
6320 // <->
6321 // nxv8i64 concat(
6322 // nxv2i64 extract_subvector(nxv16i64, 6)
6323 // nxv2i64 extract_subvector(nxv16i64, 8)
6324 // nxv2i64 extract_subvector(nxv16i64, 10)
6325 // undef)
6326 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6327 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6328 "down type's element count");
6329 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6330 EC: ElementCount::getScalable(MinVal: GCD));
6331 // Avoid recursion around e.g. nxv1i8.
6332 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6333 SmallVector<SDValue> Parts;
6334 unsigned I = 0;
6335 for (; I < VTNumElts / GCD; ++I)
6336 Parts.push_back(
6337 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6338 for (; I < WidenNumElts / GCD; ++I)
6339 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6340
6341 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6342 }
6343
6344 // Fallback to extracting through memory.
6345
6346 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6347 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6348 MachineFunction &MF = DAG.getMachineFunction();
6349 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6350 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6351
6352 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6353 PtrInfo, F: MachineMemOperand::MOStore,
6354 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6355 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6356 PtrInfo, F: MachineMemOperand::MOLoad,
6357 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6358
6359 // Write out the input vector.
6360 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6361
6362 // Build a mask to match the length of the non-widened result.
6363 SDValue Mask =
6364 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6365
6366 // Read back the sub-vector setting the remaining lanes to poison.
6367 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6368 return DAG.getMaskedLoad(
6369 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6370 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6371 }
6372
6373 // We could try widening the input to the right length but for now, extract
6374 // the original elements, fill the rest with undefs and build a vector.
6375 SmallVector<SDValue, 16> Ops(WidenNumElts);
6376 unsigned i;
6377 for (i = 0; i < VTNumElts; ++i)
6378 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6379
6380 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6381 for (; i < WidenNumElts; ++i)
6382 Ops[i] = UndefVal;
6383 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6384}
6385
6386SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6387 SDValue InOp = ModifyToType(
6388 InOp: N->getOperand(Num: 0),
6389 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6390 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6391 N2: N->getOperand(Num: 1));
6392}
6393
6394SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6395 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6396 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6397 VT: InOp.getValueType(), N1: InOp,
6398 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6399}
6400
6401/// Either return the same load or provide appropriate casts
6402/// from the load and return that.
6403static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6404 TypeSize LdWidth, TypeSize FirstVTWidth,
6405 SDLoc dl, SelectionDAG &DAG) {
6406 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6407 "Load width must be less than or equal to first value type width");
6408 TypeSize WidenWidth = WidenVT.getSizeInBits();
6409 if (!FirstVT.isVector()) {
6410 unsigned NumElts =
6411 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6412 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6413 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6414 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6415 }
6416 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6417 return LdOp;
6418}
6419
6420static std::optional<EVT> findMemType(SelectionDAG &DAG,
6421 const TargetLowering &TLI, unsigned Width,
6422 EVT WidenVT, unsigned Align,
6423 unsigned WidenEx);
6424
6425SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6426 EVT WidenVT =
6427 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6428 EVT LdVT = LD->getMemoryVT();
6429 SDLoc dl(LD);
6430 assert(LdVT.isVector() && WidenVT.isVector() && "Expected vectors");
6431 assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6432 "Must be scalable");
6433 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6434 "Expected equivalent element types");
6435
6436 // Load information
6437 SDValue Chain = LD->getChain();
6438 SDValue BasePtr = LD->getBasePtr();
6439
6440 TypeSize LdWidth = LdVT.getSizeInBits();
6441 TypeSize WidenWidth = WidenVT.getSizeInBits();
6442 TypeSize WidthDiff = WidenWidth - LdWidth;
6443
6444 // Find the vector type that can load from.
6445 std::optional<EVT> FirstVT =
6446 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6447 WidenEx: WidthDiff.getKnownMinValue());
6448
6449 if (!FirstVT)
6450 return SDValue();
6451
6452 SmallVector<EVT, 8> MemVTs;
6453 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6454
6455 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6456 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6457
6458 // Load the element with one instruction.
6459 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6460 FirstVTWidth, dl, DAG);
6461
6462 // Modified the chain - switch anything that used the old chain to use
6463 // the new one.
6464 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6465 return Result;
6466}
6467
6468SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6469 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6470 ISD::LoadExtType ExtType = LD->getExtensionType();
6471
6472 // A vector must always be stored in memory as-is, i.e. without any padding
6473 // between the elements, since various code depend on it, e.g. in the
6474 // handling of a bitcast of a vector type to int, which may be done with a
6475 // vector store followed by an integer load. A vector that does not have
6476 // elements that are byte-sized must therefore be stored as an integer
6477 // built out of the extracted vector elements.
6478 if (!LD->getMemoryVT().isByteSized()) {
6479 SDValue Value, NewChain;
6480 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6481 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6482 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6483 return SDValue();
6484 }
6485
6486 // Generate a vector-predicated load if it is custom/legal on the target. To
6487 // avoid possible recursion, only do this if the widened mask type is legal.
6488 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6489 // removed from the IR by the ExpandVectorPredication pass but we're
6490 // reintroducing them here.
6491 EVT VT = LD->getValueType(ResNo: 0);
6492 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6493 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6494
6495 if (ExtType == ISD::NON_EXTLOAD &&
6496 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6497 TLI.isTypeLegal(VT: WideMaskVT)) {
6498 SDLoc DL(N);
6499 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6500 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6501 EC: VT.getVectorElementCount());
6502 SDValue NewLoad =
6503 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6504 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6505 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6506
6507 // Modified the chain - switch anything that used the old chain to use
6508 // the new one.
6509 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6510
6511 return NewLoad;
6512 }
6513
6514 SDValue Result;
6515 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6516 if (ExtType != ISD::NON_EXTLOAD)
6517 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6518 else
6519 Result = GenWidenVectorLoads(LdChain, LD);
6520
6521 if (Result) {
6522 // If we generate a single load, we can use that for the chain. Otherwise,
6523 // build a factor node to remember the multiple loads are independent and
6524 // chain to that.
6525 SDValue NewChain;
6526 if (LdChain.size() == 1)
6527 NewChain = LdChain[0];
6528 else
6529 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6530
6531 // Modified the chain - switch anything that used the old chain to use
6532 // the new one.
6533 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6534
6535 return Result;
6536 }
6537
6538 if (VT.isVector()) {
6539 // If all else fails replace the load with a wide masked load.
6540 SDLoc DL(N);
6541 SDValue Mask =
6542 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6543
6544 SDValue NewLoad = DAG.getMaskedLoad(
6545 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6546 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6547 AM: LD->getAddressingMode(), LD->getExtensionType());
6548
6549 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6550 return NewLoad;
6551 }
6552
6553 report_fatal_error(reason: "Unable to widen vector load");
6554}
6555
6556SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6557 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6558 SDValue Mask = N->getMask();
6559 SDValue EVL = N->getVectorLength();
6560 ISD::LoadExtType ExtType = N->getExtensionType();
6561 SDLoc dl(N);
6562
6563 // The mask should be widened as well
6564 assert(getTypeAction(Mask.getValueType()) ==
6565 TargetLowering::TypeWidenVector &&
6566 "Unable to widen binary VP op");
6567 Mask = GetWidenedVector(Op: Mask);
6568 assert(Mask.getValueType().getVectorElementCount() ==
6569 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6570 .getVectorElementCount() &&
6571 "Unable to widen vector load");
6572
6573 SDValue Res =
6574 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6575 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6576 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6577 // Legalize the chain result - switch anything that used the old chain to
6578 // use the new one.
6579 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6580 return Res;
6581}
6582
6583SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6584 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6585 SDValue Mask = N->getMask();
6586 SDValue EVL = N->getVectorLength();
6587 SDLoc dl(N);
6588
6589 // The mask should be widened as well
6590 assert(getTypeAction(Mask.getValueType()) ==
6591 TargetLowering::TypeWidenVector &&
6592 "Unable to widen binary VP op");
6593 Mask = GetWidenedVector(Op: Mask);
6594 assert(Mask.getValueType().getVectorElementCount() ==
6595 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6596 .getVectorElementCount() &&
6597 "Unable to widen vector load");
6598
6599 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6600 Mask, EVL, MMO: N->getMemOperand());
6601 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6602 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6603 return Res;
6604}
6605
6606SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6607 SDLoc DL(N);
6608
6609 // The mask should be widened as well
6610 SDValue Mask = N->getMask();
6611 assert(getTypeAction(Mask.getValueType()) ==
6612 TargetLowering::TypeWidenVector &&
6613 "Unable to widen VP strided load");
6614 Mask = GetWidenedVector(Op: Mask);
6615
6616 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6617 assert(Mask.getValueType().getVectorElementCount() ==
6618 WidenVT.getVectorElementCount() &&
6619 "Data and mask vectors should have the same number of elements");
6620
6621 SDValue Res = DAG.getStridedLoadVP(
6622 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6623 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6624 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6625 IsExpanding: N->isExpandingLoad());
6626
6627 // Legalize the chain result - switch anything that used the old chain to
6628 // use the new one.
6629 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6630 return Res;
6631}
6632
6633SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6634 SDValue Vec = N->getOperand(Num: 0);
6635 SDValue Mask = N->getOperand(Num: 1);
6636 SDValue Passthru = N->getOperand(Num: 2);
6637 EVT WideVecVT =
6638 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6639 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6640 VT: Mask.getValueType().getVectorElementType(),
6641 EC: WideVecVT.getVectorElementCount());
6642
6643 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6644 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6645 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6646 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6647 N2: WideMask, N3: WidePassthru);
6648}
6649
6650SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6651 EVT VT = N->getValueType(ResNo: 0);
6652 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6653 SDValue Mask = N->getMask();
6654 EVT MaskVT = Mask.getValueType();
6655 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6656 ISD::LoadExtType ExtType = N->getExtensionType();
6657 SDLoc dl(N);
6658
6659 EVT WideMaskVT =
6660 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6661 EC: WidenVT.getVectorElementCount());
6662
6663 if (ExtType == ISD::NON_EXTLOAD &&
6664 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6665 TLI.isTypeLegal(VT: WideMaskVT) &&
6666 // If there is a passthru, we shouldn't use vp.load. However,
6667 // type legalizer will struggle on masked.load with
6668 // scalable vectors, so for scalable vectors, we still use vp.load
6669 // but manually merge the load result with the passthru using vp.select.
6670 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6671 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6672 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6673 EC: VT.getVectorElementCount());
6674 SDValue NewLoad =
6675 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6676 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6677 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6678 SDValue NewVal = NewLoad;
6679
6680 // Manually merge with vselect
6681 if (!N->getPassThru()->isUndef()) {
6682 assert(WidenVT.isScalableVector());
6683 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
6684 // The lanes past EVL are poison.
6685 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
6686 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
6687 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
6688 }
6689
6690 // Modified the chain - switch anything that used the old chain to use
6691 // the new one.
6692 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6693
6694 return NewVal;
6695 }
6696
6697 // The mask should be widened as well
6698 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6699
6700 SDValue Res = DAG.getMaskedLoad(
6701 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
6702 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
6703 ExtType, IsExpanding: N->isExpandingLoad());
6704 // Legalize the chain result - switch anything that used the old chain to
6705 // use the new one.
6706 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6707 return Res;
6708}
6709
6710SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
6711
6712 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6713 SDValue Mask = N->getMask();
6714 EVT MaskVT = Mask.getValueType();
6715 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6716 SDValue Scale = N->getScale();
6717 unsigned NumElts = WideVT.getVectorNumElements();
6718 SDLoc dl(N);
6719
6720 // The mask should be widened as well
6721 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6722 VT: MaskVT.getVectorElementType(),
6723 NumElements: WideVT.getVectorNumElements());
6724 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6725
6726 // Widen the Index operand
6727 SDValue Index = N->getIndex();
6728 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6729 VT: Index.getValueType().getScalarType(),
6730 NumElements: NumElts);
6731 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6732 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
6733 Scale };
6734
6735 // Widen the MemoryType
6736 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6737 VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
6738 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
6739 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
6740 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
6741
6742 // Legalize the chain result - switch anything that used the old chain to
6743 // use the new one.
6744 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6745 return Res;
6746}
6747
6748SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
6749 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6750 SDValue Mask = N->getMask();
6751 SDValue Scale = N->getScale();
6752 ElementCount WideEC = WideVT.getVectorElementCount();
6753 SDLoc dl(N);
6754
6755 SDValue Index = GetWidenedVector(Op: N->getIndex());
6756 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6757 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
6758 Mask = GetWidenedMask(Mask, EC: WideEC);
6759
6760 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
6761 Mask, N->getVectorLength()};
6762 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
6763 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
6764
6765 // Legalize the chain result - switch anything that used the old chain to
6766 // use the new one.
6767 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6768 return Res;
6769}
6770
6771SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
6772 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6773 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
6774}
6775
6776// Return true is this is a SETCC node or a strict version of it.
6777static inline bool isSETCCOp(unsigned Opcode) {
6778 switch (Opcode) {
6779 case ISD::SETCC:
6780 case ISD::STRICT_FSETCC:
6781 case ISD::STRICT_FSETCCS:
6782 return true;
6783 }
6784 return false;
6785}
6786
6787// Return true if this is a node that could have two SETCCs as operands.
6788static inline bool isLogicalMaskOp(unsigned Opcode) {
6789 switch (Opcode) {
6790 case ISD::AND:
6791 case ISD::OR:
6792 case ISD::XOR:
6793 return true;
6794 }
6795 return false;
6796}
6797
6798// If N is a SETCC or a strict variant of it, return the type
6799// of the compare operands.
6800static inline EVT getSETCCOperandType(SDValue N) {
6801 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
6802 return N->getOperand(Num: OpNo).getValueType();
6803}
6804
6805// This is used just for the assert in convertMask(). Check that this either
6806// a SETCC or a previously handled SETCC by convertMask().
6807#ifndef NDEBUG
6808static inline bool isSETCCorConvertedSETCC(SDValue N) {
6809 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
6810 N = N.getOperand(0);
6811 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
6812 for (unsigned i = 1; i < N->getNumOperands(); ++i)
6813 if (!N->getOperand(i)->isUndef())
6814 return false;
6815 N = N.getOperand(0);
6816 }
6817
6818 if (N.getOpcode() == ISD::TRUNCATE)
6819 N = N.getOperand(0);
6820 else if (N.getOpcode() == ISD::SIGN_EXTEND)
6821 N = N.getOperand(0);
6822
6823 if (isLogicalMaskOp(N.getOpcode()))
6824 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
6825 isSETCCorConvertedSETCC(N.getOperand(1));
6826
6827 return (isSETCCOp(N.getOpcode()) ||
6828 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
6829}
6830#endif
6831
6832// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
6833// to ToMaskVT if needed with vector extension or truncation.
6834SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
6835 EVT ToMaskVT) {
6836 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
6837 // FIXME: This code seems to be too restrictive, we might consider
6838 // generalizing it or dropping it.
6839 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
6840
6841 // Make a new Mask node, with a legal result VT.
6842 SDValue Mask;
6843 SmallVector<SDValue, 4> Ops;
6844 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
6845 Ops.push_back(Elt: InMask->getOperand(Num: i));
6846 if (InMask->isStrictFPOpcode()) {
6847 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
6848 ResultTys: { MaskVT, MVT::Other }, Ops);
6849 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
6850 }
6851 else
6852 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
6853 Flags: InMask->getFlags());
6854
6855 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
6856 // extend or truncate is needed.
6857 LLVMContext &Ctx = *DAG.getContext();
6858 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
6859 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
6860 if (MaskScalarBits < ToMaskScalBits) {
6861 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6862 NumElements: MaskVT.getVectorNumElements());
6863 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
6864 } else if (MaskScalarBits > ToMaskScalBits) {
6865 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6866 NumElements: MaskVT.getVectorNumElements());
6867 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
6868 }
6869
6870 assert(Mask->getValueType(0).getScalarSizeInBits() ==
6871 ToMaskVT.getScalarSizeInBits() &&
6872 "Mask should have the right element size by now.");
6873
6874 // Adjust Mask to the right number of elements.
6875 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
6876 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
6877 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
6878 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
6879 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
6880 EVT SubVT = Mask->getValueType(ResNo: 0);
6881 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
6882 SubOps[0] = Mask;
6883 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
6884 }
6885
6886 assert((Mask->getValueType(0) == ToMaskVT) &&
6887 "A mask of ToMaskVT should have been produced by now.");
6888
6889 return Mask;
6890}
6891
6892// This method tries to handle some special cases for the vselect mask
6893// and if needed adjusting the mask vector type to match that of the VSELECT.
6894// Without it, many cases end up with scalarization of the SETCC, with many
6895// unnecessary instructions.
6896SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
6897 LLVMContext &Ctx = *DAG.getContext();
6898 SDValue Cond = N->getOperand(Num: 0);
6899
6900 if (N->getOpcode() != ISD::VSELECT)
6901 return SDValue();
6902
6903 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
6904 return SDValue();
6905
6906 // If this is a splitted VSELECT that was previously already handled, do
6907 // nothing.
6908 EVT CondVT = Cond->getValueType(ResNo: 0);
6909 if (CondVT.getScalarSizeInBits() != 1)
6910 return SDValue();
6911
6912 EVT VSelVT = N->getValueType(ResNo: 0);
6913
6914 // This method can't handle scalable vector types.
6915 // FIXME: This support could be added in the future.
6916 if (VSelVT.isScalableVector())
6917 return SDValue();
6918
6919 // Only handle vector types which are a power of 2.
6920 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
6921 return SDValue();
6922
6923 // Don't touch if this will be scalarized.
6924 EVT FinalVT = VSelVT;
6925 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
6926 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
6927
6928 if (FinalVT.getVectorNumElements() == 1)
6929 return SDValue();
6930
6931 // If there is support for an i1 vector mask, don't touch.
6932 if (isSETCCOp(Opcode: Cond.getOpcode())) {
6933 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
6934 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
6935 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
6936 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
6937 if (SetCCResVT.getScalarSizeInBits() == 1)
6938 return SDValue();
6939 } else if (CondVT.getScalarType() == MVT::i1) {
6940 // If there is support for an i1 vector mask (or only scalar i1 conditions),
6941 // don't touch.
6942 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
6943 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
6944
6945 if (CondVT.getScalarType() == MVT::i1)
6946 return SDValue();
6947 }
6948
6949 // Widen the vselect result type if needed.
6950 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
6951 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
6952
6953 // The mask of the VSELECT should have integer elements.
6954 EVT ToMaskVT = VSelVT;
6955 if (!ToMaskVT.getScalarType().isInteger())
6956 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
6957
6958 SDValue Mask;
6959 if (isSETCCOp(Opcode: Cond->getOpcode())) {
6960 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
6961 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6962 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
6963 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
6964 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
6965 // Cond is (AND/OR/XOR (SETCC, SETCC))
6966 SDValue SETCC0 = Cond->getOperand(Num: 0);
6967 SDValue SETCC1 = Cond->getOperand(Num: 1);
6968 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
6969 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
6970 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
6971 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
6972 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
6973 EVT MaskVT;
6974 // If the two SETCCs have different VTs, either extend/truncate one of
6975 // them to the other "towards" ToMaskVT, or truncate one and extend the
6976 // other to ToMaskVT.
6977 if (ScalarBits0 != ScalarBits1) {
6978 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
6979 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
6980 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
6981 MaskVT = WideVT;
6982 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
6983 MaskVT = NarrowVT;
6984 else
6985 MaskVT = ToMaskVT;
6986 } else
6987 // If the two SETCCs have the same VT, don't change it.
6988 MaskVT = VT0;
6989
6990 // Make new SETCCs and logical nodes.
6991 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
6992 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
6993 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
6994
6995 // Convert the logical op for VSELECT if needed.
6996 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6997 } else
6998 return SDValue();
6999
7000 return Mask;
7001}
7002
7003SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7004 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7005 ElementCount WidenEC = WidenVT.getVectorElementCount();
7006
7007 SDValue Cond1 = N->getOperand(Num: 0);
7008 EVT CondVT = Cond1.getValueType();
7009 unsigned Opcode = N->getOpcode();
7010 if (CondVT.isVector()) {
7011 if (SDValue WideCond = WidenVSELECTMask(N)) {
7012 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7013 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7014 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7015 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7016 }
7017
7018 EVT CondEltVT = CondVT.getVectorElementType();
7019 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7020 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7021 Cond1 = GetWidenedVector(Op: Cond1);
7022
7023 // If we have to split the condition there is no point in widening the
7024 // select. This would result in an cycle of widening the select ->
7025 // widening the condition operand -> splitting the condition operand ->
7026 // splitting the select -> widening the select. Instead split this select
7027 // further and widen the resulting type.
7028 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7029 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7030 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7031 return Res;
7032 }
7033
7034 if (Cond1.getValueType() != CondWidenVT)
7035 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7036 }
7037
7038 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7039 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7040 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7041 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
7042 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7043 N4: N->getOperand(Num: 3));
7044 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7045}
7046
7047SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7048 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7049 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7050 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7051 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7052 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7053}
7054
7055SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7056 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7057 return DAG.getUNDEF(VT: WidenVT);
7058}
7059
7060SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7061 EVT VT = N->getValueType(ResNo: 0);
7062 SDLoc dl(N);
7063
7064 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7065 unsigned NumElts = VT.getVectorNumElements();
7066 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7067
7068 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7069 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7070
7071 // Adjust mask based on new input vector length.
7072 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7073 for (unsigned i = 0; i != NumElts; ++i) {
7074 int Idx = N->getMaskElt(Idx: i);
7075 if (Idx < (int)NumElts)
7076 NewMask[i] = Idx;
7077 else
7078 NewMask[i] = Idx - NumElts + WidenNumElts;
7079 }
7080 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7081}
7082
7083SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7084 EVT VT = N->getValueType(ResNo: 0);
7085 EVT EltVT = VT.getVectorElementType();
7086 SDLoc dl(N);
7087
7088 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7089 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7090 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7091
7092 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7093 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7094 unsigned VTNumElts = VT.getVectorMinNumElements();
7095 unsigned IdxVal = WidenNumElts - VTNumElts;
7096
7097 if (VT.isScalableVector()) {
7098 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7099 // results together, e.g.(nxv6i64 -> nxv8i64)
7100 // nxv8i64 vector_reverse
7101 // <->
7102 // nxv8i64 concat(
7103 // nxv2i64 extract_subvector(nxv8i64, 2)
7104 // nxv2i64 extract_subvector(nxv8i64, 4)
7105 // nxv2i64 extract_subvector(nxv8i64, 6)
7106 // nxv2i64 undef)
7107
7108 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7109 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7110 EC: ElementCount::getScalable(MinVal: GCD));
7111 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7112 "down type's element count");
7113 SmallVector<SDValue> Parts;
7114 unsigned i = 0;
7115 for (; i < VTNumElts / GCD; ++i)
7116 Parts.push_back(
7117 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7118 for (; i < WidenNumElts / GCD; ++i)
7119 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7120
7121 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7122 }
7123
7124 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7125 // fixed-vectors.
7126 SmallVector<int, 16> Mask(WidenNumElts, -1);
7127 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7128
7129 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7130 Mask);
7131}
7132
7133SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7134 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7135 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7136}
7137
7138SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7139 assert(N->getValueType(0).isVector() &&
7140 N->getOperand(0).getValueType().isVector() &&
7141 "Operands must be vectors");
7142 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7143 ElementCount WidenEC = WidenVT.getVectorElementCount();
7144
7145 SDValue InOp1 = N->getOperand(Num: 0);
7146 EVT InVT = InOp1.getValueType();
7147 assert(InVT.isVector() && "can not widen non-vector type");
7148 EVT WidenInVT =
7149 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7150
7151 // The input and output types often differ here, and it could be that while
7152 // we'd prefer to widen the result type, the input operands have been split.
7153 // In this case, we also need to split the result of this node as well.
7154 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7155 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7156 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7157 return Res;
7158 }
7159
7160 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7161 SDValue InOp2 = N->getOperand(Num: 1);
7162 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7163 InOp1 = GetWidenedVector(Op: InOp1);
7164 InOp2 = GetWidenedVector(Op: InOp2);
7165 } else {
7166 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7167 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7168 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7169 N2: InOp1, N3: ZeroIdx);
7170 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7171 N2: InOp2, N3: ZeroIdx);
7172 }
7173
7174 // Assume that the input and output will be widen appropriately. If not,
7175 // we will have to unroll it at some point.
7176 assert(InOp1.getValueType() == WidenInVT &&
7177 InOp2.getValueType() == WidenInVT &&
7178 "Input not widened to expected type!");
7179 (void)WidenInVT;
7180 if (N->getOpcode() == ISD::VP_SETCC) {
7181 SDValue Mask =
7182 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7183 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7184 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7185 }
7186 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7187 N3: N->getOperand(Num: 2));
7188}
7189
7190SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7191 assert(N->getValueType(0).isVector() &&
7192 N->getOperand(1).getValueType().isVector() &&
7193 "Operands must be vectors");
7194 EVT VT = N->getValueType(ResNo: 0);
7195 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7196 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7197 unsigned NumElts = VT.getVectorNumElements();
7198 EVT EltVT = VT.getVectorElementType();
7199
7200 SDLoc dl(N);
7201 SDValue Chain = N->getOperand(Num: 0);
7202 SDValue LHS = N->getOperand(Num: 1);
7203 SDValue RHS = N->getOperand(Num: 2);
7204 SDValue CC = N->getOperand(Num: 3);
7205 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7206
7207 // Fully unroll and reassemble.
7208 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7209 SmallVector<SDValue, 8> Chains(NumElts);
7210 for (unsigned i = 0; i != NumElts; ++i) {
7211 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7212 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7213
7214 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7215 Ops: {Chain, LHSElem, RHSElem, CC});
7216 Chains[i] = Scalars[i].getValue(R: 1);
7217 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7218 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7219 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7220 }
7221
7222 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7223 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7224
7225 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7226}
7227
7228//===----------------------------------------------------------------------===//
7229// Widen Vector Operand
7230//===----------------------------------------------------------------------===//
7231bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7232 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7233 SDValue Res = SDValue();
7234
7235 // See if the target wants to custom widen this node.
7236 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7237 return false;
7238
7239 switch (N->getOpcode()) {
7240 default:
7241#ifndef NDEBUG
7242 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7243 N->dump(&DAG);
7244 dbgs() << "\n";
7245#endif
7246 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7247
7248 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7249 case ISD::FAKE_USE:
7250 Res = WidenVecOp_FAKE_USE(N);
7251 break;
7252 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7253 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7254 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7255 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7256 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7257 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7258 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7259 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7260 break;
7261 case ISD::ANY_EXTEND_VECTOR_INREG:
7262 case ISD::SIGN_EXTEND_VECTOR_INREG:
7263 case ISD::ZERO_EXTEND_VECTOR_INREG:
7264 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7265 break;
7266 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7267 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7268 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7269 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7270 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7271 case ISD::STRICT_FSETCC:
7272 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7273 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7274 case ISD::FLDEXP:
7275 case ISD::FCOPYSIGN:
7276 case ISD::LROUND:
7277 case ISD::LLROUND:
7278 case ISD::LRINT:
7279 case ISD::LLRINT:
7280 Res = WidenVecOp_UnrollVectorOp(N);
7281 break;
7282 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7283
7284 case ISD::ANY_EXTEND:
7285 case ISD::SIGN_EXTEND:
7286 case ISD::ZERO_EXTEND:
7287 Res = WidenVecOp_EXTEND(N);
7288 break;
7289
7290 case ISD::SCMP:
7291 case ISD::UCMP:
7292 Res = WidenVecOp_CMP(N);
7293 break;
7294
7295 case ISD::FP_EXTEND:
7296 case ISD::STRICT_FP_EXTEND:
7297 case ISD::FP_ROUND:
7298 case ISD::STRICT_FP_ROUND:
7299 case ISD::FP_TO_SINT:
7300 case ISD::STRICT_FP_TO_SINT:
7301 case ISD::FP_TO_UINT:
7302 case ISD::STRICT_FP_TO_UINT:
7303 case ISD::SINT_TO_FP:
7304 case ISD::STRICT_SINT_TO_FP:
7305 case ISD::UINT_TO_FP:
7306 case ISD::STRICT_UINT_TO_FP:
7307 case ISD::TRUNCATE:
7308 case ISD::CONVERT_FROM_ARBITRARY_FP:
7309 Res = WidenVecOp_Convert(N);
7310 break;
7311
7312 case ISD::FP_TO_SINT_SAT:
7313 case ISD::FP_TO_UINT_SAT:
7314 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7315 break;
7316
7317 case ISD::VECREDUCE_FADD:
7318 case ISD::VECREDUCE_FMUL:
7319 case ISD::VECREDUCE_ADD:
7320 case ISD::VECREDUCE_MUL:
7321 case ISD::VECREDUCE_AND:
7322 case ISD::VECREDUCE_OR:
7323 case ISD::VECREDUCE_XOR:
7324 case ISD::VECREDUCE_SMAX:
7325 case ISD::VECREDUCE_SMIN:
7326 case ISD::VECREDUCE_UMAX:
7327 case ISD::VECREDUCE_UMIN:
7328 case ISD::VECREDUCE_FMAX:
7329 case ISD::VECREDUCE_FMIN:
7330 case ISD::VECREDUCE_FMAXIMUM:
7331 case ISD::VECREDUCE_FMINIMUM:
7332 Res = WidenVecOp_VECREDUCE(N);
7333 break;
7334 case ISD::VECREDUCE_SEQ_FADD:
7335 case ISD::VECREDUCE_SEQ_FMUL:
7336 Res = WidenVecOp_VECREDUCE_SEQ(N);
7337 break;
7338 case ISD::VP_REDUCE_FADD:
7339 case ISD::VP_REDUCE_SEQ_FADD:
7340 case ISD::VP_REDUCE_FMUL:
7341 case ISD::VP_REDUCE_SEQ_FMUL:
7342 case ISD::VP_REDUCE_ADD:
7343 case ISD::VP_REDUCE_MUL:
7344 case ISD::VP_REDUCE_AND:
7345 case ISD::VP_REDUCE_OR:
7346 case ISD::VP_REDUCE_XOR:
7347 case ISD::VP_REDUCE_SMAX:
7348 case ISD::VP_REDUCE_SMIN:
7349 case ISD::VP_REDUCE_UMAX:
7350 case ISD::VP_REDUCE_UMIN:
7351 case ISD::VP_REDUCE_FMAX:
7352 case ISD::VP_REDUCE_FMIN:
7353 case ISD::VP_REDUCE_FMAXIMUM:
7354 case ISD::VP_REDUCE_FMINIMUM:
7355 Res = WidenVecOp_VP_REDUCE(N);
7356 break;
7357 case ISD::VP_CTTZ_ELTS:
7358 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7359 Res = WidenVecOp_VP_CttzElements(N);
7360 break;
7361 case ISD::VECTOR_FIND_LAST_ACTIVE:
7362 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7363 break;
7364 }
7365
7366 // If Res is null, the sub-method took care of registering the result.
7367 if (!Res.getNode()) return false;
7368
7369 // If the result is N, the sub-method updated N in place. Tell the legalizer
7370 // core about this.
7371 if (Res.getNode() == N)
7372 return true;
7373
7374
7375 if (N->isStrictFPOpcode())
7376 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7377 "Invalid operand expansion");
7378 else
7379 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7380 "Invalid operand expansion");
7381
7382 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7383 return false;
7384}
7385
7386SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7387 SDLoc DL(N);
7388 EVT VT = N->getValueType(ResNo: 0);
7389
7390 SDValue InOp = N->getOperand(Num: 0);
7391 assert(getTypeAction(InOp.getValueType()) ==
7392 TargetLowering::TypeWidenVector &&
7393 "Unexpected type action");
7394 InOp = GetWidenedVector(Op: InOp);
7395 assert(VT.getVectorNumElements() <
7396 InOp.getValueType().getVectorNumElements() &&
7397 "Input wasn't widened!");
7398
7399 // We may need to further widen the operand until it has the same total
7400 // vector size as the result.
7401 EVT InVT = InOp.getValueType();
7402 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7403 EVT InEltVT = InVT.getVectorElementType();
7404 for (EVT FixedVT : MVT::vector_valuetypes()) {
7405 EVT FixedEltVT = FixedVT.getVectorElementType();
7406 if (TLI.isTypeLegal(VT: FixedVT) &&
7407 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7408 FixedEltVT == InEltVT) {
7409 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7410 "Not enough elements in the fixed type for the operand!");
7411 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7412 "We can't have the same type as we started with!");
7413 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7414 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7415 else
7416 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7417 break;
7418 }
7419 }
7420 InVT = InOp.getValueType();
7421 if (InVT.getSizeInBits() != VT.getSizeInBits())
7422 // We couldn't find a legal vector type that was a widening of the input
7423 // and could be extended in-register to the result type, so we have to
7424 // scalarize.
7425 return WidenVecOp_Convert(N);
7426 }
7427
7428 // Use special DAG nodes to represent the operation of extending the
7429 // low lanes.
7430 switch (N->getOpcode()) {
7431 default:
7432 llvm_unreachable("Extend legalization on extend operation!");
7433 case ISD::ANY_EXTEND:
7434 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7435 case ISD::SIGN_EXTEND:
7436 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7437 case ISD::ZERO_EXTEND:
7438 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7439 }
7440}
7441
7442SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7443 SDLoc dl(N);
7444
7445 EVT OpVT = N->getOperand(Num: 0).getValueType();
7446 EVT ResVT = N->getValueType(ResNo: 0);
7447 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7448 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7449
7450 // 1. EXTRACT_SUBVECTOR
7451 // 2. SIGN_EXTEND/ZERO_EXTEND
7452 // 3. CMP
7453 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7454 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7455
7456 // At this point the result type is guaranteed to be valid, so we can use it
7457 // as the operand type by extending it appropriately
7458 ISD::NodeType ExtendOpcode =
7459 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7460 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7461 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7462
7463 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7464}
7465
7466SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7467 // The result (and first input) is legal, but the second input is illegal.
7468 // We can't do much to fix that, so just unroll and let the extracts off of
7469 // the second input be widened as needed later.
7470 return DAG.UnrollVectorOp(N);
7471}
7472
7473SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7474 SDLoc DL(N);
7475 EVT ResultVT = N->getValueType(ResNo: 0);
7476 SDValue Test = N->getOperand(Num: 1);
7477 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7478
7479 // Process this node similarly to SETCC.
7480 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7481 if (ResultVT.getScalarType() == MVT::i1)
7482 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7483 NumElements: WideResultVT.getVectorNumElements());
7484
7485 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7486 Ops: {WideArg, Test}, Flags: N->getFlags());
7487
7488 // Extract the needed results from the result vector.
7489 EVT ResVT =
7490 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7491 NumElements: ResultVT.getVectorNumElements());
7492 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7493
7494 EVT OpVT = N->getOperand(Num: 0).getValueType();
7495 ISD::NodeType ExtendCode =
7496 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7497 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7498}
7499
7500SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7501 // Since the result is legal and the input is illegal.
7502 EVT VT = N->getValueType(ResNo: 0);
7503 EVT EltVT = VT.getVectorElementType();
7504 SDLoc dl(N);
7505 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7506 assert(getTypeAction(InOp.getValueType()) ==
7507 TargetLowering::TypeWidenVector &&
7508 "Unexpected type action");
7509 InOp = GetWidenedVector(Op: InOp);
7510 EVT InVT = InOp.getValueType();
7511 unsigned Opcode = N->getOpcode();
7512
7513 // See if a widened result type would be legal, if so widen the node.
7514 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7515 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7516 EC: InVT.getVectorElementCount());
7517 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7518 SDValue Res;
7519 if (N->isStrictFPOpcode()) {
7520 if (Opcode == ISD::STRICT_FP_ROUND)
7521 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7522 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7523 else
7524 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7525 Ops: { N->getOperand(Num: 0), InOp });
7526 // Legalize the chain result - switch anything that used the old chain to
7527 // use the new one.
7528 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7529 } else {
7530 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7531 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
7532 else
7533 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
7534 }
7535 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7536 }
7537
7538 EVT InEltVT = InVT.getVectorElementType();
7539
7540 // Unroll the convert into some scalar code and create a nasty build vector.
7541 unsigned NumElts = VT.getVectorNumElements();
7542 SmallVector<SDValue, 16> Ops(NumElts);
7543 if (N->isStrictFPOpcode()) {
7544 SmallVector<SDValue, 4> NewOps(N->ops());
7545 SmallVector<SDValue, 32> OpChains;
7546 for (unsigned i=0; i < NumElts; ++i) {
7547 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7548 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7549 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7550 }
7551 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7552 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7553 } else {
7554 for (unsigned i = 0; i < NumElts; ++i) {
7555 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7556 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7557 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT, N1: Elt, N2: N->getOperand(Num: 1));
7558 else
7559 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT, Operand: Elt);
7560 }
7561 }
7562
7563 return DAG.getBuildVector(VT, DL: dl, Ops);
7564}
7565
7566SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7567 EVT DstVT = N->getValueType(ResNo: 0);
7568 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7569 EVT SrcVT = Src.getValueType();
7570 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7571 SDLoc dl(N);
7572
7573 // See if a widened result type would be legal, if so widen the node.
7574 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7575 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7576 if (TLI.isTypeLegal(VT: WideDstVT)) {
7577 SDValue Res =
7578 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7579 return DAG.getNode(
7580 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7581 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7582 }
7583
7584 // Give up and unroll.
7585 return DAG.UnrollVectorOp(N);
7586}
7587
7588SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7589 EVT VT = N->getValueType(ResNo: 0);
7590 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7591 EVT InWidenVT = InOp.getValueType();
7592 SDLoc dl(N);
7593
7594 // Check if we can convert between two legal vector types and extract.
7595 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7596 TypeSize Size = VT.getSizeInBits();
7597 // x86mmx is not an acceptable vector element type, so don't try.
7598 if (!VT.isVector() && VT != MVT::x86mmx &&
7599 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7600 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7601 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7602 if (TLI.isTypeLegal(VT: NewVT)) {
7603 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7604 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7605 }
7606 }
7607
7608 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7609 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7610 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7611 // having to copy via memory.
7612 if (VT.isVector()) {
7613 EVT EltVT = VT.getVectorElementType();
7614 unsigned EltSize = EltVT.getFixedSizeInBits();
7615 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7616 ElementCount NewNumElts =
7617 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
7618 .divideCoefficientBy(RHS: EltSize);
7619 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
7620 if (TLI.isTypeLegal(VT: NewVT)) {
7621 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7622 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
7623 }
7624 }
7625 }
7626
7627 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
7628}
7629
7630// Vectors with sizes that are not powers of 2 need to be widened to the
7631// next largest power of 2. For example, we may get a vector of 3 32-bit
7632// integers or of 6 16-bit integers, both of which have to be widened to a
7633// 128-bit vector.
7634SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
7635 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
7636 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
7637 N2: WidenedOp);
7638}
7639
7640SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
7641 EVT VT = N->getValueType(ResNo: 0);
7642 EVT EltVT = VT.getVectorElementType();
7643 EVT InVT = N->getOperand(Num: 0).getValueType();
7644 SDLoc dl(N);
7645
7646 // If the widen width for this operand is the same as the width of the concat
7647 // and all but the first operand is undef, just use the widened operand.
7648 unsigned NumOperands = N->getNumOperands();
7649 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
7650 unsigned i;
7651 for (i = 1; i < NumOperands; ++i)
7652 if (!N->getOperand(Num: i).isUndef())
7653 break;
7654
7655 if (i == NumOperands)
7656 return GetWidenedVector(Op: N->getOperand(Num: 0));
7657 }
7658
7659 // Otherwise, fall back to a nasty build vector.
7660 unsigned NumElts = VT.getVectorNumElements();
7661 SmallVector<SDValue, 16> Ops(NumElts);
7662
7663 unsigned NumInElts = InVT.getVectorNumElements();
7664
7665 unsigned Idx = 0;
7666 for (unsigned i=0; i < NumOperands; ++i) {
7667 SDValue InOp = N->getOperand(Num: i);
7668 assert(getTypeAction(InOp.getValueType()) ==
7669 TargetLowering::TypeWidenVector &&
7670 "Unexpected type action");
7671 InOp = GetWidenedVector(Op: InOp);
7672 for (unsigned j = 0; j < NumInElts; ++j)
7673 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
7674 }
7675 return DAG.getBuildVector(VT, DL: dl, Ops);
7676}
7677
7678SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
7679 EVT VT = N->getValueType(ResNo: 0);
7680 SDValue SubVec = N->getOperand(Num: 1);
7681 SDValue InVec = N->getOperand(Num: 0);
7682
7683 EVT OrigVT = SubVec.getValueType();
7684 SubVec = GetWidenedVector(Op: SubVec);
7685 EVT SubVT = SubVec.getValueType();
7686
7687 // Whether or not all the elements of the widened SubVec will be inserted into
7688 // valid indices of VT.
7689 bool IndicesValid = false;
7690 // If we statically know that VT can fit SubVT, the indices are valid.
7691 if (VT.knownBitsGE(VT: SubVT))
7692 IndicesValid = true;
7693 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
7694 // Otherwise, if we're inserting a fixed vector into a scalable vector and
7695 // we know the minimum vscale we can work out if it's valid ourselves.
7696 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
7697 Kind: Attribute::VScaleRange);
7698 if (Attr.isValid()) {
7699 unsigned VScaleMin = Attr.getVScaleRangeMin();
7700 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
7701 SubVT.getFixedSizeInBits())
7702 IndicesValid = true;
7703 }
7704 }
7705
7706 if (!IndicesValid)
7707 report_fatal_error(
7708 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
7709
7710 SDLoc DL(N);
7711
7712 // We need to make sure that the indices are still valid, otherwise we might
7713 // widen what was previously well-defined to something undefined.
7714 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
7715 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
7716 N3: N->getOperand(Num: 2));
7717
7718 if (OrigVT.isScalableVector()) {
7719 // When the widened types match, overwriting the start of a vector is
7720 // effectively a merge operation that can be implement as a vselect.
7721 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
7722 SDValue Mask =
7723 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
7724 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
7725 }
7726
7727 // Fallback to inserting through memory.
7728 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
7729 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
7730 MachineFunction &MF = DAG.getMachineFunction();
7731 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7732 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
7733
7734 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
7735 PtrInfo, F: MachineMemOperand::MOStore,
7736 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7737 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
7738 PtrInfo, F: MachineMemOperand::MOLoad,
7739 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7740
7741 // Write out the vector being inserting into.
7742 SDValue Ch =
7743 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
7744
7745 // Build a mask to match the length of the sub-vector.
7746 SDValue Mask =
7747 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
7748
7749 // Overwrite the sub-vector at the required offset.
7750 SDValue SubVecPtr =
7751 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
7752 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
7753 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
7754 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
7755
7756 // Read back the result.
7757 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
7758 }
7759
7760 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
7761 // with a series of INSERT_VECTOR_ELT
7762 unsigned Idx = N->getConstantOperandVal(Num: 2);
7763
7764 SDValue InsertElt = InVec;
7765 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
7766 SDValue ExtractElt =
7767 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
7768 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
7769 }
7770
7771 return InsertElt;
7772}
7773
7774SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
7775 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7776 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
7777 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7778}
7779
7780SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
7781 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7782 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
7783 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7784}
7785
7786SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
7787 SDLoc DL(N);
7788 EVT ResVT = N->getValueType(ResNo: 0);
7789
7790 // Widen the input as requested by the legalizer.
7791 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7792 EVT WideInVT = WideInOp.getValueType();
7793
7794 // Simple case: if widened input is still smaller than or equal to result,
7795 // just use it directly.
7796 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
7797 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
7798
7799 // EXTEND_VECTOR_INREG requires input bits <= result bits.
7800 // If widening makes the input larger than the original result, widen the
7801 // result to match, then extract back down.
7802 EVT ResEltVT = ResVT.getVectorElementType();
7803 unsigned EltBits = ResEltVT.getSizeInBits();
7804 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
7805 "Widened input size must be a multiple of result element size");
7806
7807 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
7808 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
7809
7810 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
7811 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
7812}
7813
7814SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
7815 // We have to widen the value, but we want only to store the original
7816 // vector type.
7817 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
7818
7819 if (!ST->getMemoryVT().getScalarType().isByteSized())
7820 return TLI.scalarizeVectorStore(ST, DAG);
7821
7822 if (ST->isTruncatingStore())
7823 return TLI.scalarizeVectorStore(ST, DAG);
7824
7825 // Generate a vector-predicated store if it is custom/legal on the target.
7826 // To avoid possible recursion, only do this if the widened mask type is
7827 // legal.
7828 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
7829 // removed from the IR by the ExpandVectorPredication pass but we're
7830 // reintroducing them here.
7831 SDValue StVal = ST->getValue();
7832 EVT StVT = StVal.getValueType();
7833 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
7834 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
7835
7836 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7837 TLI.isTypeLegal(VT: WideMaskVT)) {
7838 // Widen the value.
7839 SDLoc DL(N);
7840 StVal = GetWidenedVector(Op: StVal);
7841 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
7842 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
7843 EC: StVT.getVectorElementCount());
7844 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
7845 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
7846 AM: ST->getAddressingMode());
7847 }
7848
7849 SmallVector<SDValue, 16> StChain;
7850 if (GenWidenVectorStores(StChain, ST)) {
7851 if (StChain.size() == 1)
7852 return StChain[0];
7853
7854 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
7855 }
7856
7857 if (StVT.isVector()) {
7858 // If all else fails replace the store with a wide masked store.
7859 SDLoc DL(N);
7860 SDValue WideStVal = GetWidenedVector(Op: StVal);
7861 SDValue Mask =
7862 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
7863
7864 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
7865 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
7866 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
7867 IsTruncating: ST->isTruncatingStore());
7868 }
7869
7870 report_fatal_error(reason: "Unable to widen vector store");
7871}
7872
7873SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
7874 assert((OpNo == 1 || OpNo == 3) &&
7875 "Can widen only data or mask operand of vp_store");
7876 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
7877 SDValue Mask = ST->getMask();
7878 SDValue StVal = ST->getValue();
7879 SDLoc dl(N);
7880
7881 if (OpNo == 1) {
7882 // Widen the value.
7883 StVal = GetWidenedVector(Op: StVal);
7884
7885 // We only handle the case where the mask needs widening to an
7886 // identically-sized type as the vector inputs.
7887 assert(getTypeAction(Mask.getValueType()) ==
7888 TargetLowering::TypeWidenVector &&
7889 "Unable to widen VP store");
7890 Mask = GetWidenedVector(Op: Mask);
7891 } else {
7892 Mask = GetWidenedVector(Op: Mask);
7893
7894 // We only handle the case where the stored value needs widening to an
7895 // identically-sized type as the mask.
7896 assert(getTypeAction(StVal.getValueType()) ==
7897 TargetLowering::TypeWidenVector &&
7898 "Unable to widen VP store");
7899 StVal = GetWidenedVector(Op: StVal);
7900 }
7901
7902 assert(Mask.getValueType().getVectorElementCount() ==
7903 StVal.getValueType().getVectorElementCount() &&
7904 "Mask and data vectors should have the same number of elements");
7905 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
7906 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
7907 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
7908 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
7909 IsCompressing: ST->isCompressingStore());
7910}
7911
7912SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
7913 unsigned OpNo) {
7914 assert((OpNo == 1 || OpNo == 4) &&
7915 "Can widen only data or mask operand of vp_strided_store");
7916 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
7917 SDValue Mask = SST->getMask();
7918 SDValue StVal = SST->getValue();
7919 SDLoc DL(N);
7920
7921 if (OpNo == 1)
7922 assert(getTypeAction(Mask.getValueType()) ==
7923 TargetLowering::TypeWidenVector &&
7924 "Unable to widen VP strided store");
7925 else
7926 assert(getTypeAction(StVal.getValueType()) ==
7927 TargetLowering::TypeWidenVector &&
7928 "Unable to widen VP strided store");
7929
7930 StVal = GetWidenedVector(Op: StVal);
7931 Mask = GetWidenedVector(Op: Mask);
7932
7933 assert(StVal.getValueType().getVectorElementCount() ==
7934 Mask.getValueType().getVectorElementCount() &&
7935 "Data and mask vectors should have the same number of elements");
7936
7937 return DAG.getStridedStoreVP(
7938 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
7939 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
7940 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
7941 IsCompressing: SST->isCompressingStore());
7942}
7943
7944SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
7945 assert((OpNo == 1 || OpNo == 4) &&
7946 "Can widen only data or mask operand of mstore");
7947 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
7948 SDValue Mask = MST->getMask();
7949 EVT MaskVT = Mask.getValueType();
7950 SDValue StVal = MST->getValue();
7951 EVT VT = StVal.getValueType();
7952 SDLoc dl(N);
7953
7954 EVT WideVT, WideMaskVT;
7955 if (OpNo == 1) {
7956 // Widen the value.
7957 StVal = GetWidenedVector(Op: StVal);
7958
7959 WideVT = StVal.getValueType();
7960 WideMaskVT =
7961 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7962 EC: WideVT.getVectorElementCount());
7963 } else {
7964 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
7965
7966 EVT ValueVT = StVal.getValueType();
7967 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
7968 EC: WideMaskVT.getVectorElementCount());
7969 }
7970
7971 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7972 TLI.isTypeLegal(VT: WideMaskVT)) {
7973 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7974 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7975 EC: VT.getVectorElementCount());
7976 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
7977 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
7978 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
7979 }
7980
7981 if (OpNo == 1) {
7982 // The mask should be widened as well.
7983 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7984 } else {
7985 // Widen the mask.
7986 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7987
7988 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
7989 }
7990
7991 assert(Mask.getValueType().getVectorElementCount() ==
7992 StVal.getValueType().getVectorElementCount() &&
7993 "Mask and data vectors should have the same number of elements");
7994 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
7995 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
7996 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
7997 IsTruncating: false, IsCompressing: MST->isCompressingStore());
7998}
7999
8000SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8001 assert(OpNo == 4 && "Can widen only the index of mgather");
8002 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8003 SDValue DataOp = MG->getPassThru();
8004 SDValue Mask = MG->getMask();
8005 SDValue Scale = MG->getScale();
8006
8007 // Just widen the index. It's allowed to have extra elements.
8008 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8009
8010 SDLoc dl(N);
8011 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8012 Scale};
8013 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8014 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8015 ExtTy: MG->getExtensionType());
8016 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8017 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8018 return SDValue();
8019}
8020
8021SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8022 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8023 SDValue DataOp = MSC->getValue();
8024 SDValue Mask = MSC->getMask();
8025 SDValue Index = MSC->getIndex();
8026 SDValue Scale = MSC->getScale();
8027 EVT WideMemVT = MSC->getMemoryVT();
8028
8029 if (OpNo == 1) {
8030 DataOp = GetWidenedVector(Op: DataOp);
8031 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
8032
8033 // Widen index.
8034 EVT IndexVT = Index.getValueType();
8035 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8036 VT: IndexVT.getVectorElementType(), NumElements: NumElts);
8037 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8038
8039 // The mask should be widened as well.
8040 EVT MaskVT = Mask.getValueType();
8041 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8042 VT: MaskVT.getVectorElementType(), NumElements: NumElts);
8043 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8044
8045 // Widen the MemoryType
8046 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8047 VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
8048 } else if (OpNo == 4) {
8049 // Just widen the index. It's allowed to have extra elements.
8050 Index = GetWidenedVector(Op: Index);
8051 } else
8052 llvm_unreachable("Can't widen this operand of mscatter");
8053
8054 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8055 Scale};
8056 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8057 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8058 IsTruncating: MSC->isTruncatingStore());
8059}
8060
8061SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8062 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8063 SDValue DataOp = VPSC->getValue();
8064 SDValue Mask = VPSC->getMask();
8065 SDValue Index = VPSC->getIndex();
8066 SDValue Scale = VPSC->getScale();
8067 EVT WideMemVT = VPSC->getMemoryVT();
8068
8069 if (OpNo == 1) {
8070 DataOp = GetWidenedVector(Op: DataOp);
8071 Index = GetWidenedVector(Op: Index);
8072 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8073 Mask = GetWidenedMask(Mask, EC: WideEC);
8074 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8075 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8076 } else if (OpNo == 3) {
8077 // Just widen the index. It's allowed to have extra elements.
8078 Index = GetWidenedVector(Op: Index);
8079 } else
8080 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8081
8082 SDValue Ops[] = {
8083 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8084 VPSC->getVectorLength()};
8085 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8086 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8087}
8088
8089SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8090 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8091 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8092 SDLoc dl(N);
8093 EVT VT = N->getValueType(ResNo: 0);
8094
8095 // WARNING: In this code we widen the compare instruction with garbage.
8096 // This garbage may contain denormal floats which may be slow. Is this a real
8097 // concern ? Should we zero the unused lanes if this is a float compare ?
8098
8099 // Get a new SETCC node to compare the newly widened operands.
8100 // Only some of the compared elements are legal.
8101 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8102 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8103 if (VT.getScalarType() == MVT::i1)
8104 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8105 EC: SVT.getVectorElementCount());
8106
8107 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8108 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8109
8110 // Extract the needed results from the result vector.
8111 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8112 VT: SVT.getVectorElementType(),
8113 EC: VT.getVectorElementCount());
8114 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8115
8116 EVT OpVT = N->getOperand(Num: 0).getValueType();
8117 ISD::NodeType ExtendCode =
8118 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8119 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8120}
8121
8122SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8123 SDValue Chain = N->getOperand(Num: 0);
8124 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8125 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8126 SDValue CC = N->getOperand(Num: 3);
8127 SDLoc dl(N);
8128
8129 EVT VT = N->getValueType(ResNo: 0);
8130 EVT EltVT = VT.getVectorElementType();
8131 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8132 unsigned NumElts = VT.getVectorNumElements();
8133
8134 // Unroll into a build vector.
8135 SmallVector<SDValue, 8> Scalars(NumElts);
8136 SmallVector<SDValue, 8> Chains(NumElts);
8137
8138 for (unsigned i = 0; i != NumElts; ++i) {
8139 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8140 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8141
8142 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8143 Ops: {Chain, LHSElem, RHSElem, CC});
8144 Chains[i] = Scalars[i].getValue(R: 1);
8145 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8146 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8147 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8148 }
8149
8150 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8151 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8152
8153 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8154}
8155
8156static unsigned getExtendForIntVecReduction(unsigned Opc) {
8157 switch (Opc) {
8158 default:
8159 llvm_unreachable("Expected integer vector reduction");
8160 case ISD::VECREDUCE_ADD:
8161 case ISD::VECREDUCE_MUL:
8162 case ISD::VECREDUCE_AND:
8163 case ISD::VECREDUCE_OR:
8164 case ISD::VECREDUCE_XOR:
8165 return ISD::ANY_EXTEND;
8166 case ISD::VECREDUCE_SMAX:
8167 case ISD::VECREDUCE_SMIN:
8168 return ISD::SIGN_EXTEND;
8169 case ISD::VECREDUCE_UMAX:
8170 case ISD::VECREDUCE_UMIN:
8171 return ISD::ZERO_EXTEND;
8172 }
8173}
8174
8175SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8176 SDLoc dl(N);
8177 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8178 EVT VT = N->getValueType(ResNo: 0);
8179 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8180 EVT WideVT = Op.getValueType();
8181 EVT ElemVT = OrigVT.getVectorElementType();
8182 SDNodeFlags Flags = N->getFlags();
8183
8184 unsigned Opc = N->getOpcode();
8185 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8186 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8187 assert(NeutralElem && "Neutral element must exist");
8188
8189 // Pad the vector with the neutral element.
8190 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8191 unsigned WideElts = WideVT.getVectorMinNumElements();
8192
8193 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8194 // needing to pad the source vector, because the inactive lanes can simply be
8195 // disabled and not contribute to the result.
8196 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8197 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8198 SDValue Start = NeutralElem;
8199 if (VT.isInteger())
8200 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8201 assert(Start.getValueType() == VT);
8202 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8203 EC: WideVT.getVectorElementCount());
8204 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8205 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8206 EC: OrigVT.getVectorElementCount());
8207 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8208 }
8209
8210 if (WideVT.isScalableVector()) {
8211 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8212 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8213 EC: ElementCount::getScalable(MinVal: GCD));
8214 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8215 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8216 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8217 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8218 }
8219
8220 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8221 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8222
8223 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8224}
8225
8226SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8227 SDLoc dl(N);
8228 SDValue AccOp = N->getOperand(Num: 0);
8229 SDValue VecOp = N->getOperand(Num: 1);
8230 SDValue Op = GetWidenedVector(Op: VecOp);
8231
8232 EVT VT = N->getValueType(ResNo: 0);
8233 EVT OrigVT = VecOp.getValueType();
8234 EVT WideVT = Op.getValueType();
8235 EVT ElemVT = OrigVT.getVectorElementType();
8236 SDNodeFlags Flags = N->getFlags();
8237
8238 unsigned Opc = N->getOpcode();
8239 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8240 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8241
8242 // Pad the vector with the neutral element.
8243 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8244 unsigned WideElts = WideVT.getVectorMinNumElements();
8245
8246 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8247 // needing to pad the source vector, because the inactive lanes can simply be
8248 // disabled and not contribute to the result.
8249 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8250 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8251 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8252 EC: WideVT.getVectorElementCount());
8253 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8254 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8255 EC: OrigVT.getVectorElementCount());
8256 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8257 }
8258
8259 if (WideVT.isScalableVector()) {
8260 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8261 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8262 EC: ElementCount::getScalable(MinVal: GCD));
8263 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8264 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8265 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8266 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8267 }
8268
8269 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8270 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8271
8272 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8273}
8274
8275SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8276 assert(N->isVPOpcode() && "Expected VP opcode");
8277
8278 SDLoc dl(N);
8279 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8280 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8281 EC: Op.getValueType().getVectorElementCount());
8282
8283 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8284 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8285 Flags: N->getFlags());
8286}
8287
8288SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8289 // This only gets called in the case that the left and right inputs and
8290 // result are of a legal odd vector type, and the condition is illegal i1 of
8291 // the same odd width that needs widening.
8292 EVT VT = N->getValueType(ResNo: 0);
8293 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8294
8295 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8296 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8297 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8298 SDLoc DL(N);
8299
8300 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8301 N2: LeftIn, N3: RightIn);
8302 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8303}
8304
8305SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8306 SDLoc DL(N);
8307 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8308 EVT SrcVT = Source.getValueType();
8309 SDValue Mask =
8310 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8311
8312 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8313 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8314}
8315
8316SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8317 SDLoc DL(N);
8318 SDValue Mask = N->getOperand(Num: 0);
8319 EVT OrigMaskVT = Mask.getValueType();
8320 SDValue WideMask = GetWidenedVector(Op: Mask);
8321 EVT WideMaskVT = WideMask.getValueType();
8322
8323 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8324 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8325 unsigned WideElts = WideMaskVT.getVectorNumElements();
8326 if (OrigElts != WideElts) {
8327 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8328 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8329 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8330 }
8331
8332 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8333 Operand: WideMask);
8334}
8335
8336//===----------------------------------------------------------------------===//
8337// Vector Widening Utilities
8338//===----------------------------------------------------------------------===//
8339
8340// Utility function to find the type to chop up a widen vector for load/store
8341// TLI: Target lowering used to determine legal types.
8342// Width: Width left need to load/store.
8343// WidenVT: The widen vector type to load to/store from
8344// Align: If 0, don't allow use of a wider type
8345// WidenEx: If Align is not 0, the amount additional we can load/store from.
8346
8347static std::optional<EVT> findMemType(SelectionDAG &DAG,
8348 const TargetLowering &TLI, unsigned Width,
8349 EVT WidenVT, unsigned Align = 0,
8350 unsigned WidenEx = 0) {
8351 EVT WidenEltVT = WidenVT.getVectorElementType();
8352 const bool Scalable = WidenVT.isScalableVector();
8353 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8354 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8355 unsigned AlignInBits = Align*8;
8356
8357 EVT RetVT = WidenEltVT;
8358 // Don't bother looking for an integer type if the vector is scalable, skip
8359 // to vector types.
8360 if (!Scalable) {
8361 // If we have one element to load/store, return it.
8362 if (Width == WidenEltWidth)
8363 return RetVT;
8364
8365 // See if there is larger legal integer than the element type to load/store.
8366 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8367 unsigned MemVTWidth = MemVT.getSizeInBits();
8368 if (MemVT.getSizeInBits() <= WidenEltWidth)
8369 break;
8370 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8371 if ((Action == TargetLowering::TypeLegal ||
8372 Action == TargetLowering::TypePromoteInteger) &&
8373 (WidenWidth % MemVTWidth) == 0 &&
8374 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8375 (MemVTWidth <= Width ||
8376 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8377 if (MemVTWidth == WidenWidth)
8378 return MemVT;
8379 RetVT = MemVT;
8380 break;
8381 }
8382 }
8383 }
8384
8385 // See if there is a larger vector type to load/store that has the same vector
8386 // element type and is evenly divisible with the WidenVT.
8387 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8388 // Skip vector MVTs which don't match the scalable property of WidenVT.
8389 if (Scalable != MemVT.isScalableVector())
8390 continue;
8391 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8392 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8393 if ((Action == TargetLowering::TypeLegal ||
8394 Action == TargetLowering::TypePromoteInteger) &&
8395 WidenEltVT == MemVT.getVectorElementType() &&
8396 (WidenWidth % MemVTWidth) == 0 &&
8397 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8398 (MemVTWidth <= Width ||
8399 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8400 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8401 return MemVT;
8402 }
8403 }
8404
8405 // Using element-wise loads and stores for widening operations is not
8406 // supported for scalable vectors
8407 if (Scalable)
8408 return std::nullopt;
8409
8410 return RetVT;
8411}
8412
8413// Builds a vector type from scalar loads
8414// VecTy: Resulting Vector type
8415// LDOps: Load operators to build a vector type
8416// [Start,End) the list of loads to use.
8417static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8418 SmallVectorImpl<SDValue> &LdOps,
8419 unsigned Start, unsigned End) {
8420 SDLoc dl(LdOps[Start]);
8421 EVT LdTy = LdOps[Start].getValueType();
8422 unsigned Width = VecTy.getSizeInBits();
8423 unsigned NumElts = Width / LdTy.getSizeInBits();
8424 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8425
8426 unsigned Idx = 1;
8427 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8428
8429 for (unsigned i = Start + 1; i != End; ++i) {
8430 EVT NewLdTy = LdOps[i].getValueType();
8431 if (NewLdTy != LdTy) {
8432 NumElts = Width / NewLdTy.getSizeInBits();
8433 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8434 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8435 // Readjust position and vector position based on new load type.
8436 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8437 LdTy = NewLdTy;
8438 }
8439 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8440 }
8441 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8442}
8443
8444SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8445 LoadSDNode *LD) {
8446 // The strategy assumes that we can efficiently load power-of-two widths.
8447 // The routine chops the vector into the largest vector loads with the same
8448 // element type or scalar loads and then recombines it to the widen vector
8449 // type.
8450 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8451 EVT LdVT = LD->getMemoryVT();
8452 SDLoc dl(LD);
8453 assert(LdVT.isVector() && WidenVT.isVector());
8454 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8455 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8456
8457 // Load information
8458 SDValue Chain = LD->getChain();
8459 SDValue BasePtr = LD->getBasePtr();
8460 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8461 AAMDNodes AAInfo = LD->getAAInfo();
8462
8463 TypeSize LdWidth = LdVT.getSizeInBits();
8464 TypeSize WidenWidth = WidenVT.getSizeInBits();
8465 TypeSize WidthDiff = WidenWidth - LdWidth;
8466 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8467 // and if the original load is simple.
8468 unsigned LdAlign =
8469 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8470
8471 // Find the vector type that can load from.
8472 std::optional<EVT> FirstVT =
8473 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8474 WidenEx: WidthDiff.getKnownMinValue());
8475
8476 if (!FirstVT)
8477 return SDValue();
8478
8479 SmallVector<EVT, 8> MemVTs;
8480 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8481
8482 // Unless we're able to load in one instruction we must work out how to load
8483 // the remainder.
8484 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8485 std::optional<EVT> NewVT = FirstVT;
8486 TypeSize RemainingWidth = LdWidth;
8487 TypeSize NewVTWidth = FirstVTWidth;
8488 do {
8489 RemainingWidth -= NewVTWidth;
8490 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8491 // The current type we are using is too large. Find a better size.
8492 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8493 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8494 if (!NewVT)
8495 return SDValue();
8496 NewVTWidth = NewVT->getSizeInBits();
8497 }
8498 MemVTs.push_back(Elt: *NewVT);
8499 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8500 }
8501
8502 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8503 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8504 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8505
8506 // Check if we can load the element with one instruction.
8507 if (MemVTs.empty())
8508 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8509 DAG);
8510
8511 // Load vector by using multiple loads from largest vector to scalar.
8512 SmallVector<SDValue, 16> LdOps;
8513 LdOps.push_back(Elt: LdOp);
8514
8515 uint64_t ScaledOffset = 0;
8516 MachinePointerInfo MPI = LD->getPointerInfo();
8517
8518 // First incremement past the first load.
8519 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8520 ScaledOffset: &ScaledOffset);
8521
8522 for (EVT MemVT : MemVTs) {
8523 Align NewAlign = ScaledOffset == 0
8524 ? LD->getBaseAlign()
8525 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8526 SDValue L =
8527 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8528
8529 LdOps.push_back(Elt: L);
8530 LdChain.push_back(Elt: L.getValue(R: 1));
8531 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8532 }
8533
8534 // Build the vector from the load operations.
8535 unsigned End = LdOps.size();
8536 if (!LdOps[0].getValueType().isVector())
8537 // All the loads are scalar loads.
8538 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8539
8540 // If the load contains vectors, build the vector using concat vector.
8541 // All of the vectors used to load are power-of-2, and the scalar loads can be
8542 // combined to make a power-of-2 vector.
8543 SmallVector<SDValue, 16> ConcatOps(End);
8544 int i = End - 1;
8545 int Idx = End;
8546 EVT LdTy = LdOps[i].getValueType();
8547 // First, combine the scalar loads to a vector.
8548 if (!LdTy.isVector()) {
8549 for (--i; i >= 0; --i) {
8550 LdTy = LdOps[i].getValueType();
8551 if (LdTy.isVector())
8552 break;
8553 }
8554 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8555 }
8556
8557 ConcatOps[--Idx] = LdOps[i];
8558 for (--i; i >= 0; --i) {
8559 EVT NewLdTy = LdOps[i].getValueType();
8560 if (NewLdTy != LdTy) {
8561 // Create a larger vector.
8562 TypeSize LdTySize = LdTy.getSizeInBits();
8563 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8564 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8565 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8566 unsigned NumOps =
8567 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8568 SmallVector<SDValue, 16> WidenOps(NumOps);
8569 unsigned j = 0;
8570 for (; j != End-Idx; ++j)
8571 WidenOps[j] = ConcatOps[Idx+j];
8572 for (; j != NumOps; ++j)
8573 WidenOps[j] = DAG.getPOISON(VT: LdTy);
8574
8575 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
8576 Ops: WidenOps);
8577 Idx = End - 1;
8578 LdTy = NewLdTy;
8579 }
8580 ConcatOps[--Idx] = LdOps[i];
8581 }
8582
8583 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
8584 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
8585 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
8586
8587 // We need to fill the rest with undefs to build the vector.
8588 unsigned NumOps =
8589 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
8590 SmallVector<SDValue, 16> WidenOps(NumOps);
8591 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
8592 {
8593 unsigned i = 0;
8594 for (; i != End-Idx; ++i)
8595 WidenOps[i] = ConcatOps[Idx+i];
8596 for (; i != NumOps; ++i)
8597 WidenOps[i] = UndefVal;
8598 }
8599 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
8600}
8601
8602SDValue
8603DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
8604 LoadSDNode *LD,
8605 ISD::LoadExtType ExtType) {
8606 // For extension loads, it may not be more efficient to chop up the vector
8607 // and then extend it. Instead, we unroll the load and build a new vector.
8608 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8609 EVT LdVT = LD->getMemoryVT();
8610 SDLoc dl(LD);
8611 assert(LdVT.isVector() && WidenVT.isVector());
8612 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8613
8614 // Load information
8615 SDValue Chain = LD->getChain();
8616 SDValue BasePtr = LD->getBasePtr();
8617 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8618 AAMDNodes AAInfo = LD->getAAInfo();
8619
8620 if (LdVT.isScalableVector())
8621 return SDValue();
8622
8623 EVT EltVT = WidenVT.getVectorElementType();
8624 EVT LdEltVT = LdVT.getVectorElementType();
8625 unsigned NumElts = LdVT.getVectorNumElements();
8626
8627 // Load each element and widen.
8628 unsigned WidenNumElts = WidenVT.getVectorNumElements();
8629 SmallVector<SDValue, 16> Ops(WidenNumElts);
8630 unsigned Increment = LdEltVT.getSizeInBits() / 8;
8631 Ops[0] =
8632 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8633 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8634 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
8635 unsigned i = 0, Offset = Increment;
8636 for (i=1; i < NumElts; ++i, Offset += Increment) {
8637 SDValue NewBasePtr =
8638 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
8639 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
8640 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
8641 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8642 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
8643 }
8644
8645 // Fill the rest with undefs.
8646 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8647 for (; i != WidenNumElts; ++i)
8648 Ops[i] = UndefVal;
8649
8650 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
8651}
8652
8653bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
8654 StoreSDNode *ST) {
8655 // The strategy assumes that we can efficiently store power-of-two widths.
8656 // The routine chops the vector into the largest vector stores with the same
8657 // element type or scalar stores.
8658 SDValue Chain = ST->getChain();
8659 SDValue BasePtr = ST->getBasePtr();
8660 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
8661 AAMDNodes AAInfo = ST->getAAInfo();
8662 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
8663 SDLoc dl(ST);
8664
8665 EVT StVT = ST->getMemoryVT();
8666 TypeSize StWidth = StVT.getSizeInBits();
8667 EVT ValVT = ValOp.getValueType();
8668 TypeSize ValWidth = ValVT.getSizeInBits();
8669 EVT ValEltVT = ValVT.getVectorElementType();
8670 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
8671 assert(StVT.getVectorElementType() == ValEltVT);
8672 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
8673 "Mismatch between store and value types");
8674
8675 int Idx = 0; // current index to store
8676
8677 MachinePointerInfo MPI = ST->getPointerInfo();
8678 uint64_t ScaledOffset = 0;
8679
8680 // A breakdown of how to widen this vector store. Each element of the vector
8681 // is a memory VT combined with the number of times it is to be stored to,
8682 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
8683 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
8684
8685 while (StWidth.isNonZero()) {
8686 // Find the largest vector type we can store with.
8687 std::optional<EVT> NewVT =
8688 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
8689 if (!NewVT)
8690 return false;
8691 MemVTs.push_back(Elt: {*NewVT, 0});
8692 TypeSize NewVTWidth = NewVT->getSizeInBits();
8693
8694 do {
8695 StWidth -= NewVTWidth;
8696 MemVTs.back().second++;
8697 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
8698 }
8699
8700 for (const auto &Pair : MemVTs) {
8701 EVT NewVT = Pair.first;
8702 unsigned Count = Pair.second;
8703 TypeSize NewVTWidth = NewVT.getSizeInBits();
8704
8705 if (NewVT.isVector()) {
8706 unsigned NumVTElts = NewVT.getVectorMinNumElements();
8707 do {
8708 Align NewAlign = ScaledOffset == 0
8709 ? ST->getBaseAlign()
8710 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
8711 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
8712 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
8713 MMOFlags, AAInfo);
8714 StChain.push_back(Elt: PartStore);
8715
8716 Idx += NumVTElts;
8717 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
8718 ScaledOffset: &ScaledOffset);
8719 } while (--Count);
8720 } else {
8721 // Cast the vector to the scalar type we can store.
8722 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
8723 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
8724 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
8725 // Readjust index position based on new vector type.
8726 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
8727 do {
8728 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
8729 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
8730 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
8731 StChain.push_back(Elt: PartStore);
8732
8733 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
8734 } while (--Count);
8735 // Restore index back to be relative to the original widen element type.
8736 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
8737 }
8738 }
8739
8740 return true;
8741}
8742
8743/// Modifies a vector input (widen or narrows) to a vector of NVT. The
8744/// input vector must have the same element type as NVT.
8745/// FillWithZeroes specifies that the vector should be widened with zeroes.
8746SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
8747 bool FillWithZeroes) {
8748 // Note that InOp might have been widened so it might already have
8749 // the right width or it might need be narrowed.
8750 EVT InVT = InOp.getValueType();
8751 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
8752 "input and widen element type must match");
8753 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
8754 "cannot modify scalable vectors in this way");
8755 SDLoc dl(InOp);
8756
8757 // Check if InOp already has the right width.
8758 if (InVT == NVT)
8759 return InOp;
8760
8761 ElementCount InEC = InVT.getVectorElementCount();
8762 ElementCount WidenEC = NVT.getVectorElementCount();
8763 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
8764 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
8765 SmallVector<SDValue, 16> Ops(NumConcat);
8766 SDValue FillVal =
8767 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
8768 Ops[0] = InOp;
8769 for (unsigned i = 1; i != NumConcat; ++i)
8770 Ops[i] = FillVal;
8771
8772 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
8773 }
8774
8775 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
8776 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
8777
8778 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
8779 "Scalable vectors should have been handled already.");
8780
8781 unsigned InNumElts = InEC.getFixedValue();
8782 unsigned WidenNumElts = WidenEC.getFixedValue();
8783
8784 // Fall back to extract and build (+ mask, if padding with zeros).
8785 SmallVector<SDValue, 16> Ops(WidenNumElts);
8786 EVT EltVT = NVT.getVectorElementType();
8787 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
8788 unsigned Idx;
8789 for (Idx = 0; Idx < MinNumElts; ++Idx)
8790 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
8791
8792 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8793 for (; Idx < WidenNumElts; ++Idx)
8794 Ops[Idx] = UndefVal;
8795
8796 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
8797 if (!FillWithZeroes)
8798 return Widened;
8799
8800 assert(NVT.isInteger() &&
8801 "We expect to never want to FillWithZeroes for non-integral types.");
8802
8803 SmallVector<SDValue, 16> MaskOps;
8804 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
8805 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
8806
8807 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
8808 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
8809}
8810