1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error(reason: "Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::LOOP_DEPENDENCE_WAR_MASK:
57 case ISD::LOOP_DEPENDENCE_RAW_MASK:
58 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
59 break;
60 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
61 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
62 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
63 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
64 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
65 case ISD::AssertZext:
66 case ISD::AssertSext:
67 case ISD::FPOWI:
68 case ISD::AssertNoFPClass:
69 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
70 break;
71 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
72 case ISD::ATOMIC_LOAD:
73 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
74 break;
75 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
76 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
77 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
78 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
79 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
80 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
81 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
82 case ISD::POISON:
83 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
84 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
85 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
86 case ISD::ANY_EXTEND_VECTOR_INREG:
87 case ISD::SIGN_EXTEND_VECTOR_INREG:
88 case ISD::ZERO_EXTEND_VECTOR_INREG:
89 R = ScalarizeVecRes_VecInregOp(N);
90 break;
91 case ISD::ABS:
92 case ISD::ANY_EXTEND:
93 case ISD::BITREVERSE:
94 case ISD::BSWAP:
95 case ISD::CTLZ:
96 case ISD::CTLZ_ZERO_UNDEF:
97 case ISD::CTPOP:
98 case ISD::CTTZ:
99 case ISD::CTTZ_ZERO_UNDEF:
100 case ISD::FABS:
101 case ISD::FACOS:
102 case ISD::FASIN:
103 case ISD::FATAN:
104 case ISD::FCEIL:
105 case ISD::FCOS:
106 case ISD::FCOSH:
107 case ISD::FEXP:
108 case ISD::FEXP2:
109 case ISD::FEXP10:
110 case ISD::FFLOOR:
111 case ISD::FLOG:
112 case ISD::FLOG10:
113 case ISD::FLOG2:
114 case ISD::FNEARBYINT:
115 case ISD::FNEG:
116 case ISD::FREEZE:
117 case ISD::ARITH_FENCE:
118 case ISD::FP_EXTEND:
119 case ISD::FP_TO_SINT:
120 case ISD::FP_TO_UINT:
121 case ISD::FRINT:
122 case ISD::LRINT:
123 case ISD::LLRINT:
124 case ISD::FROUND:
125 case ISD::FROUNDEVEN:
126 case ISD::LROUND:
127 case ISD::LLROUND:
128 case ISD::FSIN:
129 case ISD::FSINH:
130 case ISD::FSQRT:
131 case ISD::FTAN:
132 case ISD::FTANH:
133 case ISD::FTRUNC:
134 case ISD::SIGN_EXTEND:
135 case ISD::SINT_TO_FP:
136 case ISD::TRUNCATE:
137 case ISD::UINT_TO_FP:
138 case ISD::ZERO_EXTEND:
139 case ISD::FCANONICALIZE:
140 R = ScalarizeVecRes_UnaryOp(N);
141 break;
142 case ISD::ADDRSPACECAST:
143 R = ScalarizeVecRes_ADDRSPACECAST(N);
144 break;
145 case ISD::FMODF:
146 case ISD::FFREXP:
147 case ISD::FSINCOS:
148 case ISD::FSINCOSPI:
149 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
150 break;
151 case ISD::ADD:
152 case ISD::AND:
153 case ISD::AVGCEILS:
154 case ISD::AVGCEILU:
155 case ISD::AVGFLOORS:
156 case ISD::AVGFLOORU:
157 case ISD::FADD:
158 case ISD::FCOPYSIGN:
159 case ISD::FDIV:
160 case ISD::FMUL:
161 case ISD::FMINNUM:
162 case ISD::FMAXNUM:
163 case ISD::FMINNUM_IEEE:
164 case ISD::FMAXNUM_IEEE:
165 case ISD::FMINIMUM:
166 case ISD::FMAXIMUM:
167 case ISD::FMINIMUMNUM:
168 case ISD::FMAXIMUMNUM:
169 case ISD::FLDEXP:
170 case ISD::ABDS:
171 case ISD::ABDU:
172 case ISD::SMIN:
173 case ISD::SMAX:
174 case ISD::UMIN:
175 case ISD::UMAX:
176
177 case ISD::SADDSAT:
178 case ISD::UADDSAT:
179 case ISD::SSUBSAT:
180 case ISD::USUBSAT:
181 case ISD::SSHLSAT:
182 case ISD::USHLSAT:
183
184 case ISD::FPOW:
185 case ISD::FATAN2:
186 case ISD::FREM:
187 case ISD::FSUB:
188 case ISD::MUL:
189 case ISD::MULHS:
190 case ISD::MULHU:
191 case ISD::OR:
192 case ISD::SDIV:
193 case ISD::SREM:
194 case ISD::SUB:
195 case ISD::UDIV:
196 case ISD::UREM:
197 case ISD::XOR:
198 case ISD::SHL:
199 case ISD::SRA:
200 case ISD::SRL:
201 case ISD::ROTL:
202 case ISD::ROTR:
203 case ISD::CLMUL:
204 case ISD::CLMULR:
205 case ISD::CLMULH:
206 R = ScalarizeVecRes_BinOp(N);
207 break;
208
209 case ISD::SCMP:
210 case ISD::UCMP:
211 R = ScalarizeVecRes_CMP(N);
212 break;
213
214 case ISD::FMA:
215 case ISD::FSHL:
216 case ISD::FSHR:
217 R = ScalarizeVecRes_TernaryOp(N);
218 break;
219
220#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
221 case ISD::STRICT_##DAGN:
222#include "llvm/IR/ConstrainedOps.def"
223 R = ScalarizeVecRes_StrictFPOp(N);
224 break;
225
226 case ISD::FP_TO_UINT_SAT:
227 case ISD::FP_TO_SINT_SAT:
228 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
229 break;
230
231 case ISD::UADDO:
232 case ISD::SADDO:
233 case ISD::USUBO:
234 case ISD::SSUBO:
235 case ISD::UMULO:
236 case ISD::SMULO:
237 R = ScalarizeVecRes_OverflowOp(N, ResNo);
238 break;
239 case ISD::SMULFIX:
240 case ISD::SMULFIXSAT:
241 case ISD::UMULFIX:
242 case ISD::UMULFIXSAT:
243 case ISD::SDIVFIX:
244 case ISD::SDIVFIXSAT:
245 case ISD::UDIVFIX:
246 case ISD::UDIVFIXSAT:
247 R = ScalarizeVecRes_FIX(N);
248 break;
249 }
250
251 // If R is null, the sub-method took care of registering the result.
252 if (R.getNode())
253 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
254}
255
256SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
257 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
258 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
259 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
260 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
261}
262
263SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
264 SDLoc DL(N);
265
266 SDValue LHS = N->getOperand(Num: 0);
267 SDValue RHS = N->getOperand(Num: 1);
268 if (getTypeAction(VT: LHS.getValueType()) ==
269 TargetLowering::TypeScalarizeVector) {
270 LHS = GetScalarizedVector(Op: LHS);
271 RHS = GetScalarizedVector(Op: RHS);
272 } else {
273 EVT VT = LHS.getValueType().getVectorElementType();
274 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
275 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
276 }
277
278 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
279 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
280}
281
282SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
283 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
284 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
285 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
286 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
287 N3: Op2, Flags: N->getFlags());
288}
289
290SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
291 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
292 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
293 SDValue Op2 = N->getOperand(Num: 2);
294 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
295 N3: Op2, Flags: N->getFlags());
296}
297
298SDValue
299DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
300 unsigned ResNo) {
301 assert(N->getValueType(0).getVectorNumElements() == 1 &&
302 "Unexpected vector type!");
303 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
304
305 EVT VT0 = N->getValueType(ResNo: 0);
306 EVT VT1 = N->getValueType(ResNo: 1);
307 SDLoc dl(N);
308
309 SDNode *ScalarNode =
310 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
311 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
312 .getNode();
313
314 // Replace the other vector result not being explicitly scalarized here.
315 unsigned OtherNo = 1 - ResNo;
316 EVT OtherVT = N->getValueType(ResNo: OtherNo);
317 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
318 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
319 } else {
320 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
321 Operand: SDValue(ScalarNode, OtherNo));
322 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
323 }
324
325 return SDValue(ScalarNode, ResNo);
326}
327
328SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
329 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
330 unsigned NumOpers = N->getNumOperands();
331 SDValue Chain = N->getOperand(Num: 0);
332 EVT ValueVTs[] = {VT, MVT::Other};
333 SDLoc dl(N);
334
335 SmallVector<SDValue, 4> Opers(NumOpers);
336
337 // The Chain is the first operand.
338 Opers[0] = Chain;
339
340 // Now process the remaining operands.
341 for (unsigned i = 1; i < NumOpers; ++i) {
342 SDValue Oper = N->getOperand(Num: i);
343 EVT OperVT = Oper.getValueType();
344
345 if (OperVT.isVector()) {
346 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
347 Oper = GetScalarizedVector(Op: Oper);
348 else
349 Oper =
350 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
351 }
352
353 Opers[i] = Oper;
354 }
355
356 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
357 Ops: Opers, Flags: N->getFlags());
358
359 // Legalize the chain result - switch anything that used the old chain to
360 // use the new one.
361 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
362 return Result;
363}
364
365SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
366 unsigned ResNo) {
367 SDLoc DL(N);
368 EVT ResVT = N->getValueType(ResNo: 0);
369 EVT OvVT = N->getValueType(ResNo: 1);
370
371 SDValue ScalarLHS, ScalarRHS;
372 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
373 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
374 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
375 } else {
376 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
377 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
378 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
379 ScalarLHS = ElemsLHS[0];
380 ScalarRHS = ElemsRHS[0];
381 }
382
383 SDVTList ScalarVTs = DAG.getVTList(
384 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
385 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
386 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
387 .getNode();
388
389 // Replace the other vector result not being explicitly scalarized here.
390 unsigned OtherNo = 1 - ResNo;
391 EVT OtherVT = N->getValueType(ResNo: OtherNo);
392 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
393 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
394 } else {
395 SDValue OtherVal = DAG.getNode(
396 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
397 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
398 }
399
400 return SDValue(ScalarNode, ResNo);
401}
402
403SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
404 unsigned ResNo) {
405 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
406 return GetScalarizedVector(Op);
407}
408
409SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
410 SDLoc DL(N);
411 SDValue SourceValue = N->getOperand(Num: 0);
412 SDValue SinkValue = N->getOperand(Num: 1);
413 SDValue EltSizeInBytes = N->getOperand(Num: 2);
414 SDValue LaneOffset = N->getOperand(Num: 3);
415
416 EVT PtrVT = SourceValue->getValueType(ResNo: 0);
417 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
418
419 // Take the difference between the pointers and divided by the element size,
420 // to see how many lanes separate them.
421 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: SinkValue, N2: SourceValue);
422 if (IsReadAfterWrite)
423 Diff = DAG.getNode(Opcode: ISD::ABS, DL, VT: PtrVT, Operand: Diff);
424 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: PtrVT, N1: Diff, N2: EltSizeInBytes);
425
426 // The pointers do not alias if:
427 // * Diff <= 0 || LaneOffset < Diff (WAR_MASK)
428 // * Diff == 0 || LaneOffset < abs(Diff) (RAW_MASK)
429 // Note: If LaneOffset is zero, both cases will fold to "true".
430 EVT CmpVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
431 VT: Diff.getValueType());
432 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
433 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Diff, RHS: Zero,
434 Cond: IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
435 return DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Cmp,
436 N2: DAG.getSetCC(DL, VT: CmpVT, LHS: LaneOffset, RHS: Diff, Cond: ISD::SETULT));
437}
438
439SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
440 SDValue Op = N->getOperand(Num: 0);
441 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
442 Op = GetScalarizedVector(Op);
443 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
444 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
445 VT: NewVT, Operand: Op);
446}
447
448SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
449 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
450 SDValue InOp = N->getOperand(Num: 0);
451 // The BUILD_VECTOR operands may be of wider element types and
452 // we may need to truncate them back to the requested return type.
453 if (EltVT.isInteger())
454 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
455 return InOp;
456}
457
458SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
459 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
460 VT: N->getValueType(ResNo: 0).getVectorElementType(),
461 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
462}
463
464SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
465 SDLoc DL(N);
466 SDValue Op = N->getOperand(Num: 0);
467 EVT OpVT = Op.getValueType();
468 // The result needs scalarizing, but it's not a given that the source does.
469 // See similar logic in ScalarizeVecRes_UnaryOp.
470 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
471 Op = GetScalarizedVector(Op);
472 } else {
473 EVT VT = OpVT.getVectorElementType();
474 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
475 }
476 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
477 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
478 N2: N->getOperand(Num: 1));
479}
480
481SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
482 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
483 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
484 N2: N->getOperand(Num: 1));
485}
486
487SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
488 // The value to insert may have a wider type than the vector element type,
489 // so be sure to truncate it to the element type if necessary.
490 SDValue Op = N->getOperand(Num: 1);
491 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
492 if (Op.getValueType() != EltVT)
493 // FIXME: Can this happen for floating point types?
494 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
495 return Op;
496}
497
498SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
499 SDValue Result = DAG.getAtomicLoad(
500 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
501 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
502 MMO: N->getMemOperand());
503
504 // Legalize the chain result - switch anything that used the old chain to
505 // use the new one.
506 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
507 return Result;
508}
509
510SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
511 assert(N->isUnindexed() && "Indexed vector load?");
512
513 SDValue Result = DAG.getLoad(
514 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
515 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
516 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
517 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
518 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
519
520 // Legalize the chain result - switch anything that used the old chain to
521 // use the new one.
522 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
523 return Result;
524}
525
526SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
527 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
528 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
529 SDValue Op = N->getOperand(Num: 0);
530 EVT OpVT = Op.getValueType();
531 SDLoc DL(N);
532 // The result needs scalarizing, but it's not a given that the source does.
533 // This is a workaround for targets where it's impossible to scalarize the
534 // result of a conversion, because the source type is legal.
535 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
536 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
537 // legal and was not scalarized.
538 // See the similar logic in ScalarizeVecRes_SETCC
539 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
540 Op = GetScalarizedVector(Op);
541 } else {
542 EVT VT = OpVT.getVectorElementType();
543 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
544 }
545 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
546}
547
548SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
549 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
550 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
551 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
552 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
553 N1: LHS, N2: DAG.getValueType(ExtVT));
554}
555
556SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
557 SDLoc DL(N);
558 SDValue Op = N->getOperand(Num: 0);
559
560 EVT OpVT = Op.getValueType();
561 EVT OpEltVT = OpVT.getVectorElementType();
562 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
563
564 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
565 Op = GetScalarizedVector(Op);
566 } else {
567 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
568 }
569
570 switch (N->getOpcode()) {
571 case ISD::ANY_EXTEND_VECTOR_INREG:
572 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
573 case ISD::SIGN_EXTEND_VECTOR_INREG:
574 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
575 case ISD::ZERO_EXTEND_VECTOR_INREG:
576 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
577 }
578
579 llvm_unreachable("Illegal extend_vector_inreg opcode");
580}
581
582SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
583 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
584 SDValue Op = N->getOperand(Num: 0);
585 EVT OpVT = Op.getValueType();
586 SDLoc DL(N);
587 // The result needs scalarizing, but it's not a given that the source does.
588 // This is a workaround for targets where it's impossible to scalarize the
589 // result of a conversion, because the source type is legal.
590 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
591 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
592 // legal and was not scalarized.
593 // See the similar logic in ScalarizeVecRes_SETCC
594 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
595 Op = GetScalarizedVector(Op);
596 } else {
597 EVT VT = OpVT.getVectorElementType();
598 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
599 }
600 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
601 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
602 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
603 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
604}
605
606SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
607 // If the operand is wider than the vector element type then it is implicitly
608 // truncated. Make that explicit here.
609 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
610 SDValue InOp = N->getOperand(Num: 0);
611 if (InOp.getValueType() != EltVT)
612 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
613 return InOp;
614}
615
616SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
617 SDValue Cond = N->getOperand(Num: 0);
618 EVT OpVT = Cond.getValueType();
619 SDLoc DL(N);
620 // The vselect result and true/value operands needs scalarizing, but it's
621 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
622 // See the similar logic in ScalarizeVecRes_SETCC
623 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
624 Cond = GetScalarizedVector(Op: Cond);
625 } else {
626 EVT VT = OpVT.getVectorElementType();
627 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
628 }
629
630 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
631 TargetLowering::BooleanContent ScalarBool =
632 TLI.getBooleanContents(isVec: false, isFloat: false);
633 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
634
635 // If integer and float booleans have different contents then we can't
636 // reliably optimize in all cases. There is a full explanation for this in
637 // DAGCombiner::visitSELECT() where the same issue affects folding
638 // (select C, 0, 1) to (xor C, 1).
639 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
640 TLI.getBooleanContents(isVec: false, isFloat: true)) {
641 // At least try the common case where the boolean is generated by a
642 // comparison.
643 if (Cond->getOpcode() == ISD::SETCC) {
644 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
645 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
646 VecBool = TLI.getBooleanContents(Type: OpVT);
647 } else
648 ScalarBool = TargetLowering::UndefinedBooleanContent;
649 }
650
651 EVT CondVT = Cond.getValueType();
652 if (ScalarBool != VecBool) {
653 switch (ScalarBool) {
654 case TargetLowering::UndefinedBooleanContent:
655 break;
656 case TargetLowering::ZeroOrOneBooleanContent:
657 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
658 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
659 // Vector read from all ones, scalar expects a single 1 so mask.
660 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
661 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
662 break;
663 case TargetLowering::ZeroOrNegativeOneBooleanContent:
664 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
665 VecBool == TargetLowering::ZeroOrOneBooleanContent);
666 // Vector reads from a one, scalar from all ones so sign extend.
667 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
668 N1: Cond, N2: DAG.getValueType(MVT::i1));
669 break;
670 }
671 }
672
673 // Truncate the condition if needed
674 auto BoolVT = getSetCCResultType(VT: CondVT);
675 if (BoolVT.bitsLT(VT: CondVT))
676 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
677
678 return DAG.getSelect(DL: SDLoc(N),
679 VT: LHS.getValueType(), Cond, LHS,
680 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
681}
682
683SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
684 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
685 return DAG.getSelect(DL: SDLoc(N),
686 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
687 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
688}
689
690SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
691 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
692 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
693 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
694 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
695 N5: N->getOperand(Num: 4));
696}
697
698SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
699 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
700}
701
702SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
703 // Figure out if the scalar is the LHS or RHS and return it.
704 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
705 if (Arg.isUndef())
706 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
707 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
708 return GetScalarizedVector(Op: N->getOperand(Num: Op));
709}
710
711SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
712 SDValue Src = N->getOperand(Num: 0);
713 EVT SrcVT = Src.getValueType();
714 SDLoc dl(N);
715
716 // Handle case where result is scalarized but operand is not
717 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
718 Src = GetScalarizedVector(Op: Src);
719 else
720 Src = DAG.getNode(
721 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
722 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
723
724 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
725 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
726}
727
728SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
729 assert(N->getValueType(0).isVector() &&
730 N->getOperand(0).getValueType().isVector() &&
731 "Operand types must be vectors");
732 SDValue LHS = N->getOperand(Num: 0);
733 SDValue RHS = N->getOperand(Num: 1);
734 EVT OpVT = LHS.getValueType();
735 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
736 SDLoc DL(N);
737
738 // The result needs scalarizing, but it's not a given that the source does.
739 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
740 LHS = GetScalarizedVector(Op: LHS);
741 RHS = GetScalarizedVector(Op: RHS);
742 } else {
743 EVT VT = OpVT.getVectorElementType();
744 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
745 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
746 }
747
748 // Turn it into a scalar SETCC.
749 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
750 N3: N->getOperand(Num: 2));
751 // Vectors may have a different boolean contents to scalars. Promote the
752 // value appropriately.
753 ISD::NodeType ExtendCode =
754 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
755 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
756}
757
758SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
759 SDLoc DL(N);
760 SDValue Arg = N->getOperand(Num: 0);
761 SDValue Test = N->getOperand(Num: 1);
762 EVT ArgVT = Arg.getValueType();
763 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
764
765 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
766 Arg = GetScalarizedVector(Op: Arg);
767 } else {
768 EVT VT = ArgVT.getVectorElementType();
769 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
770 }
771
772 SDValue Res =
773 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
774 // Vectors may have a different boolean contents to scalars. Promote the
775 // value appropriately.
776 ISD::NodeType ExtendCode =
777 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
778 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
779}
780
781//===----------------------------------------------------------------------===//
782// Operand Vector Scalarization <1 x ty> -> ty.
783//===----------------------------------------------------------------------===//
784
785bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
786 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
787 N->dump(&DAG));
788 SDValue Res = SDValue();
789
790 switch (N->getOpcode()) {
791 default:
792#ifndef NDEBUG
793 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
794 N->dump(&DAG);
795 dbgs() << "\n";
796#endif
797 report_fatal_error(reason: "Do not know how to scalarize this operator's "
798 "operand!\n");
799 case ISD::BITCAST:
800 Res = ScalarizeVecOp_BITCAST(N);
801 break;
802 case ISD::FAKE_USE:
803 Res = ScalarizeVecOp_FAKE_USE(N);
804 break;
805 case ISD::ANY_EXTEND:
806 case ISD::ZERO_EXTEND:
807 case ISD::SIGN_EXTEND:
808 case ISD::TRUNCATE:
809 case ISD::FP_TO_SINT:
810 case ISD::FP_TO_UINT:
811 case ISD::SINT_TO_FP:
812 case ISD::UINT_TO_FP:
813 case ISD::LROUND:
814 case ISD::LLROUND:
815 case ISD::LRINT:
816 case ISD::LLRINT:
817 Res = ScalarizeVecOp_UnaryOp(N);
818 break;
819 case ISD::FP_TO_SINT_SAT:
820 case ISD::FP_TO_UINT_SAT:
821 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
822 break;
823 case ISD::STRICT_SINT_TO_FP:
824 case ISD::STRICT_UINT_TO_FP:
825 case ISD::STRICT_FP_TO_SINT:
826 case ISD::STRICT_FP_TO_UINT:
827 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
828 break;
829 case ISD::CONCAT_VECTORS:
830 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
831 break;
832 case ISD::INSERT_SUBVECTOR:
833 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
834 break;
835 case ISD::EXTRACT_VECTOR_ELT:
836 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
837 break;
838 case ISD::VSELECT:
839 Res = ScalarizeVecOp_VSELECT(N);
840 break;
841 case ISD::SETCC:
842 Res = ScalarizeVecOp_VSETCC(N);
843 break;
844 case ISD::STRICT_FSETCC:
845 case ISD::STRICT_FSETCCS:
846 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
847 break;
848 case ISD::STORE:
849 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
850 break;
851 case ISD::STRICT_FP_ROUND:
852 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
853 break;
854 case ISD::FP_ROUND:
855 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
856 break;
857 case ISD::STRICT_FP_EXTEND:
858 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
859 break;
860 case ISD::FP_EXTEND:
861 Res = ScalarizeVecOp_FP_EXTEND(N);
862 break;
863 case ISD::VECREDUCE_FADD:
864 case ISD::VECREDUCE_FMUL:
865 case ISD::VECREDUCE_ADD:
866 case ISD::VECREDUCE_MUL:
867 case ISD::VECREDUCE_AND:
868 case ISD::VECREDUCE_OR:
869 case ISD::VECREDUCE_XOR:
870 case ISD::VECREDUCE_SMAX:
871 case ISD::VECREDUCE_SMIN:
872 case ISD::VECREDUCE_UMAX:
873 case ISD::VECREDUCE_UMIN:
874 case ISD::VECREDUCE_FMAX:
875 case ISD::VECREDUCE_FMIN:
876 case ISD::VECREDUCE_FMAXIMUM:
877 case ISD::VECREDUCE_FMINIMUM:
878 Res = ScalarizeVecOp_VECREDUCE(N);
879 break;
880 case ISD::VECREDUCE_SEQ_FADD:
881 case ISD::VECREDUCE_SEQ_FMUL:
882 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
883 break;
884 case ISD::SCMP:
885 case ISD::UCMP:
886 Res = ScalarizeVecOp_CMP(N);
887 break;
888 }
889
890 // If the result is null, the sub-method took care of registering results etc.
891 if (!Res.getNode()) return false;
892
893 // If the result is N, the sub-method updated N in place. Tell the legalizer
894 // core about this.
895 if (Res.getNode() == N)
896 return true;
897
898 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
899 "Invalid operand expansion");
900
901 ReplaceValueWith(From: SDValue(N, 0), To: Res);
902 return false;
903}
904
905/// If the value to convert is a vector that needs to be scalarized, it must be
906/// <1 x ty>. Convert the element instead.
907SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
908 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
909 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
910 VT: N->getValueType(ResNo: 0), Operand: Elt);
911}
912
913// Need to legalize vector operands of fake uses. Must be <1 x ty>.
914SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
915 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
916 "Fake Use: Unexpected vector type!");
917 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
918 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
919}
920
921/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
922/// Do the operation on the element instead.
923SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
924 assert(N->getValueType(0).getVectorNumElements() == 1 &&
925 "Unexpected vector type!");
926 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
927 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
928 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
929 // Revectorize the result so the types line up with what the uses of this
930 // expression expect.
931 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
932}
933
934/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
935/// typesize).
936SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
937 assert(N->getValueType(0).getVectorNumElements() == 1 &&
938 "Unexpected vector type!");
939 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
940 SDValue Op =
941 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
942 N1: Elt, N2: N->getOperand(Num: 1));
943 // Revectorize the result so the types line up with what the uses of this
944 // expression expect.
945 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
946}
947
948/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
949/// Do the strict FP operation on the element instead.
950SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
951 assert(N->getValueType(0).getVectorNumElements() == 1 &&
952 "Unexpected vector type!");
953 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
954 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
955 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
956 Ops: { N->getOperand(Num: 0), Elt });
957 // Legalize the chain result - switch anything that used the old chain to
958 // use the new one.
959 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
960 // Revectorize the result so the types line up with what the uses of this
961 // expression expect.
962 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
963
964 // Do our own replacement and return SDValue() to tell the caller that we
965 // handled all replacements since caller can only handle a single result.
966 ReplaceValueWith(From: SDValue(N, 0), To: Res);
967 return SDValue();
968}
969
970/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
971SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
972 SmallVector<SDValue, 8> Ops(N->getNumOperands());
973 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
974 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
975 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
976}
977
978/// The inserted subvector is to be scalarized - use insert vector element
979/// instead.
980SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
981 unsigned OpNo) {
982 // We should not be attempting to scalarize the containing vector
983 assert(OpNo == 1);
984 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
985 SDValue ContainingVec = N->getOperand(Num: 0);
986 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
987 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
988 N3: N->getOperand(Num: 2));
989}
990
991/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
992/// so just return the element, ignoring the index.
993SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
994 EVT VT = N->getValueType(ResNo: 0);
995 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
996 if (Res.getValueType() != VT)
997 Res = VT.isFloatingPoint()
998 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
999 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1000 return Res;
1001}
1002
1003/// If the input condition is a vector that needs to be scalarized, it must be
1004/// <1 x i1>, so just convert to a normal ISD::SELECT
1005/// (still with vector output type since that was acceptable if we got here).
1006SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1007 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1008 EVT VT = N->getValueType(ResNo: 0);
1009
1010 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1011 N3: N->getOperand(Num: 2));
1012}
1013
1014/// If the operand is a vector that needs to be scalarized then the
1015/// result must be v1i1, so just convert to a scalar SETCC and wrap
1016/// with a scalar_to_vector since the res type is legal if we got here
1017SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1018 assert(N->getValueType(0).isVector() &&
1019 N->getOperand(0).getValueType().isVector() &&
1020 "Operand types must be vectors");
1021 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1022
1023 EVT VT = N->getValueType(ResNo: 0);
1024 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1025 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1026
1027 EVT OpVT = N->getOperand(Num: 0).getValueType();
1028 EVT NVT = VT.getVectorElementType();
1029 SDLoc DL(N);
1030 // Turn it into a scalar SETCC.
1031 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1032 N3: N->getOperand(Num: 2));
1033
1034 // Vectors may have a different boolean contents to scalars. Promote the
1035 // value appropriately.
1036 ISD::NodeType ExtendCode =
1037 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1038
1039 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1040
1041 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1042}
1043
1044// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1045SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1046 unsigned OpNo) {
1047 assert(OpNo == 1 && "Wrong operand for scalarization!");
1048 assert(N->getValueType(0).isVector() &&
1049 N->getOperand(1).getValueType().isVector() &&
1050 "Operand types must be vectors");
1051 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1052
1053 EVT VT = N->getValueType(ResNo: 0);
1054 SDValue Ch = N->getOperand(Num: 0);
1055 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1056 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1057 SDValue CC = N->getOperand(Num: 3);
1058
1059 EVT OpVT = N->getOperand(Num: 1).getValueType();
1060 EVT NVT = VT.getVectorElementType();
1061 SDLoc DL(N);
1062 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1063 Ops: {Ch, LHS, RHS, CC});
1064
1065 // Legalize the chain result - switch anything that used the old chain to
1066 // use the new one.
1067 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1068
1069 ISD::NodeType ExtendCode =
1070 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1071
1072 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1073 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1074
1075 // Do our own replacement and return SDValue() to tell the caller that we
1076 // handled all replacements since caller can only handle a single result.
1077 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1078 return SDValue();
1079}
1080
1081/// If the value to store is a vector that needs to be scalarized, it must be
1082/// <1 x ty>. Just store the element.
1083SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1084 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1085 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1086 SDLoc dl(N);
1087
1088 if (N->isTruncatingStore())
1089 return DAG.getTruncStore(
1090 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1091 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1092 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1093 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1094
1095 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1096 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1097 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1098}
1099
1100/// If the value to round is a vector that needs to be scalarized, it must be
1101/// <1 x ty>. Convert the element instead.
1102SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1103 assert(OpNo == 0 && "Wrong operand for scalarization!");
1104 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1105 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1106 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1107 N2: N->getOperand(Num: 1));
1108 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1109}
1110
1111SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1112 unsigned OpNo) {
1113 assert(OpNo == 1 && "Wrong operand for scalarization!");
1114 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1115 SDValue Res =
1116 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1117 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1118 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1119 // Legalize the chain result - switch anything that used the old chain to
1120 // use the new one.
1121 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1122
1123 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1124
1125 // Do our own replacement and return SDValue() to tell the caller that we
1126 // handled all replacements since caller can only handle a single result.
1127 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1128 return SDValue();
1129}
1130
1131/// If the value to extend is a vector that needs to be scalarized, it must be
1132/// <1 x ty>. Convert the element instead.
1133SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1134 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1135 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1136 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1137 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1138}
1139
1140/// If the value to extend is a vector that needs to be scalarized, it must be
1141/// <1 x ty>. Convert the element instead.
1142SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1143 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1144 SDValue Res =
1145 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1146 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1147 Ops: {N->getOperand(Num: 0), Elt});
1148 // Legalize the chain result - switch anything that used the old chain to
1149 // use the new one.
1150 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1151
1152 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1153
1154 // Do our own replacement and return SDValue() to tell the caller that we
1155 // handled all replacements since caller can only handle a single result.
1156 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1157 return SDValue();
1158}
1159
1160SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1161 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1162 // Result type may be wider than element type.
1163 if (Res.getValueType() != N->getValueType(ResNo: 0))
1164 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1165 return Res;
1166}
1167
1168SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1169 SDValue AccOp = N->getOperand(Num: 0);
1170 SDValue VecOp = N->getOperand(Num: 1);
1171
1172 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1173
1174 SDValue Op = GetScalarizedVector(Op: VecOp);
1175 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1176 N1: AccOp, N2: Op, Flags: N->getFlags());
1177}
1178
1179SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1180 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1181 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1182
1183 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1184 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1185 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1186}
1187
1188//===----------------------------------------------------------------------===//
1189// Result Vector Splitting
1190//===----------------------------------------------------------------------===//
1191
1192/// This method is called when the specified result of the specified node is
1193/// found to need vector splitting. At this point, the node may also have
1194/// invalid operands or may have other results that need legalization, we just
1195/// know that (at least) one result needs vector splitting.
1196void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1197 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1198 SDValue Lo, Hi;
1199
1200 // See if the target wants to custom expand this node.
1201 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1202 return;
1203
1204 switch (N->getOpcode()) {
1205 default:
1206#ifndef NDEBUG
1207 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1208 N->dump(&DAG);
1209 dbgs() << "\n";
1210#endif
1211 report_fatal_error(reason: "Do not know how to split the result of this "
1212 "operator!\n");
1213
1214 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1215 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1216 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1217 break;
1218 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1219 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1220 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1221 case ISD::VSELECT:
1222 case ISD::SELECT:
1223 case ISD::VP_MERGE:
1224 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1225 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1226 case ISD::POISON:
1227 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1228 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1229 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1230 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1231 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1232 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1233 case ISD::FPOWI:
1234 case ISD::FLDEXP:
1235 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1236 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1237 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1238 case ISD::SPLAT_VECTOR:
1239 case ISD::SCALAR_TO_VECTOR:
1240 SplitVecRes_ScalarOp(N, Lo, Hi);
1241 break;
1242 case ISD::STEP_VECTOR:
1243 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1244 break;
1245 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1246 case ISD::LOAD:
1247 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1248 break;
1249 case ISD::VP_LOAD:
1250 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1251 break;
1252 case ISD::VP_LOAD_FF:
1253 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1254 break;
1255 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1256 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1257 break;
1258 case ISD::MLOAD:
1259 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1260 break;
1261 case ISD::MGATHER:
1262 case ISD::VP_GATHER:
1263 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1264 break;
1265 case ISD::VECTOR_COMPRESS:
1266 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1267 break;
1268 case ISD::SETCC:
1269 case ISD::VP_SETCC:
1270 SplitVecRes_SETCC(N, Lo, Hi);
1271 break;
1272 case ISD::VECTOR_REVERSE:
1273 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1274 break;
1275 case ISD::VECTOR_SHUFFLE:
1276 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1277 break;
1278 case ISD::VECTOR_SPLICE_LEFT:
1279 case ISD::VECTOR_SPLICE_RIGHT:
1280 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1281 break;
1282 case ISD::VECTOR_DEINTERLEAVE:
1283 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1284 return;
1285 case ISD::VECTOR_INTERLEAVE:
1286 SplitVecRes_VECTOR_INTERLEAVE(N);
1287 return;
1288 case ISD::VAARG:
1289 SplitVecRes_VAARG(N, Lo, Hi);
1290 break;
1291
1292 case ISD::ANY_EXTEND_VECTOR_INREG:
1293 case ISD::SIGN_EXTEND_VECTOR_INREG:
1294 case ISD::ZERO_EXTEND_VECTOR_INREG:
1295 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1296 break;
1297
1298 case ISD::ABS:
1299 case ISD::VP_ABS:
1300 case ISD::BITREVERSE:
1301 case ISD::VP_BITREVERSE:
1302 case ISD::BSWAP:
1303 case ISD::VP_BSWAP:
1304 case ISD::CTLZ:
1305 case ISD::VP_CTLZ:
1306 case ISD::CTTZ:
1307 case ISD::VP_CTTZ:
1308 case ISD::CTLZ_ZERO_UNDEF:
1309 case ISD::VP_CTLZ_ZERO_UNDEF:
1310 case ISD::CTTZ_ZERO_UNDEF:
1311 case ISD::VP_CTTZ_ZERO_UNDEF:
1312 case ISD::CTPOP:
1313 case ISD::VP_CTPOP:
1314 case ISD::FABS: case ISD::VP_FABS:
1315 case ISD::FACOS:
1316 case ISD::FASIN:
1317 case ISD::FATAN:
1318 case ISD::FCEIL:
1319 case ISD::VP_FCEIL:
1320 case ISD::FCOS:
1321 case ISD::FCOSH:
1322 case ISD::FEXP:
1323 case ISD::FEXP2:
1324 case ISD::FEXP10:
1325 case ISD::FFLOOR:
1326 case ISD::VP_FFLOOR:
1327 case ISD::FLOG:
1328 case ISD::FLOG10:
1329 case ISD::FLOG2:
1330 case ISD::FNEARBYINT:
1331 case ISD::VP_FNEARBYINT:
1332 case ISD::FNEG: case ISD::VP_FNEG:
1333 case ISD::FREEZE:
1334 case ISD::ARITH_FENCE:
1335 case ISD::FP_EXTEND:
1336 case ISD::VP_FP_EXTEND:
1337 case ISD::FP_ROUND:
1338 case ISD::VP_FP_ROUND:
1339 case ISD::FP_TO_SINT:
1340 case ISD::VP_FP_TO_SINT:
1341 case ISD::FP_TO_UINT:
1342 case ISD::VP_FP_TO_UINT:
1343 case ISD::FRINT:
1344 case ISD::VP_FRINT:
1345 case ISD::LRINT:
1346 case ISD::VP_LRINT:
1347 case ISD::LLRINT:
1348 case ISD::VP_LLRINT:
1349 case ISD::FROUND:
1350 case ISD::VP_FROUND:
1351 case ISD::FROUNDEVEN:
1352 case ISD::VP_FROUNDEVEN:
1353 case ISD::LROUND:
1354 case ISD::LLROUND:
1355 case ISD::FSIN:
1356 case ISD::FSINH:
1357 case ISD::FSQRT: case ISD::VP_SQRT:
1358 case ISD::FTAN:
1359 case ISD::FTANH:
1360 case ISD::FTRUNC:
1361 case ISD::VP_FROUNDTOZERO:
1362 case ISD::SINT_TO_FP:
1363 case ISD::VP_SINT_TO_FP:
1364 case ISD::TRUNCATE:
1365 case ISD::VP_TRUNCATE:
1366 case ISD::UINT_TO_FP:
1367 case ISD::VP_UINT_TO_FP:
1368 case ISD::FCANONICALIZE:
1369 case ISD::AssertNoFPClass:
1370 SplitVecRes_UnaryOp(N, Lo, Hi);
1371 break;
1372 case ISD::ADDRSPACECAST:
1373 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1374 break;
1375 case ISD::FMODF:
1376 case ISD::FFREXP:
1377 case ISD::FSINCOS:
1378 case ISD::FSINCOSPI:
1379 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1380 break;
1381
1382 case ISD::ANY_EXTEND:
1383 case ISD::SIGN_EXTEND:
1384 case ISD::ZERO_EXTEND:
1385 case ISD::VP_SIGN_EXTEND:
1386 case ISD::VP_ZERO_EXTEND:
1387 SplitVecRes_ExtendOp(N, Lo, Hi);
1388 break;
1389
1390 case ISD::ADD: case ISD::VP_ADD:
1391 case ISD::SUB: case ISD::VP_SUB:
1392 case ISD::MUL: case ISD::VP_MUL:
1393 case ISD::CLMUL:
1394 case ISD::CLMULR:
1395 case ISD::CLMULH:
1396 case ISD::MULHS:
1397 case ISD::MULHU:
1398 case ISD::ABDS:
1399 case ISD::ABDU:
1400 case ISD::AVGCEILS:
1401 case ISD::AVGCEILU:
1402 case ISD::AVGFLOORS:
1403 case ISD::AVGFLOORU:
1404 case ISD::FADD: case ISD::VP_FADD:
1405 case ISD::FSUB: case ISD::VP_FSUB:
1406 case ISD::FMUL: case ISD::VP_FMUL:
1407 case ISD::FMINNUM:
1408 case ISD::FMINNUM_IEEE:
1409 case ISD::VP_FMINNUM:
1410 case ISD::FMAXNUM:
1411 case ISD::FMAXNUM_IEEE:
1412 case ISD::VP_FMAXNUM:
1413 case ISD::FMINIMUM:
1414 case ISD::VP_FMINIMUM:
1415 case ISD::FMAXIMUM:
1416 case ISD::VP_FMAXIMUM:
1417 case ISD::FMINIMUMNUM:
1418 case ISD::FMAXIMUMNUM:
1419 case ISD::SDIV: case ISD::VP_SDIV:
1420 case ISD::UDIV: case ISD::VP_UDIV:
1421 case ISD::FDIV: case ISD::VP_FDIV:
1422 case ISD::FPOW:
1423 case ISD::FATAN2:
1424 case ISD::AND: case ISD::VP_AND:
1425 case ISD::OR: case ISD::VP_OR:
1426 case ISD::XOR: case ISD::VP_XOR:
1427 case ISD::SHL: case ISD::VP_SHL:
1428 case ISD::SRA: case ISD::VP_SRA:
1429 case ISD::SRL: case ISD::VP_SRL:
1430 case ISD::UREM: case ISD::VP_UREM:
1431 case ISD::SREM: case ISD::VP_SREM:
1432 case ISD::FREM: case ISD::VP_FREM:
1433 case ISD::SMIN: case ISD::VP_SMIN:
1434 case ISD::SMAX: case ISD::VP_SMAX:
1435 case ISD::UMIN: case ISD::VP_UMIN:
1436 case ISD::UMAX: case ISD::VP_UMAX:
1437 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1438 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1439 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1440 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1441 case ISD::SSHLSAT:
1442 case ISD::USHLSAT:
1443 case ISD::ROTL:
1444 case ISD::ROTR:
1445 case ISD::VP_FCOPYSIGN:
1446 SplitVecRes_BinOp(N, Lo, Hi);
1447 break;
1448 case ISD::FMA: case ISD::VP_FMA:
1449 case ISD::FSHL:
1450 case ISD::VP_FSHL:
1451 case ISD::FSHR:
1452 case ISD::VP_FSHR:
1453 SplitVecRes_TernaryOp(N, Lo, Hi);
1454 break;
1455
1456 case ISD::SCMP: case ISD::UCMP:
1457 SplitVecRes_CMP(N, Lo, Hi);
1458 break;
1459
1460#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1461 case ISD::STRICT_##DAGN:
1462#include "llvm/IR/ConstrainedOps.def"
1463 SplitVecRes_StrictFPOp(N, Lo, Hi);
1464 break;
1465
1466 case ISD::FP_TO_UINT_SAT:
1467 case ISD::FP_TO_SINT_SAT:
1468 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1469 break;
1470
1471 case ISD::UADDO:
1472 case ISD::SADDO:
1473 case ISD::USUBO:
1474 case ISD::SSUBO:
1475 case ISD::UMULO:
1476 case ISD::SMULO:
1477 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1478 break;
1479 case ISD::SMULFIX:
1480 case ISD::SMULFIXSAT:
1481 case ISD::UMULFIX:
1482 case ISD::UMULFIXSAT:
1483 case ISD::SDIVFIX:
1484 case ISD::SDIVFIXSAT:
1485 case ISD::UDIVFIX:
1486 case ISD::UDIVFIXSAT:
1487 SplitVecRes_FIX(N, Lo, Hi);
1488 break;
1489 case ISD::EXPERIMENTAL_VP_SPLICE:
1490 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1491 break;
1492 case ISD::EXPERIMENTAL_VP_REVERSE:
1493 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1494 break;
1495 case ISD::PARTIAL_REDUCE_UMLA:
1496 case ISD::PARTIAL_REDUCE_SMLA:
1497 case ISD::PARTIAL_REDUCE_SUMLA:
1498 case ISD::PARTIAL_REDUCE_FMLA:
1499 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1500 break;
1501 case ISD::GET_ACTIVE_LANE_MASK:
1502 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1503 break;
1504 }
1505
1506 // If Lo/Hi is null, the sub-method took care of registering results etc.
1507 if (Lo.getNode())
1508 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1509}
1510
1511void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1512 MachinePointerInfo &MPI, SDValue &Ptr,
1513 uint64_t *ScaledOffset) {
1514 SDLoc DL(N);
1515 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1516
1517 if (MemVT.isScalableVector()) {
1518 SDValue BytesIncrement = DAG.getVScale(
1519 DL, VT: Ptr.getValueType(),
1520 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1521 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1522 if (ScaledOffset)
1523 *ScaledOffset += IncrementSize;
1524 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1525 Flags: SDNodeFlags::NoUnsignedWrap);
1526 } else {
1527 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1528 // Increment the pointer to the other half.
1529 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1530 }
1531}
1532
1533std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1534 return SplitMask(Mask, DL: SDLoc(Mask));
1535}
1536
1537std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1538 const SDLoc &DL) {
1539 SDValue MaskLo, MaskHi;
1540 EVT MaskVT = Mask.getValueType();
1541 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1542 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1543 else
1544 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1545 return std::make_pair(x&: MaskLo, y&: MaskHi);
1546}
1547
1548void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1549 SDValue LHSLo, LHSHi;
1550 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1551 SDValue RHSLo, RHSHi;
1552 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1553 SDLoc dl(N);
1554
1555 const SDNodeFlags Flags = N->getFlags();
1556 unsigned Opcode = N->getOpcode();
1557 if (N->getNumOperands() == 2) {
1558 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1559 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1560 return;
1561 }
1562
1563 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1564 assert(N->isVPOpcode() && "Expected VP opcode");
1565
1566 SDValue MaskLo, MaskHi;
1567 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1568
1569 SDValue EVLLo, EVLHi;
1570 std::tie(args&: EVLLo, args&: EVLHi) =
1571 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1572
1573 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1574 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1575 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1576 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1577}
1578
1579void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1580 SDValue &Hi) {
1581 SDValue Op0Lo, Op0Hi;
1582 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1583 SDValue Op1Lo, Op1Hi;
1584 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1585 SDValue Op2Lo, Op2Hi;
1586 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1587 SDLoc dl(N);
1588
1589 const SDNodeFlags Flags = N->getFlags();
1590 unsigned Opcode = N->getOpcode();
1591 if (N->getNumOperands() == 3) {
1592 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1593 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1594 return;
1595 }
1596
1597 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1598 assert(N->isVPOpcode() && "Expected VP opcode");
1599
1600 SDValue MaskLo, MaskHi;
1601 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1602
1603 SDValue EVLLo, EVLHi;
1604 std::tie(args&: EVLLo, args&: EVLHi) =
1605 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1606
1607 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1608 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1609 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1610 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1611}
1612
1613void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1614 LLVMContext &Ctxt = *DAG.getContext();
1615 SDLoc dl(N);
1616
1617 SDValue LHS = N->getOperand(Num: 0);
1618 SDValue RHS = N->getOperand(Num: 1);
1619
1620 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1621 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1622 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1623 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1624 } else {
1625 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1626 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1627 }
1628
1629 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1630 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1631 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1632}
1633
1634void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1635 SDValue LHSLo, LHSHi;
1636 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1637 SDValue RHSLo, RHSHi;
1638 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1639 SDLoc dl(N);
1640 SDValue Op2 = N->getOperand(Num: 2);
1641
1642 unsigned Opcode = N->getOpcode();
1643 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1644 Flags: N->getFlags());
1645 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1646 Flags: N->getFlags());
1647}
1648
1649void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1650 SDValue &Hi) {
1651 // We know the result is a vector. The input may be either a vector or a
1652 // scalar value.
1653 EVT LoVT, HiVT;
1654 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1655 SDLoc dl(N);
1656
1657 SDValue InOp = N->getOperand(Num: 0);
1658 EVT InVT = InOp.getValueType();
1659
1660 // Handle some special cases efficiently.
1661 switch (getTypeAction(VT: InVT)) {
1662 case TargetLowering::TypeLegal:
1663 case TargetLowering::TypePromoteInteger:
1664 case TargetLowering::TypeSoftPromoteHalf:
1665 case TargetLowering::TypeSoftenFloat:
1666 case TargetLowering::TypeScalarizeVector:
1667 case TargetLowering::TypeWidenVector:
1668 break;
1669 case TargetLowering::TypeExpandInteger:
1670 case TargetLowering::TypeExpandFloat:
1671 // A scalar to vector conversion, where the scalar needs expansion.
1672 // If the vector is being split in two then we can just convert the
1673 // expanded pieces.
1674 if (LoVT == HiVT) {
1675 GetExpandedOp(Op: InOp, Lo, Hi);
1676 if (DAG.getDataLayout().isBigEndian())
1677 std::swap(a&: Lo, b&: Hi);
1678 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1679 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1680 return;
1681 }
1682 break;
1683 case TargetLowering::TypeSplitVector:
1684 // If the input is a vector that needs to be split, convert each split
1685 // piece of the input now.
1686 GetSplitVector(Op: InOp, Lo, Hi);
1687 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1688 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1689 return;
1690 case TargetLowering::TypeScalarizeScalableVector:
1691 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1692 }
1693
1694 if (LoVT.isScalableVector()) {
1695 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1696 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1697 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1698 return;
1699 }
1700
1701 // In the general case, convert the input to an integer and split it by hand.
1702 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1703 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1704 if (DAG.getDataLayout().isBigEndian())
1705 std::swap(a&: LoIntVT, b&: HiIntVT);
1706
1707 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1708
1709 if (DAG.getDataLayout().isBigEndian())
1710 std::swap(a&: Lo, b&: Hi);
1711 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1712 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1713}
1714
1715void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1716 SDValue &Hi) {
1717 SDLoc DL(N);
1718 EVT LoVT, HiVT;
1719 SDValue PtrA = N->getOperand(Num: 0);
1720 SDValue PtrB = N->getOperand(Num: 1);
1721 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1722
1723 // The lane offset for the "Lo" half of the mask is unchanged.
1724 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1725 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1726 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1727 // The lane offset for the "Hi" half of the mask is incremented by the number
1728 // of elements in the "Lo" half.
1729 unsigned LaneOffset =
1730 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1731 // Note: The lane offset is implicitly scalable for scalable masks.
1732 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1733 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1734 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1735}
1736
1737void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1738 SDValue &Hi) {
1739 EVT LoVT, HiVT;
1740 SDLoc dl(N);
1741 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1742 unsigned LoNumElts = LoVT.getVectorNumElements();
1743 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1744 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1745
1746 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1747 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1748}
1749
1750void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1751 SDValue &Hi) {
1752 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1753 SDLoc dl(N);
1754 unsigned NumSubvectors = N->getNumOperands() / 2;
1755 if (NumSubvectors == 1) {
1756 Lo = N->getOperand(Num: 0);
1757 Hi = N->getOperand(Num: 1);
1758 return;
1759 }
1760
1761 EVT LoVT, HiVT;
1762 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1763
1764 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1765 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1766
1767 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1768 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1769}
1770
1771void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1772 SDValue &Hi) {
1773 SDValue Vec = N->getOperand(Num: 0);
1774 SDValue Idx = N->getOperand(Num: 1);
1775 SDLoc dl(N);
1776
1777 EVT LoVT, HiVT;
1778 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1779
1780 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1781 uint64_t IdxVal = Idx->getAsZExtVal();
1782 Hi = DAG.getNode(
1783 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1784 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1785}
1786
1787void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1788 SDValue &Hi) {
1789 SDValue Vec = N->getOperand(Num: 0);
1790 SDValue SubVec = N->getOperand(Num: 1);
1791 SDValue Idx = N->getOperand(Num: 2);
1792 SDLoc dl(N);
1793 GetSplitVector(Op: Vec, Lo, Hi);
1794
1795 EVT VecVT = Vec.getValueType();
1796 EVT LoVT = Lo.getValueType();
1797 EVT SubVecVT = SubVec.getValueType();
1798 unsigned VecElems = VecVT.getVectorMinNumElements();
1799 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1800 unsigned LoElems = LoVT.getVectorMinNumElements();
1801
1802 // If we know the index is in the first half, and we know the subvector
1803 // doesn't cross the boundary between the halves, we can avoid spilling the
1804 // vector, and insert into the lower half of the split vector directly.
1805 unsigned IdxVal = Idx->getAsZExtVal();
1806 if (IdxVal + SubElems <= LoElems) {
1807 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1808 return;
1809 }
1810 // Similarly if the subvector is fully in the high half, but mind that we
1811 // can't tell whether a fixed-length subvector is fully within the high half
1812 // of a scalable vector.
1813 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1814 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1815 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1816 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1817 return;
1818 }
1819
1820 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1821 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1822 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1823 if (WideSubVec.getValueType() == VecVT) {
1824 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1825 return;
1826 }
1827 }
1828
1829 // Spill the vector to the stack.
1830 // In cases where the vector is illegal it will be broken down into parts
1831 // and stored in parts - we should use the alignment for the smallest part.
1832 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1833 SDValue StackPtr =
1834 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1835 auto &MF = DAG.getMachineFunction();
1836 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1837 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1838
1839 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1840 Alignment: SmallestAlign);
1841
1842 // Store the new subvector into the specified index.
1843 SDValue SubVecPtr =
1844 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1845 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1846 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1847
1848 // Load the Lo part from the stack slot.
1849 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1850 Alignment: SmallestAlign);
1851
1852 // Increment the pointer to the other part.
1853 auto *Load = cast<LoadSDNode>(Val&: Lo);
1854 MachinePointerInfo MPI = Load->getPointerInfo();
1855 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1856
1857 // Load the Hi part from the stack slot.
1858 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1859}
1860
1861// Handle splitting an FP where the second operand does not match the first
1862// type. The second operand may be a scalar, or a vector that has exactly as
1863// many elements as the first
1864void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1865 SDValue &Hi) {
1866 SDValue LHSLo, LHSHi;
1867 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1868 SDLoc DL(N);
1869
1870 SDValue RHSLo, RHSHi;
1871 SDValue RHS = N->getOperand(Num: 1);
1872 EVT RHSVT = RHS.getValueType();
1873 if (RHSVT.isVector()) {
1874 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1875 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1876 else
1877 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1878
1879 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1880 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1881 } else {
1882 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1883 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1884 }
1885}
1886
1887void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1888 SDValue &Hi) {
1889 SDLoc DL(N);
1890 SDValue ArgLo, ArgHi;
1891 SDValue Test = N->getOperand(Num: 1);
1892 SDValue FpValue = N->getOperand(Num: 0);
1893 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1894 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1895 else
1896 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1897 EVT LoVT, HiVT;
1898 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1899
1900 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1901 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1902}
1903
1904void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1905 SDValue &Hi) {
1906 SDValue LHSLo, LHSHi;
1907 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1908 SDLoc dl(N);
1909
1910 EVT LoVT, HiVT;
1911 std::tie(args&: LoVT, args&: HiVT) =
1912 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1913
1914 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1915 N2: DAG.getValueType(LoVT));
1916 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1917 N2: DAG.getValueType(HiVT));
1918}
1919
1920void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1921 SDValue &Hi) {
1922 unsigned Opcode = N->getOpcode();
1923 SDValue N0 = N->getOperand(Num: 0);
1924
1925 SDLoc dl(N);
1926 SDValue InLo, InHi;
1927
1928 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1929 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1930 else
1931 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1932
1933 EVT InLoVT = InLo.getValueType();
1934 unsigned InNumElements = InLoVT.getVectorNumElements();
1935
1936 EVT OutLoVT, OutHiVT;
1937 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1938 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1939 assert((2 * OutNumElements) <= InNumElements &&
1940 "Illegal extend vector in reg split");
1941
1942 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1943 // input vector (i.e. we only use InLo):
1944 // OutLo will extend the first OutNumElements from InLo.
1945 // OutHi will extend the next OutNumElements from InLo.
1946
1947 // Shuffle the elements from InLo for OutHi into the bottom elements to
1948 // create a 'fake' InHi.
1949 SmallVector<int, 8> SplitHi(InNumElements, -1);
1950 for (unsigned i = 0; i != OutNumElements; ++i)
1951 SplitHi[i] = i + OutNumElements;
1952 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
1953
1954 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
1955 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
1956}
1957
1958void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1959 SDValue &Hi) {
1960 unsigned NumOps = N->getNumOperands();
1961 SDValue Chain = N->getOperand(Num: 0);
1962 EVT LoVT, HiVT;
1963 SDLoc dl(N);
1964 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1965
1966 SmallVector<SDValue, 4> OpsLo(NumOps);
1967 SmallVector<SDValue, 4> OpsHi(NumOps);
1968
1969 // The Chain is the first operand.
1970 OpsLo[0] = Chain;
1971 OpsHi[0] = Chain;
1972
1973 // Now process the remaining operands.
1974 for (unsigned i = 1; i < NumOps; ++i) {
1975 SDValue Op = N->getOperand(Num: i);
1976 SDValue OpLo = Op;
1977 SDValue OpHi = Op;
1978
1979 EVT InVT = Op.getValueType();
1980 if (InVT.isVector()) {
1981 // If the input also splits, handle it directly for a
1982 // compile time speedup. Otherwise split it by hand.
1983 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
1984 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
1985 else
1986 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
1987 }
1988
1989 OpsLo[i] = OpLo;
1990 OpsHi[i] = OpHi;
1991 }
1992
1993 EVT LoValueVTs[] = {LoVT, MVT::Other};
1994 EVT HiValueVTs[] = {HiVT, MVT::Other};
1995 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
1996 Flags: N->getFlags());
1997 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
1998 Flags: N->getFlags());
1999
2000 // Build a factor node to remember that this Op is independent of the
2001 // other one.
2002 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2003 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2004
2005 // Legalize the chain result - switch anything that used the old chain to
2006 // use the new one.
2007 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2008}
2009
2010SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2011 SDValue Chain = N->getOperand(Num: 0);
2012 EVT VT = N->getValueType(ResNo: 0);
2013 unsigned NE = VT.getVectorNumElements();
2014 EVT EltVT = VT.getVectorElementType();
2015 SDLoc dl(N);
2016
2017 SmallVector<SDValue, 8> Scalars;
2018 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2019
2020 // If ResNE is 0, fully unroll the vector op.
2021 if (ResNE == 0)
2022 ResNE = NE;
2023 else if (NE > ResNE)
2024 NE = ResNE;
2025
2026 //The results of each unrolled operation, including the chain.
2027 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2028 SmallVector<SDValue, 8> Chains;
2029
2030 unsigned i;
2031 for (i = 0; i != NE; ++i) {
2032 Operands[0] = Chain;
2033 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2034 SDValue Operand = N->getOperand(Num: j);
2035 EVT OperandVT = Operand.getValueType();
2036 if (OperandVT.isVector()) {
2037 EVT OperandEltVT = OperandVT.getVectorElementType();
2038 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2039 } else {
2040 Operands[j] = Operand;
2041 }
2042 }
2043 SDValue Scalar =
2044 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2045
2046 //Add in the scalar as well as its chain value to the
2047 //result vectors.
2048 Scalars.push_back(Elt: Scalar);
2049 Chains.push_back(Elt: Scalar.getValue(R: 1));
2050 }
2051
2052 for (; i < ResNE; ++i)
2053 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2054
2055 // Build a new factor node to connect the chain back together.
2056 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2057 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2058
2059 // Create a new BUILD_VECTOR node
2060 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2061 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2062}
2063
2064void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2065 SDValue &Lo, SDValue &Hi) {
2066 SDLoc dl(N);
2067 EVT ResVT = N->getValueType(ResNo: 0);
2068 EVT OvVT = N->getValueType(ResNo: 1);
2069 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2070 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2071 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2072
2073 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2074 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2075 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2076 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2077 } else {
2078 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2079 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2080 }
2081
2082 unsigned Opcode = N->getOpcode();
2083 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2084 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2085 SDNode *LoNode =
2086 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2087 SDNode *HiNode =
2088 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2089
2090 Lo = SDValue(LoNode, ResNo);
2091 Hi = SDValue(HiNode, ResNo);
2092
2093 // Replace the other vector result not being explicitly split here.
2094 unsigned OtherNo = 1 - ResNo;
2095 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2096 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2097 SetSplitVector(Op: SDValue(N, OtherNo),
2098 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2099 } else {
2100 SDValue OtherVal = DAG.getNode(
2101 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2102 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2103 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2104 }
2105}
2106
2107void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2108 SDValue &Hi) {
2109 SDValue Vec = N->getOperand(Num: 0);
2110 SDValue Elt = N->getOperand(Num: 1);
2111 SDValue Idx = N->getOperand(Num: 2);
2112 SDLoc dl(N);
2113 GetSplitVector(Op: Vec, Lo, Hi);
2114
2115 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2116 unsigned IdxVal = CIdx->getZExtValue();
2117 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2118 if (IdxVal < LoNumElts) {
2119 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2120 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2121 return;
2122 } else if (!Vec.getValueType().isScalableVector()) {
2123 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2124 return;
2125 }
2126 }
2127
2128 // Make the vector elements byte-addressable if they aren't already.
2129 EVT VecVT = Vec.getValueType();
2130 EVT EltVT = VecVT.getVectorElementType();
2131 if (!EltVT.isByteSized()) {
2132 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2133 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2134 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2135 // Extend the element type to match if needed.
2136 if (EltVT.bitsGT(VT: Elt.getValueType()))
2137 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2138 }
2139
2140 // Spill the vector to the stack.
2141 // In cases where the vector is illegal it will be broken down into parts
2142 // and stored in parts - we should use the alignment for the smallest part.
2143 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2144 SDValue StackPtr =
2145 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2146 auto &MF = DAG.getMachineFunction();
2147 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2148 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2149
2150 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2151 Alignment: SmallestAlign);
2152
2153 // Store the new element. This may be larger than the vector element type,
2154 // so use a truncating store.
2155 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2156 Store = DAG.getTruncStore(
2157 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2158 Alignment: commonAlignment(A: SmallestAlign,
2159 Offset: EltVT.getFixedSizeInBits() / 8));
2160
2161 EVT LoVT, HiVT;
2162 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2163
2164 // Load the Lo part from the stack slot.
2165 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2166
2167 // Increment the pointer to the other part.
2168 auto Load = cast<LoadSDNode>(Val&: Lo);
2169 MachinePointerInfo MPI = Load->getPointerInfo();
2170 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2171
2172 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2173
2174 // If we adjusted the original type, we need to truncate the results.
2175 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2176 if (LoVT != Lo.getValueType())
2177 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2178 if (HiVT != Hi.getValueType())
2179 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2180}
2181
2182void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2183 SDValue &Hi) {
2184 EVT LoVT, HiVT;
2185 SDLoc dl(N);
2186 assert(N->getValueType(0).isScalableVector() &&
2187 "Only scalable vectors are supported for STEP_VECTOR");
2188 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2189 SDValue Step = N->getOperand(Num: 0);
2190
2191 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2192
2193 // Hi = Lo + (EltCnt * Step)
2194 EVT EltVT = Step.getValueType();
2195 APInt StepVal = Step->getAsAPIntVal();
2196 SDValue StartOfHi =
2197 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2198 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2199 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2200
2201 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2202 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2203}
2204
2205void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2206 SDValue &Hi) {
2207 EVT LoVT, HiVT;
2208 SDLoc dl(N);
2209 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2210 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2211 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2212 Hi = DAG.getPOISON(VT: HiVT);
2213 } else {
2214 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2215 Hi = Lo;
2216 }
2217}
2218
2219void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2220 SDValue &Hi) {
2221 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2222 EVT LoVT, HiVT;
2223 SDLoc dl(LD);
2224 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2225
2226 ISD::LoadExtType ExtType = LD->getExtensionType();
2227 SDValue Ch = LD->getChain();
2228 SDValue Ptr = LD->getBasePtr();
2229 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2230 EVT MemoryVT = LD->getMemoryVT();
2231 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2232 AAMDNodes AAInfo = LD->getAAInfo();
2233
2234 EVT LoMemVT, HiMemVT;
2235 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2236
2237 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2238 SDValue Value, NewChain;
2239 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2240 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2241 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2242 return;
2243 }
2244
2245 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2246 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2247 AAInfo);
2248
2249 MachinePointerInfo MPI;
2250 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2251
2252 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2253 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2254
2255 // Build a factor node to remember that this load is independent of the
2256 // other one.
2257 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2258 N2: Hi.getValue(R: 1));
2259
2260 // Legalize the chain result - switch anything that used the old chain to
2261 // use the new one.
2262 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2263}
2264
2265void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2266 SDValue &Hi) {
2267 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2268 EVT LoVT, HiVT;
2269 SDLoc dl(LD);
2270 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2271
2272 ISD::LoadExtType ExtType = LD->getExtensionType();
2273 SDValue Ch = LD->getChain();
2274 SDValue Ptr = LD->getBasePtr();
2275 SDValue Offset = LD->getOffset();
2276 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2277 Align Alignment = LD->getBaseAlign();
2278 SDValue Mask = LD->getMask();
2279 SDValue EVL = LD->getVectorLength();
2280 EVT MemoryVT = LD->getMemoryVT();
2281
2282 EVT LoMemVT, HiMemVT;
2283 bool HiIsEmpty = false;
2284 std::tie(args&: LoMemVT, args&: HiMemVT) =
2285 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2286
2287 // Split Mask operand
2288 SDValue MaskLo, MaskHi;
2289 if (Mask.getOpcode() == ISD::SETCC) {
2290 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2291 } else {
2292 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2293 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2294 else
2295 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2296 }
2297
2298 // Split EVL operand
2299 SDValue EVLLo, EVLHi;
2300 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2301
2302 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2303 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2304 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2305 Ranges: LD->getRanges());
2306
2307 Lo =
2308 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2309 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2310
2311 if (HiIsEmpty) {
2312 // The hi vp_load has zero storage size. We therefore simply set it to
2313 // the low vp_load and rely on subsequent removal from the chain.
2314 Hi = Lo;
2315 } else {
2316 // Generate hi vp_load.
2317 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2318 IsCompressedMemory: LD->isExpandingLoad());
2319
2320 MachinePointerInfo MPI;
2321 if (LoMemVT.isScalableVector())
2322 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2323 else
2324 MPI = LD->getPointerInfo().getWithOffset(
2325 O: LoMemVT.getStoreSize().getFixedValue());
2326
2327 MMO = DAG.getMachineFunction().getMachineMemOperand(
2328 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2329 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2330
2331 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2332 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2333 IsExpanding: LD->isExpandingLoad());
2334 }
2335
2336 // Build a factor node to remember that this load is independent of the
2337 // other one.
2338 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2339 N2: Hi.getValue(R: 1));
2340
2341 // Legalize the chain result - switch anything that used the old chain to
2342 // use the new one.
2343 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2344}
2345
2346void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2347 SDValue &Hi) {
2348 SDLoc dl(LD);
2349 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2350
2351 SDValue Ch = LD->getChain();
2352 SDValue Ptr = LD->getBasePtr();
2353 Align Alignment = LD->getBaseAlign();
2354 SDValue Mask = LD->getMask();
2355 SDValue EVL = LD->getVectorLength();
2356
2357 // Split Mask operand
2358 SDValue MaskLo, MaskHi;
2359 if (Mask.getOpcode() == ISD::SETCC) {
2360 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2361 } else {
2362 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2363 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2364 else
2365 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2366 }
2367
2368 // Split EVL operand
2369 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2370
2371 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2372 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2373 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2374 Ranges: LD->getRanges());
2375
2376 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2377
2378 // Fill the upper half with poison.
2379 Hi = DAG.getPOISON(VT: HiVT);
2380
2381 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2382 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2383}
2384
2385void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2386 SDValue &Lo, SDValue &Hi) {
2387 assert(SLD->isUnindexed() &&
2388 "Indexed VP strided load during type legalization!");
2389 assert(SLD->getOffset().isUndef() &&
2390 "Unexpected indexed variable-length load offset");
2391
2392 SDLoc DL(SLD);
2393
2394 EVT LoVT, HiVT;
2395 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2396
2397 EVT LoMemVT, HiMemVT;
2398 bool HiIsEmpty = false;
2399 std::tie(args&: LoMemVT, args&: HiMemVT) =
2400 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2401
2402 SDValue Mask = SLD->getMask();
2403 SDValue LoMask, HiMask;
2404 if (Mask.getOpcode() == ISD::SETCC) {
2405 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2406 } else {
2407 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2408 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2409 else
2410 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2411 }
2412
2413 SDValue LoEVL, HiEVL;
2414 std::tie(args&: LoEVL, args&: HiEVL) =
2415 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2416
2417 // Generate the low vp_strided_load
2418 Lo = DAG.getStridedLoadVP(
2419 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2420 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2421 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2422
2423 if (HiIsEmpty) {
2424 // The high vp_strided_load has zero storage size. We therefore simply set
2425 // it to the low vp_strided_load and rely on subsequent removal from the
2426 // chain.
2427 Hi = Lo;
2428 } else {
2429 // Generate the high vp_strided_load.
2430 // To calculate the high base address, we need to sum to the low base
2431 // address stride number of bytes for each element already loaded by low,
2432 // that is: Ptr = Ptr + (LoEVL * Stride)
2433 EVT PtrVT = SLD->getBasePtr().getValueType();
2434 SDValue Increment =
2435 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2436 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2437 SDValue Ptr =
2438 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2439
2440 Align Alignment = SLD->getBaseAlign();
2441 if (LoMemVT.isScalableVector())
2442 Alignment = commonAlignment(
2443 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2444
2445 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2446 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2447 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2448 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2449
2450 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2451 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2452 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2453 IsExpanding: SLD->isExpandingLoad());
2454 }
2455
2456 // Build a factor node to remember that this load is independent of the
2457 // other one.
2458 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2459 N2: Hi.getValue(R: 1));
2460
2461 // Legalize the chain result - switch anything that used the old chain to
2462 // use the new one.
2463 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2464}
2465
2466void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2467 SDValue &Lo, SDValue &Hi) {
2468 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2469 EVT LoVT, HiVT;
2470 SDLoc dl(MLD);
2471 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2472
2473 SDValue Ch = MLD->getChain();
2474 SDValue Ptr = MLD->getBasePtr();
2475 SDValue Offset = MLD->getOffset();
2476 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2477 SDValue Mask = MLD->getMask();
2478 SDValue PassThru = MLD->getPassThru();
2479 Align Alignment = MLD->getBaseAlign();
2480 ISD::LoadExtType ExtType = MLD->getExtensionType();
2481 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2482
2483 // Split Mask operand
2484 SDValue MaskLo, MaskHi;
2485 if (Mask.getOpcode() == ISD::SETCC) {
2486 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2487 } else {
2488 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2489 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2490 else
2491 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2492 }
2493
2494 EVT MemoryVT = MLD->getMemoryVT();
2495 EVT LoMemVT, HiMemVT;
2496 bool HiIsEmpty = false;
2497 std::tie(args&: LoMemVT, args&: HiMemVT) =
2498 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2499
2500 SDValue PassThruLo, PassThruHi;
2501 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2502 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2503 else
2504 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2505
2506 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2507 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2508 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2509
2510 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2511 MMO, AM: MLD->getAddressingMode(), ExtType,
2512 IsExpanding: MLD->isExpandingLoad());
2513
2514 if (HiIsEmpty) {
2515 // The hi masked load has zero storage size. We therefore simply set it to
2516 // the low masked load and rely on subsequent removal from the chain.
2517 Hi = Lo;
2518 } else {
2519 // Generate hi masked load.
2520 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2521 IsCompressedMemory: MLD->isExpandingLoad());
2522
2523 MachinePointerInfo MPI;
2524 if (LoMemVT.isScalableVector())
2525 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2526 else
2527 MPI = MLD->getPointerInfo().getWithOffset(
2528 O: LoMemVT.getStoreSize().getFixedValue());
2529
2530 MMO = DAG.getMachineFunction().getMachineMemOperand(
2531 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2532 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2533
2534 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2535 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2536 IsExpanding: MLD->isExpandingLoad());
2537 }
2538
2539 // Build a factor node to remember that this load is independent of the
2540 // other one.
2541 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2542 N2: Hi.getValue(R: 1));
2543
2544 // Legalize the chain result - switch anything that used the old chain to
2545 // use the new one.
2546 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2547
2548}
2549
2550void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2551 SDValue &Hi, bool SplitSETCC) {
2552 EVT LoVT, HiVT;
2553 SDLoc dl(N);
2554 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2555
2556 SDValue Ch = N->getChain();
2557 SDValue Ptr = N->getBasePtr();
2558 struct Operands {
2559 SDValue Mask;
2560 SDValue Index;
2561 SDValue Scale;
2562 } Ops = [&]() -> Operands {
2563 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2564 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2565 }
2566 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2567 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2568 }();
2569
2570 EVT MemoryVT = N->getMemoryVT();
2571 Align Alignment = N->getBaseAlign();
2572
2573 // Split Mask operand
2574 SDValue MaskLo, MaskHi;
2575 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2576 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2577 } else {
2578 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2579 }
2580
2581 EVT LoMemVT, HiMemVT;
2582 // Split MemoryVT
2583 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2584
2585 SDValue IndexHi, IndexLo;
2586 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2587 TargetLowering::TypeSplitVector)
2588 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2589 else
2590 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2591
2592 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2593 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2594 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2595 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2596
2597 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2598 SDValue PassThru = MGT->getPassThru();
2599 SDValue PassThruLo, PassThruHi;
2600 if (getTypeAction(VT: PassThru.getValueType()) ==
2601 TargetLowering::TypeSplitVector)
2602 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2603 else
2604 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2605
2606 ISD::LoadExtType ExtType = MGT->getExtensionType();
2607 ISD::MemIndexType IndexTy = MGT->getIndexType();
2608
2609 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2610 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2611 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2612
2613 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2614 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2615 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2616 } else {
2617 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2618 SDValue EVLLo, EVLHi;
2619 std::tie(args&: EVLLo, args&: EVLHi) =
2620 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2621
2622 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2623 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2624 MMO, IndexType: VPGT->getIndexType());
2625
2626 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2627 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2628 MMO, IndexType: VPGT->getIndexType());
2629 }
2630
2631 // Build a factor node to remember that this load is independent of the
2632 // other one.
2633 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2634 N2: Hi.getValue(R: 1));
2635
2636 // Legalize the chain result - switch anything that used the old chain to
2637 // use the new one.
2638 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2639}
2640
2641void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2642 SDValue &Hi) {
2643 // This is not "trivial", as there is a dependency between the two subvectors.
2644 // Depending on the number of 1s in the mask, the elements from the Hi vector
2645 // need to be moved to the Lo vector. Passthru values make this even harder.
2646 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2647 // smaller types and passthru is undef, as it is most likely faster than the
2648 // fully expand path. Otherwise, just do the full expansion as one "big"
2649 // operation and then extract the Lo and Hi vectors from that. This gets
2650 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2651 SDLoc DL(N);
2652 EVT VecVT = N->getValueType(ResNo: 0);
2653
2654 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2655 bool HasCustomLowering = false;
2656 EVT CheckVT = LoVT;
2657 while (CheckVT.getVectorMinNumElements() > 1) {
2658 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2659 // custom lowering for illegal types. So we do the checks separately.
2660 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2661 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2662 HasCustomLowering = true;
2663 break;
2664 }
2665 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2666 }
2667
2668 SDValue Passthru = N->getOperand(Num: 2);
2669 if (!HasCustomLowering) {
2670 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2671 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2672 return;
2673 }
2674
2675 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2676 SDValue Mask = N->getOperand(Num: 1);
2677 SDValue LoMask, HiMask;
2678 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2679 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2680
2681 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2682 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2683 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2684
2685 SDValue StackPtr = DAG.CreateStackTemporary(
2686 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2687 MachineFunction &MF = DAG.getMachineFunction();
2688 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2689 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2690
2691 EVT MaskVT = LoMask.getValueType();
2692 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2693
2694 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2695 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2696 EC: MaskVT.getVectorElementCount());
2697 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2698 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2699 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2700
2701 SDValue Chain = DAG.getEntryNode();
2702 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2703 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2704 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2705
2706 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2707 if (!Passthru.isUndef()) {
2708 Compressed =
2709 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2710 }
2711 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2712}
2713
2714void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2715 assert(N->getValueType(0).isVector() &&
2716 N->getOperand(0).getValueType().isVector() &&
2717 "Operand types must be vectors");
2718
2719 EVT LoVT, HiVT;
2720 SDLoc DL(N);
2721 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2722
2723 // If the input also splits, handle it directly. Otherwise split it by hand.
2724 SDValue LL, LH, RL, RH;
2725 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2726 TargetLowering::TypeSplitVector)
2727 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2728 else
2729 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2730
2731 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2732 TargetLowering::TypeSplitVector)
2733 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2734 else
2735 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2736
2737 if (N->getOpcode() == ISD::SETCC) {
2738 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2739 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2740 } else {
2741 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2742 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2743 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2744 std::tie(args&: EVLLo, args&: EVLHi) =
2745 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2746 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2747 N5: EVLLo);
2748 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2749 N5: EVLHi);
2750 }
2751}
2752
2753void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2754 SDValue &Hi) {
2755 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2756 EVT LoVT, HiVT;
2757 SDLoc dl(N);
2758 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2759
2760 // If the input also splits, handle it directly for a compile time speedup.
2761 // Otherwise split it by hand.
2762 EVT InVT = N->getOperand(Num: 0).getValueType();
2763 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2764 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2765 else
2766 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2767
2768 const SDNodeFlags Flags = N->getFlags();
2769 unsigned Opcode = N->getOpcode();
2770 if (N->getNumOperands() <= 2) {
2771 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass) {
2772 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2773 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2774 } else {
2775 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2776 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2777 }
2778 return;
2779 }
2780
2781 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2782 assert(N->isVPOpcode() && "Expected VP opcode");
2783
2784 SDValue MaskLo, MaskHi;
2785 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2786
2787 SDValue EVLLo, EVLHi;
2788 std::tie(args&: EVLLo, args&: EVLHi) =
2789 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2790
2791 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2792 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2793}
2794
2795void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2796 SDValue &Hi) {
2797 SDLoc dl(N);
2798 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2799
2800 // If the input also splits, handle it directly for a compile time speedup.
2801 // Otherwise split it by hand.
2802 EVT InVT = N->getOperand(Num: 0).getValueType();
2803 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2804 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2805 else
2806 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2807
2808 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
2809 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2810 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2811 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
2812 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
2813}
2814
2815void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
2816 unsigned ResNo,
2817 SDValue &Lo,
2818 SDValue &Hi) {
2819 SDLoc dl(N);
2820 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2821 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2822
2823 // If the input also splits, handle it directly for a compile time speedup.
2824 // Otherwise split it by hand.
2825 EVT InVT = N->getOperand(Num: 0).getValueType();
2826 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2827 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2828 else
2829 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2830
2831 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
2832 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
2833
2834 SDNode *HiNode = Hi.getNode();
2835 SDNode *LoNode = Lo.getNode();
2836
2837 // Replace the other vector result not being explicitly split here.
2838 unsigned OtherNo = 1 - ResNo;
2839 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2840 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2841 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2842 Hi: SDValue(HiNode, OtherNo));
2843 } else {
2844 SDValue OtherVal =
2845 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2846 N2: SDValue(HiNode, OtherNo));
2847 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2848 }
2849}
2850
2851void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2852 SDValue &Hi) {
2853 SDLoc dl(N);
2854 EVT SrcVT = N->getOperand(Num: 0).getValueType();
2855 EVT DestVT = N->getValueType(ResNo: 0);
2856 EVT LoVT, HiVT;
2857 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2858
2859 // We can do better than a generic split operation if the extend is doing
2860 // more than just doubling the width of the elements and the following are
2861 // true:
2862 // - The number of vector elements is even,
2863 // - the source type is legal,
2864 // - the type of a split source is illegal,
2865 // - the type of an extended (by doubling element size) source is legal, and
2866 // - the type of that extended source when split is legal.
2867 //
2868 // This won't necessarily completely legalize the operation, but it will
2869 // more effectively move in the right direction and prevent falling down
2870 // to scalarization in many cases due to the input vector being split too
2871 // far.
2872 if (SrcVT.getVectorElementCount().isKnownEven() &&
2873 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2874 LLVMContext &Ctx = *DAG.getContext();
2875 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2876 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2877
2878 EVT SplitLoVT, SplitHiVT;
2879 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2880 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2881 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2882 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2883 N->dump(&DAG); dbgs() << "\n");
2884 if (!N->isVPOpcode()) {
2885 // Extend the source vector by one step.
2886 SDValue NewSrc =
2887 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2888 // Get the low and high halves of the new, extended one step, vector.
2889 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2890 // Extend those vector halves the rest of the way.
2891 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2892 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2893 return;
2894 }
2895
2896 // Extend the source vector by one step.
2897 SDValue NewSrc =
2898 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2899 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2900 // Get the low and high halves of the new, extended one step, vector.
2901 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2902
2903 SDValue MaskLo, MaskHi;
2904 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2905
2906 SDValue EVLLo, EVLHi;
2907 std::tie(args&: EVLLo, args&: EVLHi) =
2908 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2909 // Extend those vector halves the rest of the way.
2910 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2911 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2912 return;
2913 }
2914 }
2915 // Fall back to the generic unary operator splitting otherwise.
2916 SplitVecRes_UnaryOp(N, Lo, Hi);
2917}
2918
2919void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2920 SDValue &Lo, SDValue &Hi) {
2921 // The low and high parts of the original input give four input vectors.
2922 SDValue Inputs[4];
2923 SDLoc DL(N);
2924 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2925 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2926 EVT NewVT = Inputs[0].getValueType();
2927 unsigned NewElts = NewVT.getVectorNumElements();
2928
2929 auto &&IsConstant = [](const SDValue &N) {
2930 APInt SplatValue;
2931 return N.getResNo() == 0 &&
2932 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2933 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2934 };
2935 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2936 SDValue &Input2,
2937 ArrayRef<int> Mask) {
2938 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2939 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2940 "Expected build vector node.");
2941 EVT EltVT = NewVT.getVectorElementType();
2942 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
2943 for (unsigned I = 0; I < NewElts; ++I) {
2944 if (Mask[I] == PoisonMaskElem)
2945 continue;
2946 unsigned Idx = Mask[I];
2947 if (Idx >= NewElts)
2948 Ops[I] = Input2.getOperand(i: Idx - NewElts);
2949 else
2950 Ops[I] = Input1.getOperand(i: Idx);
2951 // Make the type of all elements the same as the element type.
2952 if (Ops[I].getValueType().bitsGT(VT: EltVT))
2953 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
2954 }
2955 return DAG.getBuildVector(VT: NewVT, DL, Ops);
2956 };
2957
2958 // If Lo or Hi uses elements from at most two of the four input vectors, then
2959 // express it as a vector shuffle of those two inputs. Otherwise extract the
2960 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2961 SmallVector<int> OrigMask(N->getMask());
2962 // Try to pack incoming shuffles/inputs.
2963 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2964 &DL](SmallVectorImpl<int> &Mask) {
2965 // Check if all inputs are shuffles of the same operands or non-shuffles.
2966 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2967 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2968 SDValue Input = Inputs[Idx];
2969 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
2970 if (!Shuffle ||
2971 Input.getOperand(i: 0).getValueType() != Input.getValueType())
2972 continue;
2973 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
2974 .push_back(Elt: Idx);
2975 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
2976 .push_back(Elt: Idx);
2977 }
2978 for (auto &P : ShufflesIdxs) {
2979 if (P.second.size() < 2)
2980 continue;
2981 // Use shuffles operands instead of shuffles themselves.
2982 // 1. Adjust mask.
2983 for (int &Idx : Mask) {
2984 if (Idx == PoisonMaskElem)
2985 continue;
2986 unsigned SrcRegIdx = Idx / NewElts;
2987 if (Inputs[SrcRegIdx].isUndef()) {
2988 Idx = PoisonMaskElem;
2989 continue;
2990 }
2991 auto *Shuffle =
2992 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
2993 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
2994 continue;
2995 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
2996 if (MaskElt == PoisonMaskElem) {
2997 Idx = PoisonMaskElem;
2998 continue;
2999 }
3000 Idx = MaskElt % NewElts +
3001 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3002 ? 0
3003 : 1] *
3004 NewElts;
3005 }
3006 // 2. Update inputs.
3007 Inputs[P.second[0]] = P.first.first;
3008 Inputs[P.second[1]] = P.first.second;
3009 // Clear the pair data.
3010 P.second.clear();
3011 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3012 }
3013 // Check if any concat_vectors can be simplified.
3014 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3015 for (int &Idx : Mask) {
3016 if (Idx == PoisonMaskElem)
3017 continue;
3018 unsigned SrcRegIdx = Idx / NewElts;
3019 if (Inputs[SrcRegIdx].isUndef()) {
3020 Idx = PoisonMaskElem;
3021 continue;
3022 }
3023 TargetLowering::LegalizeTypeAction TypeAction =
3024 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3025 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3026 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3027 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3028 (TypeAction == TargetLowering::TypeLegal ||
3029 TypeAction == TargetLowering::TypeWidenVector))
3030 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3031 }
3032 if (UsedSubVector.count() > 1) {
3033 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3034 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3035 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3036 continue;
3037 if (Pairs.empty() || Pairs.back().size() == 2)
3038 Pairs.emplace_back();
3039 if (UsedSubVector.test(Idx: 2 * I)) {
3040 Pairs.back().emplace_back(Args&: I, Args: 0);
3041 } else {
3042 assert(UsedSubVector.test(2 * I + 1) &&
3043 "Expected to be used one of the subvectors.");
3044 Pairs.back().emplace_back(Args&: I, Args: 1);
3045 }
3046 }
3047 if (!Pairs.empty() && Pairs.front().size() > 1) {
3048 // Adjust mask.
3049 for (int &Idx : Mask) {
3050 if (Idx == PoisonMaskElem)
3051 continue;
3052 unsigned SrcRegIdx = Idx / NewElts;
3053 auto *It = find_if(
3054 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3055 return Idxs.front().first == SrcRegIdx ||
3056 Idxs.back().first == SrcRegIdx;
3057 });
3058 if (It == Pairs.end())
3059 continue;
3060 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3061 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3062 }
3063 // Adjust inputs.
3064 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3065 Inputs[Idxs.front().first] = DAG.getNode(
3066 Opcode: ISD::CONCAT_VECTORS, DL,
3067 VT: Inputs[Idxs.front().first].getValueType(),
3068 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3069 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3070 }
3071 }
3072 }
3073 bool Changed;
3074 do {
3075 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3076 // reused operands.
3077 Changed = false;
3078 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3079 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3080 if (!Shuffle)
3081 continue;
3082 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3083 continue;
3084 int Op = -1;
3085 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3086 !Shuffle->isSplat()) {
3087 Op = 0;
3088 } else if (!Inputs[I].hasOneUse() &&
3089 !Shuffle->getOperand(Num: 1).isUndef()) {
3090 // Find the only used operand, if possible.
3091 for (int &Idx : Mask) {
3092 if (Idx == PoisonMaskElem)
3093 continue;
3094 unsigned SrcRegIdx = Idx / NewElts;
3095 if (SrcRegIdx != I)
3096 continue;
3097 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3098 if (MaskElt == PoisonMaskElem) {
3099 Idx = PoisonMaskElem;
3100 continue;
3101 }
3102 int OpIdx = MaskElt / NewElts;
3103 if (Op == -1) {
3104 Op = OpIdx;
3105 continue;
3106 }
3107 if (Op != OpIdx) {
3108 Op = -1;
3109 break;
3110 }
3111 }
3112 }
3113 if (Op < 0) {
3114 // Try to check if one of the shuffle operands is used already.
3115 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3116 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3117 continue;
3118 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3119 if (It == std::end(arr&: Inputs))
3120 continue;
3121 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3122 // Found that operand is used already.
3123 // 1. Fix the mask for the reused operand.
3124 for (int &Idx : Mask) {
3125 if (Idx == PoisonMaskElem)
3126 continue;
3127 unsigned SrcRegIdx = Idx / NewElts;
3128 if (SrcRegIdx != I)
3129 continue;
3130 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3131 if (MaskElt == PoisonMaskElem) {
3132 Idx = PoisonMaskElem;
3133 continue;
3134 }
3135 int MaskIdx = MaskElt / NewElts;
3136 if (OpIdx == MaskIdx)
3137 Idx = MaskElt % NewElts + FoundOp * NewElts;
3138 }
3139 // 2. Set Op to the unused OpIdx.
3140 Op = (OpIdx + 1) % 2;
3141 break;
3142 }
3143 }
3144 if (Op >= 0) {
3145 Changed = true;
3146 Inputs[I] = Shuffle->getOperand(Num: Op);
3147 // Adjust mask.
3148 for (int &Idx : Mask) {
3149 if (Idx == PoisonMaskElem)
3150 continue;
3151 unsigned SrcRegIdx = Idx / NewElts;
3152 if (SrcRegIdx != I)
3153 continue;
3154 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3155 int OpIdx = MaskElt / NewElts;
3156 if (OpIdx != Op)
3157 continue;
3158 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3159 }
3160 }
3161 }
3162 } while (Changed);
3163 };
3164 TryPeekThroughShufflesInputs(OrigMask);
3165 // Proces unique inputs.
3166 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3167 NewElts](SmallVectorImpl<int> &Mask) {
3168 SetVector<SDValue> UniqueInputs;
3169 SetVector<SDValue> UniqueConstantInputs;
3170 for (const auto &I : Inputs) {
3171 if (IsConstant(I))
3172 UniqueConstantInputs.insert(X: I);
3173 else if (!I.isUndef())
3174 UniqueInputs.insert(X: I);
3175 }
3176 // Adjust mask in case of reused inputs. Also, need to insert constant
3177 // inputs at first, otherwise it affects the final outcome.
3178 if (UniqueInputs.size() != std::size(Inputs)) {
3179 auto &&UniqueVec = UniqueInputs.takeVector();
3180 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3181 unsigned ConstNum = UniqueConstantVec.size();
3182 for (int &Idx : Mask) {
3183 if (Idx == PoisonMaskElem)
3184 continue;
3185 unsigned SrcRegIdx = Idx / NewElts;
3186 if (Inputs[SrcRegIdx].isUndef()) {
3187 Idx = PoisonMaskElem;
3188 continue;
3189 }
3190 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3191 if (It != UniqueConstantVec.end()) {
3192 Idx = (Idx % NewElts) +
3193 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3194 assert(Idx >= 0 && "Expected defined mask idx.");
3195 continue;
3196 }
3197 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3198 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3199 Idx = (Idx % NewElts) +
3200 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3201 assert(Idx >= 0 && "Expected defined mask idx.");
3202 }
3203 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3204 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3205 }
3206 };
3207 MakeUniqueInputs(OrigMask);
3208 SDValue OrigInputs[4];
3209 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3210 for (unsigned High = 0; High < 2; ++High) {
3211 SDValue &Output = High ? Hi : Lo;
3212
3213 // Build a shuffle mask for the output, discovering on the fly which
3214 // input vectors to use as shuffle operands.
3215 unsigned FirstMaskIdx = High * NewElts;
3216 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3217 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3218 assert(!Output && "Expected default initialized initial value.");
3219 TryPeekThroughShufflesInputs(Mask);
3220 MakeUniqueInputs(Mask);
3221 SDValue TmpInputs[4];
3222 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3223 // Track changes in the output registers.
3224 int UsedIdx = -1;
3225 bool SecondIteration = false;
3226 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3227 if (UsedIdx < 0) {
3228 UsedIdx = Idx;
3229 return false;
3230 }
3231 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3232 SecondIteration = true;
3233 return SecondIteration;
3234 };
3235 processShuffleMasks(
3236 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3237 /*NumOfUsedRegs=*/1,
3238 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3239 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3240 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3241 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3242 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3243 else
3244 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3245 N2: DAG.getPOISON(VT: NewVT), Mask);
3246 Inputs[Idx] = Output;
3247 },
3248 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3249 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3250 unsigned Idx2, bool /*Unused*/) {
3251 if (AccumulateResults(Idx1)) {
3252 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3253 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3254 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3255 else
3256 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3257 N2: Inputs[Idx2], Mask);
3258 } else {
3259 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3260 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3261 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3262 else
3263 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3264 N2: TmpInputs[Idx2], Mask);
3265 }
3266 Inputs[Idx1] = Output;
3267 });
3268 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3269 }
3270}
3271
3272void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3273 EVT OVT = N->getValueType(ResNo: 0);
3274 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3275 SDValue Chain = N->getOperand(Num: 0);
3276 SDValue Ptr = N->getOperand(Num: 1);
3277 SDValue SV = N->getOperand(Num: 2);
3278 SDLoc dl(N);
3279
3280 const Align Alignment =
3281 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3282
3283 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3284 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3285 Chain = Hi.getValue(R: 1);
3286
3287 // Modified the chain - switch anything that used the old chain to use
3288 // the new one.
3289 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3290}
3291
3292void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3293 SDValue &Hi) {
3294 EVT DstVTLo, DstVTHi;
3295 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3296 SDLoc dl(N);
3297
3298 SDValue SrcLo, SrcHi;
3299 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3300 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3301 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3302 else
3303 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3304
3305 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3306 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3307}
3308
3309void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3310 SDValue &Hi) {
3311 SDValue InLo, InHi;
3312 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3313 SDLoc DL(N);
3314
3315 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3316 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3317}
3318
3319void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3320 SDValue &Hi) {
3321 SDLoc DL(N);
3322
3323 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3324 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3325}
3326
3327void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3328 SDValue &Hi) {
3329 EVT VT = N->getValueType(ResNo: 0);
3330 SDValue Val = N->getOperand(Num: 0);
3331 SDValue Mask = N->getOperand(Num: 1);
3332 SDValue EVL = N->getOperand(Num: 2);
3333 SDLoc DL(N);
3334
3335 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3336 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3337
3338 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3339 EC: VT.getVectorElementCount());
3340 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3341 EVT PtrVT = StackPtr.getValueType();
3342 auto &MF = DAG.getMachineFunction();
3343 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3344 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3345
3346 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3347 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3348 BaseAlignment: Alignment);
3349 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3350 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3351 BaseAlignment: Alignment);
3352
3353 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3354 SDValue NumElemMinus1 =
3355 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3356 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3357 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3358 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3359 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3360 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3361
3362 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3363 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3364 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3365 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3366
3367 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3368
3369 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3370}
3371
3372void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3373 SDValue &Hi) {
3374 EVT VT = N->getValueType(ResNo: 0);
3375 SDValue V1 = N->getOperand(Num: 0);
3376 SDValue V2 = N->getOperand(Num: 1);
3377 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3378 SDValue Mask = N->getOperand(Num: 3);
3379 SDValue EVL1 = N->getOperand(Num: 4);
3380 SDValue EVL2 = N->getOperand(Num: 5);
3381 SDLoc DL(N);
3382
3383 // Since EVL2 is considered the real VL it gets promoted during
3384 // SelectionDAGBuilder. Promote EVL1 here if needed.
3385 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3386 EVL1 = ZExtPromotedInteger(Op: EVL1);
3387
3388 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3389
3390 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3391 EC: VT.getVectorElementCount() * 2);
3392 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3393 EVT PtrVT = StackPtr.getValueType();
3394 auto &MF = DAG.getMachineFunction();
3395 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3396 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3397
3398 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3399 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3400 BaseAlignment: Alignment);
3401 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3402 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3403 BaseAlignment: Alignment);
3404
3405 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: EVL1);
3406 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3407
3408 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3409 SDValue StoreV1 =
3410 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3411 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3412
3413 SDValue StoreV2 =
3414 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3415 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3416
3417 SDValue Load;
3418 if (Imm >= 0) {
3419 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3420 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3421 } else {
3422 uint64_t TrailingElts = -Imm;
3423 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3424 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3425
3426 // Make sure TrailingBytes doesn't exceed the size of vec1.
3427 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3428 TrailingBytes =
3429 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3430
3431 // Calculate the start address of the spliced result.
3432 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3433 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3434 }
3435
3436 EVT LoVT, HiVT;
3437 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
3438 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3439 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3440 Hi =
3441 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3442 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3443}
3444
3445void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3446 SDValue &Hi) {
3447 SDLoc DL(N);
3448 SDValue Acc = N->getOperand(Num: 0);
3449 SDValue Input1 = N->getOperand(Num: 1);
3450 SDValue Input2 = N->getOperand(Num: 2);
3451
3452 SDValue AccLo, AccHi;
3453 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3454 unsigned Opcode = N->getOpcode();
3455
3456 // If the input types don't need splitting, just accumulate into the
3457 // low part of the accumulator.
3458 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3459 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3460 Hi = AccHi;
3461 return;
3462 }
3463
3464 SDValue Input1Lo, Input1Hi;
3465 SDValue Input2Lo, Input2Hi;
3466 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3467 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3468 EVT ResultVT = AccLo.getValueType();
3469
3470 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3471 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3472}
3473
3474void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3475 SDValue &Hi) {
3476 SDLoc DL(N);
3477 SDValue Op0 = N->getOperand(Num: 0);
3478 SDValue Op1 = N->getOperand(Num: 1);
3479 EVT OpVT = Op0.getValueType();
3480
3481 EVT LoVT, HiVT;
3482 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3483
3484 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3485 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3486 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3487 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3488}
3489
3490void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3491 unsigned Factor = N->getNumOperands();
3492
3493 SmallVector<SDValue, 8> Ops(Factor * 2);
3494 for (unsigned i = 0; i != Factor; ++i) {
3495 SDValue OpLo, OpHi;
3496 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3497 Ops[i * 2] = OpLo;
3498 Ops[i * 2 + 1] = OpHi;
3499 }
3500
3501 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3502
3503 SDLoc DL(N);
3504 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3505 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3506 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3507 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3508
3509 for (unsigned i = 0; i != Factor; ++i)
3510 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3511}
3512
3513void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3514 unsigned Factor = N->getNumOperands();
3515
3516 SmallVector<SDValue, 8> Ops(Factor * 2);
3517 for (unsigned i = 0; i != Factor; ++i) {
3518 SDValue OpLo, OpHi;
3519 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3520 Ops[i] = OpLo;
3521 Ops[i + Factor] = OpHi;
3522 }
3523
3524 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3525
3526 SDLoc DL(N);
3527 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3528 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3529 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3530 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3531
3532 for (unsigned i = 0; i != Factor; ++i) {
3533 unsigned IdxLo = 2 * i;
3534 unsigned IdxHi = 2 * i + 1;
3535 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3536 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3537 }
3538}
3539
3540//===----------------------------------------------------------------------===//
3541// Operand Vector Splitting
3542//===----------------------------------------------------------------------===//
3543
3544/// This method is called when the specified operand of the specified node is
3545/// found to need vector splitting. At this point, all of the result types of
3546/// the node are known to be legal, but other operands of the node may need
3547/// legalization as well as the specified one.
3548bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3549 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3550 SDValue Res = SDValue();
3551
3552 // See if the target wants to custom split this node.
3553 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3554 return false;
3555
3556 switch (N->getOpcode()) {
3557 default:
3558#ifndef NDEBUG
3559 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3560 N->dump(&DAG);
3561 dbgs() << "\n";
3562#endif
3563 report_fatal_error(reason: "Do not know how to split this operator's "
3564 "operand!\n");
3565
3566 case ISD::VP_SETCC:
3567 case ISD::STRICT_FSETCC:
3568 case ISD::STRICT_FSETCCS:
3569 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3570 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3571 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3572 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3573 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3574 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3575 case ISD::VP_TRUNCATE:
3576 case ISD::TRUNCATE:
3577 Res = SplitVecOp_TruncateHelper(N);
3578 break;
3579 case ISD::STRICT_FP_ROUND:
3580 case ISD::VP_FP_ROUND:
3581 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
3582 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3583 case ISD::STORE:
3584 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3585 break;
3586 case ISD::VP_STORE:
3587 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3588 break;
3589 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3590 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3591 break;
3592 case ISD::MSTORE:
3593 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3594 break;
3595 case ISD::MSCATTER:
3596 case ISD::VP_SCATTER:
3597 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3598 break;
3599 case ISD::MGATHER:
3600 case ISD::VP_GATHER:
3601 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3602 break;
3603 case ISD::VSELECT:
3604 Res = SplitVecOp_VSELECT(N, OpNo);
3605 break;
3606 case ISD::VECTOR_COMPRESS:
3607 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3608 break;
3609 case ISD::STRICT_SINT_TO_FP:
3610 case ISD::STRICT_UINT_TO_FP:
3611 case ISD::SINT_TO_FP:
3612 case ISD::UINT_TO_FP:
3613 case ISD::VP_SINT_TO_FP:
3614 case ISD::VP_UINT_TO_FP:
3615 if (N->getValueType(ResNo: 0).bitsLT(
3616 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3617 Res = SplitVecOp_TruncateHelper(N);
3618 else
3619 Res = SplitVecOp_UnaryOp(N);
3620 break;
3621 case ISD::FP_TO_SINT_SAT:
3622 case ISD::FP_TO_UINT_SAT:
3623 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3624 break;
3625 case ISD::FP_TO_SINT:
3626 case ISD::FP_TO_UINT:
3627 case ISD::VP_FP_TO_SINT:
3628 case ISD::VP_FP_TO_UINT:
3629 case ISD::STRICT_FP_TO_SINT:
3630 case ISD::STRICT_FP_TO_UINT:
3631 case ISD::STRICT_FP_EXTEND:
3632 case ISD::FP_EXTEND:
3633 case ISD::SIGN_EXTEND:
3634 case ISD::ZERO_EXTEND:
3635 case ISD::ANY_EXTEND:
3636 case ISD::FTRUNC:
3637 case ISD::LROUND:
3638 case ISD::LLROUND:
3639 case ISD::LRINT:
3640 case ISD::LLRINT:
3641 Res = SplitVecOp_UnaryOp(N);
3642 break;
3643 case ISD::FLDEXP:
3644 Res = SplitVecOp_FPOpDifferentTypes(N);
3645 break;
3646
3647 case ISD::SCMP:
3648 case ISD::UCMP:
3649 Res = SplitVecOp_CMP(N);
3650 break;
3651
3652 case ISD::FAKE_USE:
3653 Res = SplitVecOp_FAKE_USE(N);
3654 break;
3655 case ISD::ANY_EXTEND_VECTOR_INREG:
3656 case ISD::SIGN_EXTEND_VECTOR_INREG:
3657 case ISD::ZERO_EXTEND_VECTOR_INREG:
3658 Res = SplitVecOp_ExtVecInRegOp(N);
3659 break;
3660
3661 case ISD::VECREDUCE_FADD:
3662 case ISD::VECREDUCE_FMUL:
3663 case ISD::VECREDUCE_ADD:
3664 case ISD::VECREDUCE_MUL:
3665 case ISD::VECREDUCE_AND:
3666 case ISD::VECREDUCE_OR:
3667 case ISD::VECREDUCE_XOR:
3668 case ISD::VECREDUCE_SMAX:
3669 case ISD::VECREDUCE_SMIN:
3670 case ISD::VECREDUCE_UMAX:
3671 case ISD::VECREDUCE_UMIN:
3672 case ISD::VECREDUCE_FMAX:
3673 case ISD::VECREDUCE_FMIN:
3674 case ISD::VECREDUCE_FMAXIMUM:
3675 case ISD::VECREDUCE_FMINIMUM:
3676 Res = SplitVecOp_VECREDUCE(N, OpNo);
3677 break;
3678 case ISD::VECREDUCE_SEQ_FADD:
3679 case ISD::VECREDUCE_SEQ_FMUL:
3680 Res = SplitVecOp_VECREDUCE_SEQ(N);
3681 break;
3682 case ISD::VP_REDUCE_FADD:
3683 case ISD::VP_REDUCE_SEQ_FADD:
3684 case ISD::VP_REDUCE_FMUL:
3685 case ISD::VP_REDUCE_SEQ_FMUL:
3686 case ISD::VP_REDUCE_ADD:
3687 case ISD::VP_REDUCE_MUL:
3688 case ISD::VP_REDUCE_AND:
3689 case ISD::VP_REDUCE_OR:
3690 case ISD::VP_REDUCE_XOR:
3691 case ISD::VP_REDUCE_SMAX:
3692 case ISD::VP_REDUCE_SMIN:
3693 case ISD::VP_REDUCE_UMAX:
3694 case ISD::VP_REDUCE_UMIN:
3695 case ISD::VP_REDUCE_FMAX:
3696 case ISD::VP_REDUCE_FMIN:
3697 case ISD::VP_REDUCE_FMAXIMUM:
3698 case ISD::VP_REDUCE_FMINIMUM:
3699 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3700 break;
3701 case ISD::VP_CTTZ_ELTS:
3702 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
3703 Res = SplitVecOp_VP_CttzElements(N);
3704 break;
3705 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3706 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3707 break;
3708 case ISD::PARTIAL_REDUCE_UMLA:
3709 case ISD::PARTIAL_REDUCE_SMLA:
3710 case ISD::PARTIAL_REDUCE_SUMLA:
3711 case ISD::PARTIAL_REDUCE_FMLA:
3712 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3713 break;
3714 }
3715
3716 // If the result is null, the sub-method took care of registering results etc.
3717 if (!Res.getNode()) return false;
3718
3719 // If the result is N, the sub-method updated N in place. Tell the legalizer
3720 // core about this.
3721 if (Res.getNode() == N)
3722 return true;
3723
3724 if (N->isStrictFPOpcode())
3725 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3726 "Invalid operand expansion");
3727 else
3728 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3729 "Invalid operand expansion");
3730
3731 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3732 return false;
3733}
3734
3735SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3736 // The only possibility for an illegal operand is the mask, since result type
3737 // legalization would have handled this node already otherwise.
3738 assert(OpNo == 0 && "Illegal operand must be mask");
3739
3740 SDValue Mask = N->getOperand(Num: 0);
3741 SDValue Src0 = N->getOperand(Num: 1);
3742 SDValue Src1 = N->getOperand(Num: 2);
3743 EVT Src0VT = Src0.getValueType();
3744 SDLoc DL(N);
3745 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3746
3747 SDValue Lo, Hi;
3748 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3749 assert(Lo.getValueType() == Hi.getValueType() &&
3750 "Lo and Hi have differing types");
3751
3752 EVT LoOpVT, HiOpVT;
3753 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3754 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3755
3756 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3757 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3758 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3759 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3760
3761 SDValue LoSelect =
3762 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3763 SDValue HiSelect =
3764 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3765
3766 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3767}
3768
3769SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
3770 // The only possibility for an illegal operand is the mask, since result type
3771 // legalization would have handled this node already otherwise.
3772 assert(OpNo == 1 && "Illegal operand must be mask");
3773
3774 // To split the mask, we need to split the result type too, so we can just
3775 // reuse that logic here.
3776 SDValue Lo, Hi;
3777 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
3778
3779 EVT VecVT = N->getValueType(ResNo: 0);
3780 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
3781}
3782
3783SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3784 EVT ResVT = N->getValueType(ResNo: 0);
3785 SDValue Lo, Hi;
3786 SDLoc dl(N);
3787
3788 SDValue VecOp = N->getOperand(Num: OpNo);
3789 EVT VecVT = VecOp.getValueType();
3790 assert(VecVT.isVector() && "Can only split reduce vector operand");
3791 GetSplitVector(Op: VecOp, Lo, Hi);
3792 EVT LoOpVT, HiOpVT;
3793 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3794
3795 // Use the appropriate scalar instruction on the split subvectors before
3796 // reducing the now partially reduced smaller vector.
3797 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3798 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3799 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3800}
3801
3802SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3803 EVT ResVT = N->getValueType(ResNo: 0);
3804 SDValue Lo, Hi;
3805 SDLoc dl(N);
3806
3807 SDValue AccOp = N->getOperand(Num: 0);
3808 SDValue VecOp = N->getOperand(Num: 1);
3809 SDNodeFlags Flags = N->getFlags();
3810
3811 EVT VecVT = VecOp.getValueType();
3812 assert(VecVT.isVector() && "Can only split reduce vector operand");
3813 GetSplitVector(Op: VecOp, Lo, Hi);
3814 EVT LoOpVT, HiOpVT;
3815 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3816
3817 // Reduce low half.
3818 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3819
3820 // Reduce high half, using low half result as initial value.
3821 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3822}
3823
3824SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3825 assert(N->isVPOpcode() && "Expected VP opcode");
3826 assert(OpNo == 1 && "Can only split reduce vector operand");
3827
3828 unsigned Opc = N->getOpcode();
3829 EVT ResVT = N->getValueType(ResNo: 0);
3830 SDValue Lo, Hi;
3831 SDLoc dl(N);
3832
3833 SDValue VecOp = N->getOperand(Num: OpNo);
3834 EVT VecVT = VecOp.getValueType();
3835 assert(VecVT.isVector() && "Can only split reduce vector operand");
3836 GetSplitVector(Op: VecOp, Lo, Hi);
3837
3838 SDValue MaskLo, MaskHi;
3839 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3840
3841 SDValue EVLLo, EVLHi;
3842 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3843
3844 const SDNodeFlags Flags = N->getFlags();
3845
3846 SDValue ResLo =
3847 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3848 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3849}
3850
3851SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3852 // The result has a legal vector type, but the input needs splitting.
3853 EVT ResVT = N->getValueType(ResNo: 0);
3854 SDValue Lo, Hi;
3855 SDLoc dl(N);
3856 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3857 EVT InVT = Lo.getValueType();
3858
3859 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3860 EC: InVT.getVectorElementCount());
3861
3862 if (N->isStrictFPOpcode()) {
3863 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3864 Ops: {N->getOperand(Num: 0), Lo});
3865 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3866 Ops: {N->getOperand(Num: 0), Hi});
3867
3868 // Build a factor node to remember that this operation is independent
3869 // of the other one.
3870 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
3871 N2: Hi.getValue(R: 1));
3872
3873 // Legalize the chain result - switch anything that used the old chain to
3874 // use the new one.
3875 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3876 } else if (N->getNumOperands() == 3) {
3877 assert(N->isVPOpcode() && "Expected VP opcode");
3878 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3879 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3880 std::tie(args&: EVLLo, args&: EVLHi) =
3881 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3882 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3883 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3884 } else {
3885 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3886 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3887 }
3888
3889 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3890}
3891
3892// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
3893SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
3894 SDValue Lo, Hi;
3895 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3896 SDValue Chain =
3897 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
3898 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
3899}
3900
3901SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3902 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3903 // end up being split all the way down to individual components. Convert the
3904 // split pieces into integers and reassemble.
3905 EVT ResVT = N->getValueType(ResNo: 0);
3906 SDValue Lo, Hi;
3907 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3908 SDLoc dl(N);
3909
3910 if (ResVT.isScalableVector()) {
3911 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
3912 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
3913 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
3914 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3915 }
3916
3917 Lo = BitConvertToInteger(Op: Lo);
3918 Hi = BitConvertToInteger(Op: Hi);
3919
3920 if (DAG.getDataLayout().isBigEndian())
3921 std::swap(a&: Lo, b&: Hi);
3922
3923 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
3924}
3925
3926SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3927 unsigned OpNo) {
3928 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3929 // We know that the result type is legal.
3930 EVT ResVT = N->getValueType(ResNo: 0);
3931
3932 SDValue Vec = N->getOperand(Num: 0);
3933 SDValue SubVec = N->getOperand(Num: 1);
3934 SDValue Idx = N->getOperand(Num: 2);
3935 SDLoc dl(N);
3936
3937 SDValue Lo, Hi;
3938 GetSplitVector(Op: SubVec, Lo, Hi);
3939
3940 uint64_t IdxVal = Idx->getAsZExtVal();
3941 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3942
3943 SDValue FirstInsertion =
3944 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
3945 SDValue SecondInsertion =
3946 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
3947 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
3948
3949 return SecondInsertion;
3950}
3951
3952SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3953 // We know that the extracted result type is legal.
3954 EVT SubVT = N->getValueType(ResNo: 0);
3955 SDValue Idx = N->getOperand(Num: 1);
3956 SDLoc dl(N);
3957 SDValue Lo, Hi;
3958
3959 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3960
3961 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
3962 // Note: For scalable vectors, the index is scaled by vscale.
3963 ElementCount IdxVal =
3964 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
3965 uint64_t IdxValMin = IdxVal.getKnownMinValue();
3966
3967 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3968 ElementCount NumResultElts = SubVT.getVectorElementCount();
3969
3970 // If the extracted elements are all in the low half, do a simple extract.
3971 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
3972 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
3973
3974 unsigned LoEltsMin = LoElts.getKnownMinValue();
3975 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
3976 SrcVT.isFixedLengthVector()) {
3977 // Extracted subvector crosses vector split, so we need to blend the two
3978 // halves.
3979 // TODO: May be able to emit partial extract_subvector.
3980 SmallVector<SDValue, 8> Elts;
3981 Elts.reserve(N: NumResultElts.getFixedValue());
3982
3983 // This is not valid for scalable vectors. If SubVT is scalable, this is the
3984 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
3985 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
3986 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
3987 /*Count=*/LoEltsMin - IdxValMin);
3988 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
3989 /*Count=*/SubVT.getVectorNumElements() -
3990 Elts.size());
3991 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
3992 }
3993
3994 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
3995 ElementCount ExtractIdx = IdxVal - LoElts;
3996 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
3997 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
3998 Idx: ExtractIdx.getKnownMinValue());
3999
4000 EVT HiVT = Hi.getValueType();
4001 assert(HiVT.isFixedLengthVector() &&
4002 "Only fixed-vector extracts are supported in this case");
4003
4004 // We cannot create an extract_subvector that isn't a multiple of the
4005 // result size, which may go out of bounds for the last elements. Shuffle
4006 // the desired elements down to 0 and do a simple 0 extract.
4007 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4008 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4009 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4010
4011 SDValue Shuffle =
4012 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4013 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4014 }
4015
4016 // After this point the DAG node only permits extracting fixed-width
4017 // subvectors from scalable vectors.
4018 assert(SubVT.isFixedLengthVector() &&
4019 "Extracting scalable subvector from fixed-width unsupported");
4020
4021 // If the element type is i1 and we're not promoting the result, then we may
4022 // end up loading the wrong data since the bits are packed tightly into
4023 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4024 // type at index 4, then we will load a byte starting at index 0.
4025 if (SubVT.getScalarType() == MVT::i1)
4026 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4027 "subvector from a scalable predicate vector");
4028
4029 // Spill the vector to the stack. We should use the alignment for
4030 // the smallest part.
4031 SDValue Vec = N->getOperand(Num: 0);
4032 EVT VecVT = Vec.getValueType();
4033 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4034 SDValue StackPtr =
4035 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4036 auto &MF = DAG.getMachineFunction();
4037 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4038 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4039
4040 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4041 Alignment: SmallestAlign);
4042
4043 // Extract the subvector by loading the correct part.
4044 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4045
4046 return DAG.getLoad(
4047 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4048 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4049}
4050
4051SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4052 SDValue Vec = N->getOperand(Num: 0);
4053 SDValue Idx = N->getOperand(Num: 1);
4054 EVT VecVT = Vec.getValueType();
4055
4056 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4057 uint64_t IdxVal = Index->getZExtValue();
4058
4059 SDValue Lo, Hi;
4060 GetSplitVector(Op: Vec, Lo, Hi);
4061
4062 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4063
4064 if (IdxVal < LoElts)
4065 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4066 else if (!Vec.getValueType().isScalableVector())
4067 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4068 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4069 VT: Idx.getValueType())), 0);
4070 }
4071
4072 // See if the target wants to custom expand this node.
4073 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4074 return SDValue();
4075
4076 // Make the vector elements byte-addressable if they aren't already.
4077 SDLoc dl(N);
4078 EVT EltVT = VecVT.getVectorElementType();
4079 if (!EltVT.isByteSized()) {
4080 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4081 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4082 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4083 SDValue NewExtract =
4084 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4085 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4086 }
4087
4088 // Store the vector to the stack.
4089 // In cases where the vector is illegal it will be broken down into parts
4090 // and stored in parts - we should use the alignment for the smallest part.
4091 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4092 SDValue StackPtr =
4093 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4094 auto &MF = DAG.getMachineFunction();
4095 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4096 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4097 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4098 Alignment: SmallestAlign);
4099
4100 // Load back the required element.
4101 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4102
4103 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4104 // type, leaving the high bits undefined. But it can't truncate.
4105 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4106
4107 return DAG.getExtLoad(
4108 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4109 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4110 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4111}
4112
4113SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4114 SDValue Lo, Hi;
4115
4116 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4117 // splitting the result has the same effect as splitting the input operand.
4118 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4119
4120 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4121}
4122
4123SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4124 (void)OpNo;
4125 SDValue Lo, Hi;
4126 SplitVecRes_Gather(N, Lo, Hi);
4127
4128 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4129 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4130 return SDValue();
4131}
4132
4133SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4134 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4135 SDValue Ch = N->getChain();
4136 SDValue Ptr = N->getBasePtr();
4137 SDValue Offset = N->getOffset();
4138 assert(Offset.isUndef() && "Unexpected VP store offset");
4139 SDValue Mask = N->getMask();
4140 SDValue EVL = N->getVectorLength();
4141 SDValue Data = N->getValue();
4142 Align Alignment = N->getBaseAlign();
4143 SDLoc DL(N);
4144
4145 SDValue DataLo, DataHi;
4146 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4147 // Split Data operand
4148 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4149 else
4150 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4151
4152 // Split Mask operand
4153 SDValue MaskLo, MaskHi;
4154 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4155 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4156 } else {
4157 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4158 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4159 else
4160 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4161 }
4162
4163 EVT MemoryVT = N->getMemoryVT();
4164 EVT LoMemVT, HiMemVT;
4165 bool HiIsEmpty = false;
4166 std::tie(args&: LoMemVT, args&: HiMemVT) =
4167 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4168
4169 // Split EVL
4170 SDValue EVLLo, EVLHi;
4171 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4172
4173 SDValue Lo, Hi;
4174 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4175 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4176 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4177 Ranges: N->getRanges());
4178
4179 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4180 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4181 IsCompressing: N->isCompressingStore());
4182
4183 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4184 if (HiIsEmpty)
4185 return Lo;
4186
4187 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4188 IsCompressedMemory: N->isCompressingStore());
4189
4190 MachinePointerInfo MPI;
4191 if (LoMemVT.isScalableVector()) {
4192 Alignment = commonAlignment(A: Alignment,
4193 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4194 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4195 } else
4196 MPI = N->getPointerInfo().getWithOffset(
4197 O: LoMemVT.getStoreSize().getFixedValue());
4198
4199 MMO = DAG.getMachineFunction().getMachineMemOperand(
4200 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4201 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4202
4203 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4204 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4205 IsCompressing: N->isCompressingStore());
4206
4207 // Build a factor node to remember that this store is independent of the
4208 // other one.
4209 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4210}
4211
4212SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4213 unsigned OpNo) {
4214 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4215 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4216
4217 SDLoc DL(N);
4218
4219 SDValue Data = N->getValue();
4220 SDValue LoData, HiData;
4221 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4222 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4223 else
4224 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4225
4226 EVT LoMemVT, HiMemVT;
4227 bool HiIsEmpty = false;
4228 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4229 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4230
4231 SDValue Mask = N->getMask();
4232 SDValue LoMask, HiMask;
4233 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4234 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4235 else if (getTypeAction(VT: Mask.getValueType()) ==
4236 TargetLowering::TypeSplitVector)
4237 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4238 else
4239 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4240
4241 SDValue LoEVL, HiEVL;
4242 std::tie(args&: LoEVL, args&: HiEVL) =
4243 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4244
4245 // Generate the low vp_strided_store
4246 SDValue Lo = DAG.getStridedStoreVP(
4247 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4248 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4249 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4250
4251 // If the high vp_strided_store has zero storage size, only the low
4252 // vp_strided_store is needed.
4253 if (HiIsEmpty)
4254 return Lo;
4255
4256 // Generate the high vp_strided_store.
4257 // To calculate the high base address, we need to sum to the low base
4258 // address stride number of bytes for each element already stored by low,
4259 // that is: Ptr = Ptr + (LoEVL * Stride)
4260 EVT PtrVT = N->getBasePtr().getValueType();
4261 SDValue Increment =
4262 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4263 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4264 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4265
4266 Align Alignment = N->getBaseAlign();
4267 if (LoMemVT.isScalableVector())
4268 Alignment = commonAlignment(A: Alignment,
4269 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4270
4271 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4272 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4273 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4274 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4275
4276 SDValue Hi = DAG.getStridedStoreVP(
4277 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4278 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4279 IsCompressing: N->isCompressingStore());
4280
4281 // Build a factor node to remember that this store is independent of the
4282 // other one.
4283 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4284}
4285
4286SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4287 unsigned OpNo) {
4288 assert(N->isUnindexed() && "Indexed masked store of vector?");
4289 SDValue Ch = N->getChain();
4290 SDValue Ptr = N->getBasePtr();
4291 SDValue Offset = N->getOffset();
4292 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4293 SDValue Mask = N->getMask();
4294 SDValue Data = N->getValue();
4295 Align Alignment = N->getBaseAlign();
4296 SDLoc DL(N);
4297
4298 SDValue DataLo, DataHi;
4299 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4300 // Split Data operand
4301 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4302 else
4303 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4304
4305 // Split Mask operand
4306 SDValue MaskLo, MaskHi;
4307 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4308 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4309 } else {
4310 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4311 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4312 else
4313 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4314 }
4315
4316 EVT MemoryVT = N->getMemoryVT();
4317 EVT LoMemVT, HiMemVT;
4318 bool HiIsEmpty = false;
4319 std::tie(args&: LoMemVT, args&: HiMemVT) =
4320 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4321
4322 SDValue Lo, Hi, Res;
4323 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4324 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4325 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4326 Ranges: N->getRanges());
4327
4328 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4329 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4330 IsCompressing: N->isCompressingStore());
4331
4332 if (HiIsEmpty) {
4333 // The hi masked store has zero storage size.
4334 // Only the lo masked store is needed.
4335 Res = Lo;
4336 } else {
4337
4338 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4339 IsCompressedMemory: N->isCompressingStore());
4340
4341 MachinePointerInfo MPI;
4342 if (LoMemVT.isScalableVector()) {
4343 Alignment = commonAlignment(
4344 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4345 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4346 } else
4347 MPI = N->getPointerInfo().getWithOffset(
4348 O: LoMemVT.getStoreSize().getFixedValue());
4349
4350 MMO = DAG.getMachineFunction().getMachineMemOperand(
4351 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4352 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4353
4354 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4355 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4356 IsCompressing: N->isCompressingStore());
4357
4358 // Build a factor node to remember that this store is independent of the
4359 // other one.
4360 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4361 }
4362
4363 return Res;
4364}
4365
4366SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4367 SDValue Ch = N->getChain();
4368 SDValue Ptr = N->getBasePtr();
4369 EVT MemoryVT = N->getMemoryVT();
4370 Align Alignment = N->getBaseAlign();
4371 SDLoc DL(N);
4372 struct Operands {
4373 SDValue Mask;
4374 SDValue Index;
4375 SDValue Scale;
4376 SDValue Data;
4377 } Ops = [&]() -> Operands {
4378 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4379 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4380 .Data: MSC->getValue()};
4381 }
4382 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4383 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4384 .Data: VPSC->getValue()};
4385 }();
4386 // Split all operands
4387
4388 EVT LoMemVT, HiMemVT;
4389 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4390
4391 SDValue DataLo, DataHi;
4392 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4393 // Split Data operand
4394 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4395 else
4396 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4397
4398 // Split Mask operand
4399 SDValue MaskLo, MaskHi;
4400 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4401 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4402 } else {
4403 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4404 }
4405
4406 SDValue IndexHi, IndexLo;
4407 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4408 TargetLowering::TypeSplitVector)
4409 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4410 else
4411 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4412
4413 SDValue Lo;
4414 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4415 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4416 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4417 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4418
4419 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4420 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4421 Lo =
4422 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4423 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4424
4425 // The order of the Scatter operation after split is well defined. The "Hi"
4426 // part comes after the "Lo". So these two operations should be chained one
4427 // after another.
4428 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4429 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4430 MMO, IndexType: MSC->getIndexType(),
4431 IsTruncating: MSC->isTruncatingStore());
4432 }
4433 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4434 SDValue EVLLo, EVLHi;
4435 std::tie(args&: EVLLo, args&: EVLHi) =
4436 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4437
4438 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4439 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4440 IndexType: VPSC->getIndexType());
4441
4442 // The order of the Scatter operation after split is well defined. The "Hi"
4443 // part comes after the "Lo". So these two operations should be chained one
4444 // after another.
4445 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4446 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4447 IndexType: VPSC->getIndexType());
4448}
4449
4450SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4451 assert(N->isUnindexed() && "Indexed store of vector?");
4452 assert(OpNo == 1 && "Can only split the stored value");
4453 SDLoc DL(N);
4454
4455 bool isTruncating = N->isTruncatingStore();
4456 SDValue Ch = N->getChain();
4457 SDValue Ptr = N->getBasePtr();
4458 EVT MemoryVT = N->getMemoryVT();
4459 Align Alignment = N->getBaseAlign();
4460 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4461 AAMDNodes AAInfo = N->getAAInfo();
4462 SDValue Lo, Hi;
4463 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4464
4465 EVT LoMemVT, HiMemVT;
4466 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4467
4468 // Scalarize if the split halves are not byte-sized.
4469 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4470 return TLI.scalarizeVectorStore(ST: N, DAG);
4471
4472 if (isTruncating)
4473 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4474 Alignment, MMOFlags, AAInfo);
4475 else
4476 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4477 AAInfo);
4478
4479 MachinePointerInfo MPI;
4480 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4481
4482 if (isTruncating)
4483 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4484 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4485 else
4486 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4487
4488 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4489}
4490
4491SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4492 SDLoc DL(N);
4493
4494 // The input operands all must have the same type, and we know the result
4495 // type is valid. Convert this to a buildvector which extracts all the
4496 // input elements.
4497 // TODO: If the input elements are power-two vectors, we could convert this to
4498 // a new CONCAT_VECTORS node with elements that are half-wide.
4499 SmallVector<SDValue, 32> Elts;
4500 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4501 for (const SDValue &Op : N->op_values()) {
4502 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4503 i != e; ++i) {
4504 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4505 }
4506 }
4507
4508 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4509}
4510
4511SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4512 // The result type is legal, but the input type is illegal. If splitting
4513 // ends up with the result type of each half still being legal, just
4514 // do that. If, however, that would result in an illegal result type,
4515 // we can try to get more clever with power-two vectors. Specifically,
4516 // split the input type, but also widen the result element size, then
4517 // concatenate the halves and truncate again. For example, consider a target
4518 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4519 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4520 // %inlo = v4i32 extract_subvector %in, 0
4521 // %inhi = v4i32 extract_subvector %in, 4
4522 // %lo16 = v4i16 trunc v4i32 %inlo
4523 // %hi16 = v4i16 trunc v4i32 %inhi
4524 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4525 // %res = v8i8 trunc v8i16 %in16
4526 //
4527 // Without this transform, the original truncate would end up being
4528 // scalarized, which is pretty much always a last resort.
4529 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4530 SDValue InVec = N->getOperand(Num: OpNo);
4531 EVT InVT = InVec->getValueType(ResNo: 0);
4532 EVT OutVT = N->getValueType(ResNo: 0);
4533 ElementCount NumElements = OutVT.getVectorElementCount();
4534 bool IsFloat = OutVT.isFloatingPoint();
4535
4536 unsigned InElementSize = InVT.getScalarSizeInBits();
4537 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4538
4539 // Determine the split output VT. If its legal we can just split dirctly.
4540 EVT LoOutVT, HiOutVT;
4541 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4542 assert(LoOutVT == HiOutVT && "Unequal split?");
4543
4544 // If the input elements are only 1/2 the width of the result elements,
4545 // just use the normal splitting. Our trick only work if there's room
4546 // to split more than once.
4547 if (isTypeLegal(VT: LoOutVT) ||
4548 InElementSize <= OutElementSize * 2)
4549 return SplitVecOp_UnaryOp(N);
4550 SDLoc DL(N);
4551
4552 // Don't touch if this will be scalarized.
4553 EVT FinalVT = InVT;
4554 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4555 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4556
4557 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4558 return SplitVecOp_UnaryOp(N);
4559
4560 // Get the split input vector.
4561 SDValue InLoVec, InHiVec;
4562 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4563
4564 // Truncate them to 1/2 the element size.
4565 //
4566 // This assumes the number of elements is a power of two; any vector that
4567 // isn't should be widened, not split.
4568 EVT HalfElementVT = IsFloat ?
4569 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4570 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4571 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4572 EC: NumElements.divideCoefficientBy(RHS: 2));
4573
4574 SDValue HalfLo;
4575 SDValue HalfHi;
4576 SDValue Chain;
4577 if (N->isStrictFPOpcode()) {
4578 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4579 Ops: {N->getOperand(Num: 0), InLoVec});
4580 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4581 Ops: {N->getOperand(Num: 0), InHiVec});
4582 // Legalize the chain result - switch anything that used the old chain to
4583 // use the new one.
4584 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4585 N2: HalfHi.getValue(R: 1));
4586 } else {
4587 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4588 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4589 }
4590
4591 // Concatenate them to get the full intermediate truncation result.
4592 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4593 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4594 N2: HalfHi);
4595 // Now finish up by truncating all the way down to the original result
4596 // type. This should normally be something that ends up being legal directly,
4597 // but in theory if a target has very wide vectors and an annoyingly
4598 // restricted set of legal types, this split can chain to build things up.
4599
4600 if (N->isStrictFPOpcode()) {
4601 SDValue Res = DAG.getNode(
4602 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4603 Ops: {Chain, InterVec,
4604 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4605 // Relink the chain
4606 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4607 return Res;
4608 }
4609
4610 return IsFloat
4611 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4612 N2: DAG.getTargetConstant(
4613 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4614 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4615}
4616
4617SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4618 unsigned Opc = N->getOpcode();
4619 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4620 assert(N->getValueType(0).isVector() &&
4621 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4622 "Operand types must be vectors");
4623 // The result has a legal vector type, but the input needs splitting.
4624 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4625 SDLoc DL(N);
4626 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4627 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4628
4629 EVT VT = N->getValueType(ResNo: 0);
4630 EVT PartResVT = Lo0.getValueType().changeElementType(Context&: *DAG.getContext(),
4631 EltVT: VT.getScalarType());
4632
4633 if (Opc == ISD::SETCC) {
4634 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4635 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4636 } else if (isStrict) {
4637 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4638 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4639 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4640 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4641 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4642 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4643 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4644 } else {
4645 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4646 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4647 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4648 std::tie(args&: EVLLo, args&: EVLHi) =
4649 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4650 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4651 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4652 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4653 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4654 }
4655
4656 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: LoRes, N2: HiRes);
4657}
4658
4659
4660SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4661 // The result has a legal vector type, but the input needs splitting.
4662 EVT ResVT = N->getValueType(ResNo: 0);
4663 SDValue Lo, Hi;
4664 SDLoc DL(N);
4665 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4666 EVT InVT = Lo.getValueType();
4667
4668 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4669 EC: InVT.getVectorElementCount());
4670
4671 if (N->isStrictFPOpcode()) {
4672 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4673 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
4674 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4675 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
4676 // Legalize the chain result - switch anything that used the old chain to
4677 // use the new one.
4678 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4679 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
4680 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4681 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4682 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4683 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4684 std::tie(args&: EVLLo, args&: EVLHi) =
4685 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
4686 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4687 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4688 } else {
4689 Lo = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4690 Hi = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4691 }
4692
4693 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4694}
4695
4696// Split a vector type in an FP binary operation where the second operand has a
4697// different type from the first.
4698//
4699// The result (and the first input) has a legal vector type, but the second
4700// input needs splitting.
4701SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4702 SDLoc DL(N);
4703
4704 EVT LHSLoVT, LHSHiVT;
4705 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4706
4707 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4708 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4709
4710 SDValue LHSLo, LHSHi;
4711 std::tie(args&: LHSLo, args&: LHSHi) =
4712 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4713
4714 SDValue RHSLo, RHSHi;
4715 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4716
4717 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4718 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4719
4720 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4721}
4722
4723SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
4724 LLVMContext &Ctxt = *DAG.getContext();
4725 SDLoc dl(N);
4726
4727 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4728 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
4729 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
4730
4731 EVT ResVT = N->getValueType(ResNo: 0);
4732 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
4733 EVT NewResVT =
4734 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
4735
4736 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
4737 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
4738
4739 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4740}
4741
4742SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4743 EVT ResVT = N->getValueType(ResNo: 0);
4744 SDValue Lo, Hi;
4745 SDLoc dl(N);
4746 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4747 EVT InVT = Lo.getValueType();
4748
4749 EVT NewResVT =
4750 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4751 EC: InVT.getVectorElementCount());
4752
4753 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4754 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4755
4756 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4757}
4758
4759SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
4760 SDLoc DL(N);
4761 EVT ResVT = N->getValueType(ResNo: 0);
4762
4763 SDValue Lo, Hi;
4764 SDValue VecOp = N->getOperand(Num: 0);
4765 GetSplitVector(Op: VecOp, Lo, Hi);
4766
4767 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
4768 auto [EVLLo, EVLHi] =
4769 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
4770 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
4771
4772 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
4773 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_UNDEF(Hi)).
4774 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4775 SDValue ResLoNotEVL =
4776 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
4777 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4778 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
4779 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
4780}
4781
4782SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
4783 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
4784 SDLoc DL(HG);
4785 SDValue Inc = HG->getInc();
4786 SDValue Ptr = HG->getBasePtr();
4787 SDValue Scale = HG->getScale();
4788 SDValue IntID = HG->getIntID();
4789 EVT MemVT = HG->getMemoryVT();
4790 MachineMemOperand *MMO = HG->getMemOperand();
4791 ISD::MemIndexType IndexType = HG->getIndexType();
4792
4793 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
4794 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
4795 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
4796 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
4797 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
4798 Ops: OpsLo, MMO, IndexType);
4799 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
4800 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
4801 MMO, IndexType);
4802}
4803
4804SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
4805 SDValue Acc = N->getOperand(Num: 0);
4806 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
4807 "Accumulator should already be a legal type, and shouldn't need "
4808 "further splitting");
4809
4810 SDLoc DL(N);
4811 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
4812 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
4813 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
4814 unsigned Opcode = N->getOpcode();
4815 EVT ResultVT = Acc.getValueType();
4816
4817 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
4818 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
4819}
4820
4821//===----------------------------------------------------------------------===//
4822// Result Vector Widening
4823//===----------------------------------------------------------------------===//
4824
4825void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
4826 unsigned WidenResNo) {
4827 unsigned NumResults = N->getNumValues();
4828 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
4829 if (ResNo == WidenResNo)
4830 continue;
4831 EVT ResVT = N->getValueType(ResNo);
4832 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
4833 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
4834 } else {
4835 SDLoc DL(N);
4836 SDValue ResVal =
4837 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
4838 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
4839 }
4840 }
4841}
4842
4843void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4844 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4845
4846 // See if the target wants to custom widen this node.
4847 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4848 return;
4849
4850 SDValue Res = SDValue();
4851
4852 auto unrollExpandedOp = [&]() {
4853 // We're going to widen this vector op to a legal type by padding with undef
4854 // elements. If the wide vector op is eventually going to be expanded to
4855 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4856 // libcalls on the undef elements.
4857 EVT VT = N->getValueType(ResNo: 0);
4858 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4859 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
4860 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
4861 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4862 if (N->getNumValues() > 1)
4863 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
4864 return true;
4865 }
4866 return false;
4867 };
4868
4869 switch (N->getOpcode()) {
4870 default:
4871#ifndef NDEBUG
4872 dbgs() << "WidenVectorResult #" << ResNo << ": ";
4873 N->dump(&DAG);
4874 dbgs() << "\n";
4875#endif
4876 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4877
4878 case ISD::LOOP_DEPENDENCE_RAW_MASK:
4879 case ISD::LOOP_DEPENDENCE_WAR_MASK:
4880 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
4881 break;
4882 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4883 case ISD::ADDRSPACECAST:
4884 Res = WidenVecRes_ADDRSPACECAST(N);
4885 break;
4886 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4887 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4888 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4889 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4890 case ISD::INSERT_SUBVECTOR:
4891 Res = WidenVecRes_INSERT_SUBVECTOR(N);
4892 break;
4893 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4894 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4895 case ISD::ATOMIC_LOAD:
4896 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
4897 break;
4898 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4899 case ISD::STEP_VECTOR:
4900 case ISD::SPLAT_VECTOR:
4901 case ISD::SCALAR_TO_VECTOR:
4902 Res = WidenVecRes_ScalarOp(N);
4903 break;
4904 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4905 case ISD::VSELECT:
4906 case ISD::SELECT:
4907 case ISD::VP_SELECT:
4908 case ISD::VP_MERGE:
4909 Res = WidenVecRes_Select(N);
4910 break;
4911 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4912 case ISD::VP_SETCC:
4913 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4914 case ISD::POISON:
4915 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4916 case ISD::VECTOR_SHUFFLE:
4917 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
4918 break;
4919 case ISD::VP_LOAD:
4920 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
4921 break;
4922 case ISD::VP_LOAD_FF:
4923 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
4924 break;
4925 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4926 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
4927 break;
4928 case ISD::VECTOR_COMPRESS:
4929 Res = WidenVecRes_VECTOR_COMPRESS(N);
4930 break;
4931 case ISD::MLOAD:
4932 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
4933 break;
4934 case ISD::MGATHER:
4935 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
4936 break;
4937 case ISD::VP_GATHER:
4938 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
4939 break;
4940 case ISD::VECTOR_REVERSE:
4941 Res = WidenVecRes_VECTOR_REVERSE(N);
4942 break;
4943 case ISD::GET_ACTIVE_LANE_MASK:
4944 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
4945 break;
4946
4947 case ISD::ADD: case ISD::VP_ADD:
4948 case ISD::AND: case ISD::VP_AND:
4949 case ISD::MUL: case ISD::VP_MUL:
4950 case ISD::MULHS:
4951 case ISD::MULHU:
4952 case ISD::ABDS:
4953 case ISD::ABDU:
4954 case ISD::OR: case ISD::VP_OR:
4955 case ISD::SUB: case ISD::VP_SUB:
4956 case ISD::XOR: case ISD::VP_XOR:
4957 case ISD::SHL: case ISD::VP_SHL:
4958 case ISD::SRA: case ISD::VP_SRA:
4959 case ISD::SRL: case ISD::VP_SRL:
4960 case ISD::CLMUL:
4961 case ISD::CLMULR:
4962 case ISD::CLMULH:
4963 case ISD::FMINNUM:
4964 case ISD::FMINNUM_IEEE:
4965 case ISD::VP_FMINNUM:
4966 case ISD::FMAXNUM:
4967 case ISD::FMAXNUM_IEEE:
4968 case ISD::VP_FMAXNUM:
4969 case ISD::FMINIMUM:
4970 case ISD::VP_FMINIMUM:
4971 case ISD::FMAXIMUM:
4972 case ISD::VP_FMAXIMUM:
4973 case ISD::FMINIMUMNUM:
4974 case ISD::FMAXIMUMNUM:
4975 case ISD::SMIN: case ISD::VP_SMIN:
4976 case ISD::SMAX: case ISD::VP_SMAX:
4977 case ISD::UMIN: case ISD::VP_UMIN:
4978 case ISD::UMAX: case ISD::VP_UMAX:
4979 case ISD::UADDSAT: case ISD::VP_UADDSAT:
4980 case ISD::SADDSAT: case ISD::VP_SADDSAT:
4981 case ISD::USUBSAT: case ISD::VP_USUBSAT:
4982 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
4983 case ISD::SSHLSAT:
4984 case ISD::USHLSAT:
4985 case ISD::ROTL:
4986 case ISD::ROTR:
4987 case ISD::AVGFLOORS:
4988 case ISD::AVGFLOORU:
4989 case ISD::AVGCEILS:
4990 case ISD::AVGCEILU:
4991 // Vector-predicated binary op widening. Note that -- unlike the
4992 // unpredicated versions -- we don't have to worry about trapping on
4993 // operations like UDIV, FADD, etc., as we pass on the original vector
4994 // length parameter. This means the widened elements containing garbage
4995 // aren't active.
4996 case ISD::VP_SDIV:
4997 case ISD::VP_UDIV:
4998 case ISD::VP_SREM:
4999 case ISD::VP_UREM:
5000 case ISD::VP_FADD:
5001 case ISD::VP_FSUB:
5002 case ISD::VP_FMUL:
5003 case ISD::VP_FDIV:
5004 case ISD::VP_FREM:
5005 case ISD::VP_FCOPYSIGN:
5006 Res = WidenVecRes_Binary(N);
5007 break;
5008
5009 case ISD::SCMP:
5010 case ISD::UCMP:
5011 Res = WidenVecRes_CMP(N);
5012 break;
5013
5014 case ISD::FPOW:
5015 case ISD::FATAN2:
5016 case ISD::FREM:
5017 if (unrollExpandedOp())
5018 break;
5019 // If the target has custom/legal support for the scalar FP intrinsic ops
5020 // (they are probably not destined to become libcalls), then widen those
5021 // like any other binary ops.
5022 [[fallthrough]];
5023
5024 case ISD::FADD:
5025 case ISD::FMUL:
5026 case ISD::FSUB:
5027 case ISD::FDIV:
5028 case ISD::SDIV:
5029 case ISD::UDIV:
5030 case ISD::SREM:
5031 case ISD::UREM:
5032 Res = WidenVecRes_BinaryCanTrap(N);
5033 break;
5034
5035 case ISD::SMULFIX:
5036 case ISD::SMULFIXSAT:
5037 case ISD::UMULFIX:
5038 case ISD::UMULFIXSAT:
5039 // These are binary operations, but with an extra operand that shouldn't
5040 // be widened (the scale).
5041 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5042 break;
5043
5044#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5045 case ISD::STRICT_##DAGN:
5046#include "llvm/IR/ConstrainedOps.def"
5047 Res = WidenVecRes_StrictFP(N);
5048 break;
5049
5050 case ISD::UADDO:
5051 case ISD::SADDO:
5052 case ISD::USUBO:
5053 case ISD::SSUBO:
5054 case ISD::UMULO:
5055 case ISD::SMULO:
5056 Res = WidenVecRes_OverflowOp(N, ResNo);
5057 break;
5058
5059 case ISD::FCOPYSIGN:
5060 Res = WidenVecRes_FCOPYSIGN(N);
5061 break;
5062
5063 case ISD::IS_FPCLASS:
5064 case ISD::FPTRUNC_ROUND:
5065 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5066 break;
5067
5068 case ISD::FLDEXP:
5069 case ISD::FPOWI:
5070 if (!unrollExpandedOp())
5071 Res = WidenVecRes_ExpOp(N);
5072 break;
5073
5074 case ISD::ANY_EXTEND_VECTOR_INREG:
5075 case ISD::SIGN_EXTEND_VECTOR_INREG:
5076 case ISD::ZERO_EXTEND_VECTOR_INREG:
5077 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5078 break;
5079
5080 case ISD::ANY_EXTEND:
5081 case ISD::FP_EXTEND:
5082 case ISD::VP_FP_EXTEND:
5083 case ISD::FP_ROUND:
5084 case ISD::VP_FP_ROUND:
5085 case ISD::FP_TO_SINT:
5086 case ISD::VP_FP_TO_SINT:
5087 case ISD::FP_TO_UINT:
5088 case ISD::VP_FP_TO_UINT:
5089 case ISD::SIGN_EXTEND:
5090 case ISD::VP_SIGN_EXTEND:
5091 case ISD::SINT_TO_FP:
5092 case ISD::VP_SINT_TO_FP:
5093 case ISD::VP_TRUNCATE:
5094 case ISD::TRUNCATE:
5095 case ISD::UINT_TO_FP:
5096 case ISD::VP_UINT_TO_FP:
5097 case ISD::ZERO_EXTEND:
5098 case ISD::VP_ZERO_EXTEND:
5099 Res = WidenVecRes_Convert(N);
5100 break;
5101
5102 case ISD::FP_TO_SINT_SAT:
5103 case ISD::FP_TO_UINT_SAT:
5104 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5105 break;
5106
5107 case ISD::LRINT:
5108 case ISD::LLRINT:
5109 case ISD::VP_LRINT:
5110 case ISD::VP_LLRINT:
5111 case ISD::LROUND:
5112 case ISD::LLROUND:
5113 Res = WidenVecRes_XROUND(N);
5114 break;
5115
5116 case ISD::FACOS:
5117 case ISD::FASIN:
5118 case ISD::FATAN:
5119 case ISD::FCEIL:
5120 case ISD::FCOS:
5121 case ISD::FCOSH:
5122 case ISD::FEXP:
5123 case ISD::FEXP2:
5124 case ISD::FEXP10:
5125 case ISD::FFLOOR:
5126 case ISD::FLOG:
5127 case ISD::FLOG10:
5128 case ISD::FLOG2:
5129 case ISD::FNEARBYINT:
5130 case ISD::FRINT:
5131 case ISD::FROUND:
5132 case ISD::FROUNDEVEN:
5133 case ISD::FSIN:
5134 case ISD::FSINH:
5135 case ISD::FSQRT:
5136 case ISD::FTAN:
5137 case ISD::FTANH:
5138 case ISD::FTRUNC:
5139 if (unrollExpandedOp())
5140 break;
5141 // If the target has custom/legal support for the scalar FP intrinsic ops
5142 // (they are probably not destined to become libcalls), then widen those
5143 // like any other unary ops.
5144 [[fallthrough]];
5145
5146 case ISD::ABS:
5147 case ISD::VP_ABS:
5148 case ISD::BITREVERSE:
5149 case ISD::VP_BITREVERSE:
5150 case ISD::BSWAP:
5151 case ISD::VP_BSWAP:
5152 case ISD::CTLZ:
5153 case ISD::VP_CTLZ:
5154 case ISD::CTLZ_ZERO_UNDEF:
5155 case ISD::VP_CTLZ_ZERO_UNDEF:
5156 case ISD::CTPOP:
5157 case ISD::VP_CTPOP:
5158 case ISD::CTTZ:
5159 case ISD::VP_CTTZ:
5160 case ISD::CTTZ_ZERO_UNDEF:
5161 case ISD::VP_CTTZ_ZERO_UNDEF:
5162 case ISD::FNEG: case ISD::VP_FNEG:
5163 case ISD::FABS: case ISD::VP_FABS:
5164 case ISD::VP_SQRT:
5165 case ISD::VP_FCEIL:
5166 case ISD::VP_FFLOOR:
5167 case ISD::VP_FRINT:
5168 case ISD::VP_FNEARBYINT:
5169 case ISD::VP_FROUND:
5170 case ISD::VP_FROUNDEVEN:
5171 case ISD::VP_FROUNDTOZERO:
5172 case ISD::FREEZE:
5173 case ISD::ARITH_FENCE:
5174 case ISD::FCANONICALIZE:
5175 case ISD::AssertNoFPClass:
5176 Res = WidenVecRes_Unary(N);
5177 break;
5178 case ISD::FMA: case ISD::VP_FMA:
5179 case ISD::FSHL:
5180 case ISD::VP_FSHL:
5181 case ISD::FSHR:
5182 case ISD::VP_FSHR:
5183 Res = WidenVecRes_Ternary(N);
5184 break;
5185 case ISD::FMODF:
5186 case ISD::FFREXP:
5187 case ISD::FSINCOS:
5188 case ISD::FSINCOSPI: {
5189 if (!unrollExpandedOp())
5190 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5191 break;
5192 }
5193 }
5194
5195 // If Res is null, the sub-method took care of registering the result.
5196 if (Res.getNode())
5197 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5198}
5199
5200SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5201 // Ternary op widening.
5202 SDLoc dl(N);
5203 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5204 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5205 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5206 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5207 if (N->getNumOperands() == 3)
5208 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5209
5210 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5211 assert(N->isVPOpcode() && "Expected VP opcode");
5212
5213 SDValue Mask =
5214 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5215 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5216 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5217}
5218
5219SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5220 // Binary op widening.
5221 SDLoc dl(N);
5222 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5223 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5224 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5225 if (N->getNumOperands() == 2)
5226 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5227 Flags: N->getFlags());
5228
5229 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5230 assert(N->isVPOpcode() && "Expected VP opcode");
5231
5232 SDValue Mask =
5233 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5234 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5235 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5236}
5237
5238SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5239 LLVMContext &Ctxt = *DAG.getContext();
5240 SDLoc dl(N);
5241
5242 SDValue LHS = N->getOperand(Num: 0);
5243 SDValue RHS = N->getOperand(Num: 1);
5244 EVT OpVT = LHS.getValueType();
5245 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5246 LHS = GetWidenedVector(Op: LHS);
5247 RHS = GetWidenedVector(Op: RHS);
5248 OpVT = LHS.getValueType();
5249 }
5250
5251 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5252 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5253 if (WidenResEC == OpVT.getVectorElementCount()) {
5254 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5255 }
5256
5257 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5258}
5259
5260SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5261 // Binary op widening, but with an extra operand that shouldn't be widened.
5262 SDLoc dl(N);
5263 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5264 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5265 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5266 SDValue InOp3 = N->getOperand(Num: 2);
5267 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5268 Flags: N->getFlags());
5269}
5270
5271// Given a vector of operations that have been broken up to widen, see
5272// if we can collect them together into the next widest legal VT. This
5273// implementation is trap-safe.
5274static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5275 SmallVectorImpl<SDValue> &ConcatOps,
5276 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5277 EVT WidenVT) {
5278 // Check to see if we have a single operation with the widen type.
5279 if (ConcatEnd == 1) {
5280 VT = ConcatOps[0].getValueType();
5281 if (VT == WidenVT)
5282 return ConcatOps[0];
5283 }
5284
5285 SDLoc dl(ConcatOps[0]);
5286 EVT WidenEltVT = WidenVT.getVectorElementType();
5287
5288 // while (Some element of ConcatOps is not of type MaxVT) {
5289 // From the end of ConcatOps, collect elements of the same type and put
5290 // them into an op of the next larger supported type
5291 // }
5292 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5293 int Idx = ConcatEnd - 1;
5294 VT = ConcatOps[Idx--].getValueType();
5295 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5296 Idx--;
5297
5298 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5299 EVT NextVT;
5300 do {
5301 NextSize *= 2;
5302 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5303 } while (!TLI.isTypeLegal(VT: NextVT));
5304
5305 if (!VT.isVector()) {
5306 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5307 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5308 unsigned NumToInsert = ConcatEnd - Idx - 1;
5309 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5310 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5311 ConcatOps[Idx+1] = VecOp;
5312 ConcatEnd = Idx + 2;
5313 } else {
5314 // Vector type, create a CONCAT_VECTORS of type NextVT
5315 SDValue undefVec = DAG.getPOISON(VT);
5316 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5317 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5318 unsigned RealVals = ConcatEnd - Idx - 1;
5319 unsigned SubConcatEnd = 0;
5320 unsigned SubConcatIdx = Idx + 1;
5321 while (SubConcatEnd < RealVals)
5322 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5323 while (SubConcatEnd < OpsToConcat)
5324 SubConcatOps[SubConcatEnd++] = undefVec;
5325 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5326 VT: NextVT, Ops: SubConcatOps);
5327 ConcatEnd = SubConcatIdx + 1;
5328 }
5329 }
5330
5331 // Check to see if we have a single operation with the widen type.
5332 if (ConcatEnd == 1) {
5333 VT = ConcatOps[0].getValueType();
5334 if (VT == WidenVT)
5335 return ConcatOps[0];
5336 }
5337
5338 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5339 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5340 if (NumOps != ConcatEnd ) {
5341 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5342 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5343 ConcatOps[j] = UndefVal;
5344 }
5345 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5346 Ops: ArrayRef(ConcatOps.data(), NumOps));
5347}
5348
5349SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5350 // Binary op widening for operations that can trap.
5351 unsigned Opcode = N->getOpcode();
5352 SDLoc dl(N);
5353 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5354 EVT WidenEltVT = WidenVT.getVectorElementType();
5355 EVT VT = WidenVT;
5356 unsigned NumElts = VT.getVectorMinNumElements();
5357 const SDNodeFlags Flags = N->getFlags();
5358 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5359 NumElts = NumElts / 2;
5360 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5361 }
5362
5363 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5364 // Operation doesn't trap so just widen as normal.
5365 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5366 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5367 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5368 }
5369
5370 // Generate a vp.op if it is custom/legal for the target. This avoids need
5371 // to split and tile the subvectors (below), because the inactive lanes can
5372 // simply be disabled. To avoid possible recursion, only do this if the
5373 // widened mask type is legal.
5374 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5375 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5376 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5377 EC: WidenVT.getVectorElementCount());
5378 TLI.isTypeLegal(VT: WideMaskVT)) {
5379 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5380 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5381 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5382 SDValue EVL =
5383 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5384 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5385 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5386 Flags);
5387 }
5388 }
5389
5390 // FIXME: Improve support for scalable vectors.
5391 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5392
5393 // No legal vector version so unroll the vector operation and then widen.
5394 if (NumElts == 1)
5395 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5396
5397 // Since the operation can trap, apply operation on the original vector.
5398 EVT MaxVT = VT;
5399 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5400 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5401 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5402
5403 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5404 unsigned ConcatEnd = 0; // Current ConcatOps index.
5405 int Idx = 0; // Current Idx into input vectors.
5406
5407 // NumElts := greatest legal vector size (at most WidenVT)
5408 // while (orig. vector has unhandled elements) {
5409 // take munches of size NumElts from the beginning and add to ConcatOps
5410 // NumElts := next smaller supported vector size or 1
5411 // }
5412 while (CurNumElts != 0) {
5413 while (CurNumElts >= NumElts) {
5414 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5415 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5416 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5417 Idx += NumElts;
5418 CurNumElts -= NumElts;
5419 }
5420 do {
5421 NumElts = NumElts / 2;
5422 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5423 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5424
5425 if (NumElts == 1) {
5426 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5427 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5428 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5429 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5430 N1: EOp1, N2: EOp2, Flags);
5431 }
5432 CurNumElts = 0;
5433 }
5434 }
5435
5436 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5437}
5438
5439SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5440 switch (N->getOpcode()) {
5441 case ISD::STRICT_FSETCC:
5442 case ISD::STRICT_FSETCCS:
5443 return WidenVecRes_STRICT_FSETCC(N);
5444 case ISD::STRICT_FP_EXTEND:
5445 case ISD::STRICT_FP_ROUND:
5446 case ISD::STRICT_FP_TO_SINT:
5447 case ISD::STRICT_FP_TO_UINT:
5448 case ISD::STRICT_SINT_TO_FP:
5449 case ISD::STRICT_UINT_TO_FP:
5450 return WidenVecRes_Convert_StrictFP(N);
5451 default:
5452 break;
5453 }
5454
5455 // StrictFP op widening for operations that can trap.
5456 unsigned NumOpers = N->getNumOperands();
5457 unsigned Opcode = N->getOpcode();
5458 SDLoc dl(N);
5459 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5460 EVT WidenEltVT = WidenVT.getVectorElementType();
5461 EVT VT = WidenVT;
5462 unsigned NumElts = VT.getVectorNumElements();
5463 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5464 NumElts = NumElts / 2;
5465 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5466 }
5467
5468 // No legal vector version so unroll the vector operation and then widen.
5469 if (NumElts == 1)
5470 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5471
5472 // Since the operation can trap, apply operation on the original vector.
5473 EVT MaxVT = VT;
5474 SmallVector<SDValue, 4> InOps;
5475 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5476
5477 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5478 SmallVector<SDValue, 16> Chains;
5479 unsigned ConcatEnd = 0; // Current ConcatOps index.
5480 int Idx = 0; // Current Idx into input vectors.
5481
5482 // The Chain is the first operand.
5483 InOps.push_back(Elt: N->getOperand(Num: 0));
5484
5485 // Now process the remaining operands.
5486 for (unsigned i = 1; i < NumOpers; ++i) {
5487 SDValue Oper = N->getOperand(Num: i);
5488
5489 EVT OpVT = Oper.getValueType();
5490 if (OpVT.isVector()) {
5491 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5492 Oper = GetWidenedVector(Op: Oper);
5493 else {
5494 EVT WideOpVT =
5495 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5496 EC: WidenVT.getVectorElementCount());
5497 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5498 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5499 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5500 }
5501 }
5502
5503 InOps.push_back(Elt: Oper);
5504 }
5505
5506 // NumElts := greatest legal vector size (at most WidenVT)
5507 // while (orig. vector has unhandled elements) {
5508 // take munches of size NumElts from the beginning and add to ConcatOps
5509 // NumElts := next smaller supported vector size or 1
5510 // }
5511 while (CurNumElts != 0) {
5512 while (CurNumElts >= NumElts) {
5513 SmallVector<SDValue, 4> EOps;
5514
5515 for (unsigned i = 0; i < NumOpers; ++i) {
5516 SDValue Op = InOps[i];
5517
5518 EVT OpVT = Op.getValueType();
5519 if (OpVT.isVector()) {
5520 EVT OpExtractVT =
5521 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5522 EC: VT.getVectorElementCount());
5523 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5524 }
5525
5526 EOps.push_back(Elt: Op);
5527 }
5528
5529 EVT OperVT[] = {VT, MVT::Other};
5530 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5531 ConcatOps[ConcatEnd++] = Oper;
5532 Chains.push_back(Elt: Oper.getValue(R: 1));
5533 Idx += NumElts;
5534 CurNumElts -= NumElts;
5535 }
5536 do {
5537 NumElts = NumElts / 2;
5538 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5539 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5540
5541 if (NumElts == 1) {
5542 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5543 SmallVector<SDValue, 4> EOps;
5544
5545 for (unsigned i = 0; i < NumOpers; ++i) {
5546 SDValue Op = InOps[i];
5547
5548 EVT OpVT = Op.getValueType();
5549 if (OpVT.isVector())
5550 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5551 Idx);
5552
5553 EOps.push_back(Elt: Op);
5554 }
5555
5556 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5557 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5558 ConcatOps[ConcatEnd++] = Oper;
5559 Chains.push_back(Elt: Oper.getValue(R: 1));
5560 }
5561 CurNumElts = 0;
5562 }
5563 }
5564
5565 // Build a factor node to remember all the Ops that have been created.
5566 SDValue NewChain;
5567 if (Chains.size() == 1)
5568 NewChain = Chains[0];
5569 else
5570 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5571 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5572
5573 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5574}
5575
5576SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5577 SDLoc DL(N);
5578 EVT ResVT = N->getValueType(ResNo: 0);
5579 EVT OvVT = N->getValueType(ResNo: 1);
5580 EVT WideResVT, WideOvVT;
5581 SDValue WideLHS, WideRHS;
5582
5583 // TODO: This might result in a widen/split loop.
5584 if (ResNo == 0) {
5585 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5586 WideOvVT = EVT::getVectorVT(
5587 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5588 NumElements: WideResVT.getVectorNumElements());
5589
5590 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5591 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5592 } else {
5593 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5594 WideResVT = EVT::getVectorVT(
5595 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5596 NumElements: WideOvVT.getVectorNumElements());
5597
5598 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5599 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5600
5601 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5602 N2: N->getOperand(Num: 0), N3: Zero);
5603 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5604 N2: N->getOperand(Num: 1), N3: Zero);
5605 }
5606
5607 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5608 SDNode *WideNode = DAG.getNode(
5609 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5610
5611 // Replace the other vector result not being explicitly widened here.
5612 unsigned OtherNo = 1 - ResNo;
5613 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5614 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5615 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5616 } else {
5617 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5618 SDValue OtherVal = DAG.getNode(
5619 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5620 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5621 }
5622
5623 return SDValue(WideNode, ResNo);
5624}
5625
5626SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5627 LLVMContext &Ctx = *DAG.getContext();
5628 SDValue InOp = N->getOperand(Num: 0);
5629 SDLoc DL(N);
5630
5631 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
5632 ElementCount WidenEC = WidenVT.getVectorElementCount();
5633
5634 EVT InVT = InOp.getValueType();
5635
5636 unsigned Opcode = N->getOpcode();
5637 const SDNodeFlags Flags = N->getFlags();
5638
5639 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
5640 // not equal that of WidenVT.
5641 if (N->getOpcode() == ISD::ZERO_EXTEND &&
5642 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
5643 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
5644 WidenVT.getScalarSizeInBits()) {
5645 InOp = ZExtPromotedInteger(Op: InOp);
5646 InVT = InOp.getValueType();
5647 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
5648 Opcode = ISD::TRUNCATE;
5649 }
5650
5651 EVT InEltVT = InVT.getVectorElementType();
5652 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
5653 ElementCount InVTEC = InVT.getVectorElementCount();
5654
5655 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5656 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5657 InVT = InOp.getValueType();
5658 InVTEC = InVT.getVectorElementCount();
5659 if (InVTEC == WidenEC) {
5660 if (N->getNumOperands() == 1)
5661 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp, Flags);
5662 if (N->getNumOperands() == 3) {
5663 assert(N->isVPOpcode() && "Expected VP opcode");
5664 SDValue Mask =
5665 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5666 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
5667 }
5668 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
5669 }
5670 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
5671 // If both input and result vector types are of same width, extend
5672 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
5673 // accepts fewer elements in the result than in the input.
5674 if (Opcode == ISD::ANY_EXTEND)
5675 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5676 if (Opcode == ISD::SIGN_EXTEND)
5677 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5678 if (Opcode == ISD::ZERO_EXTEND)
5679 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5680 }
5681
5682 // For TRUNCATE, try to widen using the legal EC of the input type instead
5683 // if the legalisation action for that intermediate type is not widening.
5684 // E.g. for trunc nxv1i64 -> nxv1i8 where
5685 // - nxv1i64 input gets widened to nxv2i64
5686 // - nxv1i8 output gets widened to nxv16i8
5687 // Then one can try widening the result to nxv2i8 (instead of going all the
5688 // way to nxv16i8) if this later allows type promotion.
5689 EVT MidResVT =
5690 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
5691 if (N->getOpcode() == ISD::TRUNCATE &&
5692 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
5693 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
5694 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
5695 }
5696 }
5697
5698 if (TLI.isTypeLegal(VT: InWidenVT)) {
5699 // Because the result and the input are different vector types, widening
5700 // the result could create a legal type but widening the input might make
5701 // it an illegal type that might lead to repeatedly splitting the input
5702 // and then widening it. To avoid this, we widen the input only if
5703 // it results in a legal type.
5704 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
5705 // Widen the input and call convert on the widened input vector.
5706 unsigned NumConcat =
5707 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
5708 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
5709 Ops[0] = InOp;
5710 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
5711 if (N->getNumOperands() == 1)
5712 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec, Flags);
5713 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
5714 }
5715
5716 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
5717 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
5718 // Extract the input and convert the shorten input vector.
5719 if (N->getNumOperands() == 1)
5720 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal, Flags);
5721 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
5722 }
5723 }
5724
5725 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5726 EVT EltVT = WidenVT.getVectorElementType();
5727 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
5728 // Use the original element count so we don't do more scalar opts than
5729 // necessary.
5730 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5731 for (unsigned i=0; i < MinElts; ++i) {
5732 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5733 if (N->getNumOperands() == 1)
5734 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val, Flags);
5735 else
5736 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
5737 }
5738
5739 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5740}
5741
5742SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
5743 SDLoc dl(N);
5744 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5745 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5746
5747 SDValue Src = N->getOperand(Num: 0);
5748 EVT SrcVT = Src.getValueType();
5749
5750 // Also widen the input.
5751 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5752 Src = GetWidenedVector(Op: Src);
5753 SrcVT = Src.getValueType();
5754 }
5755
5756 // Input and output not widened to the same size, give up.
5757 if (WidenNumElts != SrcVT.getVectorElementCount())
5758 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5759
5760 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
5761}
5762
5763SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
5764 SDLoc dl(N);
5765 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5766 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5767
5768 SDValue Src = N->getOperand(Num: 0);
5769 EVT SrcVT = Src.getValueType();
5770
5771 // Also widen the input.
5772 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5773 Src = GetWidenedVector(Op: Src);
5774 SrcVT = Src.getValueType();
5775 }
5776
5777 // Input and output not widened to the same size, give up.
5778 if (WidenNumElts != SrcVT.getVectorElementCount())
5779 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5780
5781 if (N->getNumOperands() == 1)
5782 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
5783
5784 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5785 assert(N->isVPOpcode() && "Expected VP opcode");
5786
5787 SDValue Mask =
5788 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5789 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
5790}
5791
5792SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
5793 SDValue InOp = N->getOperand(Num: 1);
5794 SDLoc DL(N);
5795 SmallVector<SDValue, 4> NewOps(N->ops());
5796
5797 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5798 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5799
5800 EVT InVT = InOp.getValueType();
5801 EVT InEltVT = InVT.getVectorElementType();
5802
5803 unsigned Opcode = N->getOpcode();
5804
5805 // FIXME: Optimizations need to be implemented here.
5806
5807 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5808 EVT EltVT = WidenVT.getVectorElementType();
5809 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
5810 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
5811 SmallVector<SDValue, 32> OpChains;
5812 // Use the original element count so we don't do more scalar opts than
5813 // necessary.
5814 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5815 for (unsigned i=0; i < MinElts; ++i) {
5816 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5817 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
5818 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
5819 }
5820 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
5821 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5822
5823 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5824}
5825
5826SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
5827 unsigned Opcode = N->getOpcode();
5828 SDValue InOp = N->getOperand(Num: 0);
5829 SDLoc DL(N);
5830
5831 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5832 EVT WidenSVT = WidenVT.getVectorElementType();
5833 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5834
5835 EVT InVT = InOp.getValueType();
5836 EVT InSVT = InVT.getVectorElementType();
5837 unsigned InVTNumElts = InVT.getVectorNumElements();
5838
5839 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5840 InOp = GetWidenedVector(Op: InOp);
5841 InVT = InOp.getValueType();
5842 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
5843 switch (Opcode) {
5844 case ISD::ANY_EXTEND_VECTOR_INREG:
5845 case ISD::SIGN_EXTEND_VECTOR_INREG:
5846 case ISD::ZERO_EXTEND_VECTOR_INREG:
5847 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
5848 }
5849 }
5850 }
5851
5852 // Unroll, extend the scalars and rebuild the vector.
5853 SmallVector<SDValue, 16> Ops;
5854 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
5855 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
5856 switch (Opcode) {
5857 case ISD::ANY_EXTEND_VECTOR_INREG:
5858 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
5859 break;
5860 case ISD::SIGN_EXTEND_VECTOR_INREG:
5861 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
5862 break;
5863 case ISD::ZERO_EXTEND_VECTOR_INREG:
5864 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
5865 break;
5866 default:
5867 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
5868 }
5869 Ops.push_back(Elt: Val);
5870 }
5871
5872 while (Ops.size() != WidenNumElts)
5873 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
5874
5875 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5876}
5877
5878SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
5879 // If this is an FCOPYSIGN with same input types, we can treat it as a
5880 // normal (can trap) binary op.
5881 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
5882 return WidenVecRes_BinaryCanTrap(N);
5883
5884 // If the types are different, fall back to unrolling.
5885 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5886 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5887}
5888
5889/// Result and first source operand are different scalar types, but must have
5890/// the same number of elements. There is an additional control argument which
5891/// should be passed through unchanged.
5892SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
5893 SDValue FpValue = N->getOperand(Num: 0);
5894 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5895 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
5896 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5897 SDValue Arg = GetWidenedVector(Op: FpValue);
5898 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
5899 Flags: N->getFlags());
5900}
5901
5902SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
5903 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5904 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5905 SDValue RHS = N->getOperand(Num: 1);
5906 EVT ExpVT = RHS.getValueType();
5907 SDValue ExpOp = RHS;
5908 if (ExpVT.isVector()) {
5909 EVT WideExpVT = WidenVT.changeVectorElementType(
5910 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
5911 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
5912 }
5913
5914 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
5915}
5916
5917SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
5918 // Unary op widening.
5919 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5920 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5921 if (N->getNumOperands() == 1)
5922 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
5923 if (N->getOpcode() == ISD::AssertNoFPClass)
5924 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
5925 N2: N->getOperand(Num: 1), Flags: N->getFlags());
5926
5927 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5928 assert(N->isVPOpcode() && "Expected VP opcode");
5929
5930 SDValue Mask =
5931 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5932 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
5933 Ops: {InOp, Mask, N->getOperand(Num: 2)});
5934}
5935
5936SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
5937 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5938 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5939 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
5940 .getVectorElementType(),
5941 NumElements: WidenVT.getVectorNumElements());
5942 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5943 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
5944 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
5945}
5946
5947SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
5948 unsigned ResNo) {
5949 EVT VT0 = N->getValueType(ResNo: 0);
5950 EVT VT1 = N->getValueType(ResNo: 1);
5951
5952 assert(VT0.isVector() && VT1.isVector() &&
5953 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
5954 "expected both results to be vectors of matching element count");
5955
5956 LLVMContext &Ctx = *DAG.getContext();
5957 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5958
5959 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
5960 ElementCount WidenEC = WidenVT.getVectorElementCount();
5961
5962 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
5963 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
5964
5965 SDNode *WidenNode =
5966 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
5967 .getNode();
5968
5969 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
5970 return SDValue(WidenNode, ResNo);
5971}
5972
5973SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
5974 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
5975 return GetWidenedVector(Op: WidenVec);
5976}
5977
5978SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
5979 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5980 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5981 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
5982
5983 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
5984 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
5985 DestAS: AddrSpaceCastN->getDestAddressSpace());
5986}
5987
5988SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
5989 SDValue InOp = N->getOperand(Num: 0);
5990 EVT InVT = InOp.getValueType();
5991 EVT VT = N->getValueType(ResNo: 0);
5992 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5993 SDLoc dl(N);
5994
5995 switch (getTypeAction(VT: InVT)) {
5996 case TargetLowering::TypeLegal:
5997 break;
5998 case TargetLowering::TypeScalarizeScalableVector:
5999 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6000 case TargetLowering::TypePromoteInteger: {
6001 // If the incoming type is a vector that is being promoted, then
6002 // we know that the elements are arranged differently and that we
6003 // must perform the conversion using a stack slot.
6004 if (InVT.isVector())
6005 break;
6006
6007 // If the InOp is promoted to the same size, convert it. Otherwise,
6008 // fall out of the switch and widen the promoted input.
6009 SDValue NInOp = GetPromotedInteger(Op: InOp);
6010 EVT NInVT = NInOp.getValueType();
6011 if (WidenVT.bitsEq(VT: NInVT)) {
6012 // For big endian targets we need to shift the input integer or the
6013 // interesting bits will end up at the wrong place.
6014 if (DAG.getDataLayout().isBigEndian()) {
6015 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6016 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6017 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6018 }
6019 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6020 }
6021 InOp = NInOp;
6022 InVT = NInVT;
6023 break;
6024 }
6025 case TargetLowering::TypeSoftenFloat:
6026 case TargetLowering::TypeSoftPromoteHalf:
6027 case TargetLowering::TypeExpandInteger:
6028 case TargetLowering::TypeExpandFloat:
6029 case TargetLowering::TypeScalarizeVector:
6030 case TargetLowering::TypeSplitVector:
6031 break;
6032 case TargetLowering::TypeWidenVector:
6033 // If the InOp is widened to the same size, convert it. Otherwise, fall
6034 // out of the switch and widen the widened input.
6035 InOp = GetWidenedVector(Op: InOp);
6036 InVT = InOp.getValueType();
6037 if (WidenVT.bitsEq(VT: InVT))
6038 // The input widens to the same size. Convert to the widen value.
6039 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6040 break;
6041 }
6042
6043 unsigned WidenSize = WidenVT.getSizeInBits();
6044 unsigned InSize = InVT.getSizeInBits();
6045 unsigned InScalarSize = InVT.getScalarSizeInBits();
6046 // x86mmx is not an acceptable vector element type, so don't try.
6047 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6048 // Determine new input vector type. The new input vector type will use
6049 // the same element type (if its a vector) or use the input type as a
6050 // vector. It is the same size as the type to widen to.
6051 EVT NewInVT;
6052 unsigned NewNumParts = WidenSize / InSize;
6053 if (InVT.isVector()) {
6054 EVT InEltVT = InVT.getVectorElementType();
6055 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6056 NumElements: WidenSize / InEltVT.getSizeInBits());
6057 } else {
6058 // For big endian systems, using the promoted input scalar type
6059 // to produce the scalar_to_vector would put the desired bits into
6060 // the least significant byte(s) of the wider element zero. This
6061 // will mean that the users of the result vector are using incorrect
6062 // bits. Use the original input type instead. Although either input
6063 // type can be used on little endian systems, for consistency we
6064 // use the original type there as well.
6065 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6066 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6067 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6068 }
6069
6070 if (TLI.isTypeLegal(VT: NewInVT)) {
6071 SDValue NewVec;
6072 if (InVT.isVector()) {
6073 // Because the result and the input are different vector types, widening
6074 // the result could create a legal type but widening the input might
6075 // make it an illegal type that might lead to repeatedly splitting the
6076 // input and then widening it. To avoid this, we widen the input only if
6077 // it results in a legal type.
6078 if (WidenSize % InSize == 0) {
6079 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6080 Ops[0] = InOp;
6081
6082 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6083 } else {
6084 SmallVector<SDValue, 16> Ops;
6085 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6086 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6087 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6088
6089 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6090 }
6091 } else {
6092 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6093 }
6094 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6095 }
6096 }
6097
6098 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6099}
6100
6101SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6102 return DAG.getNode(
6103 Opcode: N->getOpcode(), DL: SDLoc(N),
6104 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6105 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6106}
6107
6108SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6109 SDLoc dl(N);
6110 // Build a vector with poison for the new nodes.
6111 EVT VT = N->getValueType(ResNo: 0);
6112
6113 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6114 // type. The POISONs need to have the same type as the existing operands.
6115 EVT EltVT = N->getOperand(Num: 0).getValueType();
6116 unsigned NumElts = VT.getVectorNumElements();
6117
6118 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6119 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6120
6121 SmallVector<SDValue, 16> NewOps(N->ops());
6122 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6123 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6124
6125 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6126}
6127
6128SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6129 EVT InVT = N->getOperand(Num: 0).getValueType();
6130 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6131 SDLoc dl(N);
6132 unsigned NumOperands = N->getNumOperands();
6133
6134 bool InputWidened = false; // Indicates we need to widen the input.
6135 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6136 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6137 unsigned NumInElts = InVT.getVectorMinNumElements();
6138 if (WidenNumElts % NumInElts == 0) {
6139 // Add undef vectors to widen to correct length.
6140 unsigned NumConcat = WidenNumElts / NumInElts;
6141 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6142 SmallVector<SDValue, 16> Ops(NumConcat);
6143 for (unsigned i=0; i < NumOperands; ++i)
6144 Ops[i] = N->getOperand(Num: i);
6145 for (unsigned i = NumOperands; i != NumConcat; ++i)
6146 Ops[i] = UndefVal;
6147 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6148 }
6149 } else {
6150 InputWidened = true;
6151 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6152 // The inputs and the result are widen to the same value.
6153 unsigned i;
6154 for (i=1; i < NumOperands; ++i)
6155 if (!N->getOperand(Num: i).isUndef())
6156 break;
6157
6158 if (i == NumOperands)
6159 // Everything but the first operand is an UNDEF so just return the
6160 // widened first operand.
6161 return GetWidenedVector(Op: N->getOperand(Num: 0));
6162
6163 if (NumOperands == 2) {
6164 assert(!WidenVT.isScalableVector() &&
6165 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6166 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6167 unsigned NumInElts = InVT.getVectorNumElements();
6168
6169 // Replace concat of two operands with a shuffle.
6170 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6171 for (unsigned i = 0; i < NumInElts; ++i) {
6172 MaskOps[i] = i;
6173 MaskOps[i + NumInElts] = i + WidenNumElts;
6174 }
6175 return DAG.getVectorShuffle(VT: WidenVT, dl,
6176 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6177 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6178 Mask: MaskOps);
6179 }
6180 }
6181 }
6182
6183 assert(!WidenVT.isScalableVector() &&
6184 "Cannot use build vectors to widen CONCAT_VECTOR result");
6185 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6186 unsigned NumInElts = InVT.getVectorNumElements();
6187
6188 // Fall back to use extracts and build vector.
6189 EVT EltVT = WidenVT.getVectorElementType();
6190 SmallVector<SDValue, 16> Ops(WidenNumElts);
6191 unsigned Idx = 0;
6192 for (unsigned i=0; i < NumOperands; ++i) {
6193 SDValue InOp = N->getOperand(Num: i);
6194 if (InputWidened)
6195 InOp = GetWidenedVector(Op: InOp);
6196 for (unsigned j = 0; j < NumInElts; ++j)
6197 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6198 }
6199 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6200 for (; Idx < WidenNumElts; ++Idx)
6201 Ops[Idx] = UndefVal;
6202 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6203}
6204
6205SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6206 EVT VT = N->getValueType(ResNo: 0);
6207 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6208 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6209 SDValue InOp2 = N->getOperand(Num: 1);
6210 SDValue Idx = N->getOperand(Num: 2);
6211 SDLoc dl(N);
6212 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6213}
6214
6215SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6216 EVT VT = N->getValueType(ResNo: 0);
6217 EVT EltVT = VT.getVectorElementType();
6218 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6219 SDValue InOp = N->getOperand(Num: 0);
6220 SDValue Idx = N->getOperand(Num: 1);
6221 SDLoc dl(N);
6222
6223 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6224 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6225 InOp = GetWidenedVector(Op: InOp);
6226
6227 EVT InVT = InOp.getValueType();
6228
6229 // Check if we can just return the input vector after widening.
6230 uint64_t IdxVal = Idx->getAsZExtVal();
6231 if (IdxVal == 0 && InVT == WidenVT)
6232 return InOp;
6233
6234 // Check if we can extract from the vector.
6235 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6236 unsigned InNumElts = InVT.getVectorMinNumElements();
6237 unsigned VTNumElts = VT.getVectorMinNumElements();
6238 assert(IdxVal % VTNumElts == 0 &&
6239 "Expected Idx to be a multiple of subvector minimum vector length");
6240 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6241 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6242
6243 if (VT.isScalableVector()) {
6244 // Try to split the operation up into smaller extracts and concat the
6245 // results together, e.g.
6246 // nxv6i64 extract_subvector(nxv12i64, 6)
6247 // <->
6248 // nxv8i64 concat(
6249 // nxv2i64 extract_subvector(nxv16i64, 6)
6250 // nxv2i64 extract_subvector(nxv16i64, 8)
6251 // nxv2i64 extract_subvector(nxv16i64, 10)
6252 // undef)
6253 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6254 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6255 "down type's element count");
6256 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6257 EC: ElementCount::getScalable(MinVal: GCD));
6258 // Avoid recursion around e.g. nxv1i8.
6259 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6260 SmallVector<SDValue> Parts;
6261 unsigned I = 0;
6262 for (; I < VTNumElts / GCD; ++I)
6263 Parts.push_back(
6264 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6265 for (; I < WidenNumElts / GCD; ++I)
6266 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6267
6268 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6269 }
6270
6271 // Fallback to extracting through memory.
6272
6273 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6274 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6275 MachineFunction &MF = DAG.getMachineFunction();
6276 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6277 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6278
6279 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6280 PtrInfo, F: MachineMemOperand::MOStore,
6281 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6282 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6283 PtrInfo, F: MachineMemOperand::MOLoad,
6284 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6285
6286 // Write out the input vector.
6287 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6288
6289 // Build a mask to match the length of the non-widened result.
6290 SDValue Mask =
6291 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6292
6293 // Read back the sub-vector setting the remaining lanes to poison.
6294 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6295 return DAG.getMaskedLoad(
6296 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6297 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6298 }
6299
6300 // We could try widening the input to the right length but for now, extract
6301 // the original elements, fill the rest with undefs and build a vector.
6302 SmallVector<SDValue, 16> Ops(WidenNumElts);
6303 unsigned i;
6304 for (i = 0; i < VTNumElts; ++i)
6305 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6306
6307 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6308 for (; i < WidenNumElts; ++i)
6309 Ops[i] = UndefVal;
6310 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6311}
6312
6313SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6314 SDValue InOp = ModifyToType(
6315 InOp: N->getOperand(Num: 0),
6316 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6317 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6318 N2: N->getOperand(Num: 1));
6319}
6320
6321SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6322 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6323 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6324 VT: InOp.getValueType(), N1: InOp,
6325 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6326}
6327
6328/// Either return the same load or provide appropriate casts
6329/// from the load and return that.
6330static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6331 TypeSize LdWidth, TypeSize FirstVTWidth,
6332 SDLoc dl, SelectionDAG &DAG) {
6333 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6334 "Load width must be less than or equal to first value type width");
6335 TypeSize WidenWidth = WidenVT.getSizeInBits();
6336 if (!FirstVT.isVector()) {
6337 unsigned NumElts =
6338 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6339 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6340 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6341 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6342 }
6343 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6344 return LdOp;
6345}
6346
6347static std::optional<EVT> findMemType(SelectionDAG &DAG,
6348 const TargetLowering &TLI, unsigned Width,
6349 EVT WidenVT, unsigned Align,
6350 unsigned WidenEx);
6351
6352SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6353 EVT WidenVT =
6354 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6355 EVT LdVT = LD->getMemoryVT();
6356 SDLoc dl(LD);
6357 assert(LdVT.isVector() && WidenVT.isVector() && "Expected vectors");
6358 assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6359 "Must be scalable");
6360 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6361 "Expected equivalent element types");
6362
6363 // Load information
6364 SDValue Chain = LD->getChain();
6365 SDValue BasePtr = LD->getBasePtr();
6366
6367 TypeSize LdWidth = LdVT.getSizeInBits();
6368 TypeSize WidenWidth = WidenVT.getSizeInBits();
6369 TypeSize WidthDiff = WidenWidth - LdWidth;
6370
6371 // Find the vector type that can load from.
6372 std::optional<EVT> FirstVT =
6373 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6374 WidenEx: WidthDiff.getKnownMinValue());
6375
6376 if (!FirstVT)
6377 return SDValue();
6378
6379 SmallVector<EVT, 8> MemVTs;
6380 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6381
6382 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6383 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6384
6385 // Load the element with one instruction.
6386 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6387 FirstVTWidth, dl, DAG);
6388
6389 // Modified the chain - switch anything that used the old chain to use
6390 // the new one.
6391 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6392 return Result;
6393}
6394
6395SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6396 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6397 ISD::LoadExtType ExtType = LD->getExtensionType();
6398
6399 // A vector must always be stored in memory as-is, i.e. without any padding
6400 // between the elements, since various code depend on it, e.g. in the
6401 // handling of a bitcast of a vector type to int, which may be done with a
6402 // vector store followed by an integer load. A vector that does not have
6403 // elements that are byte-sized must therefore be stored as an integer
6404 // built out of the extracted vector elements.
6405 if (!LD->getMemoryVT().isByteSized()) {
6406 SDValue Value, NewChain;
6407 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6408 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6409 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6410 return SDValue();
6411 }
6412
6413 // Generate a vector-predicated load if it is custom/legal on the target. To
6414 // avoid possible recursion, only do this if the widened mask type is legal.
6415 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6416 // removed from the IR by the ExpandVectorPredication pass but we're
6417 // reintroducing them here.
6418 EVT VT = LD->getValueType(ResNo: 0);
6419 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6420 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6421
6422 if (ExtType == ISD::NON_EXTLOAD &&
6423 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6424 TLI.isTypeLegal(VT: WideMaskVT)) {
6425 SDLoc DL(N);
6426 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6427 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6428 EC: VT.getVectorElementCount());
6429 SDValue NewLoad =
6430 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6431 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6432 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6433
6434 // Modified the chain - switch anything that used the old chain to use
6435 // the new one.
6436 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6437
6438 return NewLoad;
6439 }
6440
6441 SDValue Result;
6442 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6443 if (ExtType != ISD::NON_EXTLOAD)
6444 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6445 else
6446 Result = GenWidenVectorLoads(LdChain, LD);
6447
6448 if (Result) {
6449 // If we generate a single load, we can use that for the chain. Otherwise,
6450 // build a factor node to remember the multiple loads are independent and
6451 // chain to that.
6452 SDValue NewChain;
6453 if (LdChain.size() == 1)
6454 NewChain = LdChain[0];
6455 else
6456 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6457
6458 // Modified the chain - switch anything that used the old chain to use
6459 // the new one.
6460 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6461
6462 return Result;
6463 }
6464
6465 if (VT.isVector()) {
6466 // If all else fails replace the load with a wide masked load.
6467 SDLoc DL(N);
6468 SDValue Mask =
6469 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6470
6471 SDValue NewLoad = DAG.getMaskedLoad(
6472 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6473 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6474 AM: LD->getAddressingMode(), LD->getExtensionType());
6475
6476 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6477 return NewLoad;
6478 }
6479
6480 report_fatal_error(reason: "Unable to widen vector load");
6481}
6482
6483SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6484 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6485 SDValue Mask = N->getMask();
6486 SDValue EVL = N->getVectorLength();
6487 ISD::LoadExtType ExtType = N->getExtensionType();
6488 SDLoc dl(N);
6489
6490 // The mask should be widened as well
6491 assert(getTypeAction(Mask.getValueType()) ==
6492 TargetLowering::TypeWidenVector &&
6493 "Unable to widen binary VP op");
6494 Mask = GetWidenedVector(Op: Mask);
6495 assert(Mask.getValueType().getVectorElementCount() ==
6496 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6497 .getVectorElementCount() &&
6498 "Unable to widen vector load");
6499
6500 SDValue Res =
6501 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6502 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6503 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6504 // Legalize the chain result - switch anything that used the old chain to
6505 // use the new one.
6506 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6507 return Res;
6508}
6509
6510SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6511 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6512 SDValue Mask = N->getMask();
6513 SDValue EVL = N->getVectorLength();
6514 SDLoc dl(N);
6515
6516 // The mask should be widened as well
6517 assert(getTypeAction(Mask.getValueType()) ==
6518 TargetLowering::TypeWidenVector &&
6519 "Unable to widen binary VP op");
6520 Mask = GetWidenedVector(Op: Mask);
6521 assert(Mask.getValueType().getVectorElementCount() ==
6522 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6523 .getVectorElementCount() &&
6524 "Unable to widen vector load");
6525
6526 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6527 Mask, EVL, MMO: N->getMemOperand());
6528 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6529 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6530 return Res;
6531}
6532
6533SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6534 SDLoc DL(N);
6535
6536 // The mask should be widened as well
6537 SDValue Mask = N->getMask();
6538 assert(getTypeAction(Mask.getValueType()) ==
6539 TargetLowering::TypeWidenVector &&
6540 "Unable to widen VP strided load");
6541 Mask = GetWidenedVector(Op: Mask);
6542
6543 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6544 assert(Mask.getValueType().getVectorElementCount() ==
6545 WidenVT.getVectorElementCount() &&
6546 "Data and mask vectors should have the same number of elements");
6547
6548 SDValue Res = DAG.getStridedLoadVP(
6549 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6550 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6551 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6552 IsExpanding: N->isExpandingLoad());
6553
6554 // Legalize the chain result - switch anything that used the old chain to
6555 // use the new one.
6556 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6557 return Res;
6558}
6559
6560SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6561 SDValue Vec = N->getOperand(Num: 0);
6562 SDValue Mask = N->getOperand(Num: 1);
6563 SDValue Passthru = N->getOperand(Num: 2);
6564 EVT WideVecVT =
6565 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6566 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6567 VT: Mask.getValueType().getVectorElementType(),
6568 EC: WideVecVT.getVectorElementCount());
6569
6570 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6571 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6572 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6573 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6574 N2: WideMask, N3: WidePassthru);
6575}
6576
6577SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6578 EVT VT = N->getValueType(ResNo: 0);
6579 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6580 SDValue Mask = N->getMask();
6581 EVT MaskVT = Mask.getValueType();
6582 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6583 ISD::LoadExtType ExtType = N->getExtensionType();
6584 SDLoc dl(N);
6585
6586 EVT WideMaskVT =
6587 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6588 EC: WidenVT.getVectorElementCount());
6589
6590 if (ExtType == ISD::NON_EXTLOAD &&
6591 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6592 TLI.isTypeLegal(VT: WideMaskVT) &&
6593 // If there is a passthru, we shouldn't use vp.load. However,
6594 // type legalizer will struggle on masked.load with
6595 // scalable vectors, so for scalable vectors, we still use vp.load
6596 // but manually merge the load result with the passthru using vp.select.
6597 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6598 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6599 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6600 EC: VT.getVectorElementCount());
6601 SDValue NewLoad =
6602 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6603 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6604 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6605 SDValue NewVal = NewLoad;
6606
6607 // Manually merge with vp.select
6608 if (!N->getPassThru()->isUndef()) {
6609 assert(WidenVT.isScalableVector());
6610 NewVal =
6611 DAG.getNode(Opcode: ISD::VP_SELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru, N4: EVL);
6612 }
6613
6614 // Modified the chain - switch anything that used the old chain to use
6615 // the new one.
6616 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6617
6618 return NewVal;
6619 }
6620
6621 // The mask should be widened as well
6622 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6623
6624 SDValue Res = DAG.getMaskedLoad(
6625 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
6626 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
6627 ExtType, IsExpanding: N->isExpandingLoad());
6628 // Legalize the chain result - switch anything that used the old chain to
6629 // use the new one.
6630 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6631 return Res;
6632}
6633
6634SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
6635
6636 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6637 SDValue Mask = N->getMask();
6638 EVT MaskVT = Mask.getValueType();
6639 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6640 SDValue Scale = N->getScale();
6641 unsigned NumElts = WideVT.getVectorNumElements();
6642 SDLoc dl(N);
6643
6644 // The mask should be widened as well
6645 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6646 VT: MaskVT.getVectorElementType(),
6647 NumElements: WideVT.getVectorNumElements());
6648 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6649
6650 // Widen the Index operand
6651 SDValue Index = N->getIndex();
6652 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6653 VT: Index.getValueType().getScalarType(),
6654 NumElements: NumElts);
6655 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6656 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
6657 Scale };
6658
6659 // Widen the MemoryType
6660 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6661 VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
6662 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
6663 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
6664 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
6665
6666 // Legalize the chain result - switch anything that used the old chain to
6667 // use the new one.
6668 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6669 return Res;
6670}
6671
6672SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
6673 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6674 SDValue Mask = N->getMask();
6675 SDValue Scale = N->getScale();
6676 ElementCount WideEC = WideVT.getVectorElementCount();
6677 SDLoc dl(N);
6678
6679 SDValue Index = GetWidenedVector(Op: N->getIndex());
6680 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6681 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
6682 Mask = GetWidenedMask(Mask, EC: WideEC);
6683
6684 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
6685 Mask, N->getVectorLength()};
6686 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
6687 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
6688
6689 // Legalize the chain result - switch anything that used the old chain to
6690 // use the new one.
6691 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6692 return Res;
6693}
6694
6695SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
6696 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6697 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
6698}
6699
6700// Return true is this is a SETCC node or a strict version of it.
6701static inline bool isSETCCOp(unsigned Opcode) {
6702 switch (Opcode) {
6703 case ISD::SETCC:
6704 case ISD::STRICT_FSETCC:
6705 case ISD::STRICT_FSETCCS:
6706 return true;
6707 }
6708 return false;
6709}
6710
6711// Return true if this is a node that could have two SETCCs as operands.
6712static inline bool isLogicalMaskOp(unsigned Opcode) {
6713 switch (Opcode) {
6714 case ISD::AND:
6715 case ISD::OR:
6716 case ISD::XOR:
6717 return true;
6718 }
6719 return false;
6720}
6721
6722// If N is a SETCC or a strict variant of it, return the type
6723// of the compare operands.
6724static inline EVT getSETCCOperandType(SDValue N) {
6725 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
6726 return N->getOperand(Num: OpNo).getValueType();
6727}
6728
6729// This is used just for the assert in convertMask(). Check that this either
6730// a SETCC or a previously handled SETCC by convertMask().
6731#ifndef NDEBUG
6732static inline bool isSETCCorConvertedSETCC(SDValue N) {
6733 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
6734 N = N.getOperand(0);
6735 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
6736 for (unsigned i = 1; i < N->getNumOperands(); ++i)
6737 if (!N->getOperand(i)->isUndef())
6738 return false;
6739 N = N.getOperand(0);
6740 }
6741
6742 if (N.getOpcode() == ISD::TRUNCATE)
6743 N = N.getOperand(0);
6744 else if (N.getOpcode() == ISD::SIGN_EXTEND)
6745 N = N.getOperand(0);
6746
6747 if (isLogicalMaskOp(N.getOpcode()))
6748 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
6749 isSETCCorConvertedSETCC(N.getOperand(1));
6750
6751 return (isSETCCOp(N.getOpcode()) ||
6752 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
6753}
6754#endif
6755
6756// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
6757// to ToMaskVT if needed with vector extension or truncation.
6758SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
6759 EVT ToMaskVT) {
6760 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
6761 // FIXME: This code seems to be too restrictive, we might consider
6762 // generalizing it or dropping it.
6763 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
6764
6765 // Make a new Mask node, with a legal result VT.
6766 SDValue Mask;
6767 SmallVector<SDValue, 4> Ops;
6768 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
6769 Ops.push_back(Elt: InMask->getOperand(Num: i));
6770 if (InMask->isStrictFPOpcode()) {
6771 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
6772 ResultTys: { MaskVT, MVT::Other }, Ops);
6773 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
6774 }
6775 else
6776 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops);
6777
6778 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
6779 // extend or truncate is needed.
6780 LLVMContext &Ctx = *DAG.getContext();
6781 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
6782 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
6783 if (MaskScalarBits < ToMaskScalBits) {
6784 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6785 NumElements: MaskVT.getVectorNumElements());
6786 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
6787 } else if (MaskScalarBits > ToMaskScalBits) {
6788 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6789 NumElements: MaskVT.getVectorNumElements());
6790 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
6791 }
6792
6793 assert(Mask->getValueType(0).getScalarSizeInBits() ==
6794 ToMaskVT.getScalarSizeInBits() &&
6795 "Mask should have the right element size by now.");
6796
6797 // Adjust Mask to the right number of elements.
6798 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
6799 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
6800 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
6801 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
6802 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
6803 EVT SubVT = Mask->getValueType(ResNo: 0);
6804 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
6805 SubOps[0] = Mask;
6806 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
6807 }
6808
6809 assert((Mask->getValueType(0) == ToMaskVT) &&
6810 "A mask of ToMaskVT should have been produced by now.");
6811
6812 return Mask;
6813}
6814
6815// This method tries to handle some special cases for the vselect mask
6816// and if needed adjusting the mask vector type to match that of the VSELECT.
6817// Without it, many cases end up with scalarization of the SETCC, with many
6818// unnecessary instructions.
6819SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
6820 LLVMContext &Ctx = *DAG.getContext();
6821 SDValue Cond = N->getOperand(Num: 0);
6822
6823 if (N->getOpcode() != ISD::VSELECT)
6824 return SDValue();
6825
6826 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
6827 return SDValue();
6828
6829 // If this is a splitted VSELECT that was previously already handled, do
6830 // nothing.
6831 EVT CondVT = Cond->getValueType(ResNo: 0);
6832 if (CondVT.getScalarSizeInBits() != 1)
6833 return SDValue();
6834
6835 EVT VSelVT = N->getValueType(ResNo: 0);
6836
6837 // This method can't handle scalable vector types.
6838 // FIXME: This support could be added in the future.
6839 if (VSelVT.isScalableVector())
6840 return SDValue();
6841
6842 // Only handle vector types which are a power of 2.
6843 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
6844 return SDValue();
6845
6846 // Don't touch if this will be scalarized.
6847 EVT FinalVT = VSelVT;
6848 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
6849 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
6850
6851 if (FinalVT.getVectorNumElements() == 1)
6852 return SDValue();
6853
6854 // If there is support for an i1 vector mask, don't touch.
6855 if (isSETCCOp(Opcode: Cond.getOpcode())) {
6856 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
6857 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
6858 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
6859 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
6860 if (SetCCResVT.getScalarSizeInBits() == 1)
6861 return SDValue();
6862 } else if (CondVT.getScalarType() == MVT::i1) {
6863 // If there is support for an i1 vector mask (or only scalar i1 conditions),
6864 // don't touch.
6865 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
6866 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
6867
6868 if (CondVT.getScalarType() == MVT::i1)
6869 return SDValue();
6870 }
6871
6872 // Widen the vselect result type if needed.
6873 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
6874 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
6875
6876 // The mask of the VSELECT should have integer elements.
6877 EVT ToMaskVT = VSelVT;
6878 if (!ToMaskVT.getScalarType().isInteger())
6879 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
6880
6881 SDValue Mask;
6882 if (isSETCCOp(Opcode: Cond->getOpcode())) {
6883 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
6884 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6885 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
6886 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
6887 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
6888 // Cond is (AND/OR/XOR (SETCC, SETCC))
6889 SDValue SETCC0 = Cond->getOperand(Num: 0);
6890 SDValue SETCC1 = Cond->getOperand(Num: 1);
6891 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
6892 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
6893 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
6894 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
6895 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
6896 EVT MaskVT;
6897 // If the two SETCCs have different VTs, either extend/truncate one of
6898 // them to the other "towards" ToMaskVT, or truncate one and extend the
6899 // other to ToMaskVT.
6900 if (ScalarBits0 != ScalarBits1) {
6901 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
6902 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
6903 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
6904 MaskVT = WideVT;
6905 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
6906 MaskVT = NarrowVT;
6907 else
6908 MaskVT = ToMaskVT;
6909 } else
6910 // If the two SETCCs have the same VT, don't change it.
6911 MaskVT = VT0;
6912
6913 // Make new SETCCs and logical nodes.
6914 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
6915 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
6916 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
6917
6918 // Convert the logical op for VSELECT if needed.
6919 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6920 } else
6921 return SDValue();
6922
6923 return Mask;
6924}
6925
6926SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
6927 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6928 ElementCount WidenEC = WidenVT.getVectorElementCount();
6929
6930 SDValue Cond1 = N->getOperand(Num: 0);
6931 EVT CondVT = Cond1.getValueType();
6932 unsigned Opcode = N->getOpcode();
6933 if (CondVT.isVector()) {
6934 if (SDValue WideCond = WidenVSELECTMask(N)) {
6935 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
6936 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
6937 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
6938 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
6939 }
6940
6941 EVT CondEltVT = CondVT.getVectorElementType();
6942 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
6943 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
6944 Cond1 = GetWidenedVector(Op: Cond1);
6945
6946 // If we have to split the condition there is no point in widening the
6947 // select. This would result in an cycle of widening the select ->
6948 // widening the condition operand -> splitting the condition operand ->
6949 // splitting the select -> widening the select. Instead split this select
6950 // further and widen the resulting type.
6951 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
6952 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
6953 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
6954 return Res;
6955 }
6956
6957 if (Cond1.getValueType() != CondWidenVT)
6958 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
6959 }
6960
6961 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
6962 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
6963 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
6964 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
6965 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
6966 N4: N->getOperand(Num: 3));
6967 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
6968}
6969
6970SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
6971 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
6972 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
6973 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
6974 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
6975 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
6976}
6977
6978SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
6979 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6980 return DAG.getUNDEF(VT: WidenVT);
6981}
6982
6983SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
6984 EVT VT = N->getValueType(ResNo: 0);
6985 SDLoc dl(N);
6986
6987 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6988 unsigned NumElts = VT.getVectorNumElements();
6989 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6990
6991 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6992 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
6993
6994 // Adjust mask based on new input vector length.
6995 SmallVector<int, 16> NewMask(WidenNumElts, -1);
6996 for (unsigned i = 0; i != NumElts; ++i) {
6997 int Idx = N->getMaskElt(Idx: i);
6998 if (Idx < (int)NumElts)
6999 NewMask[i] = Idx;
7000 else
7001 NewMask[i] = Idx - NumElts + WidenNumElts;
7002 }
7003 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7004}
7005
7006SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7007 EVT VT = N->getValueType(ResNo: 0);
7008 EVT EltVT = VT.getVectorElementType();
7009 SDLoc dl(N);
7010
7011 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7012 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7013 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7014
7015 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7016 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7017 unsigned VTNumElts = VT.getVectorMinNumElements();
7018 unsigned IdxVal = WidenNumElts - VTNumElts;
7019
7020 if (VT.isScalableVector()) {
7021 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7022 // results together, e.g.(nxv6i64 -> nxv8i64)
7023 // nxv8i64 vector_reverse
7024 // <->
7025 // nxv8i64 concat(
7026 // nxv2i64 extract_subvector(nxv8i64, 2)
7027 // nxv2i64 extract_subvector(nxv8i64, 4)
7028 // nxv2i64 extract_subvector(nxv8i64, 6)
7029 // nxv2i64 undef)
7030
7031 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7032 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7033 EC: ElementCount::getScalable(MinVal: GCD));
7034 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7035 "down type's element count");
7036 SmallVector<SDValue> Parts;
7037 unsigned i = 0;
7038 for (; i < VTNumElts / GCD; ++i)
7039 Parts.push_back(
7040 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7041 for (; i < WidenNumElts / GCD; ++i)
7042 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7043
7044 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7045 }
7046
7047 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7048 // fixed-vectors.
7049 SmallVector<int, 16> Mask(WidenNumElts, -1);
7050 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7051
7052 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7053 Mask);
7054}
7055
7056SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7057 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7058 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7059}
7060
7061SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7062 assert(N->getValueType(0).isVector() &&
7063 N->getOperand(0).getValueType().isVector() &&
7064 "Operands must be vectors");
7065 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7066 ElementCount WidenEC = WidenVT.getVectorElementCount();
7067
7068 SDValue InOp1 = N->getOperand(Num: 0);
7069 EVT InVT = InOp1.getValueType();
7070 assert(InVT.isVector() && "can not widen non-vector type");
7071 EVT WidenInVT =
7072 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7073
7074 // The input and output types often differ here, and it could be that while
7075 // we'd prefer to widen the result type, the input operands have been split.
7076 // In this case, we also need to split the result of this node as well.
7077 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7078 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7079 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7080 return Res;
7081 }
7082
7083 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7084 SDValue InOp2 = N->getOperand(Num: 1);
7085 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7086 InOp1 = GetWidenedVector(Op: InOp1);
7087 InOp2 = GetWidenedVector(Op: InOp2);
7088 } else {
7089 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7090 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7091 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7092 N2: InOp1, N3: ZeroIdx);
7093 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7094 N2: InOp2, N3: ZeroIdx);
7095 }
7096
7097 // Assume that the input and output will be widen appropriately. If not,
7098 // we will have to unroll it at some point.
7099 assert(InOp1.getValueType() == WidenInVT &&
7100 InOp2.getValueType() == WidenInVT &&
7101 "Input not widened to expected type!");
7102 (void)WidenInVT;
7103 if (N->getOpcode() == ISD::VP_SETCC) {
7104 SDValue Mask =
7105 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7106 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7107 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7108 }
7109 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7110 N3: N->getOperand(Num: 2));
7111}
7112
7113SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7114 assert(N->getValueType(0).isVector() &&
7115 N->getOperand(1).getValueType().isVector() &&
7116 "Operands must be vectors");
7117 EVT VT = N->getValueType(ResNo: 0);
7118 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7119 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7120 unsigned NumElts = VT.getVectorNumElements();
7121 EVT EltVT = VT.getVectorElementType();
7122
7123 SDLoc dl(N);
7124 SDValue Chain = N->getOperand(Num: 0);
7125 SDValue LHS = N->getOperand(Num: 1);
7126 SDValue RHS = N->getOperand(Num: 2);
7127 SDValue CC = N->getOperand(Num: 3);
7128 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7129
7130 // Fully unroll and reassemble.
7131 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7132 SmallVector<SDValue, 8> Chains(NumElts);
7133 for (unsigned i = 0; i != NumElts; ++i) {
7134 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7135 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7136
7137 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7138 Ops: {Chain, LHSElem, RHSElem, CC});
7139 Chains[i] = Scalars[i].getValue(R: 1);
7140 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7141 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7142 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7143 }
7144
7145 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7146 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7147
7148 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7149}
7150
7151//===----------------------------------------------------------------------===//
7152// Widen Vector Operand
7153//===----------------------------------------------------------------------===//
7154bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7155 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7156 SDValue Res = SDValue();
7157
7158 // See if the target wants to custom widen this node.
7159 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7160 return false;
7161
7162 switch (N->getOpcode()) {
7163 default:
7164#ifndef NDEBUG
7165 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7166 N->dump(&DAG);
7167 dbgs() << "\n";
7168#endif
7169 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7170
7171 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7172 case ISD::FAKE_USE:
7173 Res = WidenVecOp_FAKE_USE(N);
7174 break;
7175 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7176 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7177 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7178 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7179 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7180 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7181 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7182 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7183 break;
7184 case ISD::ANY_EXTEND_VECTOR_INREG:
7185 case ISD::SIGN_EXTEND_VECTOR_INREG:
7186 case ISD::ZERO_EXTEND_VECTOR_INREG:
7187 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7188 break;
7189 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7190 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7191 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7192 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7193 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7194 case ISD::STRICT_FSETCC:
7195 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7196 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7197 case ISD::FLDEXP:
7198 case ISD::FCOPYSIGN:
7199 case ISD::LROUND:
7200 case ISD::LLROUND:
7201 case ISD::LRINT:
7202 case ISD::LLRINT:
7203 Res = WidenVecOp_UnrollVectorOp(N);
7204 break;
7205 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7206
7207 case ISD::ANY_EXTEND:
7208 case ISD::SIGN_EXTEND:
7209 case ISD::ZERO_EXTEND:
7210 Res = WidenVecOp_EXTEND(N);
7211 break;
7212
7213 case ISD::SCMP:
7214 case ISD::UCMP:
7215 Res = WidenVecOp_CMP(N);
7216 break;
7217
7218 case ISD::FP_EXTEND:
7219 case ISD::STRICT_FP_EXTEND:
7220 case ISD::FP_ROUND:
7221 case ISD::STRICT_FP_ROUND:
7222 case ISD::FP_TO_SINT:
7223 case ISD::STRICT_FP_TO_SINT:
7224 case ISD::FP_TO_UINT:
7225 case ISD::STRICT_FP_TO_UINT:
7226 case ISD::SINT_TO_FP:
7227 case ISD::STRICT_SINT_TO_FP:
7228 case ISD::UINT_TO_FP:
7229 case ISD::STRICT_UINT_TO_FP:
7230 case ISD::TRUNCATE:
7231 Res = WidenVecOp_Convert(N);
7232 break;
7233
7234 case ISD::FP_TO_SINT_SAT:
7235 case ISD::FP_TO_UINT_SAT:
7236 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7237 break;
7238
7239 case ISD::VECREDUCE_FADD:
7240 case ISD::VECREDUCE_FMUL:
7241 case ISD::VECREDUCE_ADD:
7242 case ISD::VECREDUCE_MUL:
7243 case ISD::VECREDUCE_AND:
7244 case ISD::VECREDUCE_OR:
7245 case ISD::VECREDUCE_XOR:
7246 case ISD::VECREDUCE_SMAX:
7247 case ISD::VECREDUCE_SMIN:
7248 case ISD::VECREDUCE_UMAX:
7249 case ISD::VECREDUCE_UMIN:
7250 case ISD::VECREDUCE_FMAX:
7251 case ISD::VECREDUCE_FMIN:
7252 case ISD::VECREDUCE_FMAXIMUM:
7253 case ISD::VECREDUCE_FMINIMUM:
7254 Res = WidenVecOp_VECREDUCE(N);
7255 break;
7256 case ISD::VECREDUCE_SEQ_FADD:
7257 case ISD::VECREDUCE_SEQ_FMUL:
7258 Res = WidenVecOp_VECREDUCE_SEQ(N);
7259 break;
7260 case ISD::VP_REDUCE_FADD:
7261 case ISD::VP_REDUCE_SEQ_FADD:
7262 case ISD::VP_REDUCE_FMUL:
7263 case ISD::VP_REDUCE_SEQ_FMUL:
7264 case ISD::VP_REDUCE_ADD:
7265 case ISD::VP_REDUCE_MUL:
7266 case ISD::VP_REDUCE_AND:
7267 case ISD::VP_REDUCE_OR:
7268 case ISD::VP_REDUCE_XOR:
7269 case ISD::VP_REDUCE_SMAX:
7270 case ISD::VP_REDUCE_SMIN:
7271 case ISD::VP_REDUCE_UMAX:
7272 case ISD::VP_REDUCE_UMIN:
7273 case ISD::VP_REDUCE_FMAX:
7274 case ISD::VP_REDUCE_FMIN:
7275 case ISD::VP_REDUCE_FMAXIMUM:
7276 case ISD::VP_REDUCE_FMINIMUM:
7277 Res = WidenVecOp_VP_REDUCE(N);
7278 break;
7279 case ISD::VP_CTTZ_ELTS:
7280 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7281 Res = WidenVecOp_VP_CttzElements(N);
7282 break;
7283 case ISD::VECTOR_FIND_LAST_ACTIVE:
7284 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7285 break;
7286 }
7287
7288 // If Res is null, the sub-method took care of registering the result.
7289 if (!Res.getNode()) return false;
7290
7291 // If the result is N, the sub-method updated N in place. Tell the legalizer
7292 // core about this.
7293 if (Res.getNode() == N)
7294 return true;
7295
7296
7297 if (N->isStrictFPOpcode())
7298 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7299 "Invalid operand expansion");
7300 else
7301 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7302 "Invalid operand expansion");
7303
7304 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7305 return false;
7306}
7307
7308SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7309 SDLoc DL(N);
7310 EVT VT = N->getValueType(ResNo: 0);
7311
7312 SDValue InOp = N->getOperand(Num: 0);
7313 assert(getTypeAction(InOp.getValueType()) ==
7314 TargetLowering::TypeWidenVector &&
7315 "Unexpected type action");
7316 InOp = GetWidenedVector(Op: InOp);
7317 assert(VT.getVectorNumElements() <
7318 InOp.getValueType().getVectorNumElements() &&
7319 "Input wasn't widened!");
7320
7321 // We may need to further widen the operand until it has the same total
7322 // vector size as the result.
7323 EVT InVT = InOp.getValueType();
7324 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7325 EVT InEltVT = InVT.getVectorElementType();
7326 for (EVT FixedVT : MVT::vector_valuetypes()) {
7327 EVT FixedEltVT = FixedVT.getVectorElementType();
7328 if (TLI.isTypeLegal(VT: FixedVT) &&
7329 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7330 FixedEltVT == InEltVT) {
7331 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7332 "Not enough elements in the fixed type for the operand!");
7333 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7334 "We can't have the same type as we started with!");
7335 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7336 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7337 else
7338 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7339 break;
7340 }
7341 }
7342 InVT = InOp.getValueType();
7343 if (InVT.getSizeInBits() != VT.getSizeInBits())
7344 // We couldn't find a legal vector type that was a widening of the input
7345 // and could be extended in-register to the result type, so we have to
7346 // scalarize.
7347 return WidenVecOp_Convert(N);
7348 }
7349
7350 // Use special DAG nodes to represent the operation of extending the
7351 // low lanes.
7352 switch (N->getOpcode()) {
7353 default:
7354 llvm_unreachable("Extend legalization on extend operation!");
7355 case ISD::ANY_EXTEND:
7356 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7357 case ISD::SIGN_EXTEND:
7358 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7359 case ISD::ZERO_EXTEND:
7360 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7361 }
7362}
7363
7364SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7365 SDLoc dl(N);
7366
7367 EVT OpVT = N->getOperand(Num: 0).getValueType();
7368 EVT ResVT = N->getValueType(ResNo: 0);
7369 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7370 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7371
7372 // 1. EXTRACT_SUBVECTOR
7373 // 2. SIGN_EXTEND/ZERO_EXTEND
7374 // 3. CMP
7375 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7376 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7377
7378 // At this point the result type is guaranteed to be valid, so we can use it
7379 // as the operand type by extending it appropriately
7380 ISD::NodeType ExtendOpcode =
7381 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7382 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7383 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7384
7385 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7386}
7387
7388SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7389 // The result (and first input) is legal, but the second input is illegal.
7390 // We can't do much to fix that, so just unroll and let the extracts off of
7391 // the second input be widened as needed later.
7392 return DAG.UnrollVectorOp(N);
7393}
7394
7395SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7396 SDLoc DL(N);
7397 EVT ResultVT = N->getValueType(ResNo: 0);
7398 SDValue Test = N->getOperand(Num: 1);
7399 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7400
7401 // Process this node similarly to SETCC.
7402 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7403 if (ResultVT.getScalarType() == MVT::i1)
7404 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7405 NumElements: WideResultVT.getVectorNumElements());
7406
7407 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7408 Ops: {WideArg, Test}, Flags: N->getFlags());
7409
7410 // Extract the needed results from the result vector.
7411 EVT ResVT =
7412 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7413 NumElements: ResultVT.getVectorNumElements());
7414 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7415
7416 EVT OpVT = N->getOperand(Num: 0).getValueType();
7417 ISD::NodeType ExtendCode =
7418 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7419 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7420}
7421
7422SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7423 // Since the result is legal and the input is illegal.
7424 EVT VT = N->getValueType(ResNo: 0);
7425 EVT EltVT = VT.getVectorElementType();
7426 SDLoc dl(N);
7427 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7428 assert(getTypeAction(InOp.getValueType()) ==
7429 TargetLowering::TypeWidenVector &&
7430 "Unexpected type action");
7431 InOp = GetWidenedVector(Op: InOp);
7432 EVT InVT = InOp.getValueType();
7433 unsigned Opcode = N->getOpcode();
7434
7435 // See if a widened result type would be legal, if so widen the node.
7436 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7437 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7438 EC: InVT.getVectorElementCount());
7439 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7440 SDValue Res;
7441 if (N->isStrictFPOpcode()) {
7442 if (Opcode == ISD::STRICT_FP_ROUND)
7443 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7444 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7445 else
7446 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7447 Ops: { N->getOperand(Num: 0), InOp });
7448 // Legalize the chain result - switch anything that used the old chain to
7449 // use the new one.
7450 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7451 } else {
7452 if (Opcode == ISD::FP_ROUND)
7453 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
7454 else
7455 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
7456 }
7457 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7458 }
7459
7460 EVT InEltVT = InVT.getVectorElementType();
7461
7462 // Unroll the convert into some scalar code and create a nasty build vector.
7463 unsigned NumElts = VT.getVectorNumElements();
7464 SmallVector<SDValue, 16> Ops(NumElts);
7465 if (N->isStrictFPOpcode()) {
7466 SmallVector<SDValue, 4> NewOps(N->ops());
7467 SmallVector<SDValue, 32> OpChains;
7468 for (unsigned i=0; i < NumElts; ++i) {
7469 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7470 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7471 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7472 }
7473 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7474 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7475 } else {
7476 for (unsigned i = 0; i < NumElts; ++i)
7477 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT,
7478 Operand: DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i));
7479 }
7480
7481 return DAG.getBuildVector(VT, DL: dl, Ops);
7482}
7483
7484SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7485 EVT DstVT = N->getValueType(ResNo: 0);
7486 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7487 EVT SrcVT = Src.getValueType();
7488 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7489 SDLoc dl(N);
7490
7491 // See if a widened result type would be legal, if so widen the node.
7492 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7493 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7494 if (TLI.isTypeLegal(VT: WideDstVT)) {
7495 SDValue Res =
7496 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7497 return DAG.getNode(
7498 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7499 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7500 }
7501
7502 // Give up and unroll.
7503 return DAG.UnrollVectorOp(N);
7504}
7505
7506SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7507 EVT VT = N->getValueType(ResNo: 0);
7508 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7509 EVT InWidenVT = InOp.getValueType();
7510 SDLoc dl(N);
7511
7512 // Check if we can convert between two legal vector types and extract.
7513 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7514 TypeSize Size = VT.getSizeInBits();
7515 // x86mmx is not an acceptable vector element type, so don't try.
7516 if (!VT.isVector() && VT != MVT::x86mmx &&
7517 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7518 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7519 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7520 if (TLI.isTypeLegal(VT: NewVT)) {
7521 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7522 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7523 }
7524 }
7525
7526 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7527 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7528 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7529 // having to copy via memory.
7530 if (VT.isVector()) {
7531 EVT EltVT = VT.getVectorElementType();
7532 unsigned EltSize = EltVT.getFixedSizeInBits();
7533 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7534 ElementCount NewNumElts =
7535 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
7536 .divideCoefficientBy(RHS: EltSize);
7537 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
7538 if (TLI.isTypeLegal(VT: NewVT)) {
7539 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7540 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
7541 }
7542 }
7543 }
7544
7545 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
7546}
7547
7548// Vectors with sizes that are not powers of 2 need to be widened to the
7549// next largest power of 2. For example, we may get a vector of 3 32-bit
7550// integers or of 6 16-bit integers, both of which have to be widened to a
7551// 128-bit vector.
7552SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
7553 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
7554 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
7555 N2: WidenedOp);
7556}
7557
7558SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
7559 EVT VT = N->getValueType(ResNo: 0);
7560 EVT EltVT = VT.getVectorElementType();
7561 EVT InVT = N->getOperand(Num: 0).getValueType();
7562 SDLoc dl(N);
7563
7564 // If the widen width for this operand is the same as the width of the concat
7565 // and all but the first operand is undef, just use the widened operand.
7566 unsigned NumOperands = N->getNumOperands();
7567 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
7568 unsigned i;
7569 for (i = 1; i < NumOperands; ++i)
7570 if (!N->getOperand(Num: i).isUndef())
7571 break;
7572
7573 if (i == NumOperands)
7574 return GetWidenedVector(Op: N->getOperand(Num: 0));
7575 }
7576
7577 // Otherwise, fall back to a nasty build vector.
7578 unsigned NumElts = VT.getVectorNumElements();
7579 SmallVector<SDValue, 16> Ops(NumElts);
7580
7581 unsigned NumInElts = InVT.getVectorNumElements();
7582
7583 unsigned Idx = 0;
7584 for (unsigned i=0; i < NumOperands; ++i) {
7585 SDValue InOp = N->getOperand(Num: i);
7586 assert(getTypeAction(InOp.getValueType()) ==
7587 TargetLowering::TypeWidenVector &&
7588 "Unexpected type action");
7589 InOp = GetWidenedVector(Op: InOp);
7590 for (unsigned j = 0; j < NumInElts; ++j)
7591 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
7592 }
7593 return DAG.getBuildVector(VT, DL: dl, Ops);
7594}
7595
7596SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
7597 EVT VT = N->getValueType(ResNo: 0);
7598 SDValue SubVec = N->getOperand(Num: 1);
7599 SDValue InVec = N->getOperand(Num: 0);
7600
7601 EVT OrigVT = SubVec.getValueType();
7602 SubVec = GetWidenedVector(Op: SubVec);
7603 EVT SubVT = SubVec.getValueType();
7604
7605 // Whether or not all the elements of the widened SubVec will be inserted into
7606 // valid indices of VT.
7607 bool IndicesValid = false;
7608 // If we statically know that VT can fit SubVT, the indices are valid.
7609 if (VT.knownBitsGE(VT: SubVT))
7610 IndicesValid = true;
7611 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
7612 // Otherwise, if we're inserting a fixed vector into a scalable vector and
7613 // we know the minimum vscale we can work out if it's valid ourselves.
7614 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
7615 Kind: Attribute::VScaleRange);
7616 if (Attr.isValid()) {
7617 unsigned VScaleMin = Attr.getVScaleRangeMin();
7618 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
7619 SubVT.getFixedSizeInBits())
7620 IndicesValid = true;
7621 }
7622 }
7623
7624 if (!IndicesValid)
7625 report_fatal_error(
7626 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
7627
7628 SDLoc DL(N);
7629
7630 // We need to make sure that the indices are still valid, otherwise we might
7631 // widen what was previously well-defined to something undefined.
7632 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
7633 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
7634 N3: N->getOperand(Num: 2));
7635
7636 if (OrigVT.isScalableVector()) {
7637 // When the widened types match, overwriting the start of a vector is
7638 // effectively a merge operation that can be implement as a vselect.
7639 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
7640 SDValue Mask =
7641 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
7642 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
7643 }
7644
7645 // Fallback to inserting through memory.
7646 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
7647 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
7648 MachineFunction &MF = DAG.getMachineFunction();
7649 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7650 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
7651
7652 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
7653 PtrInfo, F: MachineMemOperand::MOStore,
7654 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7655 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
7656 PtrInfo, F: MachineMemOperand::MOLoad,
7657 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7658
7659 // Write out the vector being inserting into.
7660 SDValue Ch =
7661 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
7662
7663 // Build a mask to match the length of the sub-vector.
7664 SDValue Mask =
7665 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
7666
7667 // Overwrite the sub-vector at the required offset.
7668 SDValue SubVecPtr =
7669 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
7670 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
7671 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
7672 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
7673
7674 // Read back the result.
7675 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
7676 }
7677
7678 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
7679 // with a series of INSERT_VECTOR_ELT
7680 unsigned Idx = N->getConstantOperandVal(Num: 2);
7681
7682 SDValue InsertElt = InVec;
7683 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
7684 SDValue ExtractElt =
7685 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
7686 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
7687 }
7688
7689 return InsertElt;
7690}
7691
7692SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
7693 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7694 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
7695 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7696}
7697
7698SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
7699 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7700 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
7701 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7702}
7703
7704SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
7705 SDLoc DL(N);
7706 EVT ResVT = N->getValueType(ResNo: 0);
7707
7708 // Widen the input as requested by the legalizer.
7709 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7710 EVT WideInVT = WideInOp.getValueType();
7711
7712 // Simple case: if widened input is still smaller than or equal to result,
7713 // just use it directly.
7714 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
7715 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
7716
7717 // EXTEND_VECTOR_INREG requires input bits <= result bits.
7718 // If widening makes the input larger than the original result, widen the
7719 // result to match, then extract back down.
7720 EVT ResEltVT = ResVT.getVectorElementType();
7721 unsigned EltBits = ResEltVT.getSizeInBits();
7722 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
7723 "Widened input size must be a multiple of result element size");
7724
7725 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
7726 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
7727
7728 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
7729 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
7730}
7731
7732SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
7733 // We have to widen the value, but we want only to store the original
7734 // vector type.
7735 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
7736
7737 if (!ST->getMemoryVT().getScalarType().isByteSized())
7738 return TLI.scalarizeVectorStore(ST, DAG);
7739
7740 if (ST->isTruncatingStore())
7741 return TLI.scalarizeVectorStore(ST, DAG);
7742
7743 // Generate a vector-predicated store if it is custom/legal on the target.
7744 // To avoid possible recursion, only do this if the widened mask type is
7745 // legal.
7746 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
7747 // removed from the IR by the ExpandVectorPredication pass but we're
7748 // reintroducing them here.
7749 SDValue StVal = ST->getValue();
7750 EVT StVT = StVal.getValueType();
7751 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
7752 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
7753
7754 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7755 TLI.isTypeLegal(VT: WideMaskVT)) {
7756 // Widen the value.
7757 SDLoc DL(N);
7758 StVal = GetWidenedVector(Op: StVal);
7759 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
7760 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
7761 EC: StVT.getVectorElementCount());
7762 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
7763 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
7764 AM: ST->getAddressingMode());
7765 }
7766
7767 SmallVector<SDValue, 16> StChain;
7768 if (GenWidenVectorStores(StChain, ST)) {
7769 if (StChain.size() == 1)
7770 return StChain[0];
7771
7772 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
7773 }
7774
7775 if (StVT.isVector()) {
7776 // If all else fails replace the store with a wide masked store.
7777 SDLoc DL(N);
7778 SDValue WideStVal = GetWidenedVector(Op: StVal);
7779 SDValue Mask =
7780 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
7781
7782 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
7783 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
7784 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
7785 IsTruncating: ST->isTruncatingStore());
7786 }
7787
7788 report_fatal_error(reason: "Unable to widen vector store");
7789}
7790
7791SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
7792 assert((OpNo == 1 || OpNo == 3) &&
7793 "Can widen only data or mask operand of vp_store");
7794 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
7795 SDValue Mask = ST->getMask();
7796 SDValue StVal = ST->getValue();
7797 SDLoc dl(N);
7798
7799 if (OpNo == 1) {
7800 // Widen the value.
7801 StVal = GetWidenedVector(Op: StVal);
7802
7803 // We only handle the case where the mask needs widening to an
7804 // identically-sized type as the vector inputs.
7805 assert(getTypeAction(Mask.getValueType()) ==
7806 TargetLowering::TypeWidenVector &&
7807 "Unable to widen VP store");
7808 Mask = GetWidenedVector(Op: Mask);
7809 } else {
7810 Mask = GetWidenedVector(Op: Mask);
7811
7812 // We only handle the case where the stored value needs widening to an
7813 // identically-sized type as the mask.
7814 assert(getTypeAction(StVal.getValueType()) ==
7815 TargetLowering::TypeWidenVector &&
7816 "Unable to widen VP store");
7817 StVal = GetWidenedVector(Op: StVal);
7818 }
7819
7820 assert(Mask.getValueType().getVectorElementCount() ==
7821 StVal.getValueType().getVectorElementCount() &&
7822 "Mask and data vectors should have the same number of elements");
7823 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
7824 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
7825 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
7826 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
7827 IsCompressing: ST->isCompressingStore());
7828}
7829
7830SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
7831 unsigned OpNo) {
7832 assert((OpNo == 1 || OpNo == 4) &&
7833 "Can widen only data or mask operand of vp_strided_store");
7834 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
7835 SDValue Mask = SST->getMask();
7836 SDValue StVal = SST->getValue();
7837 SDLoc DL(N);
7838
7839 if (OpNo == 1)
7840 assert(getTypeAction(Mask.getValueType()) ==
7841 TargetLowering::TypeWidenVector &&
7842 "Unable to widen VP strided store");
7843 else
7844 assert(getTypeAction(StVal.getValueType()) ==
7845 TargetLowering::TypeWidenVector &&
7846 "Unable to widen VP strided store");
7847
7848 StVal = GetWidenedVector(Op: StVal);
7849 Mask = GetWidenedVector(Op: Mask);
7850
7851 assert(StVal.getValueType().getVectorElementCount() ==
7852 Mask.getValueType().getVectorElementCount() &&
7853 "Data and mask vectors should have the same number of elements");
7854
7855 return DAG.getStridedStoreVP(
7856 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
7857 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
7858 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
7859 IsCompressing: SST->isCompressingStore());
7860}
7861
7862SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
7863 assert((OpNo == 1 || OpNo == 4) &&
7864 "Can widen only data or mask operand of mstore");
7865 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
7866 SDValue Mask = MST->getMask();
7867 EVT MaskVT = Mask.getValueType();
7868 SDValue StVal = MST->getValue();
7869 EVT VT = StVal.getValueType();
7870 SDLoc dl(N);
7871
7872 EVT WideVT, WideMaskVT;
7873 if (OpNo == 1) {
7874 // Widen the value.
7875 StVal = GetWidenedVector(Op: StVal);
7876
7877 WideVT = StVal.getValueType();
7878 WideMaskVT =
7879 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7880 EC: WideVT.getVectorElementCount());
7881 } else {
7882 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
7883
7884 EVT ValueVT = StVal.getValueType();
7885 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
7886 EC: WideMaskVT.getVectorElementCount());
7887 }
7888
7889 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7890 TLI.isTypeLegal(VT: WideMaskVT)) {
7891 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7892 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7893 EC: VT.getVectorElementCount());
7894 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
7895 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
7896 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
7897 }
7898
7899 if (OpNo == 1) {
7900 // The mask should be widened as well.
7901 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7902 } else {
7903 // Widen the mask.
7904 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7905
7906 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
7907 }
7908
7909 assert(Mask.getValueType().getVectorElementCount() ==
7910 StVal.getValueType().getVectorElementCount() &&
7911 "Mask and data vectors should have the same number of elements");
7912 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
7913 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
7914 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
7915 IsTruncating: false, IsCompressing: MST->isCompressingStore());
7916}
7917
7918SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
7919 assert(OpNo == 4 && "Can widen only the index of mgather");
7920 auto *MG = cast<MaskedGatherSDNode>(Val: N);
7921 SDValue DataOp = MG->getPassThru();
7922 SDValue Mask = MG->getMask();
7923 SDValue Scale = MG->getScale();
7924
7925 // Just widen the index. It's allowed to have extra elements.
7926 SDValue Index = GetWidenedVector(Op: MG->getIndex());
7927
7928 SDLoc dl(N);
7929 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
7930 Scale};
7931 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
7932 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
7933 ExtTy: MG->getExtensionType());
7934 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7935 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
7936 return SDValue();
7937}
7938
7939SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
7940 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
7941 SDValue DataOp = MSC->getValue();
7942 SDValue Mask = MSC->getMask();
7943 SDValue Index = MSC->getIndex();
7944 SDValue Scale = MSC->getScale();
7945 EVT WideMemVT = MSC->getMemoryVT();
7946
7947 if (OpNo == 1) {
7948 DataOp = GetWidenedVector(Op: DataOp);
7949 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
7950
7951 // Widen index.
7952 EVT IndexVT = Index.getValueType();
7953 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7954 VT: IndexVT.getVectorElementType(), NumElements: NumElts);
7955 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7956
7957 // The mask should be widened as well.
7958 EVT MaskVT = Mask.getValueType();
7959 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7960 VT: MaskVT.getVectorElementType(), NumElements: NumElts);
7961 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7962
7963 // Widen the MemoryType
7964 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7965 VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
7966 } else if (OpNo == 4) {
7967 // Just widen the index. It's allowed to have extra elements.
7968 Index = GetWidenedVector(Op: Index);
7969 } else
7970 llvm_unreachable("Can't widen this operand of mscatter");
7971
7972 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
7973 Scale};
7974 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
7975 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
7976 IsTruncating: MSC->isTruncatingStore());
7977}
7978
7979SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
7980 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
7981 SDValue DataOp = VPSC->getValue();
7982 SDValue Mask = VPSC->getMask();
7983 SDValue Index = VPSC->getIndex();
7984 SDValue Scale = VPSC->getScale();
7985 EVT WideMemVT = VPSC->getMemoryVT();
7986
7987 if (OpNo == 1) {
7988 DataOp = GetWidenedVector(Op: DataOp);
7989 Index = GetWidenedVector(Op: Index);
7990 const auto WideEC = DataOp.getValueType().getVectorElementCount();
7991 Mask = GetWidenedMask(Mask, EC: WideEC);
7992 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7993 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
7994 } else if (OpNo == 3) {
7995 // Just widen the index. It's allowed to have extra elements.
7996 Index = GetWidenedVector(Op: Index);
7997 } else
7998 llvm_unreachable("Can't widen this operand of VP_SCATTER");
7999
8000 SDValue Ops[] = {
8001 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8002 VPSC->getVectorLength()};
8003 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8004 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8005}
8006
8007SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8008 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8009 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8010 SDLoc dl(N);
8011 EVT VT = N->getValueType(ResNo: 0);
8012
8013 // WARNING: In this code we widen the compare instruction with garbage.
8014 // This garbage may contain denormal floats which may be slow. Is this a real
8015 // concern ? Should we zero the unused lanes if this is a float compare ?
8016
8017 // Get a new SETCC node to compare the newly widened operands.
8018 // Only some of the compared elements are legal.
8019 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8020 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8021 if (VT.getScalarType() == MVT::i1)
8022 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8023 EC: SVT.getVectorElementCount());
8024
8025 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8026 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8027
8028 // Extract the needed results from the result vector.
8029 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8030 VT: SVT.getVectorElementType(),
8031 EC: VT.getVectorElementCount());
8032 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8033
8034 EVT OpVT = N->getOperand(Num: 0).getValueType();
8035 ISD::NodeType ExtendCode =
8036 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8037 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8038}
8039
8040SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8041 SDValue Chain = N->getOperand(Num: 0);
8042 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8043 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8044 SDValue CC = N->getOperand(Num: 3);
8045 SDLoc dl(N);
8046
8047 EVT VT = N->getValueType(ResNo: 0);
8048 EVT EltVT = VT.getVectorElementType();
8049 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8050 unsigned NumElts = VT.getVectorNumElements();
8051
8052 // Unroll into a build vector.
8053 SmallVector<SDValue, 8> Scalars(NumElts);
8054 SmallVector<SDValue, 8> Chains(NumElts);
8055
8056 for (unsigned i = 0; i != NumElts; ++i) {
8057 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8058 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8059
8060 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8061 Ops: {Chain, LHSElem, RHSElem, CC});
8062 Chains[i] = Scalars[i].getValue(R: 1);
8063 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8064 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8065 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8066 }
8067
8068 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8069 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8070
8071 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8072}
8073
8074static unsigned getExtendForIntVecReduction(unsigned Opc) {
8075 switch (Opc) {
8076 default:
8077 llvm_unreachable("Expected integer vector reduction");
8078 case ISD::VECREDUCE_ADD:
8079 case ISD::VECREDUCE_MUL:
8080 case ISD::VECREDUCE_AND:
8081 case ISD::VECREDUCE_OR:
8082 case ISD::VECREDUCE_XOR:
8083 return ISD::ANY_EXTEND;
8084 case ISD::VECREDUCE_SMAX:
8085 case ISD::VECREDUCE_SMIN:
8086 return ISD::SIGN_EXTEND;
8087 case ISD::VECREDUCE_UMAX:
8088 case ISD::VECREDUCE_UMIN:
8089 return ISD::ZERO_EXTEND;
8090 }
8091}
8092
8093SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8094 SDLoc dl(N);
8095 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8096 EVT VT = N->getValueType(ResNo: 0);
8097 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8098 EVT WideVT = Op.getValueType();
8099 EVT ElemVT = OrigVT.getVectorElementType();
8100 SDNodeFlags Flags = N->getFlags();
8101
8102 unsigned Opc = N->getOpcode();
8103 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8104 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8105 assert(NeutralElem && "Neutral element must exist");
8106
8107 // Pad the vector with the neutral element.
8108 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8109 unsigned WideElts = WideVT.getVectorMinNumElements();
8110
8111 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8112 // needing to pad the source vector, because the inactive lanes can simply be
8113 // disabled and not contribute to the result.
8114 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8115 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8116 SDValue Start = NeutralElem;
8117 if (VT.isInteger())
8118 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8119 assert(Start.getValueType() == VT);
8120 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8121 EC: WideVT.getVectorElementCount());
8122 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8123 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8124 EC: OrigVT.getVectorElementCount());
8125 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8126 }
8127
8128 if (WideVT.isScalableVector()) {
8129 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8130 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8131 EC: ElementCount::getScalable(MinVal: GCD));
8132 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8133 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8134 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8135 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8136 }
8137
8138 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8139 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8140
8141 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8142}
8143
8144SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8145 SDLoc dl(N);
8146 SDValue AccOp = N->getOperand(Num: 0);
8147 SDValue VecOp = N->getOperand(Num: 1);
8148 SDValue Op = GetWidenedVector(Op: VecOp);
8149
8150 EVT VT = N->getValueType(ResNo: 0);
8151 EVT OrigVT = VecOp.getValueType();
8152 EVT WideVT = Op.getValueType();
8153 EVT ElemVT = OrigVT.getVectorElementType();
8154 SDNodeFlags Flags = N->getFlags();
8155
8156 unsigned Opc = N->getOpcode();
8157 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8158 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8159
8160 // Pad the vector with the neutral element.
8161 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8162 unsigned WideElts = WideVT.getVectorMinNumElements();
8163
8164 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8165 // needing to pad the source vector, because the inactive lanes can simply be
8166 // disabled and not contribute to the result.
8167 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8168 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8169 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8170 EC: WideVT.getVectorElementCount());
8171 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8172 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8173 EC: OrigVT.getVectorElementCount());
8174 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8175 }
8176
8177 if (WideVT.isScalableVector()) {
8178 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8179 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8180 EC: ElementCount::getScalable(MinVal: GCD));
8181 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8182 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8183 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8184 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8185 }
8186
8187 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8188 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8189
8190 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8191}
8192
8193SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8194 assert(N->isVPOpcode() && "Expected VP opcode");
8195
8196 SDLoc dl(N);
8197 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8198 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8199 EC: Op.getValueType().getVectorElementCount());
8200
8201 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8202 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8203 Flags: N->getFlags());
8204}
8205
8206SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8207 // This only gets called in the case that the left and right inputs and
8208 // result are of a legal odd vector type, and the condition is illegal i1 of
8209 // the same odd width that needs widening.
8210 EVT VT = N->getValueType(ResNo: 0);
8211 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8212
8213 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8214 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8215 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8216 SDLoc DL(N);
8217
8218 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8219 N2: LeftIn, N3: RightIn);
8220 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8221}
8222
8223SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8224 SDLoc DL(N);
8225 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8226 EVT SrcVT = Source.getValueType();
8227 SDValue Mask =
8228 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8229
8230 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8231 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8232}
8233
8234SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8235 SDLoc DL(N);
8236 SDValue Mask = N->getOperand(Num: 0);
8237 EVT OrigMaskVT = Mask.getValueType();
8238 SDValue WideMask = GetWidenedVector(Op: Mask);
8239 EVT WideMaskVT = WideMask.getValueType();
8240
8241 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8242 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8243 unsigned WideElts = WideMaskVT.getVectorNumElements();
8244 if (OrigElts != WideElts) {
8245 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8246 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8247 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8248 }
8249
8250 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8251 Operand: WideMask);
8252}
8253
8254//===----------------------------------------------------------------------===//
8255// Vector Widening Utilities
8256//===----------------------------------------------------------------------===//
8257
8258// Utility function to find the type to chop up a widen vector for load/store
8259// TLI: Target lowering used to determine legal types.
8260// Width: Width left need to load/store.
8261// WidenVT: The widen vector type to load to/store from
8262// Align: If 0, don't allow use of a wider type
8263// WidenEx: If Align is not 0, the amount additional we can load/store from.
8264
8265static std::optional<EVT> findMemType(SelectionDAG &DAG,
8266 const TargetLowering &TLI, unsigned Width,
8267 EVT WidenVT, unsigned Align = 0,
8268 unsigned WidenEx = 0) {
8269 EVT WidenEltVT = WidenVT.getVectorElementType();
8270 const bool Scalable = WidenVT.isScalableVector();
8271 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8272 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8273 unsigned AlignInBits = Align*8;
8274
8275 EVT RetVT = WidenEltVT;
8276 // Don't bother looking for an integer type if the vector is scalable, skip
8277 // to vector types.
8278 if (!Scalable) {
8279 // If we have one element to load/store, return it.
8280 if (Width == WidenEltWidth)
8281 return RetVT;
8282
8283 // See if there is larger legal integer than the element type to load/store.
8284 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8285 unsigned MemVTWidth = MemVT.getSizeInBits();
8286 if (MemVT.getSizeInBits() <= WidenEltWidth)
8287 break;
8288 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8289 if ((Action == TargetLowering::TypeLegal ||
8290 Action == TargetLowering::TypePromoteInteger) &&
8291 (WidenWidth % MemVTWidth) == 0 &&
8292 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8293 (MemVTWidth <= Width ||
8294 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8295 if (MemVTWidth == WidenWidth)
8296 return MemVT;
8297 RetVT = MemVT;
8298 break;
8299 }
8300 }
8301 }
8302
8303 // See if there is a larger vector type to load/store that has the same vector
8304 // element type and is evenly divisible with the WidenVT.
8305 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8306 // Skip vector MVTs which don't match the scalable property of WidenVT.
8307 if (Scalable != MemVT.isScalableVector())
8308 continue;
8309 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8310 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8311 if ((Action == TargetLowering::TypeLegal ||
8312 Action == TargetLowering::TypePromoteInteger) &&
8313 WidenEltVT == MemVT.getVectorElementType() &&
8314 (WidenWidth % MemVTWidth) == 0 &&
8315 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8316 (MemVTWidth <= Width ||
8317 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8318 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8319 return MemVT;
8320 }
8321 }
8322
8323 // Using element-wise loads and stores for widening operations is not
8324 // supported for scalable vectors
8325 if (Scalable)
8326 return std::nullopt;
8327
8328 return RetVT;
8329}
8330
8331// Builds a vector type from scalar loads
8332// VecTy: Resulting Vector type
8333// LDOps: Load operators to build a vector type
8334// [Start,End) the list of loads to use.
8335static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8336 SmallVectorImpl<SDValue> &LdOps,
8337 unsigned Start, unsigned End) {
8338 SDLoc dl(LdOps[Start]);
8339 EVT LdTy = LdOps[Start].getValueType();
8340 unsigned Width = VecTy.getSizeInBits();
8341 unsigned NumElts = Width / LdTy.getSizeInBits();
8342 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8343
8344 unsigned Idx = 1;
8345 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8346
8347 for (unsigned i = Start + 1; i != End; ++i) {
8348 EVT NewLdTy = LdOps[i].getValueType();
8349 if (NewLdTy != LdTy) {
8350 NumElts = Width / NewLdTy.getSizeInBits();
8351 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8352 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8353 // Readjust position and vector position based on new load type.
8354 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8355 LdTy = NewLdTy;
8356 }
8357 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8358 }
8359 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8360}
8361
8362SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8363 LoadSDNode *LD) {
8364 // The strategy assumes that we can efficiently load power-of-two widths.
8365 // The routine chops the vector into the largest vector loads with the same
8366 // element type or scalar loads and then recombines it to the widen vector
8367 // type.
8368 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8369 EVT LdVT = LD->getMemoryVT();
8370 SDLoc dl(LD);
8371 assert(LdVT.isVector() && WidenVT.isVector());
8372 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8373 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8374
8375 // Load information
8376 SDValue Chain = LD->getChain();
8377 SDValue BasePtr = LD->getBasePtr();
8378 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8379 AAMDNodes AAInfo = LD->getAAInfo();
8380
8381 TypeSize LdWidth = LdVT.getSizeInBits();
8382 TypeSize WidenWidth = WidenVT.getSizeInBits();
8383 TypeSize WidthDiff = WidenWidth - LdWidth;
8384 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8385 // and if the original load is simple.
8386 unsigned LdAlign =
8387 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8388
8389 // Find the vector type that can load from.
8390 std::optional<EVT> FirstVT =
8391 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8392 WidenEx: WidthDiff.getKnownMinValue());
8393
8394 if (!FirstVT)
8395 return SDValue();
8396
8397 SmallVector<EVT, 8> MemVTs;
8398 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8399
8400 // Unless we're able to load in one instruction we must work out how to load
8401 // the remainder.
8402 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8403 std::optional<EVT> NewVT = FirstVT;
8404 TypeSize RemainingWidth = LdWidth;
8405 TypeSize NewVTWidth = FirstVTWidth;
8406 do {
8407 RemainingWidth -= NewVTWidth;
8408 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8409 // The current type we are using is too large. Find a better size.
8410 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8411 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8412 if (!NewVT)
8413 return SDValue();
8414 NewVTWidth = NewVT->getSizeInBits();
8415 }
8416 MemVTs.push_back(Elt: *NewVT);
8417 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8418 }
8419
8420 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8421 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8422 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8423
8424 // Check if we can load the element with one instruction.
8425 if (MemVTs.empty())
8426 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8427 DAG);
8428
8429 // Load vector by using multiple loads from largest vector to scalar.
8430 SmallVector<SDValue, 16> LdOps;
8431 LdOps.push_back(Elt: LdOp);
8432
8433 uint64_t ScaledOffset = 0;
8434 MachinePointerInfo MPI = LD->getPointerInfo();
8435
8436 // First incremement past the first load.
8437 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8438 ScaledOffset: &ScaledOffset);
8439
8440 for (EVT MemVT : MemVTs) {
8441 Align NewAlign = ScaledOffset == 0
8442 ? LD->getBaseAlign()
8443 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8444 SDValue L =
8445 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8446
8447 LdOps.push_back(Elt: L);
8448 LdChain.push_back(Elt: L.getValue(R: 1));
8449 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8450 }
8451
8452 // Build the vector from the load operations.
8453 unsigned End = LdOps.size();
8454 if (!LdOps[0].getValueType().isVector())
8455 // All the loads are scalar loads.
8456 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8457
8458 // If the load contains vectors, build the vector using concat vector.
8459 // All of the vectors used to load are power-of-2, and the scalar loads can be
8460 // combined to make a power-of-2 vector.
8461 SmallVector<SDValue, 16> ConcatOps(End);
8462 int i = End - 1;
8463 int Idx = End;
8464 EVT LdTy = LdOps[i].getValueType();
8465 // First, combine the scalar loads to a vector.
8466 if (!LdTy.isVector()) {
8467 for (--i; i >= 0; --i) {
8468 LdTy = LdOps[i].getValueType();
8469 if (LdTy.isVector())
8470 break;
8471 }
8472 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8473 }
8474
8475 ConcatOps[--Idx] = LdOps[i];
8476 for (--i; i >= 0; --i) {
8477 EVT NewLdTy = LdOps[i].getValueType();
8478 if (NewLdTy != LdTy) {
8479 // Create a larger vector.
8480 TypeSize LdTySize = LdTy.getSizeInBits();
8481 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8482 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8483 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8484 unsigned NumOps =
8485 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8486 SmallVector<SDValue, 16> WidenOps(NumOps);
8487 unsigned j = 0;
8488 for (; j != End-Idx; ++j)
8489 WidenOps[j] = ConcatOps[Idx+j];
8490 for (; j != NumOps; ++j)
8491 WidenOps[j] = DAG.getPOISON(VT: LdTy);
8492
8493 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
8494 Ops: WidenOps);
8495 Idx = End - 1;
8496 LdTy = NewLdTy;
8497 }
8498 ConcatOps[--Idx] = LdOps[i];
8499 }
8500
8501 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
8502 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
8503 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
8504
8505 // We need to fill the rest with undefs to build the vector.
8506 unsigned NumOps =
8507 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
8508 SmallVector<SDValue, 16> WidenOps(NumOps);
8509 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
8510 {
8511 unsigned i = 0;
8512 for (; i != End-Idx; ++i)
8513 WidenOps[i] = ConcatOps[Idx+i];
8514 for (; i != NumOps; ++i)
8515 WidenOps[i] = UndefVal;
8516 }
8517 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
8518}
8519
8520SDValue
8521DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
8522 LoadSDNode *LD,
8523 ISD::LoadExtType ExtType) {
8524 // For extension loads, it may not be more efficient to chop up the vector
8525 // and then extend it. Instead, we unroll the load and build a new vector.
8526 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8527 EVT LdVT = LD->getMemoryVT();
8528 SDLoc dl(LD);
8529 assert(LdVT.isVector() && WidenVT.isVector());
8530 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8531
8532 // Load information
8533 SDValue Chain = LD->getChain();
8534 SDValue BasePtr = LD->getBasePtr();
8535 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8536 AAMDNodes AAInfo = LD->getAAInfo();
8537
8538 if (LdVT.isScalableVector())
8539 return SDValue();
8540
8541 EVT EltVT = WidenVT.getVectorElementType();
8542 EVT LdEltVT = LdVT.getVectorElementType();
8543 unsigned NumElts = LdVT.getVectorNumElements();
8544
8545 // Load each element and widen.
8546 unsigned WidenNumElts = WidenVT.getVectorNumElements();
8547 SmallVector<SDValue, 16> Ops(WidenNumElts);
8548 unsigned Increment = LdEltVT.getSizeInBits() / 8;
8549 Ops[0] =
8550 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8551 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8552 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
8553 unsigned i = 0, Offset = Increment;
8554 for (i=1; i < NumElts; ++i, Offset += Increment) {
8555 SDValue NewBasePtr =
8556 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
8557 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
8558 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
8559 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8560 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
8561 }
8562
8563 // Fill the rest with undefs.
8564 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8565 for (; i != WidenNumElts; ++i)
8566 Ops[i] = UndefVal;
8567
8568 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
8569}
8570
8571bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
8572 StoreSDNode *ST) {
8573 // The strategy assumes that we can efficiently store power-of-two widths.
8574 // The routine chops the vector into the largest vector stores with the same
8575 // element type or scalar stores.
8576 SDValue Chain = ST->getChain();
8577 SDValue BasePtr = ST->getBasePtr();
8578 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
8579 AAMDNodes AAInfo = ST->getAAInfo();
8580 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
8581 SDLoc dl(ST);
8582
8583 EVT StVT = ST->getMemoryVT();
8584 TypeSize StWidth = StVT.getSizeInBits();
8585 EVT ValVT = ValOp.getValueType();
8586 TypeSize ValWidth = ValVT.getSizeInBits();
8587 EVT ValEltVT = ValVT.getVectorElementType();
8588 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
8589 assert(StVT.getVectorElementType() == ValEltVT);
8590 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
8591 "Mismatch between store and value types");
8592
8593 int Idx = 0; // current index to store
8594
8595 MachinePointerInfo MPI = ST->getPointerInfo();
8596 uint64_t ScaledOffset = 0;
8597
8598 // A breakdown of how to widen this vector store. Each element of the vector
8599 // is a memory VT combined with the number of times it is to be stored to,
8600 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
8601 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
8602
8603 while (StWidth.isNonZero()) {
8604 // Find the largest vector type we can store with.
8605 std::optional<EVT> NewVT =
8606 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
8607 if (!NewVT)
8608 return false;
8609 MemVTs.push_back(Elt: {*NewVT, 0});
8610 TypeSize NewVTWidth = NewVT->getSizeInBits();
8611
8612 do {
8613 StWidth -= NewVTWidth;
8614 MemVTs.back().second++;
8615 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
8616 }
8617
8618 for (const auto &Pair : MemVTs) {
8619 EVT NewVT = Pair.first;
8620 unsigned Count = Pair.second;
8621 TypeSize NewVTWidth = NewVT.getSizeInBits();
8622
8623 if (NewVT.isVector()) {
8624 unsigned NumVTElts = NewVT.getVectorMinNumElements();
8625 do {
8626 Align NewAlign = ScaledOffset == 0
8627 ? ST->getBaseAlign()
8628 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
8629 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
8630 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
8631 MMOFlags, AAInfo);
8632 StChain.push_back(Elt: PartStore);
8633
8634 Idx += NumVTElts;
8635 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
8636 ScaledOffset: &ScaledOffset);
8637 } while (--Count);
8638 } else {
8639 // Cast the vector to the scalar type we can store.
8640 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
8641 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
8642 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
8643 // Readjust index position based on new vector type.
8644 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
8645 do {
8646 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
8647 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
8648 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
8649 StChain.push_back(Elt: PartStore);
8650
8651 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
8652 } while (--Count);
8653 // Restore index back to be relative to the original widen element type.
8654 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
8655 }
8656 }
8657
8658 return true;
8659}
8660
8661/// Modifies a vector input (widen or narrows) to a vector of NVT. The
8662/// input vector must have the same element type as NVT.
8663/// FillWithZeroes specifies that the vector should be widened with zeroes.
8664SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
8665 bool FillWithZeroes) {
8666 // Note that InOp might have been widened so it might already have
8667 // the right width or it might need be narrowed.
8668 EVT InVT = InOp.getValueType();
8669 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
8670 "input and widen element type must match");
8671 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
8672 "cannot modify scalable vectors in this way");
8673 SDLoc dl(InOp);
8674
8675 // Check if InOp already has the right width.
8676 if (InVT == NVT)
8677 return InOp;
8678
8679 ElementCount InEC = InVT.getVectorElementCount();
8680 ElementCount WidenEC = NVT.getVectorElementCount();
8681 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
8682 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
8683 SmallVector<SDValue, 16> Ops(NumConcat);
8684 SDValue FillVal =
8685 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
8686 Ops[0] = InOp;
8687 for (unsigned i = 1; i != NumConcat; ++i)
8688 Ops[i] = FillVal;
8689
8690 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
8691 }
8692
8693 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
8694 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
8695
8696 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
8697 "Scalable vectors should have been handled already.");
8698
8699 unsigned InNumElts = InEC.getFixedValue();
8700 unsigned WidenNumElts = WidenEC.getFixedValue();
8701
8702 // Fall back to extract and build (+ mask, if padding with zeros).
8703 SmallVector<SDValue, 16> Ops(WidenNumElts);
8704 EVT EltVT = NVT.getVectorElementType();
8705 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
8706 unsigned Idx;
8707 for (Idx = 0; Idx < MinNumElts; ++Idx)
8708 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
8709
8710 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8711 for (; Idx < WidenNumElts; ++Idx)
8712 Ops[Idx] = UndefVal;
8713
8714 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
8715 if (!FillWithZeroes)
8716 return Widened;
8717
8718 assert(NVT.isInteger() &&
8719 "We expect to never want to FillWithZeroes for non-integral types.");
8720
8721 SmallVector<SDValue, 16> MaskOps;
8722 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
8723 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
8724
8725 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
8726 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
8727}
8728