1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 switch (N->getOpcode()) {
47 default:
48#ifndef NDEBUG
49 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
50 N->dump(&DAG);
51 dbgs() << "\n";
52#endif
53 report_fatal_error(reason: "Do not know how to scalarize the result of this "
54 "operator!\n");
55
56 case ISD::LOOP_DEPENDENCE_WAR_MASK:
57 case ISD::LOOP_DEPENDENCE_RAW_MASK:
58 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
59 break;
60 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
61 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
62 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
63 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
64 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
65 case ISD::AssertZext:
66 case ISD::AssertSext:
67 case ISD::FPOWI:
68 case ISD::AssertNoFPClass:
69 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
70 break;
71 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
72 case ISD::ATOMIC_LOAD:
73 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
74 break;
75 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
76 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
77 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
78 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
79 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
80 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
81 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
82 case ISD::POISON:
83 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
84 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
85 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
86 case ISD::ANY_EXTEND_VECTOR_INREG:
87 case ISD::SIGN_EXTEND_VECTOR_INREG:
88 case ISD::ZERO_EXTEND_VECTOR_INREG:
89 R = ScalarizeVecRes_VecInregOp(N);
90 break;
91 case ISD::ABS:
92 case ISD::ANY_EXTEND:
93 case ISD::BITREVERSE:
94 case ISD::BSWAP:
95 case ISD::CTLZ:
96 case ISD::CTLZ_ZERO_UNDEF:
97 case ISD::CTPOP:
98 case ISD::CTTZ:
99 case ISD::CTTZ_ZERO_UNDEF:
100 case ISD::FABS:
101 case ISD::FACOS:
102 case ISD::FASIN:
103 case ISD::FATAN:
104 case ISD::FCEIL:
105 case ISD::FCOS:
106 case ISD::FCOSH:
107 case ISD::FEXP:
108 case ISD::FEXP2:
109 case ISD::FEXP10:
110 case ISD::FFLOOR:
111 case ISD::FLOG:
112 case ISD::FLOG10:
113 case ISD::FLOG2:
114 case ISD::FNEARBYINT:
115 case ISD::FNEG:
116 case ISD::FREEZE:
117 case ISD::ARITH_FENCE:
118 case ISD::FP_EXTEND:
119 case ISD::FP_TO_SINT:
120 case ISD::FP_TO_UINT:
121 case ISD::FRINT:
122 case ISD::LRINT:
123 case ISD::LLRINT:
124 case ISD::FROUND:
125 case ISD::FROUNDEVEN:
126 case ISD::LROUND:
127 case ISD::LLROUND:
128 case ISD::FSIN:
129 case ISD::FSINH:
130 case ISD::FSQRT:
131 case ISD::FTAN:
132 case ISD::FTANH:
133 case ISD::FTRUNC:
134 case ISD::SIGN_EXTEND:
135 case ISD::SINT_TO_FP:
136 case ISD::TRUNCATE:
137 case ISD::UINT_TO_FP:
138 case ISD::ZERO_EXTEND:
139 case ISD::FCANONICALIZE:
140 R = ScalarizeVecRes_UnaryOp(N);
141 break;
142 case ISD::ADDRSPACECAST:
143 R = ScalarizeVecRes_ADDRSPACECAST(N);
144 break;
145 case ISD::FMODF:
146 case ISD::FFREXP:
147 case ISD::FSINCOS:
148 case ISD::FSINCOSPI:
149 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
150 break;
151 case ISD::ADD:
152 case ISD::AND:
153 case ISD::AVGCEILS:
154 case ISD::AVGCEILU:
155 case ISD::AVGFLOORS:
156 case ISD::AVGFLOORU:
157 case ISD::FADD:
158 case ISD::FCOPYSIGN:
159 case ISD::FDIV:
160 case ISD::FMUL:
161 case ISD::FMINNUM:
162 case ISD::FMAXNUM:
163 case ISD::FMINNUM_IEEE:
164 case ISD::FMAXNUM_IEEE:
165 case ISD::FMINIMUM:
166 case ISD::FMAXIMUM:
167 case ISD::FMINIMUMNUM:
168 case ISD::FMAXIMUMNUM:
169 case ISD::FLDEXP:
170 case ISD::ABDS:
171 case ISD::ABDU:
172 case ISD::SMIN:
173 case ISD::SMAX:
174 case ISD::UMIN:
175 case ISD::UMAX:
176
177 case ISD::SADDSAT:
178 case ISD::UADDSAT:
179 case ISD::SSUBSAT:
180 case ISD::USUBSAT:
181 case ISD::SSHLSAT:
182 case ISD::USHLSAT:
183
184 case ISD::FPOW:
185 case ISD::FATAN2:
186 case ISD::FREM:
187 case ISD::FSUB:
188 case ISD::MUL:
189 case ISD::MULHS:
190 case ISD::MULHU:
191 case ISD::OR:
192 case ISD::SDIV:
193 case ISD::SREM:
194 case ISD::SUB:
195 case ISD::UDIV:
196 case ISD::UREM:
197 case ISD::XOR:
198 case ISD::SHL:
199 case ISD::SRA:
200 case ISD::SRL:
201 case ISD::ROTL:
202 case ISD::ROTR:
203 case ISD::CLMUL:
204 case ISD::CLMULR:
205 case ISD::CLMULH:
206 R = ScalarizeVecRes_BinOp(N);
207 break;
208
209 case ISD::SCMP:
210 case ISD::UCMP:
211 R = ScalarizeVecRes_CMP(N);
212 break;
213
214 case ISD::FMA:
215 case ISD::FSHL:
216 case ISD::FSHR:
217 R = ScalarizeVecRes_TernaryOp(N);
218 break;
219
220#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
221 case ISD::STRICT_##DAGN:
222#include "llvm/IR/ConstrainedOps.def"
223 R = ScalarizeVecRes_StrictFPOp(N);
224 break;
225
226 case ISD::FP_TO_UINT_SAT:
227 case ISD::FP_TO_SINT_SAT:
228 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
229 break;
230
231 case ISD::UADDO:
232 case ISD::SADDO:
233 case ISD::USUBO:
234 case ISD::SSUBO:
235 case ISD::UMULO:
236 case ISD::SMULO:
237 R = ScalarizeVecRes_OverflowOp(N, ResNo);
238 break;
239 case ISD::SMULFIX:
240 case ISD::SMULFIXSAT:
241 case ISD::UMULFIX:
242 case ISD::UMULFIXSAT:
243 case ISD::SDIVFIX:
244 case ISD::SDIVFIXSAT:
245 case ISD::UDIVFIX:
246 case ISD::UDIVFIXSAT:
247 R = ScalarizeVecRes_FIX(N);
248 break;
249 }
250
251 // If R is null, the sub-method took care of registering the result.
252 if (R.getNode())
253 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
254}
255
256SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
257 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
258 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
259 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
260 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
261}
262
263SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
264 SDLoc DL(N);
265
266 SDValue LHS = N->getOperand(Num: 0);
267 SDValue RHS = N->getOperand(Num: 1);
268 if (getTypeAction(VT: LHS.getValueType()) ==
269 TargetLowering::TypeScalarizeVector) {
270 LHS = GetScalarizedVector(Op: LHS);
271 RHS = GetScalarizedVector(Op: RHS);
272 } else {
273 EVT VT = LHS.getValueType().getVectorElementType();
274 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
275 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
276 }
277
278 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
279 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
280}
281
282SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
283 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
284 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
285 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
286 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
287 N3: Op2, Flags: N->getFlags());
288}
289
290SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
291 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
292 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
293 SDValue Op2 = N->getOperand(Num: 2);
294 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
295 N3: Op2, Flags: N->getFlags());
296}
297
298SDValue
299DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
300 unsigned ResNo) {
301 assert(N->getValueType(0).getVectorNumElements() == 1 &&
302 "Unexpected vector type!");
303 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
304
305 EVT VT0 = N->getValueType(ResNo: 0);
306 EVT VT1 = N->getValueType(ResNo: 1);
307 SDLoc dl(N);
308
309 SDNode *ScalarNode =
310 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
311 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
312 .getNode();
313
314 // Replace the other vector result not being explicitly scalarized here.
315 unsigned OtherNo = 1 - ResNo;
316 EVT OtherVT = N->getValueType(ResNo: OtherNo);
317 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
318 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
319 } else {
320 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
321 Operand: SDValue(ScalarNode, OtherNo));
322 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
323 }
324
325 return SDValue(ScalarNode, ResNo);
326}
327
328SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
329 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
330 unsigned NumOpers = N->getNumOperands();
331 SDValue Chain = N->getOperand(Num: 0);
332 EVT ValueVTs[] = {VT, MVT::Other};
333 SDLoc dl(N);
334
335 SmallVector<SDValue, 4> Opers(NumOpers);
336
337 // The Chain is the first operand.
338 Opers[0] = Chain;
339
340 // Now process the remaining operands.
341 for (unsigned i = 1; i < NumOpers; ++i) {
342 SDValue Oper = N->getOperand(Num: i);
343 EVT OperVT = Oper.getValueType();
344
345 if (OperVT.isVector()) {
346 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
347 Oper = GetScalarizedVector(Op: Oper);
348 else
349 Oper =
350 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
351 }
352
353 Opers[i] = Oper;
354 }
355
356 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
357 Ops: Opers, Flags: N->getFlags());
358
359 // Legalize the chain result - switch anything that used the old chain to
360 // use the new one.
361 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
362 return Result;
363}
364
365SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
366 unsigned ResNo) {
367 SDLoc DL(N);
368 EVT ResVT = N->getValueType(ResNo: 0);
369 EVT OvVT = N->getValueType(ResNo: 1);
370
371 SDValue ScalarLHS, ScalarRHS;
372 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
373 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
374 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
375 } else {
376 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
377 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
378 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
379 ScalarLHS = ElemsLHS[0];
380 ScalarRHS = ElemsRHS[0];
381 }
382
383 SDVTList ScalarVTs = DAG.getVTList(
384 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
385 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
386 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
387 .getNode();
388
389 // Replace the other vector result not being explicitly scalarized here.
390 unsigned OtherNo = 1 - ResNo;
391 EVT OtherVT = N->getValueType(ResNo: OtherNo);
392 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
393 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
394 } else {
395 SDValue OtherVal = DAG.getNode(
396 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
397 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
398 }
399
400 return SDValue(ScalarNode, ResNo);
401}
402
403SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
404 unsigned ResNo) {
405 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
406 return GetScalarizedVector(Op);
407}
408
409SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
410 SDLoc DL(N);
411 SDValue SourceValue = N->getOperand(Num: 0);
412 SDValue SinkValue = N->getOperand(Num: 1);
413 SDValue EltSizeInBytes = N->getOperand(Num: 2);
414 SDValue LaneOffset = N->getOperand(Num: 3);
415
416 EVT PtrVT = SourceValue->getValueType(ResNo: 0);
417 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
418
419 // Take the difference between the pointers and divided by the element size,
420 // to see how many lanes separate them.
421 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: SinkValue, N2: SourceValue);
422 if (IsReadAfterWrite)
423 Diff = DAG.getNode(Opcode: ISD::ABS, DL, VT: PtrVT, Operand: Diff);
424 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: PtrVT, N1: Diff, N2: EltSizeInBytes);
425
426 // The pointers do not alias if:
427 // * Diff <= 0 || LaneOffset < Diff (WAR_MASK)
428 // * Diff == 0 || LaneOffset < abs(Diff) (RAW_MASK)
429 // Note: If LaneOffset is zero, both cases will fold to "true".
430 EVT CmpVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
431 VT: Diff.getValueType());
432 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
433 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Diff, RHS: Zero,
434 Cond: IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
435 return DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Cmp,
436 N2: DAG.getSetCC(DL, VT: CmpVT, LHS: LaneOffset, RHS: Diff, Cond: ISD::SETULT));
437}
438
439SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
440 SDValue Op = N->getOperand(Num: 0);
441 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
442 Op = GetScalarizedVector(Op);
443 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
444 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
445 VT: NewVT, Operand: Op);
446}
447
448SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
449 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
450 SDValue InOp = N->getOperand(Num: 0);
451 // The BUILD_VECTOR operands may be of wider element types and
452 // we may need to truncate them back to the requested return type.
453 if (EltVT.isInteger())
454 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
455 return InOp;
456}
457
458SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
459 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
460 VT: N->getValueType(ResNo: 0).getVectorElementType(),
461 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
462}
463
464SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
465 SDLoc DL(N);
466 SDValue Op = N->getOperand(Num: 0);
467 EVT OpVT = Op.getValueType();
468 // The result needs scalarizing, but it's not a given that the source does.
469 // See similar logic in ScalarizeVecRes_UnaryOp.
470 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
471 Op = GetScalarizedVector(Op);
472 } else {
473 EVT VT = OpVT.getVectorElementType();
474 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
475 }
476 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
477 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
478 N2: N->getOperand(Num: 1));
479}
480
481SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
482 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
483 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
484 N2: N->getOperand(Num: 1));
485}
486
487SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
488 // The value to insert may have a wider type than the vector element type,
489 // so be sure to truncate it to the element type if necessary.
490 SDValue Op = N->getOperand(Num: 1);
491 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
492 if (Op.getValueType() != EltVT)
493 // FIXME: Can this happen for floating point types?
494 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
495 return Op;
496}
497
498SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
499 SDValue Result = DAG.getAtomicLoad(
500 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
501 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
502 MMO: N->getMemOperand());
503
504 // Legalize the chain result - switch anything that used the old chain to
505 // use the new one.
506 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
507 return Result;
508}
509
510SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
511 assert(N->isUnindexed() && "Indexed vector load?");
512
513 SDValue Result = DAG.getLoad(
514 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
515 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
516 Ptr: N->getBasePtr(), Offset: DAG.getUNDEF(VT: N->getBasePtr().getValueType()),
517 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
518 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
519
520 // Legalize the chain result - switch anything that used the old chain to
521 // use the new one.
522 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
523 return Result;
524}
525
526SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
527 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
528 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
529 SDValue Op = N->getOperand(Num: 0);
530 EVT OpVT = Op.getValueType();
531 SDLoc DL(N);
532 // The result needs scalarizing, but it's not a given that the source does.
533 // This is a workaround for targets where it's impossible to scalarize the
534 // result of a conversion, because the source type is legal.
535 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
536 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
537 // legal and was not scalarized.
538 // See the similar logic in ScalarizeVecRes_SETCC
539 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
540 Op = GetScalarizedVector(Op);
541 } else {
542 EVT VT = OpVT.getVectorElementType();
543 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
544 }
545 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
546}
547
548SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
549 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
550 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
551 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
552 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
553 N1: LHS, N2: DAG.getValueType(ExtVT));
554}
555
556SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
557 SDLoc DL(N);
558 SDValue Op = N->getOperand(Num: 0);
559
560 EVT OpVT = Op.getValueType();
561 EVT OpEltVT = OpVT.getVectorElementType();
562 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
563
564 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
565 Op = GetScalarizedVector(Op);
566 } else {
567 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
568 }
569
570 switch (N->getOpcode()) {
571 case ISD::ANY_EXTEND_VECTOR_INREG:
572 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
573 case ISD::SIGN_EXTEND_VECTOR_INREG:
574 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
575 case ISD::ZERO_EXTEND_VECTOR_INREG:
576 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
577 }
578
579 llvm_unreachable("Illegal extend_vector_inreg opcode");
580}
581
582SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
583 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
584 SDValue Op = N->getOperand(Num: 0);
585 EVT OpVT = Op.getValueType();
586 SDLoc DL(N);
587 // The result needs scalarizing, but it's not a given that the source does.
588 // This is a workaround for targets where it's impossible to scalarize the
589 // result of a conversion, because the source type is legal.
590 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
591 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
592 // legal and was not scalarized.
593 // See the similar logic in ScalarizeVecRes_SETCC
594 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
595 Op = GetScalarizedVector(Op);
596 } else {
597 EVT VT = OpVT.getVectorElementType();
598 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
599 }
600 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
601 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
602 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
603 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
604}
605
606SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
607 // If the operand is wider than the vector element type then it is implicitly
608 // truncated. Make that explicit here.
609 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
610 SDValue InOp = N->getOperand(Num: 0);
611 if (InOp.getValueType() != EltVT)
612 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
613 return InOp;
614}
615
616SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
617 SDValue Cond = N->getOperand(Num: 0);
618 EVT OpVT = Cond.getValueType();
619 SDLoc DL(N);
620 // The vselect result and true/value operands needs scalarizing, but it's
621 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
622 // See the similar logic in ScalarizeVecRes_SETCC
623 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
624 Cond = GetScalarizedVector(Op: Cond);
625 } else {
626 EVT VT = OpVT.getVectorElementType();
627 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
628 }
629
630 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
631 TargetLowering::BooleanContent ScalarBool =
632 TLI.getBooleanContents(isVec: false, isFloat: false);
633 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
634
635 // If integer and float booleans have different contents then we can't
636 // reliably optimize in all cases. There is a full explanation for this in
637 // DAGCombiner::visitSELECT() where the same issue affects folding
638 // (select C, 0, 1) to (xor C, 1).
639 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
640 TLI.getBooleanContents(isVec: false, isFloat: true)) {
641 // At least try the common case where the boolean is generated by a
642 // comparison.
643 if (Cond->getOpcode() == ISD::SETCC) {
644 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
645 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
646 VecBool = TLI.getBooleanContents(Type: OpVT);
647 } else
648 ScalarBool = TargetLowering::UndefinedBooleanContent;
649 }
650
651 EVT CondVT = Cond.getValueType();
652 if (ScalarBool != VecBool) {
653 switch (ScalarBool) {
654 case TargetLowering::UndefinedBooleanContent:
655 break;
656 case TargetLowering::ZeroOrOneBooleanContent:
657 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
658 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
659 // Vector read from all ones, scalar expects a single 1 so mask.
660 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
661 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
662 break;
663 case TargetLowering::ZeroOrNegativeOneBooleanContent:
664 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
665 VecBool == TargetLowering::ZeroOrOneBooleanContent);
666 // Vector reads from a one, scalar from all ones so sign extend.
667 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
668 N1: Cond, N2: DAG.getValueType(MVT::i1));
669 break;
670 }
671 }
672
673 // Truncate the condition if needed
674 auto BoolVT = getSetCCResultType(VT: CondVT);
675 if (BoolVT.bitsLT(VT: CondVT))
676 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
677
678 return DAG.getSelect(DL: SDLoc(N),
679 VT: LHS.getValueType(), Cond, LHS,
680 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
681}
682
683SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
684 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
685 return DAG.getSelect(DL: SDLoc(N),
686 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
687 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
688}
689
690SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
691 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
692 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
693 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
694 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
695 N5: N->getOperand(Num: 4));
696}
697
698SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
699 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
700}
701
702SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
703 // Figure out if the scalar is the LHS or RHS and return it.
704 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
705 if (Arg.isUndef())
706 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
707 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
708 return GetScalarizedVector(Op: N->getOperand(Num: Op));
709}
710
711SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
712 SDValue Src = N->getOperand(Num: 0);
713 EVT SrcVT = Src.getValueType();
714 SDLoc dl(N);
715
716 // Handle case where result is scalarized but operand is not
717 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
718 Src = GetScalarizedVector(Op: Src);
719 else
720 Src = DAG.getNode(
721 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
722 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
723
724 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
725 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
726}
727
728SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
729 assert(N->getValueType(0).isVector() &&
730 N->getOperand(0).getValueType().isVector() &&
731 "Operand types must be vectors");
732 SDValue LHS = N->getOperand(Num: 0);
733 SDValue RHS = N->getOperand(Num: 1);
734 EVT OpVT = LHS.getValueType();
735 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
736 SDLoc DL(N);
737
738 // The result needs scalarizing, but it's not a given that the source does.
739 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
740 LHS = GetScalarizedVector(Op: LHS);
741 RHS = GetScalarizedVector(Op: RHS);
742 } else {
743 EVT VT = OpVT.getVectorElementType();
744 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
745 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
746 }
747
748 // Turn it into a scalar SETCC.
749 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
750 N3: N->getOperand(Num: 2));
751 // Vectors may have a different boolean contents to scalars. Promote the
752 // value appropriately.
753 ISD::NodeType ExtendCode =
754 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
755 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
756}
757
758SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
759 SDLoc DL(N);
760 SDValue Arg = N->getOperand(Num: 0);
761 SDValue Test = N->getOperand(Num: 1);
762 EVT ArgVT = Arg.getValueType();
763 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
764
765 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
766 Arg = GetScalarizedVector(Op: Arg);
767 } else {
768 EVT VT = ArgVT.getVectorElementType();
769 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
770 }
771
772 SDValue Res =
773 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
774 // Vectors may have a different boolean contents to scalars. Promote the
775 // value appropriately.
776 ISD::NodeType ExtendCode =
777 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
778 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
779}
780
781//===----------------------------------------------------------------------===//
782// Operand Vector Scalarization <1 x ty> -> ty.
783//===----------------------------------------------------------------------===//
784
785bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
786 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
787 N->dump(&DAG));
788 SDValue Res = SDValue();
789
790 switch (N->getOpcode()) {
791 default:
792#ifndef NDEBUG
793 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
794 N->dump(&DAG);
795 dbgs() << "\n";
796#endif
797 report_fatal_error(reason: "Do not know how to scalarize this operator's "
798 "operand!\n");
799 case ISD::BITCAST:
800 Res = ScalarizeVecOp_BITCAST(N);
801 break;
802 case ISD::FAKE_USE:
803 Res = ScalarizeVecOp_FAKE_USE(N);
804 break;
805 case ISD::ANY_EXTEND:
806 case ISD::ZERO_EXTEND:
807 case ISD::SIGN_EXTEND:
808 case ISD::TRUNCATE:
809 case ISD::FP_TO_SINT:
810 case ISD::FP_TO_UINT:
811 case ISD::SINT_TO_FP:
812 case ISD::UINT_TO_FP:
813 case ISD::LROUND:
814 case ISD::LLROUND:
815 case ISD::LRINT:
816 case ISD::LLRINT:
817 Res = ScalarizeVecOp_UnaryOp(N);
818 break;
819 case ISD::FP_TO_SINT_SAT:
820 case ISD::FP_TO_UINT_SAT:
821 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
822 break;
823 case ISD::STRICT_SINT_TO_FP:
824 case ISD::STRICT_UINT_TO_FP:
825 case ISD::STRICT_FP_TO_SINT:
826 case ISD::STRICT_FP_TO_UINT:
827 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
828 break;
829 case ISD::CONCAT_VECTORS:
830 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
831 break;
832 case ISD::INSERT_SUBVECTOR:
833 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
834 break;
835 case ISD::EXTRACT_VECTOR_ELT:
836 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
837 break;
838 case ISD::VSELECT:
839 Res = ScalarizeVecOp_VSELECT(N);
840 break;
841 case ISD::SETCC:
842 Res = ScalarizeVecOp_VSETCC(N);
843 break;
844 case ISD::STRICT_FSETCC:
845 case ISD::STRICT_FSETCCS:
846 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
847 break;
848 case ISD::STORE:
849 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
850 break;
851 case ISD::STRICT_FP_ROUND:
852 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
853 break;
854 case ISD::FP_ROUND:
855 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
856 break;
857 case ISD::STRICT_FP_EXTEND:
858 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
859 break;
860 case ISD::FP_EXTEND:
861 Res = ScalarizeVecOp_FP_EXTEND(N);
862 break;
863 case ISD::VECREDUCE_FADD:
864 case ISD::VECREDUCE_FMUL:
865 case ISD::VECREDUCE_ADD:
866 case ISD::VECREDUCE_MUL:
867 case ISD::VECREDUCE_AND:
868 case ISD::VECREDUCE_OR:
869 case ISD::VECREDUCE_XOR:
870 case ISD::VECREDUCE_SMAX:
871 case ISD::VECREDUCE_SMIN:
872 case ISD::VECREDUCE_UMAX:
873 case ISD::VECREDUCE_UMIN:
874 case ISD::VECREDUCE_FMAX:
875 case ISD::VECREDUCE_FMIN:
876 case ISD::VECREDUCE_FMAXIMUM:
877 case ISD::VECREDUCE_FMINIMUM:
878 Res = ScalarizeVecOp_VECREDUCE(N);
879 break;
880 case ISD::VECREDUCE_SEQ_FADD:
881 case ISD::VECREDUCE_SEQ_FMUL:
882 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
883 break;
884 case ISD::SCMP:
885 case ISD::UCMP:
886 Res = ScalarizeVecOp_CMP(N);
887 break;
888 case ISD::VECTOR_FIND_LAST_ACTIVE:
889 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
890 break;
891 }
892
893 // If the result is null, the sub-method took care of registering results etc.
894 if (!Res.getNode()) return false;
895
896 // If the result is N, the sub-method updated N in place. Tell the legalizer
897 // core about this.
898 if (Res.getNode() == N)
899 return true;
900
901 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
902 "Invalid operand expansion");
903
904 ReplaceValueWith(From: SDValue(N, 0), To: Res);
905 return false;
906}
907
908/// If the value to convert is a vector that needs to be scalarized, it must be
909/// <1 x ty>. Convert the element instead.
910SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
911 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
912 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
913 VT: N->getValueType(ResNo: 0), Operand: Elt);
914}
915
916// Need to legalize vector operands of fake uses. Must be <1 x ty>.
917SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
918 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
919 "Fake Use: Unexpected vector type!");
920 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
921 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
922}
923
924/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
925/// Do the operation on the element instead.
926SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
927 assert(N->getValueType(0).getVectorNumElements() == 1 &&
928 "Unexpected vector type!");
929 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
930 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
931 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
932 // Revectorize the result so the types line up with what the uses of this
933 // expression expect.
934 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
935}
936
937/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
938/// typesize).
939SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
940 assert(N->getValueType(0).getVectorNumElements() == 1 &&
941 "Unexpected vector type!");
942 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
943 SDValue Op =
944 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
945 N1: Elt, N2: N->getOperand(Num: 1));
946 // Revectorize the result so the types line up with what the uses of this
947 // expression expect.
948 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
949}
950
951/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
952/// Do the strict FP operation on the element instead.
953SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
954 assert(N->getValueType(0).getVectorNumElements() == 1 &&
955 "Unexpected vector type!");
956 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
957 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
958 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
959 Ops: { N->getOperand(Num: 0), Elt });
960 // Legalize the chain result - switch anything that used the old chain to
961 // use the new one.
962 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
963 // Revectorize the result so the types line up with what the uses of this
964 // expression expect.
965 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
966
967 // Do our own replacement and return SDValue() to tell the caller that we
968 // handled all replacements since caller can only handle a single result.
969 ReplaceValueWith(From: SDValue(N, 0), To: Res);
970 return SDValue();
971}
972
973/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
974SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
975 SmallVector<SDValue, 8> Ops(N->getNumOperands());
976 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
977 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
978 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
979}
980
981/// The inserted subvector is to be scalarized - use insert vector element
982/// instead.
983SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
984 unsigned OpNo) {
985 // We should not be attempting to scalarize the containing vector
986 assert(OpNo == 1);
987 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
988 SDValue ContainingVec = N->getOperand(Num: 0);
989 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
990 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
991 N3: N->getOperand(Num: 2));
992}
993
994/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
995/// so just return the element, ignoring the index.
996SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
997 EVT VT = N->getValueType(ResNo: 0);
998 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
999 if (Res.getValueType() != VT)
1000 Res = VT.isFloatingPoint()
1001 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1002 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1003 return Res;
1004}
1005
1006/// If the input condition is a vector that needs to be scalarized, it must be
1007/// <1 x i1>, so just convert to a normal ISD::SELECT
1008/// (still with vector output type since that was acceptable if we got here).
1009SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1010 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1011 EVT VT = N->getValueType(ResNo: 0);
1012
1013 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1014 N3: N->getOperand(Num: 2));
1015}
1016
1017/// If the operand is a vector that needs to be scalarized then the
1018/// result must be v1i1, so just convert to a scalar SETCC and wrap
1019/// with a scalar_to_vector since the res type is legal if we got here
1020SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1021 assert(N->getValueType(0).isVector() &&
1022 N->getOperand(0).getValueType().isVector() &&
1023 "Operand types must be vectors");
1024 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1025
1026 EVT VT = N->getValueType(ResNo: 0);
1027 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1028 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1029
1030 EVT OpVT = N->getOperand(Num: 0).getValueType();
1031 EVT NVT = VT.getVectorElementType();
1032 SDLoc DL(N);
1033 // Turn it into a scalar SETCC.
1034 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1035 N3: N->getOperand(Num: 2));
1036
1037 // Vectors may have a different boolean contents to scalars. Promote the
1038 // value appropriately.
1039 ISD::NodeType ExtendCode =
1040 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1041
1042 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1043
1044 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1045}
1046
1047// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1048SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1049 unsigned OpNo) {
1050 assert(OpNo == 1 && "Wrong operand for scalarization!");
1051 assert(N->getValueType(0).isVector() &&
1052 N->getOperand(1).getValueType().isVector() &&
1053 "Operand types must be vectors");
1054 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
1055
1056 EVT VT = N->getValueType(ResNo: 0);
1057 SDValue Ch = N->getOperand(Num: 0);
1058 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1059 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1060 SDValue CC = N->getOperand(Num: 3);
1061
1062 EVT OpVT = N->getOperand(Num: 1).getValueType();
1063 EVT NVT = VT.getVectorElementType();
1064 SDLoc DL(N);
1065 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1066 Ops: {Ch, LHS, RHS, CC});
1067
1068 // Legalize the chain result - switch anything that used the old chain to
1069 // use the new one.
1070 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1071
1072 ISD::NodeType ExtendCode =
1073 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1074
1075 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1076 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1077
1078 // Do our own replacement and return SDValue() to tell the caller that we
1079 // handled all replacements since caller can only handle a single result.
1080 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1081 return SDValue();
1082}
1083
1084/// If the value to store is a vector that needs to be scalarized, it must be
1085/// <1 x ty>. Just store the element.
1086SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1087 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1088 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1089 SDLoc dl(N);
1090
1091 if (N->isTruncatingStore())
1092 return DAG.getTruncStore(
1093 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1094 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1095 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1096 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1097
1098 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1099 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1100 MMOFlags: N->getMemOperand()->getFlags(), AAInfo: N->getAAInfo());
1101}
1102
1103/// If the value to round is a vector that needs to be scalarized, it must be
1104/// <1 x ty>. Convert the element instead.
1105SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1106 assert(OpNo == 0 && "Wrong operand for scalarization!");
1107 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1108 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1109 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1110 N2: N->getOperand(Num: 1));
1111 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1112}
1113
1114SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1115 unsigned OpNo) {
1116 assert(OpNo == 1 && "Wrong operand for scalarization!");
1117 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1118 SDValue Res =
1119 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1120 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1121 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1122 // Legalize the chain result - switch anything that used the old chain to
1123 // use the new one.
1124 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1125
1126 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1127
1128 // Do our own replacement and return SDValue() to tell the caller that we
1129 // handled all replacements since caller can only handle a single result.
1130 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1131 return SDValue();
1132}
1133
1134/// If the value to extend is a vector that needs to be scalarized, it must be
1135/// <1 x ty>. Convert the element instead.
1136SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1137 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1138 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1139 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1140 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1141}
1142
1143/// If the value to extend is a vector that needs to be scalarized, it must be
1144/// <1 x ty>. Convert the element instead.
1145SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1146 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1147 SDValue Res =
1148 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1149 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1150 Ops: {N->getOperand(Num: 0), Elt});
1151 // Legalize the chain result - switch anything that used the old chain to
1152 // use the new one.
1153 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1154
1155 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1156
1157 // Do our own replacement and return SDValue() to tell the caller that we
1158 // handled all replacements since caller can only handle a single result.
1159 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1160 return SDValue();
1161}
1162
1163SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1164 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1165 // Result type may be wider than element type.
1166 if (Res.getValueType() != N->getValueType(ResNo: 0))
1167 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1168 return Res;
1169}
1170
1171SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1172 SDValue AccOp = N->getOperand(Num: 0);
1173 SDValue VecOp = N->getOperand(Num: 1);
1174
1175 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1176
1177 SDValue Op = GetScalarizedVector(Op: VecOp);
1178 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1179 N1: AccOp, N2: Op, Flags: N->getFlags());
1180}
1181
1182SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1183 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1184 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1185
1186 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1187 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1188 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1189}
1190
1191SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1192 // Since there is no "none-active" result, the only valid return for <1 x ty>
1193 // is 0. Note: Since we check the high mask during splitting this is safe.
1194 // As e.g., a <2 x ty> operation would split to:
1195 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1196 // : `last_active(%lo_mask)`
1197 // Which then scalarizes to:
1198 // %mask[1] ? 1 : 0
1199 EVT VT = N->getValueType(ResNo: 0);
1200 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1201}
1202
1203//===----------------------------------------------------------------------===//
1204// Result Vector Splitting
1205//===----------------------------------------------------------------------===//
1206
1207/// This method is called when the specified result of the specified node is
1208/// found to need vector splitting. At this point, the node may also have
1209/// invalid operands or may have other results that need legalization, we just
1210/// know that (at least) one result needs vector splitting.
1211void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1212 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1213 SDValue Lo, Hi;
1214
1215 // See if the target wants to custom expand this node.
1216 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1217 return;
1218
1219 switch (N->getOpcode()) {
1220 default:
1221#ifndef NDEBUG
1222 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1223 N->dump(&DAG);
1224 dbgs() << "\n";
1225#endif
1226 report_fatal_error(reason: "Do not know how to split the result of this "
1227 "operator!\n");
1228
1229 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1230 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1231 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1232 break;
1233 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1234 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1235 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1236 case ISD::VSELECT:
1237 case ISD::SELECT:
1238 case ISD::VP_MERGE:
1239 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
1240 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1241 case ISD::POISON:
1242 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1243 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1244 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1245 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1246 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1247 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1248 case ISD::FPOWI:
1249 case ISD::FLDEXP:
1250 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1251 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1252 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1253 case ISD::SPLAT_VECTOR:
1254 case ISD::SCALAR_TO_VECTOR:
1255 SplitVecRes_ScalarOp(N, Lo, Hi);
1256 break;
1257 case ISD::STEP_VECTOR:
1258 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1259 break;
1260 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1261 case ISD::LOAD:
1262 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1263 break;
1264 case ISD::VP_LOAD:
1265 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1266 break;
1267 case ISD::VP_LOAD_FF:
1268 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1269 break;
1270 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1271 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1272 break;
1273 case ISD::MLOAD:
1274 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1275 break;
1276 case ISD::MGATHER:
1277 case ISD::VP_GATHER:
1278 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1279 break;
1280 case ISD::VECTOR_COMPRESS:
1281 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1282 break;
1283 case ISD::SETCC:
1284 case ISD::VP_SETCC:
1285 SplitVecRes_SETCC(N, Lo, Hi);
1286 break;
1287 case ISD::VECTOR_REVERSE:
1288 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1289 break;
1290 case ISD::VECTOR_SHUFFLE:
1291 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1292 break;
1293 case ISD::VECTOR_SPLICE_LEFT:
1294 case ISD::VECTOR_SPLICE_RIGHT:
1295 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1296 break;
1297 case ISD::VECTOR_DEINTERLEAVE:
1298 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1299 return;
1300 case ISD::VECTOR_INTERLEAVE:
1301 SplitVecRes_VECTOR_INTERLEAVE(N);
1302 return;
1303 case ISD::VAARG:
1304 SplitVecRes_VAARG(N, Lo, Hi);
1305 break;
1306
1307 case ISD::ANY_EXTEND_VECTOR_INREG:
1308 case ISD::SIGN_EXTEND_VECTOR_INREG:
1309 case ISD::ZERO_EXTEND_VECTOR_INREG:
1310 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1311 break;
1312
1313 case ISD::ABS:
1314 case ISD::VP_ABS:
1315 case ISD::BITREVERSE:
1316 case ISD::VP_BITREVERSE:
1317 case ISD::BSWAP:
1318 case ISD::VP_BSWAP:
1319 case ISD::CTLZ:
1320 case ISD::VP_CTLZ:
1321 case ISD::CTTZ:
1322 case ISD::VP_CTTZ:
1323 case ISD::CTLZ_ZERO_UNDEF:
1324 case ISD::VP_CTLZ_ZERO_UNDEF:
1325 case ISD::CTTZ_ZERO_UNDEF:
1326 case ISD::VP_CTTZ_ZERO_UNDEF:
1327 case ISD::CTPOP:
1328 case ISD::VP_CTPOP:
1329 case ISD::FABS: case ISD::VP_FABS:
1330 case ISD::FACOS:
1331 case ISD::FASIN:
1332 case ISD::FATAN:
1333 case ISD::FCEIL:
1334 case ISD::VP_FCEIL:
1335 case ISD::FCOS:
1336 case ISD::FCOSH:
1337 case ISD::FEXP:
1338 case ISD::FEXP2:
1339 case ISD::FEXP10:
1340 case ISD::FFLOOR:
1341 case ISD::VP_FFLOOR:
1342 case ISD::FLOG:
1343 case ISD::FLOG10:
1344 case ISD::FLOG2:
1345 case ISD::FNEARBYINT:
1346 case ISD::VP_FNEARBYINT:
1347 case ISD::FNEG: case ISD::VP_FNEG:
1348 case ISD::FREEZE:
1349 case ISD::ARITH_FENCE:
1350 case ISD::FP_EXTEND:
1351 case ISD::VP_FP_EXTEND:
1352 case ISD::FP_ROUND:
1353 case ISD::VP_FP_ROUND:
1354 case ISD::FP_TO_SINT:
1355 case ISD::VP_FP_TO_SINT:
1356 case ISD::FP_TO_UINT:
1357 case ISD::VP_FP_TO_UINT:
1358 case ISD::FRINT:
1359 case ISD::VP_FRINT:
1360 case ISD::LRINT:
1361 case ISD::VP_LRINT:
1362 case ISD::LLRINT:
1363 case ISD::VP_LLRINT:
1364 case ISD::FROUND:
1365 case ISD::VP_FROUND:
1366 case ISD::FROUNDEVEN:
1367 case ISD::VP_FROUNDEVEN:
1368 case ISD::LROUND:
1369 case ISD::LLROUND:
1370 case ISD::FSIN:
1371 case ISD::FSINH:
1372 case ISD::FSQRT: case ISD::VP_SQRT:
1373 case ISD::FTAN:
1374 case ISD::FTANH:
1375 case ISD::FTRUNC:
1376 case ISD::VP_FROUNDTOZERO:
1377 case ISD::SINT_TO_FP:
1378 case ISD::VP_SINT_TO_FP:
1379 case ISD::TRUNCATE:
1380 case ISD::VP_TRUNCATE:
1381 case ISD::UINT_TO_FP:
1382 case ISD::VP_UINT_TO_FP:
1383 case ISD::FCANONICALIZE:
1384 case ISD::AssertNoFPClass:
1385 SplitVecRes_UnaryOp(N, Lo, Hi);
1386 break;
1387 case ISD::ADDRSPACECAST:
1388 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1389 break;
1390 case ISD::FMODF:
1391 case ISD::FFREXP:
1392 case ISD::FSINCOS:
1393 case ISD::FSINCOSPI:
1394 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1395 break;
1396
1397 case ISD::ANY_EXTEND:
1398 case ISD::SIGN_EXTEND:
1399 case ISD::ZERO_EXTEND:
1400 case ISD::VP_SIGN_EXTEND:
1401 case ISD::VP_ZERO_EXTEND:
1402 SplitVecRes_ExtendOp(N, Lo, Hi);
1403 break;
1404
1405 case ISD::ADD: case ISD::VP_ADD:
1406 case ISD::SUB: case ISD::VP_SUB:
1407 case ISD::MUL: case ISD::VP_MUL:
1408 case ISD::CLMUL:
1409 case ISD::CLMULR:
1410 case ISD::CLMULH:
1411 case ISD::MULHS:
1412 case ISD::MULHU:
1413 case ISD::ABDS:
1414 case ISD::ABDU:
1415 case ISD::AVGCEILS:
1416 case ISD::AVGCEILU:
1417 case ISD::AVGFLOORS:
1418 case ISD::AVGFLOORU:
1419 case ISD::FADD: case ISD::VP_FADD:
1420 case ISD::FSUB: case ISD::VP_FSUB:
1421 case ISD::FMUL: case ISD::VP_FMUL:
1422 case ISD::FMINNUM:
1423 case ISD::FMINNUM_IEEE:
1424 case ISD::VP_FMINNUM:
1425 case ISD::FMAXNUM:
1426 case ISD::FMAXNUM_IEEE:
1427 case ISD::VP_FMAXNUM:
1428 case ISD::FMINIMUM:
1429 case ISD::VP_FMINIMUM:
1430 case ISD::FMAXIMUM:
1431 case ISD::VP_FMAXIMUM:
1432 case ISD::FMINIMUMNUM:
1433 case ISD::FMAXIMUMNUM:
1434 case ISD::SDIV: case ISD::VP_SDIV:
1435 case ISD::UDIV: case ISD::VP_UDIV:
1436 case ISD::FDIV: case ISD::VP_FDIV:
1437 case ISD::FPOW:
1438 case ISD::FATAN2:
1439 case ISD::AND: case ISD::VP_AND:
1440 case ISD::OR: case ISD::VP_OR:
1441 case ISD::XOR: case ISD::VP_XOR:
1442 case ISD::SHL: case ISD::VP_SHL:
1443 case ISD::SRA: case ISD::VP_SRA:
1444 case ISD::SRL: case ISD::VP_SRL:
1445 case ISD::UREM: case ISD::VP_UREM:
1446 case ISD::SREM: case ISD::VP_SREM:
1447 case ISD::FREM: case ISD::VP_FREM:
1448 case ISD::SMIN: case ISD::VP_SMIN:
1449 case ISD::SMAX: case ISD::VP_SMAX:
1450 case ISD::UMIN: case ISD::VP_UMIN:
1451 case ISD::UMAX: case ISD::VP_UMAX:
1452 case ISD::SADDSAT: case ISD::VP_SADDSAT:
1453 case ISD::UADDSAT: case ISD::VP_UADDSAT:
1454 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
1455 case ISD::USUBSAT: case ISD::VP_USUBSAT:
1456 case ISD::SSHLSAT:
1457 case ISD::USHLSAT:
1458 case ISD::ROTL:
1459 case ISD::ROTR:
1460 case ISD::VP_FCOPYSIGN:
1461 SplitVecRes_BinOp(N, Lo, Hi);
1462 break;
1463 case ISD::FMA: case ISD::VP_FMA:
1464 case ISD::FSHL:
1465 case ISD::VP_FSHL:
1466 case ISD::FSHR:
1467 case ISD::VP_FSHR:
1468 SplitVecRes_TernaryOp(N, Lo, Hi);
1469 break;
1470
1471 case ISD::SCMP: case ISD::UCMP:
1472 SplitVecRes_CMP(N, Lo, Hi);
1473 break;
1474
1475#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1476 case ISD::STRICT_##DAGN:
1477#include "llvm/IR/ConstrainedOps.def"
1478 SplitVecRes_StrictFPOp(N, Lo, Hi);
1479 break;
1480
1481 case ISD::FP_TO_UINT_SAT:
1482 case ISD::FP_TO_SINT_SAT:
1483 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1484 break;
1485
1486 case ISD::UADDO:
1487 case ISD::SADDO:
1488 case ISD::USUBO:
1489 case ISD::SSUBO:
1490 case ISD::UMULO:
1491 case ISD::SMULO:
1492 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1493 break;
1494 case ISD::SMULFIX:
1495 case ISD::SMULFIXSAT:
1496 case ISD::UMULFIX:
1497 case ISD::UMULFIXSAT:
1498 case ISD::SDIVFIX:
1499 case ISD::SDIVFIXSAT:
1500 case ISD::UDIVFIX:
1501 case ISD::UDIVFIXSAT:
1502 SplitVecRes_FIX(N, Lo, Hi);
1503 break;
1504 case ISD::EXPERIMENTAL_VP_SPLICE:
1505 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1506 break;
1507 case ISD::EXPERIMENTAL_VP_REVERSE:
1508 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1509 break;
1510 case ISD::PARTIAL_REDUCE_UMLA:
1511 case ISD::PARTIAL_REDUCE_SMLA:
1512 case ISD::PARTIAL_REDUCE_SUMLA:
1513 case ISD::PARTIAL_REDUCE_FMLA:
1514 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1515 break;
1516 case ISD::GET_ACTIVE_LANE_MASK:
1517 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1518 break;
1519 }
1520
1521 // If Lo/Hi is null, the sub-method took care of registering results etc.
1522 if (Lo.getNode())
1523 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1524}
1525
1526void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1527 MachinePointerInfo &MPI, SDValue &Ptr,
1528 uint64_t *ScaledOffset) {
1529 SDLoc DL(N);
1530 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1531
1532 if (MemVT.isScalableVector()) {
1533 SDValue BytesIncrement = DAG.getVScale(
1534 DL, VT: Ptr.getValueType(),
1535 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1536 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1537 if (ScaledOffset)
1538 *ScaledOffset += IncrementSize;
1539 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1540 Flags: SDNodeFlags::NoUnsignedWrap);
1541 } else {
1542 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1543 // Increment the pointer to the other half.
1544 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1545 }
1546}
1547
1548std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1549 return SplitMask(Mask, DL: SDLoc(Mask));
1550}
1551
1552std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1553 const SDLoc &DL) {
1554 SDValue MaskLo, MaskHi;
1555 EVT MaskVT = Mask.getValueType();
1556 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1557 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1558 else
1559 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1560 return std::make_pair(x&: MaskLo, y&: MaskHi);
1561}
1562
1563void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1564 SDValue LHSLo, LHSHi;
1565 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1566 SDValue RHSLo, RHSHi;
1567 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1568 SDLoc dl(N);
1569
1570 const SDNodeFlags Flags = N->getFlags();
1571 unsigned Opcode = N->getOpcode();
1572 if (N->getNumOperands() == 2) {
1573 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1574 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1575 return;
1576 }
1577
1578 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1579 assert(N->isVPOpcode() && "Expected VP opcode");
1580
1581 SDValue MaskLo, MaskHi;
1582 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1583
1584 SDValue EVLLo, EVLHi;
1585 std::tie(args&: EVLLo, args&: EVLHi) =
1586 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1587
1588 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1589 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1590 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1591 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1592}
1593
1594void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1595 SDValue &Hi) {
1596 SDValue Op0Lo, Op0Hi;
1597 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1598 SDValue Op1Lo, Op1Hi;
1599 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1600 SDValue Op2Lo, Op2Hi;
1601 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1602 SDLoc dl(N);
1603
1604 const SDNodeFlags Flags = N->getFlags();
1605 unsigned Opcode = N->getOpcode();
1606 if (N->getNumOperands() == 3) {
1607 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1608 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1609 return;
1610 }
1611
1612 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1613 assert(N->isVPOpcode() && "Expected VP opcode");
1614
1615 SDValue MaskLo, MaskHi;
1616 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
1617
1618 SDValue EVLLo, EVLHi;
1619 std::tie(args&: EVLLo, args&: EVLHi) =
1620 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL: dl);
1621
1622 Lo = DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(),
1623 Ops: {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1624 Hi = DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(),
1625 Ops: {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1626}
1627
1628void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1629 LLVMContext &Ctxt = *DAG.getContext();
1630 SDLoc dl(N);
1631
1632 SDValue LHS = N->getOperand(Num: 0);
1633 SDValue RHS = N->getOperand(Num: 1);
1634
1635 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1636 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1637 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1638 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1639 } else {
1640 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1641 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1642 }
1643
1644 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1645 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1646 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1647}
1648
1649void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1650 SDValue LHSLo, LHSHi;
1651 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1652 SDValue RHSLo, RHSHi;
1653 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1654 SDLoc dl(N);
1655 SDValue Op2 = N->getOperand(Num: 2);
1656
1657 unsigned Opcode = N->getOpcode();
1658 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1659 Flags: N->getFlags());
1660 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1661 Flags: N->getFlags());
1662}
1663
1664void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1665 SDValue &Hi) {
1666 // We know the result is a vector. The input may be either a vector or a
1667 // scalar value.
1668 EVT LoVT, HiVT;
1669 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1670 SDLoc dl(N);
1671
1672 SDValue InOp = N->getOperand(Num: 0);
1673 EVT InVT = InOp.getValueType();
1674
1675 // Handle some special cases efficiently.
1676 switch (getTypeAction(VT: InVT)) {
1677 case TargetLowering::TypeLegal:
1678 case TargetLowering::TypePromoteInteger:
1679 case TargetLowering::TypeSoftPromoteHalf:
1680 case TargetLowering::TypeSoftenFloat:
1681 case TargetLowering::TypeScalarizeVector:
1682 case TargetLowering::TypeWidenVector:
1683 break;
1684 case TargetLowering::TypeExpandInteger:
1685 case TargetLowering::TypeExpandFloat:
1686 // A scalar to vector conversion, where the scalar needs expansion.
1687 // If the vector is being split in two then we can just convert the
1688 // expanded pieces.
1689 if (LoVT == HiVT) {
1690 GetExpandedOp(Op: InOp, Lo, Hi);
1691 if (DAG.getDataLayout().isBigEndian())
1692 std::swap(a&: Lo, b&: Hi);
1693 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1694 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1695 return;
1696 }
1697 break;
1698 case TargetLowering::TypeSplitVector:
1699 // If the input is a vector that needs to be split, convert each split
1700 // piece of the input now.
1701 GetSplitVector(Op: InOp, Lo, Hi);
1702 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1703 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1704 return;
1705 case TargetLowering::TypeScalarizeScalableVector:
1706 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1707 }
1708
1709 if (LoVT.isScalableVector()) {
1710 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1711 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1712 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1713 return;
1714 }
1715
1716 // In the general case, convert the input to an integer and split it by hand.
1717 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1718 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1719 if (DAG.getDataLayout().isBigEndian())
1720 std::swap(a&: LoIntVT, b&: HiIntVT);
1721
1722 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1723
1724 if (DAG.getDataLayout().isBigEndian())
1725 std::swap(a&: Lo, b&: Hi);
1726 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1727 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1728}
1729
1730void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 SDLoc DL(N);
1733 EVT LoVT, HiVT;
1734 SDValue PtrA = N->getOperand(Num: 0);
1735 SDValue PtrB = N->getOperand(Num: 1);
1736 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1737
1738 // The lane offset for the "Lo" half of the mask is unchanged.
1739 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1740 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1741 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1742 // The lane offset for the "Hi" half of the mask is incremented by the number
1743 // of elements in the "Lo" half.
1744 unsigned LaneOffset =
1745 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1746 // Note: The lane offset is implicitly scalable for scalable masks.
1747 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1748 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1749 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1750}
1751
1752void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1753 SDValue &Hi) {
1754 EVT LoVT, HiVT;
1755 SDLoc dl(N);
1756 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1757 unsigned LoNumElts = LoVT.getVectorNumElements();
1758 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1759 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1760
1761 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1762 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1763}
1764
1765void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1766 SDValue &Hi) {
1767 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1768 SDLoc dl(N);
1769 unsigned NumSubvectors = N->getNumOperands() / 2;
1770 if (NumSubvectors == 1) {
1771 Lo = N->getOperand(Num: 0);
1772 Hi = N->getOperand(Num: 1);
1773 return;
1774 }
1775
1776 EVT LoVT, HiVT;
1777 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1778
1779 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1780 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1781
1782 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1783 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1784}
1785
1786void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1787 SDValue &Hi) {
1788 SDValue Vec = N->getOperand(Num: 0);
1789 SDValue Idx = N->getOperand(Num: 1);
1790 SDLoc dl(N);
1791
1792 EVT LoVT, HiVT;
1793 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1794
1795 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1796 uint64_t IdxVal = Idx->getAsZExtVal();
1797 Hi = DAG.getNode(
1798 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1799 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1800}
1801
1802void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1803 SDValue &Hi) {
1804 SDValue Vec = N->getOperand(Num: 0);
1805 SDValue SubVec = N->getOperand(Num: 1);
1806 SDValue Idx = N->getOperand(Num: 2);
1807 SDLoc dl(N);
1808 GetSplitVector(Op: Vec, Lo, Hi);
1809
1810 EVT VecVT = Vec.getValueType();
1811 EVT LoVT = Lo.getValueType();
1812 EVT SubVecVT = SubVec.getValueType();
1813 unsigned VecElems = VecVT.getVectorMinNumElements();
1814 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1815 unsigned LoElems = LoVT.getVectorMinNumElements();
1816
1817 // If we know the index is in the first half, and we know the subvector
1818 // doesn't cross the boundary between the halves, we can avoid spilling the
1819 // vector, and insert into the lower half of the split vector directly.
1820 unsigned IdxVal = Idx->getAsZExtVal();
1821 if (IdxVal + SubElems <= LoElems) {
1822 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1823 return;
1824 }
1825 // Similarly if the subvector is fully in the high half, but mind that we
1826 // can't tell whether a fixed-length subvector is fully within the high half
1827 // of a scalable vector.
1828 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1829 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1830 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1831 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1832 return;
1833 }
1834
1835 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1836 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1837 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1838 if (WideSubVec.getValueType() == VecVT) {
1839 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1840 return;
1841 }
1842 }
1843
1844 // Spill the vector to the stack.
1845 // In cases where the vector is illegal it will be broken down into parts
1846 // and stored in parts - we should use the alignment for the smallest part.
1847 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
1848 SDValue StackPtr =
1849 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
1850 auto &MF = DAG.getMachineFunction();
1851 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1852 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
1853
1854 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1855 Alignment: SmallestAlign);
1856
1857 // Store the new subvector into the specified index.
1858 SDValue SubVecPtr =
1859 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
1860 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
1861 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
1862
1863 // Load the Lo part from the stack slot.
1864 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
1865 Alignment: SmallestAlign);
1866
1867 // Increment the pointer to the other part.
1868 auto *Load = cast<LoadSDNode>(Val&: Lo);
1869 MachinePointerInfo MPI = Load->getPointerInfo();
1870 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
1871
1872 // Load the Hi part from the stack slot.
1873 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
1874}
1875
1876// Handle splitting an FP where the second operand does not match the first
1877// type. The second operand may be a scalar, or a vector that has exactly as
1878// many elements as the first
1879void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1880 SDValue &Hi) {
1881 SDValue LHSLo, LHSHi;
1882 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1883 SDLoc DL(N);
1884
1885 SDValue RHSLo, RHSHi;
1886 SDValue RHS = N->getOperand(Num: 1);
1887 EVT RHSVT = RHS.getValueType();
1888 if (RHSVT.isVector()) {
1889 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
1890 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1891 else
1892 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
1893
1894 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
1895 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
1896 } else {
1897 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
1898 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
1899 }
1900}
1901
1902void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1903 SDValue &Hi) {
1904 SDLoc DL(N);
1905 SDValue ArgLo, ArgHi;
1906 SDValue Test = N->getOperand(Num: 1);
1907 SDValue FpValue = N->getOperand(Num: 0);
1908 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1909 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
1910 else
1911 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
1912 EVT LoVT, HiVT;
1913 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1914
1915 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
1916 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
1917}
1918
1919void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1920 SDValue &Hi) {
1921 SDValue LHSLo, LHSHi;
1922 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1923 SDLoc dl(N);
1924
1925 EVT LoVT, HiVT;
1926 std::tie(args&: LoVT, args&: HiVT) =
1927 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
1928
1929 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
1930 N2: DAG.getValueType(LoVT));
1931 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
1932 N2: DAG.getValueType(HiVT));
1933}
1934
1935void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1936 SDValue &Hi) {
1937 unsigned Opcode = N->getOpcode();
1938 SDValue N0 = N->getOperand(Num: 0);
1939
1940 SDLoc dl(N);
1941 SDValue InLo, InHi;
1942
1943 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
1944 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
1945 else
1946 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
1947
1948 EVT InLoVT = InLo.getValueType();
1949 unsigned InNumElements = InLoVT.getVectorNumElements();
1950
1951 EVT OutLoVT, OutHiVT;
1952 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1953 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1954 assert((2 * OutNumElements) <= InNumElements &&
1955 "Illegal extend vector in reg split");
1956
1957 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1958 // input vector (i.e. we only use InLo):
1959 // OutLo will extend the first OutNumElements from InLo.
1960 // OutHi will extend the next OutNumElements from InLo.
1961
1962 // Shuffle the elements from InLo for OutHi into the bottom elements to
1963 // create a 'fake' InHi.
1964 SmallVector<int, 8> SplitHi(InNumElements, -1);
1965 for (unsigned i = 0; i != OutNumElements; ++i)
1966 SplitHi[i] = i + OutNumElements;
1967 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
1968
1969 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
1970 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
1971}
1972
1973void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1974 SDValue &Hi) {
1975 unsigned NumOps = N->getNumOperands();
1976 SDValue Chain = N->getOperand(Num: 0);
1977 EVT LoVT, HiVT;
1978 SDLoc dl(N);
1979 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1980
1981 SmallVector<SDValue, 4> OpsLo(NumOps);
1982 SmallVector<SDValue, 4> OpsHi(NumOps);
1983
1984 // The Chain is the first operand.
1985 OpsLo[0] = Chain;
1986 OpsHi[0] = Chain;
1987
1988 // Now process the remaining operands.
1989 for (unsigned i = 1; i < NumOps; ++i) {
1990 SDValue Op = N->getOperand(Num: i);
1991 SDValue OpLo = Op;
1992 SDValue OpHi = Op;
1993
1994 EVT InVT = Op.getValueType();
1995 if (InVT.isVector()) {
1996 // If the input also splits, handle it directly for a
1997 // compile time speedup. Otherwise split it by hand.
1998 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
1999 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2000 else
2001 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2002 }
2003
2004 OpsLo[i] = OpLo;
2005 OpsHi[i] = OpHi;
2006 }
2007
2008 EVT LoValueVTs[] = {LoVT, MVT::Other};
2009 EVT HiValueVTs[] = {HiVT, MVT::Other};
2010 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2011 Flags: N->getFlags());
2012 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2013 Flags: N->getFlags());
2014
2015 // Build a factor node to remember that this Op is independent of the
2016 // other one.
2017 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2018 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2019
2020 // Legalize the chain result - switch anything that used the old chain to
2021 // use the new one.
2022 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2023}
2024
2025SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2026 SDValue Chain = N->getOperand(Num: 0);
2027 EVT VT = N->getValueType(ResNo: 0);
2028 unsigned NE = VT.getVectorNumElements();
2029 EVT EltVT = VT.getVectorElementType();
2030 SDLoc dl(N);
2031
2032 SmallVector<SDValue, 8> Scalars;
2033 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2034
2035 // If ResNE is 0, fully unroll the vector op.
2036 if (ResNE == 0)
2037 ResNE = NE;
2038 else if (NE > ResNE)
2039 NE = ResNE;
2040
2041 //The results of each unrolled operation, including the chain.
2042 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2043 SmallVector<SDValue, 8> Chains;
2044
2045 unsigned i;
2046 for (i = 0; i != NE; ++i) {
2047 Operands[0] = Chain;
2048 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2049 SDValue Operand = N->getOperand(Num: j);
2050 EVT OperandVT = Operand.getValueType();
2051 if (OperandVT.isVector()) {
2052 EVT OperandEltVT = OperandVT.getVectorElementType();
2053 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2054 } else {
2055 Operands[j] = Operand;
2056 }
2057 }
2058 SDValue Scalar =
2059 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2060
2061 //Add in the scalar as well as its chain value to the
2062 //result vectors.
2063 Scalars.push_back(Elt: Scalar);
2064 Chains.push_back(Elt: Scalar.getValue(R: 1));
2065 }
2066
2067 for (; i < ResNE; ++i)
2068 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2069
2070 // Build a new factor node to connect the chain back together.
2071 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2072 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2073
2074 // Create a new BUILD_VECTOR node
2075 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2076 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2077}
2078
2079void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2080 SDValue &Lo, SDValue &Hi) {
2081 SDLoc dl(N);
2082 EVT ResVT = N->getValueType(ResNo: 0);
2083 EVT OvVT = N->getValueType(ResNo: 1);
2084 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2085 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2086 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2087
2088 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2089 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2090 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2091 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2092 } else {
2093 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2094 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2095 }
2096
2097 unsigned Opcode = N->getOpcode();
2098 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2099 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2100 SDNode *LoNode =
2101 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2102 SDNode *HiNode =
2103 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2104
2105 Lo = SDValue(LoNode, ResNo);
2106 Hi = SDValue(HiNode, ResNo);
2107
2108 // Replace the other vector result not being explicitly split here.
2109 unsigned OtherNo = 1 - ResNo;
2110 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2111 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2112 SetSplitVector(Op: SDValue(N, OtherNo),
2113 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2114 } else {
2115 SDValue OtherVal = DAG.getNode(
2116 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2117 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2118 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2119 }
2120}
2121
2122void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2123 SDValue &Hi) {
2124 SDValue Vec = N->getOperand(Num: 0);
2125 SDValue Elt = N->getOperand(Num: 1);
2126 SDValue Idx = N->getOperand(Num: 2);
2127 SDLoc dl(N);
2128 GetSplitVector(Op: Vec, Lo, Hi);
2129
2130 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2131 unsigned IdxVal = CIdx->getZExtValue();
2132 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2133 if (IdxVal < LoNumElts) {
2134 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2135 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2136 return;
2137 } else if (!Vec.getValueType().isScalableVector()) {
2138 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2139 return;
2140 }
2141 }
2142
2143 // Make the vector elements byte-addressable if they aren't already.
2144 EVT VecVT = Vec.getValueType();
2145 EVT EltVT = VecVT.getVectorElementType();
2146 if (!EltVT.isByteSized()) {
2147 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2148 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2149 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2150 // Extend the element type to match if needed.
2151 if (EltVT.bitsGT(VT: Elt.getValueType()))
2152 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2153 }
2154
2155 // Spill the vector to the stack.
2156 // In cases where the vector is illegal it will be broken down into parts
2157 // and stored in parts - we should use the alignment for the smallest part.
2158 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2159 SDValue StackPtr =
2160 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2161 auto &MF = DAG.getMachineFunction();
2162 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2163 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2164
2165 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2166 Alignment: SmallestAlign);
2167
2168 // Store the new element. This may be larger than the vector element type,
2169 // so use a truncating store.
2170 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2171 Store = DAG.getTruncStore(
2172 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2173 Alignment: commonAlignment(A: SmallestAlign,
2174 Offset: EltVT.getFixedSizeInBits() / 8));
2175
2176 EVT LoVT, HiVT;
2177 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2178
2179 // Load the Lo part from the stack slot.
2180 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2181
2182 // Increment the pointer to the other part.
2183 auto Load = cast<LoadSDNode>(Val&: Lo);
2184 MachinePointerInfo MPI = Load->getPointerInfo();
2185 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2186
2187 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2188
2189 // If we adjusted the original type, we need to truncate the results.
2190 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2191 if (LoVT != Lo.getValueType())
2192 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2193 if (HiVT != Hi.getValueType())
2194 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2195}
2196
2197void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2198 SDValue &Hi) {
2199 EVT LoVT, HiVT;
2200 SDLoc dl(N);
2201 assert(N->getValueType(0).isScalableVector() &&
2202 "Only scalable vectors are supported for STEP_VECTOR");
2203 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2204 SDValue Step = N->getOperand(Num: 0);
2205
2206 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2207
2208 // Hi = Lo + (EltCnt * Step)
2209 EVT EltVT = Step.getValueType();
2210 APInt StepVal = Step->getAsAPIntVal();
2211 SDValue StartOfHi =
2212 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2213 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2214 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2215
2216 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2217 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2218}
2219
2220void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2221 SDValue &Hi) {
2222 EVT LoVT, HiVT;
2223 SDLoc dl(N);
2224 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2225 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2226 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2227 Hi = DAG.getPOISON(VT: HiVT);
2228 } else {
2229 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2230 Hi = Lo;
2231 }
2232}
2233
2234void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2235 SDValue &Hi) {
2236 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2237 EVT LoVT, HiVT;
2238 SDLoc dl(LD);
2239 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2240
2241 ISD::LoadExtType ExtType = LD->getExtensionType();
2242 SDValue Ch = LD->getChain();
2243 SDValue Ptr = LD->getBasePtr();
2244 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
2245 EVT MemoryVT = LD->getMemoryVT();
2246 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2247 AAMDNodes AAInfo = LD->getAAInfo();
2248
2249 EVT LoMemVT, HiMemVT;
2250 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2251
2252 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2253 SDValue Value, NewChain;
2254 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2255 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2256 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2257 return;
2258 }
2259
2260 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2261 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2262 AAInfo);
2263
2264 MachinePointerInfo MPI;
2265 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2266
2267 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2268 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
2269
2270 // Build a factor node to remember that this load is independent of the
2271 // other one.
2272 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2273 N2: Hi.getValue(R: 1));
2274
2275 // Legalize the chain result - switch anything that used the old chain to
2276 // use the new one.
2277 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2278}
2279
2280void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2281 SDValue &Hi) {
2282 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2283 EVT LoVT, HiVT;
2284 SDLoc dl(LD);
2285 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2286
2287 ISD::LoadExtType ExtType = LD->getExtensionType();
2288 SDValue Ch = LD->getChain();
2289 SDValue Ptr = LD->getBasePtr();
2290 SDValue Offset = LD->getOffset();
2291 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2292 Align Alignment = LD->getBaseAlign();
2293 SDValue Mask = LD->getMask();
2294 SDValue EVL = LD->getVectorLength();
2295 EVT MemoryVT = LD->getMemoryVT();
2296
2297 EVT LoMemVT, HiMemVT;
2298 bool HiIsEmpty = false;
2299 std::tie(args&: LoMemVT, args&: HiMemVT) =
2300 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2301
2302 // Split Mask operand
2303 SDValue MaskLo, MaskHi;
2304 if (Mask.getOpcode() == ISD::SETCC) {
2305 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2306 } else {
2307 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2308 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2309 else
2310 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2311 }
2312
2313 // Split EVL operand
2314 SDValue EVLLo, EVLHi;
2315 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2316
2317 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2318 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2319 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2320 Ranges: LD->getRanges());
2321
2322 Lo =
2323 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2324 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2325
2326 if (HiIsEmpty) {
2327 // The hi vp_load has zero storage size. We therefore simply set it to
2328 // the low vp_load and rely on subsequent removal from the chain.
2329 Hi = Lo;
2330 } else {
2331 // Generate hi vp_load.
2332 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2333 IsCompressedMemory: LD->isExpandingLoad());
2334
2335 MachinePointerInfo MPI;
2336 if (LoMemVT.isScalableVector())
2337 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2338 else
2339 MPI = LD->getPointerInfo().getWithOffset(
2340 O: LoMemVT.getStoreSize().getFixedValue());
2341
2342 MMO = DAG.getMachineFunction().getMachineMemOperand(
2343 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2344 BaseAlignment: Alignment, AAInfo: LD->getAAInfo(), Ranges: LD->getRanges());
2345
2346 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2347 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2348 IsExpanding: LD->isExpandingLoad());
2349 }
2350
2351 // Build a factor node to remember that this load is independent of the
2352 // other one.
2353 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2354 N2: Hi.getValue(R: 1));
2355
2356 // Legalize the chain result - switch anything that used the old chain to
2357 // use the new one.
2358 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2359}
2360
2361void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2362 SDValue &Hi) {
2363 SDLoc dl(LD);
2364 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2365
2366 SDValue Ch = LD->getChain();
2367 SDValue Ptr = LD->getBasePtr();
2368 Align Alignment = LD->getBaseAlign();
2369 SDValue Mask = LD->getMask();
2370 SDValue EVL = LD->getVectorLength();
2371
2372 // Split Mask operand
2373 SDValue MaskLo, MaskHi;
2374 if (Mask.getOpcode() == ISD::SETCC) {
2375 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2376 } else {
2377 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2378 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2379 else
2380 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2381 }
2382
2383 // Split EVL operand
2384 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2385
2386 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2387 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2388 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: LD->getAAInfo(),
2389 Ranges: LD->getRanges());
2390
2391 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2392
2393 // Fill the upper half with poison.
2394 Hi = DAG.getPOISON(VT: HiVT);
2395
2396 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2397 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2398}
2399
2400void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2401 SDValue &Lo, SDValue &Hi) {
2402 assert(SLD->isUnindexed() &&
2403 "Indexed VP strided load during type legalization!");
2404 assert(SLD->getOffset().isUndef() &&
2405 "Unexpected indexed variable-length load offset");
2406
2407 SDLoc DL(SLD);
2408
2409 EVT LoVT, HiVT;
2410 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2411
2412 EVT LoMemVT, HiMemVT;
2413 bool HiIsEmpty = false;
2414 std::tie(args&: LoMemVT, args&: HiMemVT) =
2415 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2416
2417 SDValue Mask = SLD->getMask();
2418 SDValue LoMask, HiMask;
2419 if (Mask.getOpcode() == ISD::SETCC) {
2420 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2421 } else {
2422 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2423 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2424 else
2425 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2426 }
2427
2428 SDValue LoEVL, HiEVL;
2429 std::tie(args&: LoEVL, args&: HiEVL) =
2430 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2431
2432 // Generate the low vp_strided_load
2433 Lo = DAG.getStridedLoadVP(
2434 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2435 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2436 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2437
2438 if (HiIsEmpty) {
2439 // The high vp_strided_load has zero storage size. We therefore simply set
2440 // it to the low vp_strided_load and rely on subsequent removal from the
2441 // chain.
2442 Hi = Lo;
2443 } else {
2444 // Generate the high vp_strided_load.
2445 // To calculate the high base address, we need to sum to the low base
2446 // address stride number of bytes for each element already loaded by low,
2447 // that is: Ptr = Ptr + (LoEVL * Stride)
2448 EVT PtrVT = SLD->getBasePtr().getValueType();
2449 SDValue Increment =
2450 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2451 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2452 SDValue Ptr =
2453 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2454
2455 Align Alignment = SLD->getBaseAlign();
2456 if (LoMemVT.isScalableVector())
2457 Alignment = commonAlignment(
2458 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2459
2460 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2461 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2462 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2463 BaseAlignment: Alignment, AAInfo: SLD->getAAInfo(), Ranges: SLD->getRanges());
2464
2465 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2466 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2467 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2468 IsExpanding: SLD->isExpandingLoad());
2469 }
2470
2471 // Build a factor node to remember that this load is independent of the
2472 // other one.
2473 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2474 N2: Hi.getValue(R: 1));
2475
2476 // Legalize the chain result - switch anything that used the old chain to
2477 // use the new one.
2478 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2479}
2480
2481void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2482 SDValue &Lo, SDValue &Hi) {
2483 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2484 EVT LoVT, HiVT;
2485 SDLoc dl(MLD);
2486 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2487
2488 SDValue Ch = MLD->getChain();
2489 SDValue Ptr = MLD->getBasePtr();
2490 SDValue Offset = MLD->getOffset();
2491 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2492 SDValue Mask = MLD->getMask();
2493 SDValue PassThru = MLD->getPassThru();
2494 Align Alignment = MLD->getBaseAlign();
2495 ISD::LoadExtType ExtType = MLD->getExtensionType();
2496 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2497
2498 // Split Mask operand
2499 SDValue MaskLo, MaskHi;
2500 if (Mask.getOpcode() == ISD::SETCC) {
2501 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2502 } else {
2503 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2504 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2505 else
2506 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2507 }
2508
2509 EVT MemoryVT = MLD->getMemoryVT();
2510 EVT LoMemVT, HiMemVT;
2511 bool HiIsEmpty = false;
2512 std::tie(args&: LoMemVT, args&: HiMemVT) =
2513 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2514
2515 SDValue PassThruLo, PassThruHi;
2516 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2517 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2518 else
2519 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2520
2521 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2522 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2523 BaseAlignment: Alignment, AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2524
2525 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2526 MMO, AM: MLD->getAddressingMode(), ExtType,
2527 IsExpanding: MLD->isExpandingLoad());
2528
2529 if (HiIsEmpty) {
2530 // The hi masked load has zero storage size. We therefore simply set it to
2531 // the low masked load and rely on subsequent removal from the chain.
2532 Hi = Lo;
2533 } else {
2534 // Generate hi masked load.
2535 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2536 IsCompressedMemory: MLD->isExpandingLoad());
2537
2538 MachinePointerInfo MPI;
2539 if (LoMemVT.isScalableVector())
2540 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2541 else
2542 MPI = MLD->getPointerInfo().getWithOffset(
2543 O: LoMemVT.getStoreSize().getFixedValue());
2544
2545 MMO = DAG.getMachineFunction().getMachineMemOperand(
2546 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2547 AAInfo: MLD->getAAInfo(), Ranges: MLD->getRanges());
2548
2549 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2550 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2551 IsExpanding: MLD->isExpandingLoad());
2552 }
2553
2554 // Build a factor node to remember that this load is independent of the
2555 // other one.
2556 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2557 N2: Hi.getValue(R: 1));
2558
2559 // Legalize the chain result - switch anything that used the old chain to
2560 // use the new one.
2561 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2562
2563}
2564
2565void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2566 SDValue &Hi, bool SplitSETCC) {
2567 EVT LoVT, HiVT;
2568 SDLoc dl(N);
2569 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2570
2571 SDValue Ch = N->getChain();
2572 SDValue Ptr = N->getBasePtr();
2573 struct Operands {
2574 SDValue Mask;
2575 SDValue Index;
2576 SDValue Scale;
2577 } Ops = [&]() -> Operands {
2578 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2579 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2580 }
2581 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2582 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2583 }();
2584
2585 EVT MemoryVT = N->getMemoryVT();
2586 Align Alignment = N->getBaseAlign();
2587
2588 // Split Mask operand
2589 SDValue MaskLo, MaskHi;
2590 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2591 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2592 } else {
2593 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2594 }
2595
2596 EVT LoMemVT, HiMemVT;
2597 // Split MemoryVT
2598 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2599
2600 SDValue IndexHi, IndexLo;
2601 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2602 TargetLowering::TypeSplitVector)
2603 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2604 else
2605 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2606
2607 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2608 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2609 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2610 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
2611
2612 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2613 SDValue PassThru = MGT->getPassThru();
2614 SDValue PassThruLo, PassThruHi;
2615 if (getTypeAction(VT: PassThru.getValueType()) ==
2616 TargetLowering::TypeSplitVector)
2617 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2618 else
2619 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2620
2621 ISD::LoadExtType ExtType = MGT->getExtensionType();
2622 ISD::MemIndexType IndexTy = MGT->getIndexType();
2623
2624 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2625 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2626 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2627
2628 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2629 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2630 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2631 } else {
2632 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2633 SDValue EVLLo, EVLHi;
2634 std::tie(args&: EVLLo, args&: EVLHi) =
2635 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2636
2637 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2638 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2639 MMO, IndexType: VPGT->getIndexType());
2640
2641 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2642 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2643 MMO, IndexType: VPGT->getIndexType());
2644 }
2645
2646 // Build a factor node to remember that this load is independent of the
2647 // other one.
2648 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2649 N2: Hi.getValue(R: 1));
2650
2651 // Legalize the chain result - switch anything that used the old chain to
2652 // use the new one.
2653 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2654}
2655
2656void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2657 SDValue &Hi) {
2658 // This is not "trivial", as there is a dependency between the two subvectors.
2659 // Depending on the number of 1s in the mask, the elements from the Hi vector
2660 // need to be moved to the Lo vector. Passthru values make this even harder.
2661 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2662 // smaller types and passthru is undef, as it is most likely faster than the
2663 // fully expand path. Otherwise, just do the full expansion as one "big"
2664 // operation and then extract the Lo and Hi vectors from that. This gets
2665 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2666 SDLoc DL(N);
2667 EVT VecVT = N->getValueType(ResNo: 0);
2668
2669 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2670 bool HasCustomLowering = false;
2671 EVT CheckVT = LoVT;
2672 while (CheckVT.getVectorMinNumElements() > 1) {
2673 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2674 // custom lowering for illegal types. So we do the checks separately.
2675 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2676 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2677 HasCustomLowering = true;
2678 break;
2679 }
2680 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2681 }
2682
2683 SDValue Passthru = N->getOperand(Num: 2);
2684 if (!HasCustomLowering) {
2685 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2686 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2687 return;
2688 }
2689
2690 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2691 SDValue Mask = N->getOperand(Num: 1);
2692 SDValue LoMask, HiMask;
2693 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2694 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2695
2696 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2697 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2698 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2699
2700 SDValue StackPtr = DAG.CreateStackTemporary(
2701 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2702 MachineFunction &MF = DAG.getMachineFunction();
2703 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2704 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2705
2706 EVT MaskVT = LoMask.getValueType();
2707 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2708
2709 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2710 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2711 EC: MaskVT.getVectorElementCount());
2712 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2713 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2714 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2715
2716 SDValue Chain = DAG.getEntryNode();
2717 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2718 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2719 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2720
2721 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2722 if (!Passthru.isUndef()) {
2723 Compressed =
2724 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2725 }
2726 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2727}
2728
2729void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2730 assert(N->getValueType(0).isVector() &&
2731 N->getOperand(0).getValueType().isVector() &&
2732 "Operand types must be vectors");
2733
2734 EVT LoVT, HiVT;
2735 SDLoc DL(N);
2736 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2737
2738 // If the input also splits, handle it directly. Otherwise split it by hand.
2739 SDValue LL, LH, RL, RH;
2740 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2741 TargetLowering::TypeSplitVector)
2742 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2743 else
2744 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2745
2746 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2747 TargetLowering::TypeSplitVector)
2748 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2749 else
2750 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2751
2752 if (N->getOpcode() == ISD::SETCC) {
2753 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2754 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2755 } else {
2756 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2757 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2758 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
2759 std::tie(args&: EVLLo, args&: EVLHi) =
2760 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
2761 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2), N4: MaskLo,
2762 N5: EVLLo);
2763 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2), N4: MaskHi,
2764 N5: EVLHi);
2765 }
2766}
2767
2768void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2769 SDValue &Hi) {
2770 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2771 EVT LoVT, HiVT;
2772 SDLoc dl(N);
2773 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2774
2775 // If the input also splits, handle it directly for a compile time speedup.
2776 // Otherwise split it by hand.
2777 EVT InVT = N->getOperand(Num: 0).getValueType();
2778 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2779 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2780 else
2781 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2782
2783 const SDNodeFlags Flags = N->getFlags();
2784 unsigned Opcode = N->getOpcode();
2785 if (N->getNumOperands() <= 2) {
2786 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass) {
2787 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2788 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2789 } else {
2790 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2791 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2792 }
2793 return;
2794 }
2795
2796 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2797 assert(N->isVPOpcode() && "Expected VP opcode");
2798
2799 SDValue MaskLo, MaskHi;
2800 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2801
2802 SDValue EVLLo, EVLHi;
2803 std::tie(args&: EVLLo, args&: EVLHi) =
2804 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2805
2806 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo}, Flags);
2807 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi}, Flags);
2808}
2809
2810void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2811 SDValue &Hi) {
2812 SDLoc dl(N);
2813 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2814
2815 // If the input also splits, handle it directly for a compile time speedup.
2816 // Otherwise split it by hand.
2817 EVT InVT = N->getOperand(Num: 0).getValueType();
2818 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2819 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2820 else
2821 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2822
2823 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
2824 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2825 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2826 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
2827 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
2828}
2829
2830void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
2831 unsigned ResNo,
2832 SDValue &Lo,
2833 SDValue &Hi) {
2834 SDLoc dl(N);
2835 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2836 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
2837
2838 // If the input also splits, handle it directly for a compile time speedup.
2839 // Otherwise split it by hand.
2840 EVT InVT = N->getOperand(Num: 0).getValueType();
2841 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2842 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2843 else
2844 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2845
2846 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
2847 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
2848
2849 SDNode *HiNode = Hi.getNode();
2850 SDNode *LoNode = Lo.getNode();
2851
2852 // Replace the other vector result not being explicitly split here.
2853 unsigned OtherNo = 1 - ResNo;
2854 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2855 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2856 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
2857 Hi: SDValue(HiNode, OtherNo));
2858 } else {
2859 SDValue OtherVal =
2860 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
2861 N2: SDValue(HiNode, OtherNo));
2862 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2863 }
2864}
2865
2866void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2867 SDValue &Hi) {
2868 SDLoc dl(N);
2869 EVT SrcVT = N->getOperand(Num: 0).getValueType();
2870 EVT DestVT = N->getValueType(ResNo: 0);
2871 EVT LoVT, HiVT;
2872 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
2873
2874 // We can do better than a generic split operation if the extend is doing
2875 // more than just doubling the width of the elements and the following are
2876 // true:
2877 // - The number of vector elements is even,
2878 // - the source type is legal,
2879 // - the type of a split source is illegal,
2880 // - the type of an extended (by doubling element size) source is legal, and
2881 // - the type of that extended source when split is legal.
2882 //
2883 // This won't necessarily completely legalize the operation, but it will
2884 // more effectively move in the right direction and prevent falling down
2885 // to scalarization in many cases due to the input vector being split too
2886 // far.
2887 if (SrcVT.getVectorElementCount().isKnownEven() &&
2888 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2889 LLVMContext &Ctx = *DAG.getContext();
2890 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
2891 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
2892
2893 EVT SplitLoVT, SplitHiVT;
2894 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
2895 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
2896 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
2897 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2898 N->dump(&DAG); dbgs() << "\n");
2899 if (!N->isVPOpcode()) {
2900 // Extend the source vector by one step.
2901 SDValue NewSrc =
2902 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
2903 // Get the low and high halves of the new, extended one step, vector.
2904 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2905 // Extend those vector halves the rest of the way.
2906 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
2907 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
2908 return;
2909 }
2910
2911 // Extend the source vector by one step.
2912 SDValue NewSrc =
2913 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, N1: N->getOperand(Num: 0),
2914 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2915 // Get the low and high halves of the new, extended one step, vector.
2916 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
2917
2918 SDValue MaskLo, MaskHi;
2919 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
2920
2921 SDValue EVLLo, EVLHi;
2922 std::tie(args&: EVLLo, args&: EVLHi) =
2923 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
2924 // Extend those vector halves the rest of the way.
2925 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Ops: {Lo, MaskLo, EVLLo});
2926 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Ops: {Hi, MaskHi, EVLHi});
2927 return;
2928 }
2929 }
2930 // Fall back to the generic unary operator splitting otherwise.
2931 SplitVecRes_UnaryOp(N, Lo, Hi);
2932}
2933
2934void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2935 SDValue &Lo, SDValue &Hi) {
2936 // The low and high parts of the original input give four input vectors.
2937 SDValue Inputs[4];
2938 SDLoc DL(N);
2939 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
2940 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
2941 EVT NewVT = Inputs[0].getValueType();
2942 unsigned NewElts = NewVT.getVectorNumElements();
2943
2944 auto &&IsConstant = [](const SDValue &N) {
2945 APInt SplatValue;
2946 return N.getResNo() == 0 &&
2947 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
2948 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
2949 };
2950 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2951 SDValue &Input2,
2952 ArrayRef<int> Mask) {
2953 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2954 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2955 "Expected build vector node.");
2956 EVT EltVT = NewVT.getVectorElementType();
2957 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
2958 for (unsigned I = 0; I < NewElts; ++I) {
2959 if (Mask[I] == PoisonMaskElem)
2960 continue;
2961 unsigned Idx = Mask[I];
2962 if (Idx >= NewElts)
2963 Ops[I] = Input2.getOperand(i: Idx - NewElts);
2964 else
2965 Ops[I] = Input1.getOperand(i: Idx);
2966 // Make the type of all elements the same as the element type.
2967 if (Ops[I].getValueType().bitsGT(VT: EltVT))
2968 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
2969 }
2970 return DAG.getBuildVector(VT: NewVT, DL, Ops);
2971 };
2972
2973 // If Lo or Hi uses elements from at most two of the four input vectors, then
2974 // express it as a vector shuffle of those two inputs. Otherwise extract the
2975 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2976 SmallVector<int> OrigMask(N->getMask());
2977 // Try to pack incoming shuffles/inputs.
2978 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2979 &DL](SmallVectorImpl<int> &Mask) {
2980 // Check if all inputs are shuffles of the same operands or non-shuffles.
2981 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2982 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2983 SDValue Input = Inputs[Idx];
2984 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
2985 if (!Shuffle ||
2986 Input.getOperand(i: 0).getValueType() != Input.getValueType())
2987 continue;
2988 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
2989 .push_back(Elt: Idx);
2990 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
2991 .push_back(Elt: Idx);
2992 }
2993 for (auto &P : ShufflesIdxs) {
2994 if (P.second.size() < 2)
2995 continue;
2996 // Use shuffles operands instead of shuffles themselves.
2997 // 1. Adjust mask.
2998 for (int &Idx : Mask) {
2999 if (Idx == PoisonMaskElem)
3000 continue;
3001 unsigned SrcRegIdx = Idx / NewElts;
3002 if (Inputs[SrcRegIdx].isUndef()) {
3003 Idx = PoisonMaskElem;
3004 continue;
3005 }
3006 auto *Shuffle =
3007 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3008 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3009 continue;
3010 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3011 if (MaskElt == PoisonMaskElem) {
3012 Idx = PoisonMaskElem;
3013 continue;
3014 }
3015 Idx = MaskElt % NewElts +
3016 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3017 ? 0
3018 : 1] *
3019 NewElts;
3020 }
3021 // 2. Update inputs.
3022 Inputs[P.second[0]] = P.first.first;
3023 Inputs[P.second[1]] = P.first.second;
3024 // Clear the pair data.
3025 P.second.clear();
3026 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3027 }
3028 // Check if any concat_vectors can be simplified.
3029 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3030 for (int &Idx : Mask) {
3031 if (Idx == PoisonMaskElem)
3032 continue;
3033 unsigned SrcRegIdx = Idx / NewElts;
3034 if (Inputs[SrcRegIdx].isUndef()) {
3035 Idx = PoisonMaskElem;
3036 continue;
3037 }
3038 TargetLowering::LegalizeTypeAction TypeAction =
3039 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3040 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3041 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3042 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3043 (TypeAction == TargetLowering::TypeLegal ||
3044 TypeAction == TargetLowering::TypeWidenVector))
3045 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3046 }
3047 if (UsedSubVector.count() > 1) {
3048 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3049 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3050 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3051 continue;
3052 if (Pairs.empty() || Pairs.back().size() == 2)
3053 Pairs.emplace_back();
3054 if (UsedSubVector.test(Idx: 2 * I)) {
3055 Pairs.back().emplace_back(Args&: I, Args: 0);
3056 } else {
3057 assert(UsedSubVector.test(2 * I + 1) &&
3058 "Expected to be used one of the subvectors.");
3059 Pairs.back().emplace_back(Args&: I, Args: 1);
3060 }
3061 }
3062 if (!Pairs.empty() && Pairs.front().size() > 1) {
3063 // Adjust mask.
3064 for (int &Idx : Mask) {
3065 if (Idx == PoisonMaskElem)
3066 continue;
3067 unsigned SrcRegIdx = Idx / NewElts;
3068 auto *It = find_if(
3069 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3070 return Idxs.front().first == SrcRegIdx ||
3071 Idxs.back().first == SrcRegIdx;
3072 });
3073 if (It == Pairs.end())
3074 continue;
3075 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3076 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3077 }
3078 // Adjust inputs.
3079 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3080 Inputs[Idxs.front().first] = DAG.getNode(
3081 Opcode: ISD::CONCAT_VECTORS, DL,
3082 VT: Inputs[Idxs.front().first].getValueType(),
3083 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3084 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3085 }
3086 }
3087 }
3088 bool Changed;
3089 do {
3090 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3091 // reused operands.
3092 Changed = false;
3093 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3094 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3095 if (!Shuffle)
3096 continue;
3097 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3098 continue;
3099 int Op = -1;
3100 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3101 !Shuffle->isSplat()) {
3102 Op = 0;
3103 } else if (!Inputs[I].hasOneUse() &&
3104 !Shuffle->getOperand(Num: 1).isUndef()) {
3105 // Find the only used operand, if possible.
3106 for (int &Idx : Mask) {
3107 if (Idx == PoisonMaskElem)
3108 continue;
3109 unsigned SrcRegIdx = Idx / NewElts;
3110 if (SrcRegIdx != I)
3111 continue;
3112 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3113 if (MaskElt == PoisonMaskElem) {
3114 Idx = PoisonMaskElem;
3115 continue;
3116 }
3117 int OpIdx = MaskElt / NewElts;
3118 if (Op == -1) {
3119 Op = OpIdx;
3120 continue;
3121 }
3122 if (Op != OpIdx) {
3123 Op = -1;
3124 break;
3125 }
3126 }
3127 }
3128 if (Op < 0) {
3129 // Try to check if one of the shuffle operands is used already.
3130 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3131 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3132 continue;
3133 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3134 if (It == std::end(arr&: Inputs))
3135 continue;
3136 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3137 // Found that operand is used already.
3138 // 1. Fix the mask for the reused operand.
3139 for (int &Idx : Mask) {
3140 if (Idx == PoisonMaskElem)
3141 continue;
3142 unsigned SrcRegIdx = Idx / NewElts;
3143 if (SrcRegIdx != I)
3144 continue;
3145 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3146 if (MaskElt == PoisonMaskElem) {
3147 Idx = PoisonMaskElem;
3148 continue;
3149 }
3150 int MaskIdx = MaskElt / NewElts;
3151 if (OpIdx == MaskIdx)
3152 Idx = MaskElt % NewElts + FoundOp * NewElts;
3153 }
3154 // 2. Set Op to the unused OpIdx.
3155 Op = (OpIdx + 1) % 2;
3156 break;
3157 }
3158 }
3159 if (Op >= 0) {
3160 Changed = true;
3161 Inputs[I] = Shuffle->getOperand(Num: Op);
3162 // Adjust mask.
3163 for (int &Idx : Mask) {
3164 if (Idx == PoisonMaskElem)
3165 continue;
3166 unsigned SrcRegIdx = Idx / NewElts;
3167 if (SrcRegIdx != I)
3168 continue;
3169 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3170 int OpIdx = MaskElt / NewElts;
3171 if (OpIdx != Op)
3172 continue;
3173 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3174 }
3175 }
3176 }
3177 } while (Changed);
3178 };
3179 TryPeekThroughShufflesInputs(OrigMask);
3180 // Proces unique inputs.
3181 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3182 NewElts](SmallVectorImpl<int> &Mask) {
3183 SetVector<SDValue> UniqueInputs;
3184 SetVector<SDValue> UniqueConstantInputs;
3185 for (const auto &I : Inputs) {
3186 if (IsConstant(I))
3187 UniqueConstantInputs.insert(X: I);
3188 else if (!I.isUndef())
3189 UniqueInputs.insert(X: I);
3190 }
3191 // Adjust mask in case of reused inputs. Also, need to insert constant
3192 // inputs at first, otherwise it affects the final outcome.
3193 if (UniqueInputs.size() != std::size(Inputs)) {
3194 auto &&UniqueVec = UniqueInputs.takeVector();
3195 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3196 unsigned ConstNum = UniqueConstantVec.size();
3197 for (int &Idx : Mask) {
3198 if (Idx == PoisonMaskElem)
3199 continue;
3200 unsigned SrcRegIdx = Idx / NewElts;
3201 if (Inputs[SrcRegIdx].isUndef()) {
3202 Idx = PoisonMaskElem;
3203 continue;
3204 }
3205 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3206 if (It != UniqueConstantVec.end()) {
3207 Idx = (Idx % NewElts) +
3208 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3209 assert(Idx >= 0 && "Expected defined mask idx.");
3210 continue;
3211 }
3212 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3213 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3214 Idx = (Idx % NewElts) +
3215 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3216 assert(Idx >= 0 && "Expected defined mask idx.");
3217 }
3218 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3219 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3220 }
3221 };
3222 MakeUniqueInputs(OrigMask);
3223 SDValue OrigInputs[4];
3224 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3225 for (unsigned High = 0; High < 2; ++High) {
3226 SDValue &Output = High ? Hi : Lo;
3227
3228 // Build a shuffle mask for the output, discovering on the fly which
3229 // input vectors to use as shuffle operands.
3230 unsigned FirstMaskIdx = High * NewElts;
3231 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3232 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3233 assert(!Output && "Expected default initialized initial value.");
3234 TryPeekThroughShufflesInputs(Mask);
3235 MakeUniqueInputs(Mask);
3236 SDValue TmpInputs[4];
3237 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3238 // Track changes in the output registers.
3239 int UsedIdx = -1;
3240 bool SecondIteration = false;
3241 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3242 if (UsedIdx < 0) {
3243 UsedIdx = Idx;
3244 return false;
3245 }
3246 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3247 SecondIteration = true;
3248 return SecondIteration;
3249 };
3250 processShuffleMasks(
3251 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3252 /*NumOfUsedRegs=*/1,
3253 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3254 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3255 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3256 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3257 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3258 else
3259 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3260 N2: DAG.getPOISON(VT: NewVT), Mask);
3261 Inputs[Idx] = Output;
3262 },
3263 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3264 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3265 unsigned Idx2, bool /*Unused*/) {
3266 if (AccumulateResults(Idx1)) {
3267 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3268 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3269 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3270 else
3271 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3272 N2: Inputs[Idx2], Mask);
3273 } else {
3274 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3275 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3276 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3277 else
3278 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3279 N2: TmpInputs[Idx2], Mask);
3280 }
3281 Inputs[Idx1] = Output;
3282 });
3283 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3284 }
3285}
3286
3287void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3288 EVT OVT = N->getValueType(ResNo: 0);
3289 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3290 SDValue Chain = N->getOperand(Num: 0);
3291 SDValue Ptr = N->getOperand(Num: 1);
3292 SDValue SV = N->getOperand(Num: 2);
3293 SDLoc dl(N);
3294
3295 const Align Alignment =
3296 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3297
3298 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3299 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3300 Chain = Hi.getValue(R: 1);
3301
3302 // Modified the chain - switch anything that used the old chain to use
3303 // the new one.
3304 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3305}
3306
3307void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3308 SDValue &Hi) {
3309 EVT DstVTLo, DstVTHi;
3310 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3311 SDLoc dl(N);
3312
3313 SDValue SrcLo, SrcHi;
3314 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3315 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3316 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3317 else
3318 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3319
3320 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3321 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3322}
3323
3324void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3325 SDValue &Hi) {
3326 SDValue InLo, InHi;
3327 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3328 SDLoc DL(N);
3329
3330 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3331 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3332}
3333
3334void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3335 SDValue &Hi) {
3336 SDLoc DL(N);
3337
3338 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3339 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3340}
3341
3342void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3343 SDValue &Hi) {
3344 EVT VT = N->getValueType(ResNo: 0);
3345 SDValue Val = N->getOperand(Num: 0);
3346 SDValue Mask = N->getOperand(Num: 1);
3347 SDValue EVL = N->getOperand(Num: 2);
3348 SDLoc DL(N);
3349
3350 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3351 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3352
3353 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3354 EC: VT.getVectorElementCount());
3355 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3356 EVT PtrVT = StackPtr.getValueType();
3357 auto &MF = DAG.getMachineFunction();
3358 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3359 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3360
3361 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3362 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3363 BaseAlignment: Alignment);
3364 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3365 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3366 BaseAlignment: Alignment);
3367
3368 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3369 SDValue NumElemMinus1 =
3370 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3371 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3372 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3373 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3374 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3375 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3376
3377 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3378 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3379 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3380 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3381
3382 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3383
3384 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3385}
3386
3387void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3388 SDValue &Hi) {
3389 EVT VT = N->getValueType(ResNo: 0);
3390 SDValue V1 = N->getOperand(Num: 0);
3391 SDValue V2 = N->getOperand(Num: 1);
3392 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3393 SDValue Mask = N->getOperand(Num: 3);
3394 SDValue EVL1 = N->getOperand(Num: 4);
3395 SDValue EVL2 = N->getOperand(Num: 5);
3396 SDLoc DL(N);
3397
3398 // Since EVL2 is considered the real VL it gets promoted during
3399 // SelectionDAGBuilder. Promote EVL1 here if needed.
3400 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3401 EVL1 = ZExtPromotedInteger(Op: EVL1);
3402
3403 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3404
3405 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3406 EC: VT.getVectorElementCount() * 2);
3407 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3408 EVT PtrVT = StackPtr.getValueType();
3409 auto &MF = DAG.getMachineFunction();
3410 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3411 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3412
3413 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3414 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3415 BaseAlignment: Alignment);
3416 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3417 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3418 BaseAlignment: Alignment);
3419
3420 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: EVL1);
3421 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3422
3423 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3424 SDValue StoreV1 =
3425 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3426 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3427
3428 SDValue StoreV2 =
3429 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3430 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3431
3432 SDValue Load;
3433 if (Imm >= 0) {
3434 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3435 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3436 } else {
3437 uint64_t TrailingElts = -Imm;
3438 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3439 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3440
3441 // Make sure TrailingBytes doesn't exceed the size of vec1.
3442 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3443 TrailingBytes =
3444 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3445
3446 // Calculate the start address of the spliced result.
3447 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3448 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3449 }
3450
3451 EVT LoVT, HiVT;
3452 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
3453 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3454 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3455 Hi =
3456 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3457 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3458}
3459
3460void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3461 SDValue &Hi) {
3462 SDLoc DL(N);
3463 SDValue Acc = N->getOperand(Num: 0);
3464 SDValue Input1 = N->getOperand(Num: 1);
3465 SDValue Input2 = N->getOperand(Num: 2);
3466
3467 SDValue AccLo, AccHi;
3468 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3469 unsigned Opcode = N->getOpcode();
3470
3471 // If the input types don't need splitting, just accumulate into the
3472 // low part of the accumulator.
3473 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3474 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3475 Hi = AccHi;
3476 return;
3477 }
3478
3479 SDValue Input1Lo, Input1Hi;
3480 SDValue Input2Lo, Input2Hi;
3481 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3482 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3483 EVT ResultVT = AccLo.getValueType();
3484
3485 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3486 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3487}
3488
3489void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3490 SDValue &Hi) {
3491 SDLoc DL(N);
3492 SDValue Op0 = N->getOperand(Num: 0);
3493 SDValue Op1 = N->getOperand(Num: 1);
3494 EVT OpVT = Op0.getValueType();
3495
3496 EVT LoVT, HiVT;
3497 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3498
3499 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3500 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3501 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3502 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3503}
3504
3505void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3506 unsigned Factor = N->getNumOperands();
3507
3508 SmallVector<SDValue, 8> Ops(Factor * 2);
3509 for (unsigned i = 0; i != Factor; ++i) {
3510 SDValue OpLo, OpHi;
3511 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3512 Ops[i * 2] = OpLo;
3513 Ops[i * 2 + 1] = OpHi;
3514 }
3515
3516 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3517
3518 SDLoc DL(N);
3519 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3520 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3521 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3522 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3523
3524 for (unsigned i = 0; i != Factor; ++i)
3525 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3526}
3527
3528void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3529 unsigned Factor = N->getNumOperands();
3530
3531 SmallVector<SDValue, 8> Ops(Factor * 2);
3532 for (unsigned i = 0; i != Factor; ++i) {
3533 SDValue OpLo, OpHi;
3534 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3535 Ops[i] = OpLo;
3536 Ops[i + Factor] = OpHi;
3537 }
3538
3539 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3540
3541 SDLoc DL(N);
3542 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3543 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3544 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3545 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3546
3547 for (unsigned i = 0; i != Factor; ++i) {
3548 unsigned IdxLo = 2 * i;
3549 unsigned IdxHi = 2 * i + 1;
3550 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3551 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3552 }
3553}
3554
3555//===----------------------------------------------------------------------===//
3556// Operand Vector Splitting
3557//===----------------------------------------------------------------------===//
3558
3559/// This method is called when the specified operand of the specified node is
3560/// found to need vector splitting. At this point, all of the result types of
3561/// the node are known to be legal, but other operands of the node may need
3562/// legalization as well as the specified one.
3563bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3564 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3565 SDValue Res = SDValue();
3566
3567 // See if the target wants to custom split this node.
3568 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3569 return false;
3570
3571 switch (N->getOpcode()) {
3572 default:
3573#ifndef NDEBUG
3574 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3575 N->dump(&DAG);
3576 dbgs() << "\n";
3577#endif
3578 report_fatal_error(reason: "Do not know how to split this operator's "
3579 "operand!\n");
3580
3581 case ISD::VP_SETCC:
3582 case ISD::STRICT_FSETCC:
3583 case ISD::STRICT_FSETCCS:
3584 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3585 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3586 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3587 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3588 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3589 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3590 case ISD::VECTOR_FIND_LAST_ACTIVE:
3591 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3592 break;
3593 case ISD::VP_TRUNCATE:
3594 case ISD::TRUNCATE:
3595 Res = SplitVecOp_TruncateHelper(N);
3596 break;
3597 case ISD::STRICT_FP_ROUND:
3598 case ISD::VP_FP_ROUND:
3599 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
3600 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3601 case ISD::STORE:
3602 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3603 break;
3604 case ISD::VP_STORE:
3605 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3606 break;
3607 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3608 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3609 break;
3610 case ISD::MSTORE:
3611 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3612 break;
3613 case ISD::MSCATTER:
3614 case ISD::VP_SCATTER:
3615 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3616 break;
3617 case ISD::MGATHER:
3618 case ISD::VP_GATHER:
3619 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3620 break;
3621 case ISD::VSELECT:
3622 Res = SplitVecOp_VSELECT(N, OpNo);
3623 break;
3624 case ISD::VECTOR_COMPRESS:
3625 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3626 break;
3627 case ISD::STRICT_SINT_TO_FP:
3628 case ISD::STRICT_UINT_TO_FP:
3629 case ISD::SINT_TO_FP:
3630 case ISD::UINT_TO_FP:
3631 case ISD::VP_SINT_TO_FP:
3632 case ISD::VP_UINT_TO_FP:
3633 if (N->getValueType(ResNo: 0).bitsLT(
3634 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3635 Res = SplitVecOp_TruncateHelper(N);
3636 else
3637 Res = SplitVecOp_UnaryOp(N);
3638 break;
3639 case ISD::FP_TO_SINT_SAT:
3640 case ISD::FP_TO_UINT_SAT:
3641 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3642 break;
3643 case ISD::FP_TO_SINT:
3644 case ISD::FP_TO_UINT:
3645 case ISD::VP_FP_TO_SINT:
3646 case ISD::VP_FP_TO_UINT:
3647 case ISD::STRICT_FP_TO_SINT:
3648 case ISD::STRICT_FP_TO_UINT:
3649 case ISD::STRICT_FP_EXTEND:
3650 case ISD::FP_EXTEND:
3651 case ISD::SIGN_EXTEND:
3652 case ISD::ZERO_EXTEND:
3653 case ISD::ANY_EXTEND:
3654 case ISD::FTRUNC:
3655 case ISD::LROUND:
3656 case ISD::LLROUND:
3657 case ISD::LRINT:
3658 case ISD::LLRINT:
3659 Res = SplitVecOp_UnaryOp(N);
3660 break;
3661 case ISD::FLDEXP:
3662 Res = SplitVecOp_FPOpDifferentTypes(N);
3663 break;
3664
3665 case ISD::SCMP:
3666 case ISD::UCMP:
3667 Res = SplitVecOp_CMP(N);
3668 break;
3669
3670 case ISD::FAKE_USE:
3671 Res = SplitVecOp_FAKE_USE(N);
3672 break;
3673 case ISD::ANY_EXTEND_VECTOR_INREG:
3674 case ISD::SIGN_EXTEND_VECTOR_INREG:
3675 case ISD::ZERO_EXTEND_VECTOR_INREG:
3676 Res = SplitVecOp_ExtVecInRegOp(N);
3677 break;
3678
3679 case ISD::VECREDUCE_FADD:
3680 case ISD::VECREDUCE_FMUL:
3681 case ISD::VECREDUCE_ADD:
3682 case ISD::VECREDUCE_MUL:
3683 case ISD::VECREDUCE_AND:
3684 case ISD::VECREDUCE_OR:
3685 case ISD::VECREDUCE_XOR:
3686 case ISD::VECREDUCE_SMAX:
3687 case ISD::VECREDUCE_SMIN:
3688 case ISD::VECREDUCE_UMAX:
3689 case ISD::VECREDUCE_UMIN:
3690 case ISD::VECREDUCE_FMAX:
3691 case ISD::VECREDUCE_FMIN:
3692 case ISD::VECREDUCE_FMAXIMUM:
3693 case ISD::VECREDUCE_FMINIMUM:
3694 Res = SplitVecOp_VECREDUCE(N, OpNo);
3695 break;
3696 case ISD::VECREDUCE_SEQ_FADD:
3697 case ISD::VECREDUCE_SEQ_FMUL:
3698 Res = SplitVecOp_VECREDUCE_SEQ(N);
3699 break;
3700 case ISD::VP_REDUCE_FADD:
3701 case ISD::VP_REDUCE_SEQ_FADD:
3702 case ISD::VP_REDUCE_FMUL:
3703 case ISD::VP_REDUCE_SEQ_FMUL:
3704 case ISD::VP_REDUCE_ADD:
3705 case ISD::VP_REDUCE_MUL:
3706 case ISD::VP_REDUCE_AND:
3707 case ISD::VP_REDUCE_OR:
3708 case ISD::VP_REDUCE_XOR:
3709 case ISD::VP_REDUCE_SMAX:
3710 case ISD::VP_REDUCE_SMIN:
3711 case ISD::VP_REDUCE_UMAX:
3712 case ISD::VP_REDUCE_UMIN:
3713 case ISD::VP_REDUCE_FMAX:
3714 case ISD::VP_REDUCE_FMIN:
3715 case ISD::VP_REDUCE_FMAXIMUM:
3716 case ISD::VP_REDUCE_FMINIMUM:
3717 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3718 break;
3719 case ISD::VP_CTTZ_ELTS:
3720 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
3721 Res = SplitVecOp_VP_CttzElements(N);
3722 break;
3723 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3724 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3725 break;
3726 case ISD::PARTIAL_REDUCE_UMLA:
3727 case ISD::PARTIAL_REDUCE_SMLA:
3728 case ISD::PARTIAL_REDUCE_SUMLA:
3729 case ISD::PARTIAL_REDUCE_FMLA:
3730 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3731 break;
3732 }
3733
3734 // If the result is null, the sub-method took care of registering results etc.
3735 if (!Res.getNode()) return false;
3736
3737 // If the result is N, the sub-method updated N in place. Tell the legalizer
3738 // core about this.
3739 if (Res.getNode() == N)
3740 return true;
3741
3742 if (N->isStrictFPOpcode())
3743 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3744 "Invalid operand expansion");
3745 else
3746 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3747 "Invalid operand expansion");
3748
3749 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3750 return false;
3751}
3752
3753SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3754 SDLoc DL(N);
3755
3756 SDValue LoMask, HiMask;
3757 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
3758
3759 EVT VT = N->getValueType(ResNo: 0);
3760 EVT SplitVT = LoMask.getValueType();
3761 ElementCount SplitEC = SplitVT.getVectorElementCount();
3762
3763 // Find the last active in both the low and the high masks.
3764 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
3765 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
3766
3767 // Check if any lane is active in the high mask.
3768 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
3769 // sentinel value for "none active".
3770 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
3771 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
3772 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
3773
3774 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
3775 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
3776 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
3777 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
3778 N3: LoFind);
3779}
3780
3781SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3782 // The only possibility for an illegal operand is the mask, since result type
3783 // legalization would have handled this node already otherwise.
3784 assert(OpNo == 0 && "Illegal operand must be mask");
3785
3786 SDValue Mask = N->getOperand(Num: 0);
3787 SDValue Src0 = N->getOperand(Num: 1);
3788 SDValue Src1 = N->getOperand(Num: 2);
3789 EVT Src0VT = Src0.getValueType();
3790 SDLoc DL(N);
3791 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3792
3793 SDValue Lo, Hi;
3794 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3795 assert(Lo.getValueType() == Hi.getValueType() &&
3796 "Lo and Hi have differing types");
3797
3798 EVT LoOpVT, HiOpVT;
3799 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
3800 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3801
3802 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3803 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
3804 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
3805 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
3806
3807 SDValue LoSelect =
3808 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
3809 SDValue HiSelect =
3810 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
3811
3812 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
3813}
3814
3815SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
3816 // The only possibility for an illegal operand is the mask, since result type
3817 // legalization would have handled this node already otherwise.
3818 assert(OpNo == 1 && "Illegal operand must be mask");
3819
3820 // To split the mask, we need to split the result type too, so we can just
3821 // reuse that logic here.
3822 SDValue Lo, Hi;
3823 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
3824
3825 EVT VecVT = N->getValueType(ResNo: 0);
3826 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
3827}
3828
3829SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3830 EVT ResVT = N->getValueType(ResNo: 0);
3831 SDValue Lo, Hi;
3832 SDLoc dl(N);
3833
3834 SDValue VecOp = N->getOperand(Num: OpNo);
3835 EVT VecVT = VecOp.getValueType();
3836 assert(VecVT.isVector() && "Can only split reduce vector operand");
3837 GetSplitVector(Op: VecOp, Lo, Hi);
3838 EVT LoOpVT, HiOpVT;
3839 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3840
3841 // Use the appropriate scalar instruction on the split subvectors before
3842 // reducing the now partially reduced smaller vector.
3843 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
3844 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
3845 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
3846}
3847
3848SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3849 EVT ResVT = N->getValueType(ResNo: 0);
3850 SDValue Lo, Hi;
3851 SDLoc dl(N);
3852
3853 SDValue AccOp = N->getOperand(Num: 0);
3854 SDValue VecOp = N->getOperand(Num: 1);
3855 SDNodeFlags Flags = N->getFlags();
3856
3857 EVT VecVT = VecOp.getValueType();
3858 assert(VecVT.isVector() && "Can only split reduce vector operand");
3859 GetSplitVector(Op: VecOp, Lo, Hi);
3860 EVT LoOpVT, HiOpVT;
3861 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
3862
3863 // Reduce low half.
3864 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
3865
3866 // Reduce high half, using low half result as initial value.
3867 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
3868}
3869
3870SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3871 assert(N->isVPOpcode() && "Expected VP opcode");
3872 assert(OpNo == 1 && "Can only split reduce vector operand");
3873
3874 unsigned Opc = N->getOpcode();
3875 EVT ResVT = N->getValueType(ResNo: 0);
3876 SDValue Lo, Hi;
3877 SDLoc dl(N);
3878
3879 SDValue VecOp = N->getOperand(Num: OpNo);
3880 EVT VecVT = VecOp.getValueType();
3881 assert(VecVT.isVector() && "Can only split reduce vector operand");
3882 GetSplitVector(Op: VecOp, Lo, Hi);
3883
3884 SDValue MaskLo, MaskHi;
3885 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
3886
3887 SDValue EVLLo, EVLHi;
3888 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
3889
3890 const SDNodeFlags Flags = N->getFlags();
3891
3892 SDValue ResLo =
3893 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
3894 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
3895}
3896
3897SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3898 // The result has a legal vector type, but the input needs splitting.
3899 EVT ResVT = N->getValueType(ResNo: 0);
3900 SDValue Lo, Hi;
3901 SDLoc dl(N);
3902 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3903 EVT InVT = Lo.getValueType();
3904
3905 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
3906 EC: InVT.getVectorElementCount());
3907
3908 if (N->isStrictFPOpcode()) {
3909 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3910 Ops: {N->getOperand(Num: 0), Lo});
3911 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
3912 Ops: {N->getOperand(Num: 0), Hi});
3913
3914 // Build a factor node to remember that this operation is independent
3915 // of the other one.
3916 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
3917 N2: Hi.getValue(R: 1));
3918
3919 // Legalize the chain result - switch anything that used the old chain to
3920 // use the new one.
3921 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3922 } else if (N->getNumOperands() == 3) {
3923 assert(N->isVPOpcode() && "Expected VP opcode");
3924 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3925 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
3926 std::tie(args&: EVLLo, args&: EVLHi) =
3927 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
3928 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
3929 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
3930 } else {
3931 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
3932 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
3933 }
3934
3935 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3936}
3937
3938// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
3939SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
3940 SDValue Lo, Hi;
3941 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
3942 SDValue Chain =
3943 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
3944 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
3945}
3946
3947SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3948 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3949 // end up being split all the way down to individual components. Convert the
3950 // split pieces into integers and reassemble.
3951 EVT ResVT = N->getValueType(ResNo: 0);
3952 SDValue Lo, Hi;
3953 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3954 SDLoc dl(N);
3955
3956 if (ResVT.isScalableVector()) {
3957 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
3958 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
3959 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
3960 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
3961 }
3962
3963 Lo = BitConvertToInteger(Op: Lo);
3964 Hi = BitConvertToInteger(Op: Hi);
3965
3966 if (DAG.getDataLayout().isBigEndian())
3967 std::swap(a&: Lo, b&: Hi);
3968
3969 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
3970}
3971
3972SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3973 unsigned OpNo) {
3974 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3975 // We know that the result type is legal.
3976 EVT ResVT = N->getValueType(ResNo: 0);
3977
3978 SDValue Vec = N->getOperand(Num: 0);
3979 SDValue SubVec = N->getOperand(Num: 1);
3980 SDValue Idx = N->getOperand(Num: 2);
3981 SDLoc dl(N);
3982
3983 SDValue Lo, Hi;
3984 GetSplitVector(Op: SubVec, Lo, Hi);
3985
3986 uint64_t IdxVal = Idx->getAsZExtVal();
3987 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3988
3989 SDValue FirstInsertion =
3990 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
3991 SDValue SecondInsertion =
3992 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
3993 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
3994
3995 return SecondInsertion;
3996}
3997
3998SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3999 // We know that the extracted result type is legal.
4000 EVT SubVT = N->getValueType(ResNo: 0);
4001 SDValue Idx = N->getOperand(Num: 1);
4002 SDLoc dl(N);
4003 SDValue Lo, Hi;
4004
4005 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4006
4007 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4008 // Note: For scalable vectors, the index is scaled by vscale.
4009 ElementCount IdxVal =
4010 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4011 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4012
4013 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4014 ElementCount NumResultElts = SubVT.getVectorElementCount();
4015
4016 // If the extracted elements are all in the low half, do a simple extract.
4017 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4018 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4019
4020 unsigned LoEltsMin = LoElts.getKnownMinValue();
4021 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4022 SrcVT.isFixedLengthVector()) {
4023 // Extracted subvector crosses vector split, so we need to blend the two
4024 // halves.
4025 // TODO: May be able to emit partial extract_subvector.
4026 SmallVector<SDValue, 8> Elts;
4027 Elts.reserve(N: NumResultElts.getFixedValue());
4028
4029 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4030 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4031 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4032 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4033 /*Count=*/LoEltsMin - IdxValMin);
4034 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4035 /*Count=*/SubVT.getVectorNumElements() -
4036 Elts.size());
4037 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4038 }
4039
4040 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4041 ElementCount ExtractIdx = IdxVal - LoElts;
4042 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4043 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4044 Idx: ExtractIdx.getKnownMinValue());
4045
4046 EVT HiVT = Hi.getValueType();
4047 assert(HiVT.isFixedLengthVector() &&
4048 "Only fixed-vector extracts are supported in this case");
4049
4050 // We cannot create an extract_subvector that isn't a multiple of the
4051 // result size, which may go out of bounds for the last elements. Shuffle
4052 // the desired elements down to 0 and do a simple 0 extract.
4053 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4054 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4055 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4056
4057 SDValue Shuffle =
4058 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4059 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4060 }
4061
4062 // After this point the DAG node only permits extracting fixed-width
4063 // subvectors from scalable vectors.
4064 assert(SubVT.isFixedLengthVector() &&
4065 "Extracting scalable subvector from fixed-width unsupported");
4066
4067 // If the element type is i1 and we're not promoting the result, then we may
4068 // end up loading the wrong data since the bits are packed tightly into
4069 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4070 // type at index 4, then we will load a byte starting at index 0.
4071 if (SubVT.getScalarType() == MVT::i1)
4072 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4073 "subvector from a scalable predicate vector");
4074
4075 // Spill the vector to the stack. We should use the alignment for
4076 // the smallest part.
4077 SDValue Vec = N->getOperand(Num: 0);
4078 EVT VecVT = Vec.getValueType();
4079 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4080 SDValue StackPtr =
4081 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4082 auto &MF = DAG.getMachineFunction();
4083 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4084 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4085
4086 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4087 Alignment: SmallestAlign);
4088
4089 // Extract the subvector by loading the correct part.
4090 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4091
4092 return DAG.getLoad(
4093 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4094 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4095}
4096
4097SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4098 SDValue Vec = N->getOperand(Num: 0);
4099 SDValue Idx = N->getOperand(Num: 1);
4100 EVT VecVT = Vec.getValueType();
4101
4102 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4103 uint64_t IdxVal = Index->getZExtValue();
4104
4105 SDValue Lo, Hi;
4106 GetSplitVector(Op: Vec, Lo, Hi);
4107
4108 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4109
4110 if (IdxVal < LoElts)
4111 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4112 else if (!Vec.getValueType().isScalableVector())
4113 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4114 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4115 VT: Idx.getValueType())), 0);
4116 }
4117
4118 // See if the target wants to custom expand this node.
4119 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4120 return SDValue();
4121
4122 // Make the vector elements byte-addressable if they aren't already.
4123 SDLoc dl(N);
4124 EVT EltVT = VecVT.getVectorElementType();
4125 if (!EltVT.isByteSized()) {
4126 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4127 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4128 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4129 SDValue NewExtract =
4130 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4131 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4132 }
4133
4134 // Store the vector to the stack.
4135 // In cases where the vector is illegal it will be broken down into parts
4136 // and stored in parts - we should use the alignment for the smallest part.
4137 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4138 SDValue StackPtr =
4139 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4140 auto &MF = DAG.getMachineFunction();
4141 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4142 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4143 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4144 Alignment: SmallestAlign);
4145
4146 // Load back the required element.
4147 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4148
4149 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4150 // type, leaving the high bits undefined. But it can't truncate.
4151 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4152
4153 return DAG.getExtLoad(
4154 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4155 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4156 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4157}
4158
4159SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4160 SDValue Lo, Hi;
4161
4162 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4163 // splitting the result has the same effect as splitting the input operand.
4164 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4165
4166 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4167}
4168
4169SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4170 (void)OpNo;
4171 SDValue Lo, Hi;
4172 SplitVecRes_Gather(N, Lo, Hi);
4173
4174 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4175 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4176 return SDValue();
4177}
4178
4179SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4180 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4181 SDValue Ch = N->getChain();
4182 SDValue Ptr = N->getBasePtr();
4183 SDValue Offset = N->getOffset();
4184 assert(Offset.isUndef() && "Unexpected VP store offset");
4185 SDValue Mask = N->getMask();
4186 SDValue EVL = N->getVectorLength();
4187 SDValue Data = N->getValue();
4188 Align Alignment = N->getBaseAlign();
4189 SDLoc DL(N);
4190
4191 SDValue DataLo, DataHi;
4192 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4193 // Split Data operand
4194 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4195 else
4196 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4197
4198 // Split Mask operand
4199 SDValue MaskLo, MaskHi;
4200 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4201 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4202 } else {
4203 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4204 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4205 else
4206 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4207 }
4208
4209 EVT MemoryVT = N->getMemoryVT();
4210 EVT LoMemVT, HiMemVT;
4211 bool HiIsEmpty = false;
4212 std::tie(args&: LoMemVT, args&: HiMemVT) =
4213 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4214
4215 // Split EVL
4216 SDValue EVLLo, EVLHi;
4217 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4218
4219 SDValue Lo, Hi;
4220 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4221 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4222 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4223 Ranges: N->getRanges());
4224
4225 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4226 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4227 IsCompressing: N->isCompressingStore());
4228
4229 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4230 if (HiIsEmpty)
4231 return Lo;
4232
4233 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4234 IsCompressedMemory: N->isCompressingStore());
4235
4236 MachinePointerInfo MPI;
4237 if (LoMemVT.isScalableVector()) {
4238 Alignment = commonAlignment(A: Alignment,
4239 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4240 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4241 } else
4242 MPI = N->getPointerInfo().getWithOffset(
4243 O: LoMemVT.getStoreSize().getFixedValue());
4244
4245 MMO = DAG.getMachineFunction().getMachineMemOperand(
4246 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4247 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4248
4249 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4250 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4251 IsCompressing: N->isCompressingStore());
4252
4253 // Build a factor node to remember that this store is independent of the
4254 // other one.
4255 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4256}
4257
4258SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4259 unsigned OpNo) {
4260 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4261 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4262
4263 SDLoc DL(N);
4264
4265 SDValue Data = N->getValue();
4266 SDValue LoData, HiData;
4267 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4268 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4269 else
4270 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4271
4272 EVT LoMemVT, HiMemVT;
4273 bool HiIsEmpty = false;
4274 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4275 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4276
4277 SDValue Mask = N->getMask();
4278 SDValue LoMask, HiMask;
4279 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4280 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4281 else if (getTypeAction(VT: Mask.getValueType()) ==
4282 TargetLowering::TypeSplitVector)
4283 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4284 else
4285 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4286
4287 SDValue LoEVL, HiEVL;
4288 std::tie(args&: LoEVL, args&: HiEVL) =
4289 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4290
4291 // Generate the low vp_strided_store
4292 SDValue Lo = DAG.getStridedStoreVP(
4293 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4294 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4295 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4296
4297 // If the high vp_strided_store has zero storage size, only the low
4298 // vp_strided_store is needed.
4299 if (HiIsEmpty)
4300 return Lo;
4301
4302 // Generate the high vp_strided_store.
4303 // To calculate the high base address, we need to sum to the low base
4304 // address stride number of bytes for each element already stored by low,
4305 // that is: Ptr = Ptr + (LoEVL * Stride)
4306 EVT PtrVT = N->getBasePtr().getValueType();
4307 SDValue Increment =
4308 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4309 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4310 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4311
4312 Align Alignment = N->getBaseAlign();
4313 if (LoMemVT.isScalableVector())
4314 Alignment = commonAlignment(A: Alignment,
4315 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4316
4317 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4318 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4319 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4320 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4321
4322 SDValue Hi = DAG.getStridedStoreVP(
4323 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4324 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4325 IsCompressing: N->isCompressingStore());
4326
4327 // Build a factor node to remember that this store is independent of the
4328 // other one.
4329 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4330}
4331
4332SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4333 unsigned OpNo) {
4334 assert(N->isUnindexed() && "Indexed masked store of vector?");
4335 SDValue Ch = N->getChain();
4336 SDValue Ptr = N->getBasePtr();
4337 SDValue Offset = N->getOffset();
4338 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4339 SDValue Mask = N->getMask();
4340 SDValue Data = N->getValue();
4341 Align Alignment = N->getBaseAlign();
4342 SDLoc DL(N);
4343
4344 SDValue DataLo, DataHi;
4345 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4346 // Split Data operand
4347 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4348 else
4349 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4350
4351 // Split Mask operand
4352 SDValue MaskLo, MaskHi;
4353 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4354 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4355 } else {
4356 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4357 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4358 else
4359 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4360 }
4361
4362 EVT MemoryVT = N->getMemoryVT();
4363 EVT LoMemVT, HiMemVT;
4364 bool HiIsEmpty = false;
4365 std::tie(args&: LoMemVT, args&: HiMemVT) =
4366 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4367
4368 SDValue Lo, Hi, Res;
4369 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4370 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4371 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: N->getAAInfo(),
4372 Ranges: N->getRanges());
4373
4374 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4375 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4376 IsCompressing: N->isCompressingStore());
4377
4378 if (HiIsEmpty) {
4379 // The hi masked store has zero storage size.
4380 // Only the lo masked store is needed.
4381 Res = Lo;
4382 } else {
4383
4384 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4385 IsCompressedMemory: N->isCompressingStore());
4386
4387 MachinePointerInfo MPI;
4388 if (LoMemVT.isScalableVector()) {
4389 Alignment = commonAlignment(
4390 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4391 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4392 } else
4393 MPI = N->getPointerInfo().getWithOffset(
4394 O: LoMemVT.getStoreSize().getFixedValue());
4395
4396 MMO = DAG.getMachineFunction().getMachineMemOperand(
4397 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4398 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4399
4400 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4401 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4402 IsCompressing: N->isCompressingStore());
4403
4404 // Build a factor node to remember that this store is independent of the
4405 // other one.
4406 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4407 }
4408
4409 return Res;
4410}
4411
4412SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4413 SDValue Ch = N->getChain();
4414 SDValue Ptr = N->getBasePtr();
4415 EVT MemoryVT = N->getMemoryVT();
4416 Align Alignment = N->getBaseAlign();
4417 SDLoc DL(N);
4418 struct Operands {
4419 SDValue Mask;
4420 SDValue Index;
4421 SDValue Scale;
4422 SDValue Data;
4423 } Ops = [&]() -> Operands {
4424 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4425 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4426 .Data: MSC->getValue()};
4427 }
4428 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4429 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4430 .Data: VPSC->getValue()};
4431 }();
4432 // Split all operands
4433
4434 EVT LoMemVT, HiMemVT;
4435 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4436
4437 SDValue DataLo, DataHi;
4438 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4439 // Split Data operand
4440 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4441 else
4442 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4443
4444 // Split Mask operand
4445 SDValue MaskLo, MaskHi;
4446 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4447 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4448 } else {
4449 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4450 }
4451
4452 SDValue IndexHi, IndexLo;
4453 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4454 TargetLowering::TypeSplitVector)
4455 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4456 else
4457 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4458
4459 SDValue Lo;
4460 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4461 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4462 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4463 BaseAlignment: Alignment, AAInfo: N->getAAInfo(), Ranges: N->getRanges());
4464
4465 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4466 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4467 Lo =
4468 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4469 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4470
4471 // The order of the Scatter operation after split is well defined. The "Hi"
4472 // part comes after the "Lo". So these two operations should be chained one
4473 // after another.
4474 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4475 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4476 MMO, IndexType: MSC->getIndexType(),
4477 IsTruncating: MSC->isTruncatingStore());
4478 }
4479 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4480 SDValue EVLLo, EVLHi;
4481 std::tie(args&: EVLLo, args&: EVLHi) =
4482 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4483
4484 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4485 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4486 IndexType: VPSC->getIndexType());
4487
4488 // The order of the Scatter operation after split is well defined. The "Hi"
4489 // part comes after the "Lo". So these two operations should be chained one
4490 // after another.
4491 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4492 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4493 IndexType: VPSC->getIndexType());
4494}
4495
4496SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4497 assert(N->isUnindexed() && "Indexed store of vector?");
4498 assert(OpNo == 1 && "Can only split the stored value");
4499 SDLoc DL(N);
4500
4501 bool isTruncating = N->isTruncatingStore();
4502 SDValue Ch = N->getChain();
4503 SDValue Ptr = N->getBasePtr();
4504 EVT MemoryVT = N->getMemoryVT();
4505 Align Alignment = N->getBaseAlign();
4506 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4507 AAMDNodes AAInfo = N->getAAInfo();
4508 SDValue Lo, Hi;
4509 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4510
4511 EVT LoMemVT, HiMemVT;
4512 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4513
4514 // Scalarize if the split halves are not byte-sized.
4515 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4516 return TLI.scalarizeVectorStore(ST: N, DAG);
4517
4518 if (isTruncating)
4519 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4520 Alignment, MMOFlags, AAInfo);
4521 else
4522 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4523 AAInfo);
4524
4525 MachinePointerInfo MPI;
4526 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4527
4528 if (isTruncating)
4529 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4530 SVT: HiMemVT, Alignment, MMOFlags, AAInfo);
4531 else
4532 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, AAInfo);
4533
4534 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4535}
4536
4537SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4538 SDLoc DL(N);
4539
4540 // The input operands all must have the same type, and we know the result
4541 // type is valid. Convert this to a buildvector which extracts all the
4542 // input elements.
4543 // TODO: If the input elements are power-two vectors, we could convert this to
4544 // a new CONCAT_VECTORS node with elements that are half-wide.
4545 SmallVector<SDValue, 32> Elts;
4546 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4547 for (const SDValue &Op : N->op_values()) {
4548 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4549 i != e; ++i) {
4550 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4551 }
4552 }
4553
4554 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4555}
4556
4557SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4558 // The result type is legal, but the input type is illegal. If splitting
4559 // ends up with the result type of each half still being legal, just
4560 // do that. If, however, that would result in an illegal result type,
4561 // we can try to get more clever with power-two vectors. Specifically,
4562 // split the input type, but also widen the result element size, then
4563 // concatenate the halves and truncate again. For example, consider a target
4564 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4565 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4566 // %inlo = v4i32 extract_subvector %in, 0
4567 // %inhi = v4i32 extract_subvector %in, 4
4568 // %lo16 = v4i16 trunc v4i32 %inlo
4569 // %hi16 = v4i16 trunc v4i32 %inhi
4570 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4571 // %res = v8i8 trunc v8i16 %in16
4572 //
4573 // Without this transform, the original truncate would end up being
4574 // scalarized, which is pretty much always a last resort.
4575 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4576 SDValue InVec = N->getOperand(Num: OpNo);
4577 EVT InVT = InVec->getValueType(ResNo: 0);
4578 EVT OutVT = N->getValueType(ResNo: 0);
4579 ElementCount NumElements = OutVT.getVectorElementCount();
4580 bool IsFloat = OutVT.isFloatingPoint();
4581
4582 unsigned InElementSize = InVT.getScalarSizeInBits();
4583 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4584
4585 // Determine the split output VT. If its legal we can just split dirctly.
4586 EVT LoOutVT, HiOutVT;
4587 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4588 assert(LoOutVT == HiOutVT && "Unequal split?");
4589
4590 // If the input elements are only 1/2 the width of the result elements,
4591 // just use the normal splitting. Our trick only work if there's room
4592 // to split more than once.
4593 if (isTypeLegal(VT: LoOutVT) ||
4594 InElementSize <= OutElementSize * 2)
4595 return SplitVecOp_UnaryOp(N);
4596 SDLoc DL(N);
4597
4598 // Don't touch if this will be scalarized.
4599 EVT FinalVT = InVT;
4600 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4601 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4602
4603 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4604 return SplitVecOp_UnaryOp(N);
4605
4606 // Get the split input vector.
4607 SDValue InLoVec, InHiVec;
4608 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4609
4610 // Truncate them to 1/2 the element size.
4611 //
4612 // This assumes the number of elements is a power of two; any vector that
4613 // isn't should be widened, not split.
4614 EVT HalfElementVT = IsFloat ?
4615 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4616 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4617 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4618 EC: NumElements.divideCoefficientBy(RHS: 2));
4619
4620 SDValue HalfLo;
4621 SDValue HalfHi;
4622 SDValue Chain;
4623 if (N->isStrictFPOpcode()) {
4624 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4625 Ops: {N->getOperand(Num: 0), InLoVec});
4626 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4627 Ops: {N->getOperand(Num: 0), InHiVec});
4628 // Legalize the chain result - switch anything that used the old chain to
4629 // use the new one.
4630 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4631 N2: HalfHi.getValue(R: 1));
4632 } else {
4633 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4634 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4635 }
4636
4637 // Concatenate them to get the full intermediate truncation result.
4638 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4639 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4640 N2: HalfHi);
4641 // Now finish up by truncating all the way down to the original result
4642 // type. This should normally be something that ends up being legal directly,
4643 // but in theory if a target has very wide vectors and an annoyingly
4644 // restricted set of legal types, this split can chain to build things up.
4645
4646 if (N->isStrictFPOpcode()) {
4647 SDValue Res = DAG.getNode(
4648 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4649 Ops: {Chain, InterVec,
4650 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4651 // Relink the chain
4652 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4653 return Res;
4654 }
4655
4656 return IsFloat
4657 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4658 N2: DAG.getTargetConstant(
4659 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4660 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4661}
4662
4663SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4664 unsigned Opc = N->getOpcode();
4665 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4666 assert(N->getValueType(0).isVector() &&
4667 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4668 "Operand types must be vectors");
4669 // The result has a legal vector type, but the input needs splitting.
4670 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4671 SDLoc DL(N);
4672 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4673 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4674
4675 EVT VT = N->getValueType(ResNo: 0);
4676 EVT PartResVT = Lo0.getValueType().changeElementType(Context&: *DAG.getContext(),
4677 EltVT: VT.getScalarType());
4678
4679 if (Opc == ISD::SETCC) {
4680 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4681 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4682 } else if (isStrict) {
4683 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4684 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4685 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4686 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4687 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4688 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4689 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4690 } else {
4691 assert(Opc == ISD::VP_SETCC && "Expected VP_SETCC opcode");
4692 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4693 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 3));
4694 std::tie(args&: EVLLo, args&: EVLHi) =
4695 DAG.SplitEVL(N: N->getOperand(Num: 4), VecVT: N->getValueType(ResNo: 0), DL);
4696 LoRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1,
4697 N3: N->getOperand(Num: 2), N4: MaskLo, N5: EVLLo);
4698 HiRes = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1,
4699 N3: N->getOperand(Num: 2), N4: MaskHi, N5: EVLHi);
4700 }
4701
4702 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: LoRes, N2: HiRes);
4703}
4704
4705
4706SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4707 // The result has a legal vector type, but the input needs splitting.
4708 EVT ResVT = N->getValueType(ResNo: 0);
4709 SDValue Lo, Hi;
4710 SDLoc DL(N);
4711 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4712 EVT InVT = Lo.getValueType();
4713
4714 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4715 EC: InVT.getVectorElementCount());
4716
4717 if (N->isStrictFPOpcode()) {
4718 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4719 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
4720 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4721 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
4722 // Legalize the chain result - switch anything that used the old chain to
4723 // use the new one.
4724 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4725 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
4726 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4727 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
4728 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
4729 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
4730 std::tie(args&: EVLLo, args&: EVLHi) =
4731 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL);
4732 Lo = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4733 Hi = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL, VT: OutVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4734 } else {
4735 Lo = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4736 Hi = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4737 }
4738
4739 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4740}
4741
4742// Split a vector type in an FP binary operation where the second operand has a
4743// different type from the first.
4744//
4745// The result (and the first input) has a legal vector type, but the second
4746// input needs splitting.
4747SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
4748 SDLoc DL(N);
4749
4750 EVT LHSLoVT, LHSHiVT;
4751 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
4752
4753 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
4754 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
4755
4756 SDValue LHSLo, LHSHi;
4757 std::tie(args&: LHSLo, args&: LHSHi) =
4758 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
4759
4760 SDValue RHSLo, RHSHi;
4761 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4762
4763 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
4764 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
4765
4766 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4767}
4768
4769SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
4770 LLVMContext &Ctxt = *DAG.getContext();
4771 SDLoc dl(N);
4772
4773 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4774 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
4775 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
4776
4777 EVT ResVT = N->getValueType(ResNo: 0);
4778 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
4779 EVT NewResVT =
4780 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
4781
4782 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
4783 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
4784
4785 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4786}
4787
4788SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4789 EVT ResVT = N->getValueType(ResNo: 0);
4790 SDValue Lo, Hi;
4791 SDLoc dl(N);
4792 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4793 EVT InVT = Lo.getValueType();
4794
4795 EVT NewResVT =
4796 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4797 EC: InVT.getVectorElementCount());
4798
4799 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
4800 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
4801
4802 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4803}
4804
4805SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
4806 SDLoc DL(N);
4807 EVT ResVT = N->getValueType(ResNo: 0);
4808
4809 SDValue Lo, Hi;
4810 SDValue VecOp = N->getOperand(Num: 0);
4811 GetSplitVector(Op: VecOp, Lo, Hi);
4812
4813 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
4814 auto [EVLLo, EVLHi] =
4815 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
4816 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
4817
4818 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
4819 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_UNDEF(Hi)).
4820 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
4821 SDValue ResLoNotEVL =
4822 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
4823 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
4824 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
4825 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
4826}
4827
4828SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
4829 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
4830 SDLoc DL(HG);
4831 SDValue Inc = HG->getInc();
4832 SDValue Ptr = HG->getBasePtr();
4833 SDValue Scale = HG->getScale();
4834 SDValue IntID = HG->getIntID();
4835 EVT MemVT = HG->getMemoryVT();
4836 MachineMemOperand *MMO = HG->getMemOperand();
4837 ISD::MemIndexType IndexType = HG->getIndexType();
4838
4839 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
4840 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
4841 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
4842 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
4843 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
4844 Ops: OpsLo, MMO, IndexType);
4845 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
4846 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
4847 MMO, IndexType);
4848}
4849
4850SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
4851 SDValue Acc = N->getOperand(Num: 0);
4852 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
4853 "Accumulator should already be a legal type, and shouldn't need "
4854 "further splitting");
4855
4856 SDLoc DL(N);
4857 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
4858 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
4859 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
4860 unsigned Opcode = N->getOpcode();
4861 EVT ResultVT = Acc.getValueType();
4862
4863 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
4864 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
4865}
4866
4867//===----------------------------------------------------------------------===//
4868// Result Vector Widening
4869//===----------------------------------------------------------------------===//
4870
4871void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
4872 unsigned WidenResNo) {
4873 unsigned NumResults = N->getNumValues();
4874 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
4875 if (ResNo == WidenResNo)
4876 continue;
4877 EVT ResVT = N->getValueType(ResNo);
4878 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
4879 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
4880 } else {
4881 SDLoc DL(N);
4882 SDValue ResVal =
4883 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
4884 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
4885 }
4886 }
4887}
4888
4889void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4890 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4891
4892 // See if the target wants to custom widen this node.
4893 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
4894 return;
4895
4896 SDValue Res = SDValue();
4897
4898 auto unrollExpandedOp = [&]() {
4899 // We're going to widen this vector op to a legal type by padding with undef
4900 // elements. If the wide vector op is eventually going to be expanded to
4901 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4902 // libcalls on the undef elements.
4903 EVT VT = N->getValueType(ResNo: 0);
4904 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4905 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
4906 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
4907 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
4908 if (N->getNumValues() > 1)
4909 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
4910 return true;
4911 }
4912 return false;
4913 };
4914
4915 switch (N->getOpcode()) {
4916 default:
4917#ifndef NDEBUG
4918 dbgs() << "WidenVectorResult #" << ResNo << ": ";
4919 N->dump(&DAG);
4920 dbgs() << "\n";
4921#endif
4922 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
4923
4924 case ISD::LOOP_DEPENDENCE_RAW_MASK:
4925 case ISD::LOOP_DEPENDENCE_WAR_MASK:
4926 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
4927 break;
4928 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4929 case ISD::ADDRSPACECAST:
4930 Res = WidenVecRes_ADDRSPACECAST(N);
4931 break;
4932 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4933 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4934 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4935 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4936 case ISD::INSERT_SUBVECTOR:
4937 Res = WidenVecRes_INSERT_SUBVECTOR(N);
4938 break;
4939 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4940 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4941 case ISD::ATOMIC_LOAD:
4942 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
4943 break;
4944 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4945 case ISD::STEP_VECTOR:
4946 case ISD::SPLAT_VECTOR:
4947 case ISD::SCALAR_TO_VECTOR:
4948 Res = WidenVecRes_ScalarOp(N);
4949 break;
4950 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4951 case ISD::VSELECT:
4952 case ISD::SELECT:
4953 case ISD::VP_SELECT:
4954 case ISD::VP_MERGE:
4955 Res = WidenVecRes_Select(N);
4956 break;
4957 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4958 case ISD::VP_SETCC:
4959 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4960 case ISD::POISON:
4961 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4962 case ISD::VECTOR_SHUFFLE:
4963 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
4964 break;
4965 case ISD::VP_LOAD:
4966 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
4967 break;
4968 case ISD::VP_LOAD_FF:
4969 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
4970 break;
4971 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4972 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
4973 break;
4974 case ISD::VECTOR_COMPRESS:
4975 Res = WidenVecRes_VECTOR_COMPRESS(N);
4976 break;
4977 case ISD::MLOAD:
4978 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
4979 break;
4980 case ISD::MGATHER:
4981 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
4982 break;
4983 case ISD::VP_GATHER:
4984 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
4985 break;
4986 case ISD::VECTOR_REVERSE:
4987 Res = WidenVecRes_VECTOR_REVERSE(N);
4988 break;
4989 case ISD::GET_ACTIVE_LANE_MASK:
4990 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
4991 break;
4992
4993 case ISD::ADD: case ISD::VP_ADD:
4994 case ISD::AND: case ISD::VP_AND:
4995 case ISD::MUL: case ISD::VP_MUL:
4996 case ISD::MULHS:
4997 case ISD::MULHU:
4998 case ISD::ABDS:
4999 case ISD::ABDU:
5000 case ISD::OR: case ISD::VP_OR:
5001 case ISD::SUB: case ISD::VP_SUB:
5002 case ISD::XOR: case ISD::VP_XOR:
5003 case ISD::SHL: case ISD::VP_SHL:
5004 case ISD::SRA: case ISD::VP_SRA:
5005 case ISD::SRL: case ISD::VP_SRL:
5006 case ISD::CLMUL:
5007 case ISD::CLMULR:
5008 case ISD::CLMULH:
5009 case ISD::FMINNUM:
5010 case ISD::FMINNUM_IEEE:
5011 case ISD::VP_FMINNUM:
5012 case ISD::FMAXNUM:
5013 case ISD::FMAXNUM_IEEE:
5014 case ISD::VP_FMAXNUM:
5015 case ISD::FMINIMUM:
5016 case ISD::VP_FMINIMUM:
5017 case ISD::FMAXIMUM:
5018 case ISD::VP_FMAXIMUM:
5019 case ISD::FMINIMUMNUM:
5020 case ISD::FMAXIMUMNUM:
5021 case ISD::SMIN: case ISD::VP_SMIN:
5022 case ISD::SMAX: case ISD::VP_SMAX:
5023 case ISD::UMIN: case ISD::VP_UMIN:
5024 case ISD::UMAX: case ISD::VP_UMAX:
5025 case ISD::UADDSAT: case ISD::VP_UADDSAT:
5026 case ISD::SADDSAT: case ISD::VP_SADDSAT:
5027 case ISD::USUBSAT: case ISD::VP_USUBSAT:
5028 case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
5029 case ISD::SSHLSAT:
5030 case ISD::USHLSAT:
5031 case ISD::ROTL:
5032 case ISD::ROTR:
5033 case ISD::AVGFLOORS:
5034 case ISD::AVGFLOORU:
5035 case ISD::AVGCEILS:
5036 case ISD::AVGCEILU:
5037 // Vector-predicated binary op widening. Note that -- unlike the
5038 // unpredicated versions -- we don't have to worry about trapping on
5039 // operations like UDIV, FADD, etc., as we pass on the original vector
5040 // length parameter. This means the widened elements containing garbage
5041 // aren't active.
5042 case ISD::VP_SDIV:
5043 case ISD::VP_UDIV:
5044 case ISD::VP_SREM:
5045 case ISD::VP_UREM:
5046 case ISD::VP_FADD:
5047 case ISD::VP_FSUB:
5048 case ISD::VP_FMUL:
5049 case ISD::VP_FDIV:
5050 case ISD::VP_FREM:
5051 case ISD::VP_FCOPYSIGN:
5052 Res = WidenVecRes_Binary(N);
5053 break;
5054
5055 case ISD::SCMP:
5056 case ISD::UCMP:
5057 Res = WidenVecRes_CMP(N);
5058 break;
5059
5060 case ISD::FPOW:
5061 case ISD::FATAN2:
5062 case ISD::FREM:
5063 if (unrollExpandedOp())
5064 break;
5065 // If the target has custom/legal support for the scalar FP intrinsic ops
5066 // (they are probably not destined to become libcalls), then widen those
5067 // like any other binary ops.
5068 [[fallthrough]];
5069
5070 case ISD::FADD:
5071 case ISD::FMUL:
5072 case ISD::FSUB:
5073 case ISD::FDIV:
5074 case ISD::SDIV:
5075 case ISD::UDIV:
5076 case ISD::SREM:
5077 case ISD::UREM:
5078 Res = WidenVecRes_BinaryCanTrap(N);
5079 break;
5080
5081 case ISD::SMULFIX:
5082 case ISD::SMULFIXSAT:
5083 case ISD::UMULFIX:
5084 case ISD::UMULFIXSAT:
5085 // These are binary operations, but with an extra operand that shouldn't
5086 // be widened (the scale).
5087 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5088 break;
5089
5090#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5091 case ISD::STRICT_##DAGN:
5092#include "llvm/IR/ConstrainedOps.def"
5093 Res = WidenVecRes_StrictFP(N);
5094 break;
5095
5096 case ISD::UADDO:
5097 case ISD::SADDO:
5098 case ISD::USUBO:
5099 case ISD::SSUBO:
5100 case ISD::UMULO:
5101 case ISD::SMULO:
5102 Res = WidenVecRes_OverflowOp(N, ResNo);
5103 break;
5104
5105 case ISD::FCOPYSIGN:
5106 Res = WidenVecRes_FCOPYSIGN(N);
5107 break;
5108
5109 case ISD::IS_FPCLASS:
5110 case ISD::FPTRUNC_ROUND:
5111 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5112 break;
5113
5114 case ISD::FLDEXP:
5115 case ISD::FPOWI:
5116 if (!unrollExpandedOp())
5117 Res = WidenVecRes_ExpOp(N);
5118 break;
5119
5120 case ISD::ANY_EXTEND_VECTOR_INREG:
5121 case ISD::SIGN_EXTEND_VECTOR_INREG:
5122 case ISD::ZERO_EXTEND_VECTOR_INREG:
5123 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5124 break;
5125
5126 case ISD::ANY_EXTEND:
5127 case ISD::FP_EXTEND:
5128 case ISD::VP_FP_EXTEND:
5129 case ISD::FP_ROUND:
5130 case ISD::VP_FP_ROUND:
5131 case ISD::FP_TO_SINT:
5132 case ISD::VP_FP_TO_SINT:
5133 case ISD::FP_TO_UINT:
5134 case ISD::VP_FP_TO_UINT:
5135 case ISD::SIGN_EXTEND:
5136 case ISD::VP_SIGN_EXTEND:
5137 case ISD::SINT_TO_FP:
5138 case ISD::VP_SINT_TO_FP:
5139 case ISD::VP_TRUNCATE:
5140 case ISD::TRUNCATE:
5141 case ISD::UINT_TO_FP:
5142 case ISD::VP_UINT_TO_FP:
5143 case ISD::ZERO_EXTEND:
5144 case ISD::VP_ZERO_EXTEND:
5145 Res = WidenVecRes_Convert(N);
5146 break;
5147
5148 case ISD::FP_TO_SINT_SAT:
5149 case ISD::FP_TO_UINT_SAT:
5150 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5151 break;
5152
5153 case ISD::LRINT:
5154 case ISD::LLRINT:
5155 case ISD::VP_LRINT:
5156 case ISD::VP_LLRINT:
5157 case ISD::LROUND:
5158 case ISD::LLROUND:
5159 Res = WidenVecRes_XROUND(N);
5160 break;
5161
5162 case ISD::FACOS:
5163 case ISD::FASIN:
5164 case ISD::FATAN:
5165 case ISD::FCEIL:
5166 case ISD::FCOS:
5167 case ISD::FCOSH:
5168 case ISD::FEXP:
5169 case ISD::FEXP2:
5170 case ISD::FEXP10:
5171 case ISD::FFLOOR:
5172 case ISD::FLOG:
5173 case ISD::FLOG10:
5174 case ISD::FLOG2:
5175 case ISD::FNEARBYINT:
5176 case ISD::FRINT:
5177 case ISD::FROUND:
5178 case ISD::FROUNDEVEN:
5179 case ISD::FSIN:
5180 case ISD::FSINH:
5181 case ISD::FSQRT:
5182 case ISD::FTAN:
5183 case ISD::FTANH:
5184 case ISD::FTRUNC:
5185 if (unrollExpandedOp())
5186 break;
5187 // If the target has custom/legal support for the scalar FP intrinsic ops
5188 // (they are probably not destined to become libcalls), then widen those
5189 // like any other unary ops.
5190 [[fallthrough]];
5191
5192 case ISD::ABS:
5193 case ISD::VP_ABS:
5194 case ISD::BITREVERSE:
5195 case ISD::VP_BITREVERSE:
5196 case ISD::BSWAP:
5197 case ISD::VP_BSWAP:
5198 case ISD::CTLZ:
5199 case ISD::VP_CTLZ:
5200 case ISD::CTLZ_ZERO_UNDEF:
5201 case ISD::VP_CTLZ_ZERO_UNDEF:
5202 case ISD::CTPOP:
5203 case ISD::VP_CTPOP:
5204 case ISD::CTTZ:
5205 case ISD::VP_CTTZ:
5206 case ISD::CTTZ_ZERO_UNDEF:
5207 case ISD::VP_CTTZ_ZERO_UNDEF:
5208 case ISD::FNEG: case ISD::VP_FNEG:
5209 case ISD::FABS: case ISD::VP_FABS:
5210 case ISD::VP_SQRT:
5211 case ISD::VP_FCEIL:
5212 case ISD::VP_FFLOOR:
5213 case ISD::VP_FRINT:
5214 case ISD::VP_FNEARBYINT:
5215 case ISD::VP_FROUND:
5216 case ISD::VP_FROUNDEVEN:
5217 case ISD::VP_FROUNDTOZERO:
5218 case ISD::FREEZE:
5219 case ISD::ARITH_FENCE:
5220 case ISD::FCANONICALIZE:
5221 case ISD::AssertNoFPClass:
5222 Res = WidenVecRes_Unary(N);
5223 break;
5224 case ISD::FMA: case ISD::VP_FMA:
5225 case ISD::FSHL:
5226 case ISD::VP_FSHL:
5227 case ISD::FSHR:
5228 case ISD::VP_FSHR:
5229 Res = WidenVecRes_Ternary(N);
5230 break;
5231 case ISD::FMODF:
5232 case ISD::FFREXP:
5233 case ISD::FSINCOS:
5234 case ISD::FSINCOSPI: {
5235 if (!unrollExpandedOp())
5236 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5237 break;
5238 }
5239 }
5240
5241 // If Res is null, the sub-method took care of registering the result.
5242 if (Res.getNode())
5243 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5244}
5245
5246SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5247 // Ternary op widening.
5248 SDLoc dl(N);
5249 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5250 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5251 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5252 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5253 if (N->getNumOperands() == 3)
5254 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5255
5256 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
5257 assert(N->isVPOpcode() && "Expected VP opcode");
5258
5259 SDValue Mask =
5260 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
5261 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5262 Ops: {InOp1, InOp2, InOp3, Mask, N->getOperand(Num: 4)});
5263}
5264
5265SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5266 // Binary op widening.
5267 SDLoc dl(N);
5268 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5269 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5270 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5271 if (N->getNumOperands() == 2)
5272 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5273 Flags: N->getFlags());
5274
5275 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5276 assert(N->isVPOpcode() && "Expected VP opcode");
5277
5278 SDValue Mask =
5279 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5280 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5281 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5282}
5283
5284SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5285 LLVMContext &Ctxt = *DAG.getContext();
5286 SDLoc dl(N);
5287
5288 SDValue LHS = N->getOperand(Num: 0);
5289 SDValue RHS = N->getOperand(Num: 1);
5290 EVT OpVT = LHS.getValueType();
5291 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5292 LHS = GetWidenedVector(Op: LHS);
5293 RHS = GetWidenedVector(Op: RHS);
5294 OpVT = LHS.getValueType();
5295 }
5296
5297 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5298 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5299 if (WidenResEC == OpVT.getVectorElementCount()) {
5300 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5301 }
5302
5303 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5304}
5305
5306SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5307 // Binary op widening, but with an extra operand that shouldn't be widened.
5308 SDLoc dl(N);
5309 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5310 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5311 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5312 SDValue InOp3 = N->getOperand(Num: 2);
5313 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5314 Flags: N->getFlags());
5315}
5316
5317// Given a vector of operations that have been broken up to widen, see
5318// if we can collect them together into the next widest legal VT. This
5319// implementation is trap-safe.
5320static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5321 SmallVectorImpl<SDValue> &ConcatOps,
5322 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5323 EVT WidenVT) {
5324 // Check to see if we have a single operation with the widen type.
5325 if (ConcatEnd == 1) {
5326 VT = ConcatOps[0].getValueType();
5327 if (VT == WidenVT)
5328 return ConcatOps[0];
5329 }
5330
5331 SDLoc dl(ConcatOps[0]);
5332 EVT WidenEltVT = WidenVT.getVectorElementType();
5333
5334 // while (Some element of ConcatOps is not of type MaxVT) {
5335 // From the end of ConcatOps, collect elements of the same type and put
5336 // them into an op of the next larger supported type
5337 // }
5338 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5339 int Idx = ConcatEnd - 1;
5340 VT = ConcatOps[Idx--].getValueType();
5341 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5342 Idx--;
5343
5344 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5345 EVT NextVT;
5346 do {
5347 NextSize *= 2;
5348 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5349 } while (!TLI.isTypeLegal(VT: NextVT));
5350
5351 if (!VT.isVector()) {
5352 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5353 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5354 unsigned NumToInsert = ConcatEnd - Idx - 1;
5355 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5356 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5357 ConcatOps[Idx+1] = VecOp;
5358 ConcatEnd = Idx + 2;
5359 } else {
5360 // Vector type, create a CONCAT_VECTORS of type NextVT
5361 SDValue undefVec = DAG.getPOISON(VT);
5362 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5363 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5364 unsigned RealVals = ConcatEnd - Idx - 1;
5365 unsigned SubConcatEnd = 0;
5366 unsigned SubConcatIdx = Idx + 1;
5367 while (SubConcatEnd < RealVals)
5368 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5369 while (SubConcatEnd < OpsToConcat)
5370 SubConcatOps[SubConcatEnd++] = undefVec;
5371 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5372 VT: NextVT, Ops: SubConcatOps);
5373 ConcatEnd = SubConcatIdx + 1;
5374 }
5375 }
5376
5377 // Check to see if we have a single operation with the widen type.
5378 if (ConcatEnd == 1) {
5379 VT = ConcatOps[0].getValueType();
5380 if (VT == WidenVT)
5381 return ConcatOps[0];
5382 }
5383
5384 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5385 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5386 if (NumOps != ConcatEnd ) {
5387 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5388 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5389 ConcatOps[j] = UndefVal;
5390 }
5391 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5392 Ops: ArrayRef(ConcatOps.data(), NumOps));
5393}
5394
5395SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5396 // Binary op widening for operations that can trap.
5397 unsigned Opcode = N->getOpcode();
5398 SDLoc dl(N);
5399 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5400 EVT WidenEltVT = WidenVT.getVectorElementType();
5401 EVT VT = WidenVT;
5402 unsigned NumElts = VT.getVectorMinNumElements();
5403 const SDNodeFlags Flags = N->getFlags();
5404 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5405 NumElts = NumElts / 2;
5406 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5407 }
5408
5409 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5410 // Operation doesn't trap so just widen as normal.
5411 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5412 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5413 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5414 }
5415
5416 // Generate a vp.op if it is custom/legal for the target. This avoids need
5417 // to split and tile the subvectors (below), because the inactive lanes can
5418 // simply be disabled. To avoid possible recursion, only do this if the
5419 // widened mask type is legal.
5420 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5421 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5422 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5423 EC: WidenVT.getVectorElementCount());
5424 TLI.isTypeLegal(VT: WideMaskVT)) {
5425 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5426 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5427 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5428 SDValue EVL =
5429 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5430 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5431 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5432 Flags);
5433 }
5434 }
5435
5436 // FIXME: Improve support for scalable vectors.
5437 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5438
5439 // No legal vector version so unroll the vector operation and then widen.
5440 if (NumElts == 1)
5441 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5442
5443 // Since the operation can trap, apply operation on the original vector.
5444 EVT MaxVT = VT;
5445 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5446 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5447 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5448
5449 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5450 unsigned ConcatEnd = 0; // Current ConcatOps index.
5451 int Idx = 0; // Current Idx into input vectors.
5452
5453 // NumElts := greatest legal vector size (at most WidenVT)
5454 // while (orig. vector has unhandled elements) {
5455 // take munches of size NumElts from the beginning and add to ConcatOps
5456 // NumElts := next smaller supported vector size or 1
5457 // }
5458 while (CurNumElts != 0) {
5459 while (CurNumElts >= NumElts) {
5460 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5461 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5462 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5463 Idx += NumElts;
5464 CurNumElts -= NumElts;
5465 }
5466 do {
5467 NumElts = NumElts / 2;
5468 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5469 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5470
5471 if (NumElts == 1) {
5472 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5473 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5474 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5475 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5476 N1: EOp1, N2: EOp2, Flags);
5477 }
5478 CurNumElts = 0;
5479 }
5480 }
5481
5482 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5483}
5484
5485SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5486 switch (N->getOpcode()) {
5487 case ISD::STRICT_FSETCC:
5488 case ISD::STRICT_FSETCCS:
5489 return WidenVecRes_STRICT_FSETCC(N);
5490 case ISD::STRICT_FP_EXTEND:
5491 case ISD::STRICT_FP_ROUND:
5492 case ISD::STRICT_FP_TO_SINT:
5493 case ISD::STRICT_FP_TO_UINT:
5494 case ISD::STRICT_SINT_TO_FP:
5495 case ISD::STRICT_UINT_TO_FP:
5496 return WidenVecRes_Convert_StrictFP(N);
5497 default:
5498 break;
5499 }
5500
5501 // StrictFP op widening for operations that can trap.
5502 unsigned NumOpers = N->getNumOperands();
5503 unsigned Opcode = N->getOpcode();
5504 SDLoc dl(N);
5505 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5506 EVT WidenEltVT = WidenVT.getVectorElementType();
5507 EVT VT = WidenVT;
5508 unsigned NumElts = VT.getVectorNumElements();
5509 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5510 NumElts = NumElts / 2;
5511 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5512 }
5513
5514 // No legal vector version so unroll the vector operation and then widen.
5515 if (NumElts == 1)
5516 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5517
5518 // Since the operation can trap, apply operation on the original vector.
5519 EVT MaxVT = VT;
5520 SmallVector<SDValue, 4> InOps;
5521 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5522
5523 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5524 SmallVector<SDValue, 16> Chains;
5525 unsigned ConcatEnd = 0; // Current ConcatOps index.
5526 int Idx = 0; // Current Idx into input vectors.
5527
5528 // The Chain is the first operand.
5529 InOps.push_back(Elt: N->getOperand(Num: 0));
5530
5531 // Now process the remaining operands.
5532 for (unsigned i = 1; i < NumOpers; ++i) {
5533 SDValue Oper = N->getOperand(Num: i);
5534
5535 EVT OpVT = Oper.getValueType();
5536 if (OpVT.isVector()) {
5537 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5538 Oper = GetWidenedVector(Op: Oper);
5539 else {
5540 EVT WideOpVT =
5541 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5542 EC: WidenVT.getVectorElementCount());
5543 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5544 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5545 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5546 }
5547 }
5548
5549 InOps.push_back(Elt: Oper);
5550 }
5551
5552 // NumElts := greatest legal vector size (at most WidenVT)
5553 // while (orig. vector has unhandled elements) {
5554 // take munches of size NumElts from the beginning and add to ConcatOps
5555 // NumElts := next smaller supported vector size or 1
5556 // }
5557 while (CurNumElts != 0) {
5558 while (CurNumElts >= NumElts) {
5559 SmallVector<SDValue, 4> EOps;
5560
5561 for (unsigned i = 0; i < NumOpers; ++i) {
5562 SDValue Op = InOps[i];
5563
5564 EVT OpVT = Op.getValueType();
5565 if (OpVT.isVector()) {
5566 EVT OpExtractVT =
5567 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5568 EC: VT.getVectorElementCount());
5569 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5570 }
5571
5572 EOps.push_back(Elt: Op);
5573 }
5574
5575 EVT OperVT[] = {VT, MVT::Other};
5576 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5577 ConcatOps[ConcatEnd++] = Oper;
5578 Chains.push_back(Elt: Oper.getValue(R: 1));
5579 Idx += NumElts;
5580 CurNumElts -= NumElts;
5581 }
5582 do {
5583 NumElts = NumElts / 2;
5584 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5585 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5586
5587 if (NumElts == 1) {
5588 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5589 SmallVector<SDValue, 4> EOps;
5590
5591 for (unsigned i = 0; i < NumOpers; ++i) {
5592 SDValue Op = InOps[i];
5593
5594 EVT OpVT = Op.getValueType();
5595 if (OpVT.isVector())
5596 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5597 Idx);
5598
5599 EOps.push_back(Elt: Op);
5600 }
5601
5602 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5603 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5604 ConcatOps[ConcatEnd++] = Oper;
5605 Chains.push_back(Elt: Oper.getValue(R: 1));
5606 }
5607 CurNumElts = 0;
5608 }
5609 }
5610
5611 // Build a factor node to remember all the Ops that have been created.
5612 SDValue NewChain;
5613 if (Chains.size() == 1)
5614 NewChain = Chains[0];
5615 else
5616 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5617 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5618
5619 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5620}
5621
5622SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5623 SDLoc DL(N);
5624 EVT ResVT = N->getValueType(ResNo: 0);
5625 EVT OvVT = N->getValueType(ResNo: 1);
5626 EVT WideResVT, WideOvVT;
5627 SDValue WideLHS, WideRHS;
5628
5629 // TODO: This might result in a widen/split loop.
5630 if (ResNo == 0) {
5631 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5632 WideOvVT = EVT::getVectorVT(
5633 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5634 NumElements: WideResVT.getVectorNumElements());
5635
5636 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5637 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5638 } else {
5639 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5640 WideResVT = EVT::getVectorVT(
5641 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5642 NumElements: WideOvVT.getVectorNumElements());
5643
5644 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5645 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5646
5647 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5648 N2: N->getOperand(Num: 0), N3: Zero);
5649 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5650 N2: N->getOperand(Num: 1), N3: Zero);
5651 }
5652
5653 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5654 SDNode *WideNode = DAG.getNode(
5655 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5656
5657 // Replace the other vector result not being explicitly widened here.
5658 unsigned OtherNo = 1 - ResNo;
5659 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5660 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5661 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5662 } else {
5663 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5664 SDValue OtherVal = DAG.getNode(
5665 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5666 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5667 }
5668
5669 return SDValue(WideNode, ResNo);
5670}
5671
5672SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5673 LLVMContext &Ctx = *DAG.getContext();
5674 SDValue InOp = N->getOperand(Num: 0);
5675 SDLoc DL(N);
5676
5677 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
5678 ElementCount WidenEC = WidenVT.getVectorElementCount();
5679
5680 EVT InVT = InOp.getValueType();
5681
5682 unsigned Opcode = N->getOpcode();
5683 const SDNodeFlags Flags = N->getFlags();
5684
5685 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
5686 // not equal that of WidenVT.
5687 if (N->getOpcode() == ISD::ZERO_EXTEND &&
5688 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
5689 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
5690 WidenVT.getScalarSizeInBits()) {
5691 InOp = ZExtPromotedInteger(Op: InOp);
5692 InVT = InOp.getValueType();
5693 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
5694 Opcode = ISD::TRUNCATE;
5695 }
5696
5697 EVT InEltVT = InVT.getVectorElementType();
5698 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
5699 ElementCount InVTEC = InVT.getVectorElementCount();
5700
5701 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5702 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5703 InVT = InOp.getValueType();
5704 InVTEC = InVT.getVectorElementCount();
5705 if (InVTEC == WidenEC) {
5706 if (N->getNumOperands() == 1)
5707 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp, Flags);
5708 if (N->getNumOperands() == 3) {
5709 assert(N->isVPOpcode() && "Expected VP opcode");
5710 SDValue Mask =
5711 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5712 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: Mask, N3: N->getOperand(Num: 2));
5713 }
5714 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1), Flags);
5715 }
5716 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
5717 // If both input and result vector types are of same width, extend
5718 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
5719 // accepts fewer elements in the result than in the input.
5720 if (Opcode == ISD::ANY_EXTEND)
5721 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5722 if (Opcode == ISD::SIGN_EXTEND)
5723 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5724 if (Opcode == ISD::ZERO_EXTEND)
5725 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
5726 }
5727
5728 // For TRUNCATE, try to widen using the legal EC of the input type instead
5729 // if the legalisation action for that intermediate type is not widening.
5730 // E.g. for trunc nxv1i64 -> nxv1i8 where
5731 // - nxv1i64 input gets widened to nxv2i64
5732 // - nxv1i8 output gets widened to nxv16i8
5733 // Then one can try widening the result to nxv2i8 (instead of going all the
5734 // way to nxv16i8) if this later allows type promotion.
5735 EVT MidResVT =
5736 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
5737 if (N->getOpcode() == ISD::TRUNCATE &&
5738 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
5739 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
5740 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
5741 }
5742 }
5743
5744 if (TLI.isTypeLegal(VT: InWidenVT)) {
5745 // Because the result and the input are different vector types, widening
5746 // the result could create a legal type but widening the input might make
5747 // it an illegal type that might lead to repeatedly splitting the input
5748 // and then widening it. To avoid this, we widen the input only if
5749 // it results in a legal type.
5750 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
5751 // Widen the input and call convert on the widened input vector.
5752 unsigned NumConcat =
5753 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
5754 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
5755 Ops[0] = InOp;
5756 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
5757 if (N->getNumOperands() == 1)
5758 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVec, Flags);
5759 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVec, N2: N->getOperand(Num: 1), Flags);
5760 }
5761
5762 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
5763 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
5764 // Extract the input and convert the shorten input vector.
5765 if (N->getNumOperands() == 1)
5766 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InVal, Flags);
5767 return DAG.getNode(Opcode, DL, VT: WidenVT, N1: InVal, N2: N->getOperand(Num: 1), Flags);
5768 }
5769 }
5770
5771 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5772 EVT EltVT = WidenVT.getVectorElementType();
5773 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
5774 // Use the original element count so we don't do more scalar opts than
5775 // necessary.
5776 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5777 for (unsigned i=0; i < MinElts; ++i) {
5778 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5779 if (N->getNumOperands() == 1)
5780 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Val, Flags);
5781 else
5782 Ops[i] = DAG.getNode(Opcode, DL, VT: EltVT, N1: Val, N2: N->getOperand(Num: 1), Flags);
5783 }
5784
5785 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5786}
5787
5788SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
5789 SDLoc dl(N);
5790 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5791 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5792
5793 SDValue Src = N->getOperand(Num: 0);
5794 EVT SrcVT = Src.getValueType();
5795
5796 // Also widen the input.
5797 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5798 Src = GetWidenedVector(Op: Src);
5799 SrcVT = Src.getValueType();
5800 }
5801
5802 // Input and output not widened to the same size, give up.
5803 if (WidenNumElts != SrcVT.getVectorElementCount())
5804 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5805
5806 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
5807}
5808
5809SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
5810 SDLoc dl(N);
5811 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5812 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
5813
5814 SDValue Src = N->getOperand(Num: 0);
5815 EVT SrcVT = Src.getValueType();
5816
5817 // Also widen the input.
5818 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
5819 Src = GetWidenedVector(Op: Src);
5820 SrcVT = Src.getValueType();
5821 }
5822
5823 // Input and output not widened to the same size, give up.
5824 if (WidenNumElts != SrcVT.getVectorElementCount())
5825 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
5826
5827 if (N->getNumOperands() == 1)
5828 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
5829
5830 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5831 assert(N->isVPOpcode() && "Expected VP opcode");
5832
5833 SDValue Mask =
5834 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5835 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: Mask, N3: N->getOperand(Num: 2));
5836}
5837
5838SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
5839 SDValue InOp = N->getOperand(Num: 1);
5840 SDLoc DL(N);
5841 SmallVector<SDValue, 4> NewOps(N->ops());
5842
5843 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5844 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5845
5846 EVT InVT = InOp.getValueType();
5847 EVT InEltVT = InVT.getVectorElementType();
5848
5849 unsigned Opcode = N->getOpcode();
5850
5851 // FIXME: Optimizations need to be implemented here.
5852
5853 // Otherwise unroll into some nasty scalar code and rebuild the vector.
5854 EVT EltVT = WidenVT.getVectorElementType();
5855 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
5856 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
5857 SmallVector<SDValue, 32> OpChains;
5858 // Use the original element count so we don't do more scalar opts than
5859 // necessary.
5860 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
5861 for (unsigned i=0; i < MinElts; ++i) {
5862 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
5863 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
5864 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
5865 }
5866 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
5867 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5868
5869 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5870}
5871
5872SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
5873 unsigned Opcode = N->getOpcode();
5874 SDValue InOp = N->getOperand(Num: 0);
5875 SDLoc DL(N);
5876
5877 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5878 EVT WidenSVT = WidenVT.getVectorElementType();
5879 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5880
5881 EVT InVT = InOp.getValueType();
5882 EVT InSVT = InVT.getVectorElementType();
5883 unsigned InVTNumElts = InVT.getVectorNumElements();
5884
5885 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
5886 InOp = GetWidenedVector(Op: InOp);
5887 InVT = InOp.getValueType();
5888 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
5889 switch (Opcode) {
5890 case ISD::ANY_EXTEND_VECTOR_INREG:
5891 case ISD::SIGN_EXTEND_VECTOR_INREG:
5892 case ISD::ZERO_EXTEND_VECTOR_INREG:
5893 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
5894 }
5895 }
5896 }
5897
5898 // Unroll, extend the scalars and rebuild the vector.
5899 SmallVector<SDValue, 16> Ops;
5900 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
5901 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
5902 switch (Opcode) {
5903 case ISD::ANY_EXTEND_VECTOR_INREG:
5904 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
5905 break;
5906 case ISD::SIGN_EXTEND_VECTOR_INREG:
5907 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
5908 break;
5909 case ISD::ZERO_EXTEND_VECTOR_INREG:
5910 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
5911 break;
5912 default:
5913 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
5914 }
5915 Ops.push_back(Elt: Val);
5916 }
5917
5918 while (Ops.size() != WidenNumElts)
5919 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
5920
5921 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
5922}
5923
5924SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
5925 // If this is an FCOPYSIGN with same input types, we can treat it as a
5926 // normal (can trap) binary op.
5927 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
5928 return WidenVecRes_BinaryCanTrap(N);
5929
5930 // If the types are different, fall back to unrolling.
5931 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5932 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5933}
5934
5935/// Result and first source operand are different scalar types, but must have
5936/// the same number of elements. There is an additional control argument which
5937/// should be passed through unchanged.
5938SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
5939 SDValue FpValue = N->getOperand(Num: 0);
5940 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5941 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
5942 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5943 SDValue Arg = GetWidenedVector(Op: FpValue);
5944 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
5945 Flags: N->getFlags());
5946}
5947
5948SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
5949 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5950 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5951 SDValue RHS = N->getOperand(Num: 1);
5952 EVT ExpVT = RHS.getValueType();
5953 SDValue ExpOp = RHS;
5954 if (ExpVT.isVector()) {
5955 EVT WideExpVT = WidenVT.changeVectorElementType(
5956 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
5957 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
5958 }
5959
5960 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
5961}
5962
5963SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
5964 // Unary op widening.
5965 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5966 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
5967 if (N->getNumOperands() == 1)
5968 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
5969 if (N->getOpcode() == ISD::AssertNoFPClass)
5970 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp,
5971 N2: N->getOperand(Num: 1), Flags: N->getFlags());
5972
5973 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5974 assert(N->isVPOpcode() && "Expected VP opcode");
5975
5976 SDValue Mask =
5977 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: WidenVT.getVectorElementCount());
5978 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT,
5979 Ops: {InOp, Mask, N->getOperand(Num: 2)});
5980}
5981
5982SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
5983 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5984 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
5985 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT()
5986 .getVectorElementType(),
5987 NumElements: WidenVT.getVectorNumElements());
5988 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5989 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
5990 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
5991}
5992
5993SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
5994 unsigned ResNo) {
5995 EVT VT0 = N->getValueType(ResNo: 0);
5996 EVT VT1 = N->getValueType(ResNo: 1);
5997
5998 assert(VT0.isVector() && VT1.isVector() &&
5999 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6000 "expected both results to be vectors of matching element count");
6001
6002 LLVMContext &Ctx = *DAG.getContext();
6003 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6004
6005 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6006 ElementCount WidenEC = WidenVT.getVectorElementCount();
6007
6008 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6009 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6010
6011 SDNode *WidenNode =
6012 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6013 .getNode();
6014
6015 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6016 return SDValue(WidenNode, ResNo);
6017}
6018
6019SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6020 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6021 return GetWidenedVector(Op: WidenVec);
6022}
6023
6024SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6025 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6026 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6027 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6028
6029 return DAG.getAddrSpaceCast(dl: SDLoc(N), VT: WidenVT, Ptr: InOp,
6030 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6031 DestAS: AddrSpaceCastN->getDestAddressSpace());
6032}
6033
6034SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6035 SDValue InOp = N->getOperand(Num: 0);
6036 EVT InVT = InOp.getValueType();
6037 EVT VT = N->getValueType(ResNo: 0);
6038 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6039 SDLoc dl(N);
6040
6041 switch (getTypeAction(VT: InVT)) {
6042 case TargetLowering::TypeLegal:
6043 break;
6044 case TargetLowering::TypeScalarizeScalableVector:
6045 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6046 case TargetLowering::TypePromoteInteger: {
6047 // If the incoming type is a vector that is being promoted, then
6048 // we know that the elements are arranged differently and that we
6049 // must perform the conversion using a stack slot.
6050 if (InVT.isVector())
6051 break;
6052
6053 // If the InOp is promoted to the same size, convert it. Otherwise,
6054 // fall out of the switch and widen the promoted input.
6055 SDValue NInOp = GetPromotedInteger(Op: InOp);
6056 EVT NInVT = NInOp.getValueType();
6057 if (WidenVT.bitsEq(VT: NInVT)) {
6058 // For big endian targets we need to shift the input integer or the
6059 // interesting bits will end up at the wrong place.
6060 if (DAG.getDataLayout().isBigEndian()) {
6061 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6062 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6063 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6064 }
6065 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6066 }
6067 InOp = NInOp;
6068 InVT = NInVT;
6069 break;
6070 }
6071 case TargetLowering::TypeSoftenFloat:
6072 case TargetLowering::TypeSoftPromoteHalf:
6073 case TargetLowering::TypeExpandInteger:
6074 case TargetLowering::TypeExpandFloat:
6075 case TargetLowering::TypeScalarizeVector:
6076 case TargetLowering::TypeSplitVector:
6077 break;
6078 case TargetLowering::TypeWidenVector:
6079 // If the InOp is widened to the same size, convert it. Otherwise, fall
6080 // out of the switch and widen the widened input.
6081 InOp = GetWidenedVector(Op: InOp);
6082 InVT = InOp.getValueType();
6083 if (WidenVT.bitsEq(VT: InVT))
6084 // The input widens to the same size. Convert to the widen value.
6085 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6086 break;
6087 }
6088
6089 unsigned WidenSize = WidenVT.getSizeInBits();
6090 unsigned InSize = InVT.getSizeInBits();
6091 unsigned InScalarSize = InVT.getScalarSizeInBits();
6092 // x86mmx is not an acceptable vector element type, so don't try.
6093 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6094 // Determine new input vector type. The new input vector type will use
6095 // the same element type (if its a vector) or use the input type as a
6096 // vector. It is the same size as the type to widen to.
6097 EVT NewInVT;
6098 unsigned NewNumParts = WidenSize / InSize;
6099 if (InVT.isVector()) {
6100 EVT InEltVT = InVT.getVectorElementType();
6101 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6102 NumElements: WidenSize / InEltVT.getSizeInBits());
6103 } else {
6104 // For big endian systems, using the promoted input scalar type
6105 // to produce the scalar_to_vector would put the desired bits into
6106 // the least significant byte(s) of the wider element zero. This
6107 // will mean that the users of the result vector are using incorrect
6108 // bits. Use the original input type instead. Although either input
6109 // type can be used on little endian systems, for consistency we
6110 // use the original type there as well.
6111 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6112 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6113 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6114 }
6115
6116 if (TLI.isTypeLegal(VT: NewInVT)) {
6117 SDValue NewVec;
6118 if (InVT.isVector()) {
6119 // Because the result and the input are different vector types, widening
6120 // the result could create a legal type but widening the input might
6121 // make it an illegal type that might lead to repeatedly splitting the
6122 // input and then widening it. To avoid this, we widen the input only if
6123 // it results in a legal type.
6124 if (WidenSize % InSize == 0) {
6125 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6126 Ops[0] = InOp;
6127
6128 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6129 } else {
6130 SmallVector<SDValue, 16> Ops;
6131 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6132 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6133 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6134
6135 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6136 }
6137 } else {
6138 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6139 }
6140 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6141 }
6142 }
6143
6144 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6145}
6146
6147SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6148 return DAG.getNode(
6149 Opcode: N->getOpcode(), DL: SDLoc(N),
6150 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6151 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6152}
6153
6154SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6155 SDLoc dl(N);
6156 // Build a vector with poison for the new nodes.
6157 EVT VT = N->getValueType(ResNo: 0);
6158
6159 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6160 // type. The POISONs need to have the same type as the existing operands.
6161 EVT EltVT = N->getOperand(Num: 0).getValueType();
6162 unsigned NumElts = VT.getVectorNumElements();
6163
6164 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6165 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6166
6167 SmallVector<SDValue, 16> NewOps(N->ops());
6168 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6169 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6170
6171 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6172}
6173
6174SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6175 EVT InVT = N->getOperand(Num: 0).getValueType();
6176 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6177 SDLoc dl(N);
6178 unsigned NumOperands = N->getNumOperands();
6179
6180 bool InputWidened = false; // Indicates we need to widen the input.
6181 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6182 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6183 unsigned NumInElts = InVT.getVectorMinNumElements();
6184 if (WidenNumElts % NumInElts == 0) {
6185 // Add undef vectors to widen to correct length.
6186 unsigned NumConcat = WidenNumElts / NumInElts;
6187 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6188 SmallVector<SDValue, 16> Ops(NumConcat);
6189 for (unsigned i=0; i < NumOperands; ++i)
6190 Ops[i] = N->getOperand(Num: i);
6191 for (unsigned i = NumOperands; i != NumConcat; ++i)
6192 Ops[i] = UndefVal;
6193 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6194 }
6195 } else {
6196 InputWidened = true;
6197 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6198 // The inputs and the result are widen to the same value.
6199 unsigned i;
6200 for (i=1; i < NumOperands; ++i)
6201 if (!N->getOperand(Num: i).isUndef())
6202 break;
6203
6204 if (i == NumOperands)
6205 // Everything but the first operand is an UNDEF so just return the
6206 // widened first operand.
6207 return GetWidenedVector(Op: N->getOperand(Num: 0));
6208
6209 if (NumOperands == 2) {
6210 assert(!WidenVT.isScalableVector() &&
6211 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6212 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6213 unsigned NumInElts = InVT.getVectorNumElements();
6214
6215 // Replace concat of two operands with a shuffle.
6216 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6217 for (unsigned i = 0; i < NumInElts; ++i) {
6218 MaskOps[i] = i;
6219 MaskOps[i + NumInElts] = i + WidenNumElts;
6220 }
6221 return DAG.getVectorShuffle(VT: WidenVT, dl,
6222 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6223 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6224 Mask: MaskOps);
6225 }
6226 }
6227 }
6228
6229 assert(!WidenVT.isScalableVector() &&
6230 "Cannot use build vectors to widen CONCAT_VECTOR result");
6231 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6232 unsigned NumInElts = InVT.getVectorNumElements();
6233
6234 // Fall back to use extracts and build vector.
6235 EVT EltVT = WidenVT.getVectorElementType();
6236 SmallVector<SDValue, 16> Ops(WidenNumElts);
6237 unsigned Idx = 0;
6238 for (unsigned i=0; i < NumOperands; ++i) {
6239 SDValue InOp = N->getOperand(Num: i);
6240 if (InputWidened)
6241 InOp = GetWidenedVector(Op: InOp);
6242 for (unsigned j = 0; j < NumInElts; ++j)
6243 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6244 }
6245 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6246 for (; Idx < WidenNumElts; ++Idx)
6247 Ops[Idx] = UndefVal;
6248 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6249}
6250
6251SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6252 EVT VT = N->getValueType(ResNo: 0);
6253 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6254 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6255 SDValue InOp2 = N->getOperand(Num: 1);
6256 SDValue Idx = N->getOperand(Num: 2);
6257 SDLoc dl(N);
6258 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6259}
6260
6261SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6262 EVT VT = N->getValueType(ResNo: 0);
6263 EVT EltVT = VT.getVectorElementType();
6264 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6265 SDValue InOp = N->getOperand(Num: 0);
6266 SDValue Idx = N->getOperand(Num: 1);
6267 SDLoc dl(N);
6268
6269 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6270 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6271 InOp = GetWidenedVector(Op: InOp);
6272
6273 EVT InVT = InOp.getValueType();
6274
6275 // Check if we can just return the input vector after widening.
6276 uint64_t IdxVal = Idx->getAsZExtVal();
6277 if (IdxVal == 0 && InVT == WidenVT)
6278 return InOp;
6279
6280 // Check if we can extract from the vector.
6281 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6282 unsigned InNumElts = InVT.getVectorMinNumElements();
6283 unsigned VTNumElts = VT.getVectorMinNumElements();
6284 assert(IdxVal % VTNumElts == 0 &&
6285 "Expected Idx to be a multiple of subvector minimum vector length");
6286 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6287 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6288
6289 if (VT.isScalableVector()) {
6290 // Try to split the operation up into smaller extracts and concat the
6291 // results together, e.g.
6292 // nxv6i64 extract_subvector(nxv12i64, 6)
6293 // <->
6294 // nxv8i64 concat(
6295 // nxv2i64 extract_subvector(nxv16i64, 6)
6296 // nxv2i64 extract_subvector(nxv16i64, 8)
6297 // nxv2i64 extract_subvector(nxv16i64, 10)
6298 // undef)
6299 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6300 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6301 "down type's element count");
6302 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6303 EC: ElementCount::getScalable(MinVal: GCD));
6304 // Avoid recursion around e.g. nxv1i8.
6305 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6306 SmallVector<SDValue> Parts;
6307 unsigned I = 0;
6308 for (; I < VTNumElts / GCD; ++I)
6309 Parts.push_back(
6310 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6311 for (; I < WidenNumElts / GCD; ++I)
6312 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6313
6314 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6315 }
6316
6317 // Fallback to extracting through memory.
6318
6319 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6320 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6321 MachineFunction &MF = DAG.getMachineFunction();
6322 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6323 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6324
6325 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6326 PtrInfo, F: MachineMemOperand::MOStore,
6327 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6328 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6329 PtrInfo, F: MachineMemOperand::MOLoad,
6330 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6331
6332 // Write out the input vector.
6333 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6334
6335 // Build a mask to match the length of the non-widened result.
6336 SDValue Mask =
6337 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6338
6339 // Read back the sub-vector setting the remaining lanes to poison.
6340 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6341 return DAG.getMaskedLoad(
6342 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6343 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6344 }
6345
6346 // We could try widening the input to the right length but for now, extract
6347 // the original elements, fill the rest with undefs and build a vector.
6348 SmallVector<SDValue, 16> Ops(WidenNumElts);
6349 unsigned i;
6350 for (i = 0; i < VTNumElts; ++i)
6351 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6352
6353 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6354 for (; i < WidenNumElts; ++i)
6355 Ops[i] = UndefVal;
6356 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6357}
6358
6359SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6360 SDValue InOp = ModifyToType(
6361 InOp: N->getOperand(Num: 0),
6362 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6363 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6364 N2: N->getOperand(Num: 1));
6365}
6366
6367SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6368 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6369 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6370 VT: InOp.getValueType(), N1: InOp,
6371 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6372}
6373
6374/// Either return the same load or provide appropriate casts
6375/// from the load and return that.
6376static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6377 TypeSize LdWidth, TypeSize FirstVTWidth,
6378 SDLoc dl, SelectionDAG &DAG) {
6379 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6380 "Load width must be less than or equal to first value type width");
6381 TypeSize WidenWidth = WidenVT.getSizeInBits();
6382 if (!FirstVT.isVector()) {
6383 unsigned NumElts =
6384 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6385 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6386 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6387 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6388 }
6389 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6390 return LdOp;
6391}
6392
6393static std::optional<EVT> findMemType(SelectionDAG &DAG,
6394 const TargetLowering &TLI, unsigned Width,
6395 EVT WidenVT, unsigned Align,
6396 unsigned WidenEx);
6397
6398SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6399 EVT WidenVT =
6400 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6401 EVT LdVT = LD->getMemoryVT();
6402 SDLoc dl(LD);
6403 assert(LdVT.isVector() && WidenVT.isVector() && "Expected vectors");
6404 assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6405 "Must be scalable");
6406 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6407 "Expected equivalent element types");
6408
6409 // Load information
6410 SDValue Chain = LD->getChain();
6411 SDValue BasePtr = LD->getBasePtr();
6412
6413 TypeSize LdWidth = LdVT.getSizeInBits();
6414 TypeSize WidenWidth = WidenVT.getSizeInBits();
6415 TypeSize WidthDiff = WidenWidth - LdWidth;
6416
6417 // Find the vector type that can load from.
6418 std::optional<EVT> FirstVT =
6419 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6420 WidenEx: WidthDiff.getKnownMinValue());
6421
6422 if (!FirstVT)
6423 return SDValue();
6424
6425 SmallVector<EVT, 8> MemVTs;
6426 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6427
6428 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6429 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6430
6431 // Load the element with one instruction.
6432 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6433 FirstVTWidth, dl, DAG);
6434
6435 // Modified the chain - switch anything that used the old chain to use
6436 // the new one.
6437 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6438 return Result;
6439}
6440
6441SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6442 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6443 ISD::LoadExtType ExtType = LD->getExtensionType();
6444
6445 // A vector must always be stored in memory as-is, i.e. without any padding
6446 // between the elements, since various code depend on it, e.g. in the
6447 // handling of a bitcast of a vector type to int, which may be done with a
6448 // vector store followed by an integer load. A vector that does not have
6449 // elements that are byte-sized must therefore be stored as an integer
6450 // built out of the extracted vector elements.
6451 if (!LD->getMemoryVT().isByteSized()) {
6452 SDValue Value, NewChain;
6453 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6454 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6455 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6456 return SDValue();
6457 }
6458
6459 // Generate a vector-predicated load if it is custom/legal on the target. To
6460 // avoid possible recursion, only do this if the widened mask type is legal.
6461 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6462 // removed from the IR by the ExpandVectorPredication pass but we're
6463 // reintroducing them here.
6464 EVT VT = LD->getValueType(ResNo: 0);
6465 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6466 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6467
6468 if (ExtType == ISD::NON_EXTLOAD &&
6469 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6470 TLI.isTypeLegal(VT: WideMaskVT)) {
6471 SDLoc DL(N);
6472 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6473 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6474 EC: VT.getVectorElementCount());
6475 SDValue NewLoad =
6476 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6477 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6478 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6479
6480 // Modified the chain - switch anything that used the old chain to use
6481 // the new one.
6482 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6483
6484 return NewLoad;
6485 }
6486
6487 SDValue Result;
6488 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6489 if (ExtType != ISD::NON_EXTLOAD)
6490 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6491 else
6492 Result = GenWidenVectorLoads(LdChain, LD);
6493
6494 if (Result) {
6495 // If we generate a single load, we can use that for the chain. Otherwise,
6496 // build a factor node to remember the multiple loads are independent and
6497 // chain to that.
6498 SDValue NewChain;
6499 if (LdChain.size() == 1)
6500 NewChain = LdChain[0];
6501 else
6502 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6503
6504 // Modified the chain - switch anything that used the old chain to use
6505 // the new one.
6506 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6507
6508 return Result;
6509 }
6510
6511 if (VT.isVector()) {
6512 // If all else fails replace the load with a wide masked load.
6513 SDLoc DL(N);
6514 SDValue Mask =
6515 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6516
6517 SDValue NewLoad = DAG.getMaskedLoad(
6518 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6519 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6520 AM: LD->getAddressingMode(), LD->getExtensionType());
6521
6522 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6523 return NewLoad;
6524 }
6525
6526 report_fatal_error(reason: "Unable to widen vector load");
6527}
6528
6529SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6530 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6531 SDValue Mask = N->getMask();
6532 SDValue EVL = N->getVectorLength();
6533 ISD::LoadExtType ExtType = N->getExtensionType();
6534 SDLoc dl(N);
6535
6536 // The mask should be widened as well
6537 assert(getTypeAction(Mask.getValueType()) ==
6538 TargetLowering::TypeWidenVector &&
6539 "Unable to widen binary VP op");
6540 Mask = GetWidenedVector(Op: Mask);
6541 assert(Mask.getValueType().getVectorElementCount() ==
6542 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6543 .getVectorElementCount() &&
6544 "Unable to widen vector load");
6545
6546 SDValue Res =
6547 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6548 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6549 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6550 // Legalize the chain result - switch anything that used the old chain to
6551 // use the new one.
6552 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6553 return Res;
6554}
6555
6556SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6557 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6558 SDValue Mask = N->getMask();
6559 SDValue EVL = N->getVectorLength();
6560 SDLoc dl(N);
6561
6562 // The mask should be widened as well
6563 assert(getTypeAction(Mask.getValueType()) ==
6564 TargetLowering::TypeWidenVector &&
6565 "Unable to widen binary VP op");
6566 Mask = GetWidenedVector(Op: Mask);
6567 assert(Mask.getValueType().getVectorElementCount() ==
6568 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6569 .getVectorElementCount() &&
6570 "Unable to widen vector load");
6571
6572 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6573 Mask, EVL, MMO: N->getMemOperand());
6574 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6575 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6576 return Res;
6577}
6578
6579SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6580 SDLoc DL(N);
6581
6582 // The mask should be widened as well
6583 SDValue Mask = N->getMask();
6584 assert(getTypeAction(Mask.getValueType()) ==
6585 TargetLowering::TypeWidenVector &&
6586 "Unable to widen VP strided load");
6587 Mask = GetWidenedVector(Op: Mask);
6588
6589 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6590 assert(Mask.getValueType().getVectorElementCount() ==
6591 WidenVT.getVectorElementCount() &&
6592 "Data and mask vectors should have the same number of elements");
6593
6594 SDValue Res = DAG.getStridedLoadVP(
6595 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6596 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6597 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6598 IsExpanding: N->isExpandingLoad());
6599
6600 // Legalize the chain result - switch anything that used the old chain to
6601 // use the new one.
6602 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6603 return Res;
6604}
6605
6606SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6607 SDValue Vec = N->getOperand(Num: 0);
6608 SDValue Mask = N->getOperand(Num: 1);
6609 SDValue Passthru = N->getOperand(Num: 2);
6610 EVT WideVecVT =
6611 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6612 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6613 VT: Mask.getValueType().getVectorElementType(),
6614 EC: WideVecVT.getVectorElementCount());
6615
6616 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6617 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6618 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6619 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6620 N2: WideMask, N3: WidePassthru);
6621}
6622
6623SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6624 EVT VT = N->getValueType(ResNo: 0);
6625 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6626 SDValue Mask = N->getMask();
6627 EVT MaskVT = Mask.getValueType();
6628 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6629 ISD::LoadExtType ExtType = N->getExtensionType();
6630 SDLoc dl(N);
6631
6632 EVT WideMaskVT =
6633 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6634 EC: WidenVT.getVectorElementCount());
6635
6636 if (ExtType == ISD::NON_EXTLOAD &&
6637 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6638 TLI.isTypeLegal(VT: WideMaskVT) &&
6639 // If there is a passthru, we shouldn't use vp.load. However,
6640 // type legalizer will struggle on masked.load with
6641 // scalable vectors, so for scalable vectors, we still use vp.load
6642 // but manually merge the load result with the passthru using vp.select.
6643 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6644 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6645 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6646 EC: VT.getVectorElementCount());
6647 SDValue NewLoad =
6648 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6649 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6650 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6651 SDValue NewVal = NewLoad;
6652
6653 // Manually merge with vselect
6654 if (!N->getPassThru()->isUndef()) {
6655 assert(WidenVT.isScalableVector());
6656 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
6657 // The lanes past EVL are poison.
6658 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
6659 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
6660 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
6661 }
6662
6663 // Modified the chain - switch anything that used the old chain to use
6664 // the new one.
6665 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6666
6667 return NewVal;
6668 }
6669
6670 // The mask should be widened as well
6671 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6672
6673 SDValue Res = DAG.getMaskedLoad(
6674 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
6675 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
6676 ExtType, IsExpanding: N->isExpandingLoad());
6677 // Legalize the chain result - switch anything that used the old chain to
6678 // use the new one.
6679 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6680 return Res;
6681}
6682
6683SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
6684
6685 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6686 SDValue Mask = N->getMask();
6687 EVT MaskVT = Mask.getValueType();
6688 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6689 SDValue Scale = N->getScale();
6690 unsigned NumElts = WideVT.getVectorNumElements();
6691 SDLoc dl(N);
6692
6693 // The mask should be widened as well
6694 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6695 VT: MaskVT.getVectorElementType(),
6696 NumElements: WideVT.getVectorNumElements());
6697 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6698
6699 // Widen the Index operand
6700 SDValue Index = N->getIndex();
6701 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6702 VT: Index.getValueType().getScalarType(),
6703 NumElements: NumElts);
6704 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
6705 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
6706 Scale };
6707
6708 // Widen the MemoryType
6709 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6710 VT: N->getMemoryVT().getScalarType(), NumElements: NumElts);
6711 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
6712 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
6713 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
6714
6715 // Legalize the chain result - switch anything that used the old chain to
6716 // use the new one.
6717 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6718 return Res;
6719}
6720
6721SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
6722 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6723 SDValue Mask = N->getMask();
6724 SDValue Scale = N->getScale();
6725 ElementCount WideEC = WideVT.getVectorElementCount();
6726 SDLoc dl(N);
6727
6728 SDValue Index = GetWidenedVector(Op: N->getIndex());
6729 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6730 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
6731 Mask = GetWidenedMask(Mask, EC: WideEC);
6732
6733 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
6734 Mask, N->getVectorLength()};
6735 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
6736 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
6737
6738 // Legalize the chain result - switch anything that used the old chain to
6739 // use the new one.
6740 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6741 return Res;
6742}
6743
6744SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
6745 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6746 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
6747}
6748
6749// Return true is this is a SETCC node or a strict version of it.
6750static inline bool isSETCCOp(unsigned Opcode) {
6751 switch (Opcode) {
6752 case ISD::SETCC:
6753 case ISD::STRICT_FSETCC:
6754 case ISD::STRICT_FSETCCS:
6755 return true;
6756 }
6757 return false;
6758}
6759
6760// Return true if this is a node that could have two SETCCs as operands.
6761static inline bool isLogicalMaskOp(unsigned Opcode) {
6762 switch (Opcode) {
6763 case ISD::AND:
6764 case ISD::OR:
6765 case ISD::XOR:
6766 return true;
6767 }
6768 return false;
6769}
6770
6771// If N is a SETCC or a strict variant of it, return the type
6772// of the compare operands.
6773static inline EVT getSETCCOperandType(SDValue N) {
6774 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
6775 return N->getOperand(Num: OpNo).getValueType();
6776}
6777
6778// This is used just for the assert in convertMask(). Check that this either
6779// a SETCC or a previously handled SETCC by convertMask().
6780#ifndef NDEBUG
6781static inline bool isSETCCorConvertedSETCC(SDValue N) {
6782 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
6783 N = N.getOperand(0);
6784 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
6785 for (unsigned i = 1; i < N->getNumOperands(); ++i)
6786 if (!N->getOperand(i)->isUndef())
6787 return false;
6788 N = N.getOperand(0);
6789 }
6790
6791 if (N.getOpcode() == ISD::TRUNCATE)
6792 N = N.getOperand(0);
6793 else if (N.getOpcode() == ISD::SIGN_EXTEND)
6794 N = N.getOperand(0);
6795
6796 if (isLogicalMaskOp(N.getOpcode()))
6797 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
6798 isSETCCorConvertedSETCC(N.getOperand(1));
6799
6800 return (isSETCCOp(N.getOpcode()) ||
6801 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
6802}
6803#endif
6804
6805// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
6806// to ToMaskVT if needed with vector extension or truncation.
6807SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
6808 EVT ToMaskVT) {
6809 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
6810 // FIXME: This code seems to be too restrictive, we might consider
6811 // generalizing it or dropping it.
6812 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
6813
6814 // Make a new Mask node, with a legal result VT.
6815 SDValue Mask;
6816 SmallVector<SDValue, 4> Ops;
6817 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
6818 Ops.push_back(Elt: InMask->getOperand(Num: i));
6819 if (InMask->isStrictFPOpcode()) {
6820 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
6821 ResultTys: { MaskVT, MVT::Other }, Ops);
6822 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
6823 }
6824 else
6825 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
6826 Flags: InMask->getFlags());
6827
6828 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
6829 // extend or truncate is needed.
6830 LLVMContext &Ctx = *DAG.getContext();
6831 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
6832 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
6833 if (MaskScalarBits < ToMaskScalBits) {
6834 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6835 NumElements: MaskVT.getVectorNumElements());
6836 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
6837 } else if (MaskScalarBits > ToMaskScalBits) {
6838 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
6839 NumElements: MaskVT.getVectorNumElements());
6840 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
6841 }
6842
6843 assert(Mask->getValueType(0).getScalarSizeInBits() ==
6844 ToMaskVT.getScalarSizeInBits() &&
6845 "Mask should have the right element size by now.");
6846
6847 // Adjust Mask to the right number of elements.
6848 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
6849 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
6850 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
6851 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
6852 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
6853 EVT SubVT = Mask->getValueType(ResNo: 0);
6854 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
6855 SubOps[0] = Mask;
6856 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
6857 }
6858
6859 assert((Mask->getValueType(0) == ToMaskVT) &&
6860 "A mask of ToMaskVT should have been produced by now.");
6861
6862 return Mask;
6863}
6864
6865// This method tries to handle some special cases for the vselect mask
6866// and if needed adjusting the mask vector type to match that of the VSELECT.
6867// Without it, many cases end up with scalarization of the SETCC, with many
6868// unnecessary instructions.
6869SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
6870 LLVMContext &Ctx = *DAG.getContext();
6871 SDValue Cond = N->getOperand(Num: 0);
6872
6873 if (N->getOpcode() != ISD::VSELECT)
6874 return SDValue();
6875
6876 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
6877 return SDValue();
6878
6879 // If this is a splitted VSELECT that was previously already handled, do
6880 // nothing.
6881 EVT CondVT = Cond->getValueType(ResNo: 0);
6882 if (CondVT.getScalarSizeInBits() != 1)
6883 return SDValue();
6884
6885 EVT VSelVT = N->getValueType(ResNo: 0);
6886
6887 // This method can't handle scalable vector types.
6888 // FIXME: This support could be added in the future.
6889 if (VSelVT.isScalableVector())
6890 return SDValue();
6891
6892 // Only handle vector types which are a power of 2.
6893 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
6894 return SDValue();
6895
6896 // Don't touch if this will be scalarized.
6897 EVT FinalVT = VSelVT;
6898 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
6899 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
6900
6901 if (FinalVT.getVectorNumElements() == 1)
6902 return SDValue();
6903
6904 // If there is support for an i1 vector mask, don't touch.
6905 if (isSETCCOp(Opcode: Cond.getOpcode())) {
6906 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
6907 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
6908 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
6909 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
6910 if (SetCCResVT.getScalarSizeInBits() == 1)
6911 return SDValue();
6912 } else if (CondVT.getScalarType() == MVT::i1) {
6913 // If there is support for an i1 vector mask (or only scalar i1 conditions),
6914 // don't touch.
6915 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
6916 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
6917
6918 if (CondVT.getScalarType() == MVT::i1)
6919 return SDValue();
6920 }
6921
6922 // Widen the vselect result type if needed.
6923 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
6924 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
6925
6926 // The mask of the VSELECT should have integer elements.
6927 EVT ToMaskVT = VSelVT;
6928 if (!ToMaskVT.getScalarType().isInteger())
6929 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
6930
6931 SDValue Mask;
6932 if (isSETCCOp(Opcode: Cond->getOpcode())) {
6933 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
6934 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6935 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
6936 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
6937 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
6938 // Cond is (AND/OR/XOR (SETCC, SETCC))
6939 SDValue SETCC0 = Cond->getOperand(Num: 0);
6940 SDValue SETCC1 = Cond->getOperand(Num: 1);
6941 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
6942 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
6943 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
6944 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
6945 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
6946 EVT MaskVT;
6947 // If the two SETCCs have different VTs, either extend/truncate one of
6948 // them to the other "towards" ToMaskVT, or truncate one and extend the
6949 // other to ToMaskVT.
6950 if (ScalarBits0 != ScalarBits1) {
6951 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
6952 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
6953 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
6954 MaskVT = WideVT;
6955 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
6956 MaskVT = NarrowVT;
6957 else
6958 MaskVT = ToMaskVT;
6959 } else
6960 // If the two SETCCs have the same VT, don't change it.
6961 MaskVT = VT0;
6962
6963 // Make new SETCCs and logical nodes.
6964 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
6965 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
6966 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
6967
6968 // Convert the logical op for VSELECT if needed.
6969 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
6970 } else
6971 return SDValue();
6972
6973 return Mask;
6974}
6975
6976SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
6977 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6978 ElementCount WidenEC = WidenVT.getVectorElementCount();
6979
6980 SDValue Cond1 = N->getOperand(Num: 0);
6981 EVT CondVT = Cond1.getValueType();
6982 unsigned Opcode = N->getOpcode();
6983 if (CondVT.isVector()) {
6984 if (SDValue WideCond = WidenVSELECTMask(N)) {
6985 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
6986 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
6987 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
6988 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
6989 }
6990
6991 EVT CondEltVT = CondVT.getVectorElementType();
6992 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
6993 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
6994 Cond1 = GetWidenedVector(Op: Cond1);
6995
6996 // If we have to split the condition there is no point in widening the
6997 // select. This would result in an cycle of widening the select ->
6998 // widening the condition operand -> splitting the condition operand ->
6999 // splitting the select -> widening the select. Instead split this select
7000 // further and widen the resulting type.
7001 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7002 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7003 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7004 return Res;
7005 }
7006
7007 if (Cond1.getValueType() != CondWidenVT)
7008 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7009 }
7010
7011 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7012 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7013 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7014 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
7015 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7016 N4: N->getOperand(Num: 3));
7017 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7018}
7019
7020SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7021 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7022 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7023 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7024 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7025 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7026}
7027
7028SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7029 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7030 return DAG.getUNDEF(VT: WidenVT);
7031}
7032
7033SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7034 EVT VT = N->getValueType(ResNo: 0);
7035 SDLoc dl(N);
7036
7037 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7038 unsigned NumElts = VT.getVectorNumElements();
7039 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7040
7041 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7042 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7043
7044 // Adjust mask based on new input vector length.
7045 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7046 for (unsigned i = 0; i != NumElts; ++i) {
7047 int Idx = N->getMaskElt(Idx: i);
7048 if (Idx < (int)NumElts)
7049 NewMask[i] = Idx;
7050 else
7051 NewMask[i] = Idx - NumElts + WidenNumElts;
7052 }
7053 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7054}
7055
7056SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7057 EVT VT = N->getValueType(ResNo: 0);
7058 EVT EltVT = VT.getVectorElementType();
7059 SDLoc dl(N);
7060
7061 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7062 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7063 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7064
7065 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7066 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7067 unsigned VTNumElts = VT.getVectorMinNumElements();
7068 unsigned IdxVal = WidenNumElts - VTNumElts;
7069
7070 if (VT.isScalableVector()) {
7071 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7072 // results together, e.g.(nxv6i64 -> nxv8i64)
7073 // nxv8i64 vector_reverse
7074 // <->
7075 // nxv8i64 concat(
7076 // nxv2i64 extract_subvector(nxv8i64, 2)
7077 // nxv2i64 extract_subvector(nxv8i64, 4)
7078 // nxv2i64 extract_subvector(nxv8i64, 6)
7079 // nxv2i64 undef)
7080
7081 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7082 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7083 EC: ElementCount::getScalable(MinVal: GCD));
7084 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7085 "down type's element count");
7086 SmallVector<SDValue> Parts;
7087 unsigned i = 0;
7088 for (; i < VTNumElts / GCD; ++i)
7089 Parts.push_back(
7090 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7091 for (; i < WidenNumElts / GCD; ++i)
7092 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7093
7094 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7095 }
7096
7097 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7098 // fixed-vectors.
7099 SmallVector<int, 16> Mask(WidenNumElts, -1);
7100 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7101
7102 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7103 Mask);
7104}
7105
7106SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7107 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7108 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7109}
7110
7111SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7112 assert(N->getValueType(0).isVector() &&
7113 N->getOperand(0).getValueType().isVector() &&
7114 "Operands must be vectors");
7115 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7116 ElementCount WidenEC = WidenVT.getVectorElementCount();
7117
7118 SDValue InOp1 = N->getOperand(Num: 0);
7119 EVT InVT = InOp1.getValueType();
7120 assert(InVT.isVector() && "can not widen non-vector type");
7121 EVT WidenInVT =
7122 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7123
7124 // The input and output types often differ here, and it could be that while
7125 // we'd prefer to widen the result type, the input operands have been split.
7126 // In this case, we also need to split the result of this node as well.
7127 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7128 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7129 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7130 return Res;
7131 }
7132
7133 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7134 SDValue InOp2 = N->getOperand(Num: 1);
7135 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7136 InOp1 = GetWidenedVector(Op: InOp1);
7137 InOp2 = GetWidenedVector(Op: InOp2);
7138 } else {
7139 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7140 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7141 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7142 N2: InOp1, N3: ZeroIdx);
7143 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7144 N2: InOp2, N3: ZeroIdx);
7145 }
7146
7147 // Assume that the input and output will be widen appropriately. If not,
7148 // we will have to unroll it at some point.
7149 assert(InOp1.getValueType() == WidenInVT &&
7150 InOp2.getValueType() == WidenInVT &&
7151 "Input not widened to expected type!");
7152 (void)WidenInVT;
7153 if (N->getOpcode() == ISD::VP_SETCC) {
7154 SDValue Mask =
7155 GetWidenedMask(Mask: N->getOperand(Num: 3), EC: WidenVT.getVectorElementCount());
7156 return DAG.getNode(Opcode: ISD::VP_SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7157 N3: N->getOperand(Num: 2), N4: Mask, N5: N->getOperand(Num: 4));
7158 }
7159 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7160 N3: N->getOperand(Num: 2));
7161}
7162
7163SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7164 assert(N->getValueType(0).isVector() &&
7165 N->getOperand(1).getValueType().isVector() &&
7166 "Operands must be vectors");
7167 EVT VT = N->getValueType(ResNo: 0);
7168 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7169 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7170 unsigned NumElts = VT.getVectorNumElements();
7171 EVT EltVT = VT.getVectorElementType();
7172
7173 SDLoc dl(N);
7174 SDValue Chain = N->getOperand(Num: 0);
7175 SDValue LHS = N->getOperand(Num: 1);
7176 SDValue RHS = N->getOperand(Num: 2);
7177 SDValue CC = N->getOperand(Num: 3);
7178 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7179
7180 // Fully unroll and reassemble.
7181 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7182 SmallVector<SDValue, 8> Chains(NumElts);
7183 for (unsigned i = 0; i != NumElts; ++i) {
7184 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7185 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7186
7187 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7188 Ops: {Chain, LHSElem, RHSElem, CC});
7189 Chains[i] = Scalars[i].getValue(R: 1);
7190 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7191 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7192 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7193 }
7194
7195 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7196 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7197
7198 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7199}
7200
7201//===----------------------------------------------------------------------===//
7202// Widen Vector Operand
7203//===----------------------------------------------------------------------===//
7204bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7205 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7206 SDValue Res = SDValue();
7207
7208 // See if the target wants to custom widen this node.
7209 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7210 return false;
7211
7212 switch (N->getOpcode()) {
7213 default:
7214#ifndef NDEBUG
7215 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7216 N->dump(&DAG);
7217 dbgs() << "\n";
7218#endif
7219 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7220
7221 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7222 case ISD::FAKE_USE:
7223 Res = WidenVecOp_FAKE_USE(N);
7224 break;
7225 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7226 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7227 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7228 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7229 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7230 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7231 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7232 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7233 break;
7234 case ISD::ANY_EXTEND_VECTOR_INREG:
7235 case ISD::SIGN_EXTEND_VECTOR_INREG:
7236 case ISD::ZERO_EXTEND_VECTOR_INREG:
7237 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7238 break;
7239 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7240 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7241 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7242 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7243 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7244 case ISD::STRICT_FSETCC:
7245 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7246 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7247 case ISD::FLDEXP:
7248 case ISD::FCOPYSIGN:
7249 case ISD::LROUND:
7250 case ISD::LLROUND:
7251 case ISD::LRINT:
7252 case ISD::LLRINT:
7253 Res = WidenVecOp_UnrollVectorOp(N);
7254 break;
7255 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7256
7257 case ISD::ANY_EXTEND:
7258 case ISD::SIGN_EXTEND:
7259 case ISD::ZERO_EXTEND:
7260 Res = WidenVecOp_EXTEND(N);
7261 break;
7262
7263 case ISD::SCMP:
7264 case ISD::UCMP:
7265 Res = WidenVecOp_CMP(N);
7266 break;
7267
7268 case ISD::FP_EXTEND:
7269 case ISD::STRICT_FP_EXTEND:
7270 case ISD::FP_ROUND:
7271 case ISD::STRICT_FP_ROUND:
7272 case ISD::FP_TO_SINT:
7273 case ISD::STRICT_FP_TO_SINT:
7274 case ISD::FP_TO_UINT:
7275 case ISD::STRICT_FP_TO_UINT:
7276 case ISD::SINT_TO_FP:
7277 case ISD::STRICT_SINT_TO_FP:
7278 case ISD::UINT_TO_FP:
7279 case ISD::STRICT_UINT_TO_FP:
7280 case ISD::TRUNCATE:
7281 Res = WidenVecOp_Convert(N);
7282 break;
7283
7284 case ISD::FP_TO_SINT_SAT:
7285 case ISD::FP_TO_UINT_SAT:
7286 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7287 break;
7288
7289 case ISD::VECREDUCE_FADD:
7290 case ISD::VECREDUCE_FMUL:
7291 case ISD::VECREDUCE_ADD:
7292 case ISD::VECREDUCE_MUL:
7293 case ISD::VECREDUCE_AND:
7294 case ISD::VECREDUCE_OR:
7295 case ISD::VECREDUCE_XOR:
7296 case ISD::VECREDUCE_SMAX:
7297 case ISD::VECREDUCE_SMIN:
7298 case ISD::VECREDUCE_UMAX:
7299 case ISD::VECREDUCE_UMIN:
7300 case ISD::VECREDUCE_FMAX:
7301 case ISD::VECREDUCE_FMIN:
7302 case ISD::VECREDUCE_FMAXIMUM:
7303 case ISD::VECREDUCE_FMINIMUM:
7304 Res = WidenVecOp_VECREDUCE(N);
7305 break;
7306 case ISD::VECREDUCE_SEQ_FADD:
7307 case ISD::VECREDUCE_SEQ_FMUL:
7308 Res = WidenVecOp_VECREDUCE_SEQ(N);
7309 break;
7310 case ISD::VP_REDUCE_FADD:
7311 case ISD::VP_REDUCE_SEQ_FADD:
7312 case ISD::VP_REDUCE_FMUL:
7313 case ISD::VP_REDUCE_SEQ_FMUL:
7314 case ISD::VP_REDUCE_ADD:
7315 case ISD::VP_REDUCE_MUL:
7316 case ISD::VP_REDUCE_AND:
7317 case ISD::VP_REDUCE_OR:
7318 case ISD::VP_REDUCE_XOR:
7319 case ISD::VP_REDUCE_SMAX:
7320 case ISD::VP_REDUCE_SMIN:
7321 case ISD::VP_REDUCE_UMAX:
7322 case ISD::VP_REDUCE_UMIN:
7323 case ISD::VP_REDUCE_FMAX:
7324 case ISD::VP_REDUCE_FMIN:
7325 case ISD::VP_REDUCE_FMAXIMUM:
7326 case ISD::VP_REDUCE_FMINIMUM:
7327 Res = WidenVecOp_VP_REDUCE(N);
7328 break;
7329 case ISD::VP_CTTZ_ELTS:
7330 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7331 Res = WidenVecOp_VP_CttzElements(N);
7332 break;
7333 case ISD::VECTOR_FIND_LAST_ACTIVE:
7334 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7335 break;
7336 }
7337
7338 // If Res is null, the sub-method took care of registering the result.
7339 if (!Res.getNode()) return false;
7340
7341 // If the result is N, the sub-method updated N in place. Tell the legalizer
7342 // core about this.
7343 if (Res.getNode() == N)
7344 return true;
7345
7346
7347 if (N->isStrictFPOpcode())
7348 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7349 "Invalid operand expansion");
7350 else
7351 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7352 "Invalid operand expansion");
7353
7354 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7355 return false;
7356}
7357
7358SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7359 SDLoc DL(N);
7360 EVT VT = N->getValueType(ResNo: 0);
7361
7362 SDValue InOp = N->getOperand(Num: 0);
7363 assert(getTypeAction(InOp.getValueType()) ==
7364 TargetLowering::TypeWidenVector &&
7365 "Unexpected type action");
7366 InOp = GetWidenedVector(Op: InOp);
7367 assert(VT.getVectorNumElements() <
7368 InOp.getValueType().getVectorNumElements() &&
7369 "Input wasn't widened!");
7370
7371 // We may need to further widen the operand until it has the same total
7372 // vector size as the result.
7373 EVT InVT = InOp.getValueType();
7374 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7375 EVT InEltVT = InVT.getVectorElementType();
7376 for (EVT FixedVT : MVT::vector_valuetypes()) {
7377 EVT FixedEltVT = FixedVT.getVectorElementType();
7378 if (TLI.isTypeLegal(VT: FixedVT) &&
7379 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7380 FixedEltVT == InEltVT) {
7381 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7382 "Not enough elements in the fixed type for the operand!");
7383 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7384 "We can't have the same type as we started with!");
7385 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7386 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7387 else
7388 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7389 break;
7390 }
7391 }
7392 InVT = InOp.getValueType();
7393 if (InVT.getSizeInBits() != VT.getSizeInBits())
7394 // We couldn't find a legal vector type that was a widening of the input
7395 // and could be extended in-register to the result type, so we have to
7396 // scalarize.
7397 return WidenVecOp_Convert(N);
7398 }
7399
7400 // Use special DAG nodes to represent the operation of extending the
7401 // low lanes.
7402 switch (N->getOpcode()) {
7403 default:
7404 llvm_unreachable("Extend legalization on extend operation!");
7405 case ISD::ANY_EXTEND:
7406 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7407 case ISD::SIGN_EXTEND:
7408 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7409 case ISD::ZERO_EXTEND:
7410 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7411 }
7412}
7413
7414SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7415 SDLoc dl(N);
7416
7417 EVT OpVT = N->getOperand(Num: 0).getValueType();
7418 EVT ResVT = N->getValueType(ResNo: 0);
7419 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7420 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7421
7422 // 1. EXTRACT_SUBVECTOR
7423 // 2. SIGN_EXTEND/ZERO_EXTEND
7424 // 3. CMP
7425 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7426 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7427
7428 // At this point the result type is guaranteed to be valid, so we can use it
7429 // as the operand type by extending it appropriately
7430 ISD::NodeType ExtendOpcode =
7431 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7432 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7433 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7434
7435 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7436}
7437
7438SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7439 // The result (and first input) is legal, but the second input is illegal.
7440 // We can't do much to fix that, so just unroll and let the extracts off of
7441 // the second input be widened as needed later.
7442 return DAG.UnrollVectorOp(N);
7443}
7444
7445SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7446 SDLoc DL(N);
7447 EVT ResultVT = N->getValueType(ResNo: 0);
7448 SDValue Test = N->getOperand(Num: 1);
7449 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7450
7451 // Process this node similarly to SETCC.
7452 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7453 if (ResultVT.getScalarType() == MVT::i1)
7454 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7455 NumElements: WideResultVT.getVectorNumElements());
7456
7457 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7458 Ops: {WideArg, Test}, Flags: N->getFlags());
7459
7460 // Extract the needed results from the result vector.
7461 EVT ResVT =
7462 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7463 NumElements: ResultVT.getVectorNumElements());
7464 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7465
7466 EVT OpVT = N->getOperand(Num: 0).getValueType();
7467 ISD::NodeType ExtendCode =
7468 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7469 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7470}
7471
7472SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7473 // Since the result is legal and the input is illegal.
7474 EVT VT = N->getValueType(ResNo: 0);
7475 EVT EltVT = VT.getVectorElementType();
7476 SDLoc dl(N);
7477 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7478 assert(getTypeAction(InOp.getValueType()) ==
7479 TargetLowering::TypeWidenVector &&
7480 "Unexpected type action");
7481 InOp = GetWidenedVector(Op: InOp);
7482 EVT InVT = InOp.getValueType();
7483 unsigned Opcode = N->getOpcode();
7484
7485 // See if a widened result type would be legal, if so widen the node.
7486 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7487 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7488 EC: InVT.getVectorElementCount());
7489 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7490 SDValue Res;
7491 if (N->isStrictFPOpcode()) {
7492 if (Opcode == ISD::STRICT_FP_ROUND)
7493 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7494 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7495 else
7496 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7497 Ops: { N->getOperand(Num: 0), InOp });
7498 // Legalize the chain result - switch anything that used the old chain to
7499 // use the new one.
7500 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7501 } else {
7502 if (Opcode == ISD::FP_ROUND)
7503 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, N1: InOp, N2: N->getOperand(Num: 1));
7504 else
7505 Res = DAG.getNode(Opcode, DL: dl, VT: WideVT, Operand: InOp);
7506 }
7507 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7508 }
7509
7510 EVT InEltVT = InVT.getVectorElementType();
7511
7512 // Unroll the convert into some scalar code and create a nasty build vector.
7513 unsigned NumElts = VT.getVectorNumElements();
7514 SmallVector<SDValue, 16> Ops(NumElts);
7515 if (N->isStrictFPOpcode()) {
7516 SmallVector<SDValue, 4> NewOps(N->ops());
7517 SmallVector<SDValue, 32> OpChains;
7518 for (unsigned i=0; i < NumElts; ++i) {
7519 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7520 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7521 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7522 }
7523 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7524 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7525 } else {
7526 for (unsigned i = 0; i < NumElts; ++i)
7527 Ops[i] = DAG.getNode(Opcode, DL: dl, VT: EltVT,
7528 Operand: DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i));
7529 }
7530
7531 return DAG.getBuildVector(VT, DL: dl, Ops);
7532}
7533
7534SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7535 EVT DstVT = N->getValueType(ResNo: 0);
7536 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7537 EVT SrcVT = Src.getValueType();
7538 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7539 SDLoc dl(N);
7540
7541 // See if a widened result type would be legal, if so widen the node.
7542 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7543 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7544 if (TLI.isTypeLegal(VT: WideDstVT)) {
7545 SDValue Res =
7546 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7547 return DAG.getNode(
7548 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7549 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7550 }
7551
7552 // Give up and unroll.
7553 return DAG.UnrollVectorOp(N);
7554}
7555
7556SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7557 EVT VT = N->getValueType(ResNo: 0);
7558 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7559 EVT InWidenVT = InOp.getValueType();
7560 SDLoc dl(N);
7561
7562 // Check if we can convert between two legal vector types and extract.
7563 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7564 TypeSize Size = VT.getSizeInBits();
7565 // x86mmx is not an acceptable vector element type, so don't try.
7566 if (!VT.isVector() && VT != MVT::x86mmx &&
7567 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7568 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7569 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7570 if (TLI.isTypeLegal(VT: NewVT)) {
7571 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7572 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7573 }
7574 }
7575
7576 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7577 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7578 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7579 // having to copy via memory.
7580 if (VT.isVector()) {
7581 EVT EltVT = VT.getVectorElementType();
7582 unsigned EltSize = EltVT.getFixedSizeInBits();
7583 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7584 ElementCount NewNumElts =
7585 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
7586 .divideCoefficientBy(RHS: EltSize);
7587 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
7588 if (TLI.isTypeLegal(VT: NewVT)) {
7589 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7590 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
7591 }
7592 }
7593 }
7594
7595 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
7596}
7597
7598// Vectors with sizes that are not powers of 2 need to be widened to the
7599// next largest power of 2. For example, we may get a vector of 3 32-bit
7600// integers or of 6 16-bit integers, both of which have to be widened to a
7601// 128-bit vector.
7602SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
7603 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
7604 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
7605 N2: WidenedOp);
7606}
7607
7608SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
7609 EVT VT = N->getValueType(ResNo: 0);
7610 EVT EltVT = VT.getVectorElementType();
7611 EVT InVT = N->getOperand(Num: 0).getValueType();
7612 SDLoc dl(N);
7613
7614 // If the widen width for this operand is the same as the width of the concat
7615 // and all but the first operand is undef, just use the widened operand.
7616 unsigned NumOperands = N->getNumOperands();
7617 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
7618 unsigned i;
7619 for (i = 1; i < NumOperands; ++i)
7620 if (!N->getOperand(Num: i).isUndef())
7621 break;
7622
7623 if (i == NumOperands)
7624 return GetWidenedVector(Op: N->getOperand(Num: 0));
7625 }
7626
7627 // Otherwise, fall back to a nasty build vector.
7628 unsigned NumElts = VT.getVectorNumElements();
7629 SmallVector<SDValue, 16> Ops(NumElts);
7630
7631 unsigned NumInElts = InVT.getVectorNumElements();
7632
7633 unsigned Idx = 0;
7634 for (unsigned i=0; i < NumOperands; ++i) {
7635 SDValue InOp = N->getOperand(Num: i);
7636 assert(getTypeAction(InOp.getValueType()) ==
7637 TargetLowering::TypeWidenVector &&
7638 "Unexpected type action");
7639 InOp = GetWidenedVector(Op: InOp);
7640 for (unsigned j = 0; j < NumInElts; ++j)
7641 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
7642 }
7643 return DAG.getBuildVector(VT, DL: dl, Ops);
7644}
7645
7646SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
7647 EVT VT = N->getValueType(ResNo: 0);
7648 SDValue SubVec = N->getOperand(Num: 1);
7649 SDValue InVec = N->getOperand(Num: 0);
7650
7651 EVT OrigVT = SubVec.getValueType();
7652 SubVec = GetWidenedVector(Op: SubVec);
7653 EVT SubVT = SubVec.getValueType();
7654
7655 // Whether or not all the elements of the widened SubVec will be inserted into
7656 // valid indices of VT.
7657 bool IndicesValid = false;
7658 // If we statically know that VT can fit SubVT, the indices are valid.
7659 if (VT.knownBitsGE(VT: SubVT))
7660 IndicesValid = true;
7661 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
7662 // Otherwise, if we're inserting a fixed vector into a scalable vector and
7663 // we know the minimum vscale we can work out if it's valid ourselves.
7664 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
7665 Kind: Attribute::VScaleRange);
7666 if (Attr.isValid()) {
7667 unsigned VScaleMin = Attr.getVScaleRangeMin();
7668 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
7669 SubVT.getFixedSizeInBits())
7670 IndicesValid = true;
7671 }
7672 }
7673
7674 if (!IndicesValid)
7675 report_fatal_error(
7676 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
7677
7678 SDLoc DL(N);
7679
7680 // We need to make sure that the indices are still valid, otherwise we might
7681 // widen what was previously well-defined to something undefined.
7682 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
7683 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
7684 N3: N->getOperand(Num: 2));
7685
7686 if (OrigVT.isScalableVector()) {
7687 // When the widened types match, overwriting the start of a vector is
7688 // effectively a merge operation that can be implement as a vselect.
7689 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
7690 SDValue Mask =
7691 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
7692 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
7693 }
7694
7695 // Fallback to inserting through memory.
7696 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
7697 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
7698 MachineFunction &MF = DAG.getMachineFunction();
7699 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7700 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
7701
7702 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
7703 PtrInfo, F: MachineMemOperand::MOStore,
7704 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7705 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
7706 PtrInfo, F: MachineMemOperand::MOLoad,
7707 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
7708
7709 // Write out the vector being inserting into.
7710 SDValue Ch =
7711 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
7712
7713 // Build a mask to match the length of the sub-vector.
7714 SDValue Mask =
7715 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
7716
7717 // Overwrite the sub-vector at the required offset.
7718 SDValue SubVecPtr =
7719 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
7720 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
7721 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
7722 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
7723
7724 // Read back the result.
7725 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
7726 }
7727
7728 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
7729 // with a series of INSERT_VECTOR_ELT
7730 unsigned Idx = N->getConstantOperandVal(Num: 2);
7731
7732 SDValue InsertElt = InVec;
7733 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
7734 SDValue ExtractElt =
7735 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
7736 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
7737 }
7738
7739 return InsertElt;
7740}
7741
7742SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
7743 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7744 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
7745 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7746}
7747
7748SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
7749 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7750 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
7751 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
7752}
7753
7754SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
7755 SDLoc DL(N);
7756 EVT ResVT = N->getValueType(ResNo: 0);
7757
7758 // Widen the input as requested by the legalizer.
7759 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7760 EVT WideInVT = WideInOp.getValueType();
7761
7762 // Simple case: if widened input is still smaller than or equal to result,
7763 // just use it directly.
7764 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
7765 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
7766
7767 // EXTEND_VECTOR_INREG requires input bits <= result bits.
7768 // If widening makes the input larger than the original result, widen the
7769 // result to match, then extract back down.
7770 EVT ResEltVT = ResVT.getVectorElementType();
7771 unsigned EltBits = ResEltVT.getSizeInBits();
7772 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
7773 "Widened input size must be a multiple of result element size");
7774
7775 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
7776 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
7777
7778 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
7779 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
7780}
7781
7782SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
7783 // We have to widen the value, but we want only to store the original
7784 // vector type.
7785 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
7786
7787 if (!ST->getMemoryVT().getScalarType().isByteSized())
7788 return TLI.scalarizeVectorStore(ST, DAG);
7789
7790 if (ST->isTruncatingStore())
7791 return TLI.scalarizeVectorStore(ST, DAG);
7792
7793 // Generate a vector-predicated store if it is custom/legal on the target.
7794 // To avoid possible recursion, only do this if the widened mask type is
7795 // legal.
7796 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
7797 // removed from the IR by the ExpandVectorPredication pass but we're
7798 // reintroducing them here.
7799 SDValue StVal = ST->getValue();
7800 EVT StVT = StVal.getValueType();
7801 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
7802 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
7803
7804 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7805 TLI.isTypeLegal(VT: WideMaskVT)) {
7806 // Widen the value.
7807 SDLoc DL(N);
7808 StVal = GetWidenedVector(Op: StVal);
7809 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
7810 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
7811 EC: StVT.getVectorElementCount());
7812 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
7813 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
7814 AM: ST->getAddressingMode());
7815 }
7816
7817 SmallVector<SDValue, 16> StChain;
7818 if (GenWidenVectorStores(StChain, ST)) {
7819 if (StChain.size() == 1)
7820 return StChain[0];
7821
7822 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
7823 }
7824
7825 if (StVT.isVector()) {
7826 // If all else fails replace the store with a wide masked store.
7827 SDLoc DL(N);
7828 SDValue WideStVal = GetWidenedVector(Op: StVal);
7829 SDValue Mask =
7830 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
7831
7832 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
7833 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
7834 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
7835 IsTruncating: ST->isTruncatingStore());
7836 }
7837
7838 report_fatal_error(reason: "Unable to widen vector store");
7839}
7840
7841SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
7842 assert((OpNo == 1 || OpNo == 3) &&
7843 "Can widen only data or mask operand of vp_store");
7844 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
7845 SDValue Mask = ST->getMask();
7846 SDValue StVal = ST->getValue();
7847 SDLoc dl(N);
7848
7849 if (OpNo == 1) {
7850 // Widen the value.
7851 StVal = GetWidenedVector(Op: StVal);
7852
7853 // We only handle the case where the mask needs widening to an
7854 // identically-sized type as the vector inputs.
7855 assert(getTypeAction(Mask.getValueType()) ==
7856 TargetLowering::TypeWidenVector &&
7857 "Unable to widen VP store");
7858 Mask = GetWidenedVector(Op: Mask);
7859 } else {
7860 Mask = GetWidenedVector(Op: Mask);
7861
7862 // We only handle the case where the stored value needs widening to an
7863 // identically-sized type as the mask.
7864 assert(getTypeAction(StVal.getValueType()) ==
7865 TargetLowering::TypeWidenVector &&
7866 "Unable to widen VP store");
7867 StVal = GetWidenedVector(Op: StVal);
7868 }
7869
7870 assert(Mask.getValueType().getVectorElementCount() ==
7871 StVal.getValueType().getVectorElementCount() &&
7872 "Mask and data vectors should have the same number of elements");
7873 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
7874 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
7875 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
7876 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
7877 IsCompressing: ST->isCompressingStore());
7878}
7879
7880SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
7881 unsigned OpNo) {
7882 assert((OpNo == 1 || OpNo == 4) &&
7883 "Can widen only data or mask operand of vp_strided_store");
7884 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
7885 SDValue Mask = SST->getMask();
7886 SDValue StVal = SST->getValue();
7887 SDLoc DL(N);
7888
7889 if (OpNo == 1)
7890 assert(getTypeAction(Mask.getValueType()) ==
7891 TargetLowering::TypeWidenVector &&
7892 "Unable to widen VP strided store");
7893 else
7894 assert(getTypeAction(StVal.getValueType()) ==
7895 TargetLowering::TypeWidenVector &&
7896 "Unable to widen VP strided store");
7897
7898 StVal = GetWidenedVector(Op: StVal);
7899 Mask = GetWidenedVector(Op: Mask);
7900
7901 assert(StVal.getValueType().getVectorElementCount() ==
7902 Mask.getValueType().getVectorElementCount() &&
7903 "Data and mask vectors should have the same number of elements");
7904
7905 return DAG.getStridedStoreVP(
7906 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
7907 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
7908 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
7909 IsCompressing: SST->isCompressingStore());
7910}
7911
7912SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
7913 assert((OpNo == 1 || OpNo == 4) &&
7914 "Can widen only data or mask operand of mstore");
7915 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
7916 SDValue Mask = MST->getMask();
7917 EVT MaskVT = Mask.getValueType();
7918 SDValue StVal = MST->getValue();
7919 EVT VT = StVal.getValueType();
7920 SDLoc dl(N);
7921
7922 EVT WideVT, WideMaskVT;
7923 if (OpNo == 1) {
7924 // Widen the value.
7925 StVal = GetWidenedVector(Op: StVal);
7926
7927 WideVT = StVal.getValueType();
7928 WideMaskVT =
7929 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7930 EC: WideVT.getVectorElementCount());
7931 } else {
7932 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
7933
7934 EVT ValueVT = StVal.getValueType();
7935 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
7936 EC: WideMaskVT.getVectorElementCount());
7937 }
7938
7939 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
7940 TLI.isTypeLegal(VT: WideMaskVT)) {
7941 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7942 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7943 EC: VT.getVectorElementCount());
7944 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
7945 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
7946 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
7947 }
7948
7949 if (OpNo == 1) {
7950 // The mask should be widened as well.
7951 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7952 } else {
7953 // Widen the mask.
7954 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7955
7956 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
7957 }
7958
7959 assert(Mask.getValueType().getVectorElementCount() ==
7960 StVal.getValueType().getVectorElementCount() &&
7961 "Mask and data vectors should have the same number of elements");
7962 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
7963 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
7964 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
7965 IsTruncating: false, IsCompressing: MST->isCompressingStore());
7966}
7967
7968SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
7969 assert(OpNo == 4 && "Can widen only the index of mgather");
7970 auto *MG = cast<MaskedGatherSDNode>(Val: N);
7971 SDValue DataOp = MG->getPassThru();
7972 SDValue Mask = MG->getMask();
7973 SDValue Scale = MG->getScale();
7974
7975 // Just widen the index. It's allowed to have extra elements.
7976 SDValue Index = GetWidenedVector(Op: MG->getIndex());
7977
7978 SDLoc dl(N);
7979 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
7980 Scale};
7981 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
7982 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
7983 ExtTy: MG->getExtensionType());
7984 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7985 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
7986 return SDValue();
7987}
7988
7989SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
7990 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
7991 SDValue DataOp = MSC->getValue();
7992 SDValue Mask = MSC->getMask();
7993 SDValue Index = MSC->getIndex();
7994 SDValue Scale = MSC->getScale();
7995 EVT WideMemVT = MSC->getMemoryVT();
7996
7997 if (OpNo == 1) {
7998 DataOp = GetWidenedVector(Op: DataOp);
7999 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
8000
8001 // Widen index.
8002 EVT IndexVT = Index.getValueType();
8003 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8004 VT: IndexVT.getVectorElementType(), NumElements: NumElts);
8005 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8006
8007 // The mask should be widened as well.
8008 EVT MaskVT = Mask.getValueType();
8009 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8010 VT: MaskVT.getVectorElementType(), NumElements: NumElts);
8011 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8012
8013 // Widen the MemoryType
8014 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8015 VT: MSC->getMemoryVT().getScalarType(), NumElements: NumElts);
8016 } else if (OpNo == 4) {
8017 // Just widen the index. It's allowed to have extra elements.
8018 Index = GetWidenedVector(Op: Index);
8019 } else
8020 llvm_unreachable("Can't widen this operand of mscatter");
8021
8022 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8023 Scale};
8024 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8025 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8026 IsTruncating: MSC->isTruncatingStore());
8027}
8028
8029SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8030 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8031 SDValue DataOp = VPSC->getValue();
8032 SDValue Mask = VPSC->getMask();
8033 SDValue Index = VPSC->getIndex();
8034 SDValue Scale = VPSC->getScale();
8035 EVT WideMemVT = VPSC->getMemoryVT();
8036
8037 if (OpNo == 1) {
8038 DataOp = GetWidenedVector(Op: DataOp);
8039 Index = GetWidenedVector(Op: Index);
8040 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8041 Mask = GetWidenedMask(Mask, EC: WideEC);
8042 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8043 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8044 } else if (OpNo == 3) {
8045 // Just widen the index. It's allowed to have extra elements.
8046 Index = GetWidenedVector(Op: Index);
8047 } else
8048 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8049
8050 SDValue Ops[] = {
8051 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8052 VPSC->getVectorLength()};
8053 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8054 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8055}
8056
8057SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8058 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8059 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8060 SDLoc dl(N);
8061 EVT VT = N->getValueType(ResNo: 0);
8062
8063 // WARNING: In this code we widen the compare instruction with garbage.
8064 // This garbage may contain denormal floats which may be slow. Is this a real
8065 // concern ? Should we zero the unused lanes if this is a float compare ?
8066
8067 // Get a new SETCC node to compare the newly widened operands.
8068 // Only some of the compared elements are legal.
8069 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8070 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8071 if (VT.getScalarType() == MVT::i1)
8072 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8073 EC: SVT.getVectorElementCount());
8074
8075 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8076 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8077
8078 // Extract the needed results from the result vector.
8079 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8080 VT: SVT.getVectorElementType(),
8081 EC: VT.getVectorElementCount());
8082 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8083
8084 EVT OpVT = N->getOperand(Num: 0).getValueType();
8085 ISD::NodeType ExtendCode =
8086 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8087 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8088}
8089
8090SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8091 SDValue Chain = N->getOperand(Num: 0);
8092 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8093 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8094 SDValue CC = N->getOperand(Num: 3);
8095 SDLoc dl(N);
8096
8097 EVT VT = N->getValueType(ResNo: 0);
8098 EVT EltVT = VT.getVectorElementType();
8099 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8100 unsigned NumElts = VT.getVectorNumElements();
8101
8102 // Unroll into a build vector.
8103 SmallVector<SDValue, 8> Scalars(NumElts);
8104 SmallVector<SDValue, 8> Chains(NumElts);
8105
8106 for (unsigned i = 0; i != NumElts; ++i) {
8107 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8108 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8109
8110 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8111 Ops: {Chain, LHSElem, RHSElem, CC});
8112 Chains[i] = Scalars[i].getValue(R: 1);
8113 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8114 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8115 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8116 }
8117
8118 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8119 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8120
8121 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8122}
8123
8124static unsigned getExtendForIntVecReduction(unsigned Opc) {
8125 switch (Opc) {
8126 default:
8127 llvm_unreachable("Expected integer vector reduction");
8128 case ISD::VECREDUCE_ADD:
8129 case ISD::VECREDUCE_MUL:
8130 case ISD::VECREDUCE_AND:
8131 case ISD::VECREDUCE_OR:
8132 case ISD::VECREDUCE_XOR:
8133 return ISD::ANY_EXTEND;
8134 case ISD::VECREDUCE_SMAX:
8135 case ISD::VECREDUCE_SMIN:
8136 return ISD::SIGN_EXTEND;
8137 case ISD::VECREDUCE_UMAX:
8138 case ISD::VECREDUCE_UMIN:
8139 return ISD::ZERO_EXTEND;
8140 }
8141}
8142
8143SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8144 SDLoc dl(N);
8145 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8146 EVT VT = N->getValueType(ResNo: 0);
8147 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8148 EVT WideVT = Op.getValueType();
8149 EVT ElemVT = OrigVT.getVectorElementType();
8150 SDNodeFlags Flags = N->getFlags();
8151
8152 unsigned Opc = N->getOpcode();
8153 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8154 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8155 assert(NeutralElem && "Neutral element must exist");
8156
8157 // Pad the vector with the neutral element.
8158 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8159 unsigned WideElts = WideVT.getVectorMinNumElements();
8160
8161 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8162 // needing to pad the source vector, because the inactive lanes can simply be
8163 // disabled and not contribute to the result.
8164 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8165 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8166 SDValue Start = NeutralElem;
8167 if (VT.isInteger())
8168 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8169 assert(Start.getValueType() == VT);
8170 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8171 EC: WideVT.getVectorElementCount());
8172 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8173 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8174 EC: OrigVT.getVectorElementCount());
8175 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8176 }
8177
8178 if (WideVT.isScalableVector()) {
8179 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8180 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8181 EC: ElementCount::getScalable(MinVal: GCD));
8182 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8183 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8184 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8185 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8186 }
8187
8188 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8189 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8190
8191 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8192}
8193
8194SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8195 SDLoc dl(N);
8196 SDValue AccOp = N->getOperand(Num: 0);
8197 SDValue VecOp = N->getOperand(Num: 1);
8198 SDValue Op = GetWidenedVector(Op: VecOp);
8199
8200 EVT VT = N->getValueType(ResNo: 0);
8201 EVT OrigVT = VecOp.getValueType();
8202 EVT WideVT = Op.getValueType();
8203 EVT ElemVT = OrigVT.getVectorElementType();
8204 SDNodeFlags Flags = N->getFlags();
8205
8206 unsigned Opc = N->getOpcode();
8207 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8208 SDValue NeutralElem = DAG.getNeutralElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8209
8210 // Pad the vector with the neutral element.
8211 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8212 unsigned WideElts = WideVT.getVectorMinNumElements();
8213
8214 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8215 // needing to pad the source vector, because the inactive lanes can simply be
8216 // disabled and not contribute to the result.
8217 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8218 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8219 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8220 EC: WideVT.getVectorElementCount());
8221 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8222 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8223 EC: OrigVT.getVectorElementCount());
8224 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8225 }
8226
8227 if (WideVT.isScalableVector()) {
8228 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8229 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8230 EC: ElementCount::getScalable(MinVal: GCD));
8231 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8232 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8233 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8234 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8235 }
8236
8237 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8238 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8239
8240 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8241}
8242
8243SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8244 assert(N->isVPOpcode() && "Expected VP opcode");
8245
8246 SDLoc dl(N);
8247 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8248 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8249 EC: Op.getValueType().getVectorElementCount());
8250
8251 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8252 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8253 Flags: N->getFlags());
8254}
8255
8256SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8257 // This only gets called in the case that the left and right inputs and
8258 // result are of a legal odd vector type, and the condition is illegal i1 of
8259 // the same odd width that needs widening.
8260 EVT VT = N->getValueType(ResNo: 0);
8261 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8262
8263 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8264 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8265 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8266 SDLoc DL(N);
8267
8268 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8269 N2: LeftIn, N3: RightIn);
8270 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8271}
8272
8273SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8274 SDLoc DL(N);
8275 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8276 EVT SrcVT = Source.getValueType();
8277 SDValue Mask =
8278 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8279
8280 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8281 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8282}
8283
8284SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8285 SDLoc DL(N);
8286 SDValue Mask = N->getOperand(Num: 0);
8287 EVT OrigMaskVT = Mask.getValueType();
8288 SDValue WideMask = GetWidenedVector(Op: Mask);
8289 EVT WideMaskVT = WideMask.getValueType();
8290
8291 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8292 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8293 unsigned WideElts = WideMaskVT.getVectorNumElements();
8294 if (OrigElts != WideElts) {
8295 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8296 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8297 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8298 }
8299
8300 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8301 Operand: WideMask);
8302}
8303
8304//===----------------------------------------------------------------------===//
8305// Vector Widening Utilities
8306//===----------------------------------------------------------------------===//
8307
8308// Utility function to find the type to chop up a widen vector for load/store
8309// TLI: Target lowering used to determine legal types.
8310// Width: Width left need to load/store.
8311// WidenVT: The widen vector type to load to/store from
8312// Align: If 0, don't allow use of a wider type
8313// WidenEx: If Align is not 0, the amount additional we can load/store from.
8314
8315static std::optional<EVT> findMemType(SelectionDAG &DAG,
8316 const TargetLowering &TLI, unsigned Width,
8317 EVT WidenVT, unsigned Align = 0,
8318 unsigned WidenEx = 0) {
8319 EVT WidenEltVT = WidenVT.getVectorElementType();
8320 const bool Scalable = WidenVT.isScalableVector();
8321 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8322 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8323 unsigned AlignInBits = Align*8;
8324
8325 EVT RetVT = WidenEltVT;
8326 // Don't bother looking for an integer type if the vector is scalable, skip
8327 // to vector types.
8328 if (!Scalable) {
8329 // If we have one element to load/store, return it.
8330 if (Width == WidenEltWidth)
8331 return RetVT;
8332
8333 // See if there is larger legal integer than the element type to load/store.
8334 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8335 unsigned MemVTWidth = MemVT.getSizeInBits();
8336 if (MemVT.getSizeInBits() <= WidenEltWidth)
8337 break;
8338 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8339 if ((Action == TargetLowering::TypeLegal ||
8340 Action == TargetLowering::TypePromoteInteger) &&
8341 (WidenWidth % MemVTWidth) == 0 &&
8342 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8343 (MemVTWidth <= Width ||
8344 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8345 if (MemVTWidth == WidenWidth)
8346 return MemVT;
8347 RetVT = MemVT;
8348 break;
8349 }
8350 }
8351 }
8352
8353 // See if there is a larger vector type to load/store that has the same vector
8354 // element type and is evenly divisible with the WidenVT.
8355 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8356 // Skip vector MVTs which don't match the scalable property of WidenVT.
8357 if (Scalable != MemVT.isScalableVector())
8358 continue;
8359 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8360 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8361 if ((Action == TargetLowering::TypeLegal ||
8362 Action == TargetLowering::TypePromoteInteger) &&
8363 WidenEltVT == MemVT.getVectorElementType() &&
8364 (WidenWidth % MemVTWidth) == 0 &&
8365 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8366 (MemVTWidth <= Width ||
8367 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8368 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8369 return MemVT;
8370 }
8371 }
8372
8373 // Using element-wise loads and stores for widening operations is not
8374 // supported for scalable vectors
8375 if (Scalable)
8376 return std::nullopt;
8377
8378 return RetVT;
8379}
8380
8381// Builds a vector type from scalar loads
8382// VecTy: Resulting Vector type
8383// LDOps: Load operators to build a vector type
8384// [Start,End) the list of loads to use.
8385static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8386 SmallVectorImpl<SDValue> &LdOps,
8387 unsigned Start, unsigned End) {
8388 SDLoc dl(LdOps[Start]);
8389 EVT LdTy = LdOps[Start].getValueType();
8390 unsigned Width = VecTy.getSizeInBits();
8391 unsigned NumElts = Width / LdTy.getSizeInBits();
8392 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8393
8394 unsigned Idx = 1;
8395 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8396
8397 for (unsigned i = Start + 1; i != End; ++i) {
8398 EVT NewLdTy = LdOps[i].getValueType();
8399 if (NewLdTy != LdTy) {
8400 NumElts = Width / NewLdTy.getSizeInBits();
8401 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8402 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8403 // Readjust position and vector position based on new load type.
8404 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8405 LdTy = NewLdTy;
8406 }
8407 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8408 }
8409 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8410}
8411
8412SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8413 LoadSDNode *LD) {
8414 // The strategy assumes that we can efficiently load power-of-two widths.
8415 // The routine chops the vector into the largest vector loads with the same
8416 // element type or scalar loads and then recombines it to the widen vector
8417 // type.
8418 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8419 EVT LdVT = LD->getMemoryVT();
8420 SDLoc dl(LD);
8421 assert(LdVT.isVector() && WidenVT.isVector());
8422 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8423 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8424
8425 // Load information
8426 SDValue Chain = LD->getChain();
8427 SDValue BasePtr = LD->getBasePtr();
8428 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8429 AAMDNodes AAInfo = LD->getAAInfo();
8430
8431 TypeSize LdWidth = LdVT.getSizeInBits();
8432 TypeSize WidenWidth = WidenVT.getSizeInBits();
8433 TypeSize WidthDiff = WidenWidth - LdWidth;
8434 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8435 // and if the original load is simple.
8436 unsigned LdAlign =
8437 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8438
8439 // Find the vector type that can load from.
8440 std::optional<EVT> FirstVT =
8441 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8442 WidenEx: WidthDiff.getKnownMinValue());
8443
8444 if (!FirstVT)
8445 return SDValue();
8446
8447 SmallVector<EVT, 8> MemVTs;
8448 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8449
8450 // Unless we're able to load in one instruction we must work out how to load
8451 // the remainder.
8452 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8453 std::optional<EVT> NewVT = FirstVT;
8454 TypeSize RemainingWidth = LdWidth;
8455 TypeSize NewVTWidth = FirstVTWidth;
8456 do {
8457 RemainingWidth -= NewVTWidth;
8458 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8459 // The current type we are using is too large. Find a better size.
8460 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8461 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8462 if (!NewVT)
8463 return SDValue();
8464 NewVTWidth = NewVT->getSizeInBits();
8465 }
8466 MemVTs.push_back(Elt: *NewVT);
8467 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8468 }
8469
8470 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8471 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8472 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8473
8474 // Check if we can load the element with one instruction.
8475 if (MemVTs.empty())
8476 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8477 DAG);
8478
8479 // Load vector by using multiple loads from largest vector to scalar.
8480 SmallVector<SDValue, 16> LdOps;
8481 LdOps.push_back(Elt: LdOp);
8482
8483 uint64_t ScaledOffset = 0;
8484 MachinePointerInfo MPI = LD->getPointerInfo();
8485
8486 // First incremement past the first load.
8487 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
8488 ScaledOffset: &ScaledOffset);
8489
8490 for (EVT MemVT : MemVTs) {
8491 Align NewAlign = ScaledOffset == 0
8492 ? LD->getBaseAlign()
8493 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
8494 SDValue L =
8495 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, AAInfo);
8496
8497 LdOps.push_back(Elt: L);
8498 LdChain.push_back(Elt: L.getValue(R: 1));
8499 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
8500 }
8501
8502 // Build the vector from the load operations.
8503 unsigned End = LdOps.size();
8504 if (!LdOps[0].getValueType().isVector())
8505 // All the loads are scalar loads.
8506 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
8507
8508 // If the load contains vectors, build the vector using concat vector.
8509 // All of the vectors used to load are power-of-2, and the scalar loads can be
8510 // combined to make a power-of-2 vector.
8511 SmallVector<SDValue, 16> ConcatOps(End);
8512 int i = End - 1;
8513 int Idx = End;
8514 EVT LdTy = LdOps[i].getValueType();
8515 // First, combine the scalar loads to a vector.
8516 if (!LdTy.isVector()) {
8517 for (--i; i >= 0; --i) {
8518 LdTy = LdOps[i].getValueType();
8519 if (LdTy.isVector())
8520 break;
8521 }
8522 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
8523 }
8524
8525 ConcatOps[--Idx] = LdOps[i];
8526 for (--i; i >= 0; --i) {
8527 EVT NewLdTy = LdOps[i].getValueType();
8528 if (NewLdTy != LdTy) {
8529 // Create a larger vector.
8530 TypeSize LdTySize = LdTy.getSizeInBits();
8531 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
8532 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
8533 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
8534 unsigned NumOps =
8535 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
8536 SmallVector<SDValue, 16> WidenOps(NumOps);
8537 unsigned j = 0;
8538 for (; j != End-Idx; ++j)
8539 WidenOps[j] = ConcatOps[Idx+j];
8540 for (; j != NumOps; ++j)
8541 WidenOps[j] = DAG.getPOISON(VT: LdTy);
8542
8543 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
8544 Ops: WidenOps);
8545 Idx = End - 1;
8546 LdTy = NewLdTy;
8547 }
8548 ConcatOps[--Idx] = LdOps[i];
8549 }
8550
8551 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
8552 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
8553 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
8554
8555 // We need to fill the rest with undefs to build the vector.
8556 unsigned NumOps =
8557 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
8558 SmallVector<SDValue, 16> WidenOps(NumOps);
8559 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
8560 {
8561 unsigned i = 0;
8562 for (; i != End-Idx; ++i)
8563 WidenOps[i] = ConcatOps[Idx+i];
8564 for (; i != NumOps; ++i)
8565 WidenOps[i] = UndefVal;
8566 }
8567 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
8568}
8569
8570SDValue
8571DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
8572 LoadSDNode *LD,
8573 ISD::LoadExtType ExtType) {
8574 // For extension loads, it may not be more efficient to chop up the vector
8575 // and then extend it. Instead, we unroll the load and build a new vector.
8576 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8577 EVT LdVT = LD->getMemoryVT();
8578 SDLoc dl(LD);
8579 assert(LdVT.isVector() && WidenVT.isVector());
8580 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8581
8582 // Load information
8583 SDValue Chain = LD->getChain();
8584 SDValue BasePtr = LD->getBasePtr();
8585 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8586 AAMDNodes AAInfo = LD->getAAInfo();
8587
8588 if (LdVT.isScalableVector())
8589 return SDValue();
8590
8591 EVT EltVT = WidenVT.getVectorElementType();
8592 EVT LdEltVT = LdVT.getVectorElementType();
8593 unsigned NumElts = LdVT.getVectorNumElements();
8594
8595 // Load each element and widen.
8596 unsigned WidenNumElts = WidenVT.getVectorNumElements();
8597 SmallVector<SDValue, 16> Ops(WidenNumElts);
8598 unsigned Increment = LdEltVT.getSizeInBits() / 8;
8599 Ops[0] =
8600 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8601 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8602 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
8603 unsigned i = 0, Offset = Increment;
8604 for (i=1; i < NumElts; ++i, Offset += Increment) {
8605 SDValue NewBasePtr =
8606 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
8607 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
8608 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
8609 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
8610 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
8611 }
8612
8613 // Fill the rest with undefs.
8614 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8615 for (; i != WidenNumElts; ++i)
8616 Ops[i] = UndefVal;
8617
8618 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
8619}
8620
8621bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
8622 StoreSDNode *ST) {
8623 // The strategy assumes that we can efficiently store power-of-two widths.
8624 // The routine chops the vector into the largest vector stores with the same
8625 // element type or scalar stores.
8626 SDValue Chain = ST->getChain();
8627 SDValue BasePtr = ST->getBasePtr();
8628 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
8629 AAMDNodes AAInfo = ST->getAAInfo();
8630 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
8631 SDLoc dl(ST);
8632
8633 EVT StVT = ST->getMemoryVT();
8634 TypeSize StWidth = StVT.getSizeInBits();
8635 EVT ValVT = ValOp.getValueType();
8636 TypeSize ValWidth = ValVT.getSizeInBits();
8637 EVT ValEltVT = ValVT.getVectorElementType();
8638 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
8639 assert(StVT.getVectorElementType() == ValEltVT);
8640 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
8641 "Mismatch between store and value types");
8642
8643 int Idx = 0; // current index to store
8644
8645 MachinePointerInfo MPI = ST->getPointerInfo();
8646 uint64_t ScaledOffset = 0;
8647
8648 // A breakdown of how to widen this vector store. Each element of the vector
8649 // is a memory VT combined with the number of times it is to be stored to,
8650 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
8651 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
8652
8653 while (StWidth.isNonZero()) {
8654 // Find the largest vector type we can store with.
8655 std::optional<EVT> NewVT =
8656 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
8657 if (!NewVT)
8658 return false;
8659 MemVTs.push_back(Elt: {*NewVT, 0});
8660 TypeSize NewVTWidth = NewVT->getSizeInBits();
8661
8662 do {
8663 StWidth -= NewVTWidth;
8664 MemVTs.back().second++;
8665 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
8666 }
8667
8668 for (const auto &Pair : MemVTs) {
8669 EVT NewVT = Pair.first;
8670 unsigned Count = Pair.second;
8671 TypeSize NewVTWidth = NewVT.getSizeInBits();
8672
8673 if (NewVT.isVector()) {
8674 unsigned NumVTElts = NewVT.getVectorMinNumElements();
8675 do {
8676 Align NewAlign = ScaledOffset == 0
8677 ? ST->getBaseAlign()
8678 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
8679 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
8680 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
8681 MMOFlags, AAInfo);
8682 StChain.push_back(Elt: PartStore);
8683
8684 Idx += NumVTElts;
8685 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
8686 ScaledOffset: &ScaledOffset);
8687 } while (--Count);
8688 } else {
8689 // Cast the vector to the scalar type we can store.
8690 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
8691 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
8692 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
8693 // Readjust index position based on new vector type.
8694 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
8695 do {
8696 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
8697 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
8698 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
8699 StChain.push_back(Elt: PartStore);
8700
8701 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
8702 } while (--Count);
8703 // Restore index back to be relative to the original widen element type.
8704 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
8705 }
8706 }
8707
8708 return true;
8709}
8710
8711/// Modifies a vector input (widen or narrows) to a vector of NVT. The
8712/// input vector must have the same element type as NVT.
8713/// FillWithZeroes specifies that the vector should be widened with zeroes.
8714SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
8715 bool FillWithZeroes) {
8716 // Note that InOp might have been widened so it might already have
8717 // the right width or it might need be narrowed.
8718 EVT InVT = InOp.getValueType();
8719 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
8720 "input and widen element type must match");
8721 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
8722 "cannot modify scalable vectors in this way");
8723 SDLoc dl(InOp);
8724
8725 // Check if InOp already has the right width.
8726 if (InVT == NVT)
8727 return InOp;
8728
8729 ElementCount InEC = InVT.getVectorElementCount();
8730 ElementCount WidenEC = NVT.getVectorElementCount();
8731 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
8732 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
8733 SmallVector<SDValue, 16> Ops(NumConcat);
8734 SDValue FillVal =
8735 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
8736 Ops[0] = InOp;
8737 for (unsigned i = 1; i != NumConcat; ++i)
8738 Ops[i] = FillVal;
8739
8740 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
8741 }
8742
8743 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
8744 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
8745
8746 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
8747 "Scalable vectors should have been handled already.");
8748
8749 unsigned InNumElts = InEC.getFixedValue();
8750 unsigned WidenNumElts = WidenEC.getFixedValue();
8751
8752 // Fall back to extract and build (+ mask, if padding with zeros).
8753 SmallVector<SDValue, 16> Ops(WidenNumElts);
8754 EVT EltVT = NVT.getVectorElementType();
8755 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
8756 unsigned Idx;
8757 for (Idx = 0; Idx < MinNumElts; ++Idx)
8758 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
8759
8760 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
8761 for (; Idx < WidenNumElts; ++Idx)
8762 Ops[Idx] = UndefVal;
8763
8764 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
8765 if (!FillWithZeroes)
8766 return Widened;
8767
8768 assert(NVT.isInteger() &&
8769 "We expect to never want to FillWithZeroes for non-integral types.");
8770
8771 SmallVector<SDValue, 16> MaskOps;
8772 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
8773 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
8774
8775 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
8776 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
8777}
8778