1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/STLFunctionalExtras.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/Analysis/TargetLibraryInfo.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/SelectionDAG.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetLowering.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/CodeGenTypes/MachineValueType.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <iterator>
48#include <utility>
49
50using namespace llvm;
51
52#define DEBUG_TYPE "legalizevectorops"
53
54namespace {
55
56class VectorLegalizer {
57 SelectionDAG& DAG;
58 const TargetLowering &TLI;
59 bool Changed = false; // Keep track of whether anything changed
60
61 /// For nodes that are of legal width, and that have more than one use, this
62 /// map indicates what regularized operand to use. This allows us to avoid
63 /// legalizing the same thing more than once.
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65
66 /// Adds a node to the translation cache.
67 void AddLegalizedOperand(SDValue From, SDValue To) {
68 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
69 // If someone requests legalization of the new node, return itself.
70 if (From != To)
71 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
72 }
73
74 /// Legalizes the given node.
75 SDValue LegalizeOp(SDValue Op);
76
77 /// Assuming the node is legal, "legalize" the results.
78 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79
80 /// Make sure Results are legal and update the translation cache.
81 SDValue RecursivelyLegalizeResults(SDValue Op,
82 MutableArrayRef<SDValue> Results);
83
84 /// Wrapper to interface LowerOperation with a vector of Results.
85 /// Returns false if the target wants to use default expansion. Otherwise
86 /// returns true. If return is true and the Results are empty, then the
87 /// target wants to keep the input node as is.
88 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89
90 /// Implements unrolling a VSETCC.
91 SDValue UnrollVSETCC(SDNode *Node);
92
93 /// Implement expand-based legalization of vector operations.
94 ///
95 /// This is just a high-level routine to dispatch to specific code paths for
96 /// operations to legalize them.
97 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98
99 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100 /// FP_TO_SINT isn't legal.
101 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102
103 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106
107 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108 SDValue ExpandSEXTINREG(SDNode *Node);
109
110 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111 ///
112 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113 /// type. The contents of the bits in the extended part of each element are
114 /// undef.
115 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116
117 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118 ///
119 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120 /// type, then shifts left and arithmetic shifts right to introduce a sign
121 /// extension.
122 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123
124 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125 ///
126 /// Shuffles the low lanes of the operand into place and blends zeros into
127 /// the remaining lanes, finally bitcasting to the proper type.
128 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129
130 /// Expand bswap of vectors into a shuffle if legal.
131 SDValue ExpandBSWAP(SDNode *Node);
132
133 /// Implement vselect in terms of XOR, AND, OR when blend is not
134 /// supported by the target.
135 SDValue ExpandVSELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
139 SDValue ExpandMASK_BEFOREFIRST(SDNode *N);
140 SDValue ExpandMaskedBinOp(SDNode *N);
141 SDValue ExpandSELECT(SDNode *Node);
142 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
143 SDValue ExpandStore(SDNode *N);
144 SDValue ExpandSubByteInsertExtractVectorElt(SDNode *Node);
145 SDValue ExpandFNEG(SDNode *Node);
146 SDValue ExpandFABS(SDNode *Node);
147 SDValue ExpandFCOPYSIGN(SDNode *Node);
148 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 SDValue ExpandBITREVERSE(SDNode *Node);
151 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 bool tryExpandVecMathCall(SDNode *Node,
159 function_ref<RTLIB::Libcall(EVT)> GetLibcall,
160 SmallVectorImpl<SDValue> &Results);
161
162 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 /// Implements vector promotion.
165 ///
166 /// This is essentially just bitcasting the operands to a different type and
167 /// bitcasting the result back to the original type.
168 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
169
170 /// Implements [SU]INT_TO_FP vector promotion.
171 ///
172 /// This is a [zs]ext of the input operand to a larger integer type.
173 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
174
175 /// Implements FP_TO_[SU]INT vector promotion of the result type.
176 ///
177 /// It is promoted to a larger integer type. The result is then
178 /// truncated back to the original type.
179 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
180
181 /// Implements vector setcc operation promotion.
182 ///
183 /// All vector operands are promoted to a vector type with larger element
184 /// type.
185 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
188
189 /// Calculate the reduction using a type of higher precision and round the
190 /// result to match the original type. Setting NonArithmetic signifies the
191 /// rounding of the result does not affect its value.
192 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
193 bool NonArithmetic);
194
195 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
196
197public:
198 VectorLegalizer(SelectionDAG& dag) :
199 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
200
201 /// Begin legalizer the vector operations in the DAG.
202 bool Run();
203};
204
205} // end anonymous namespace
206
207bool VectorLegalizer::Run() {
208 // Before we start legalizing vector nodes, check if there are any vectors.
209 bool HasVectors = false;
210 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
211 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
212 // Check if the values of the nodes contain vectors. We don't need to check
213 // the operands because we are going to check their values at some point.
214 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
215
216 // If we found a vector node we can start the legalization.
217 if (HasVectors)
218 break;
219 }
220
221 // If this basic block has no vectors then no need to legalize vectors.
222 if (!HasVectors)
223 return false;
224
225 // The legalize process is inherently a bottom-up recursive process (users
226 // legalize their uses before themselves). Given infinite stack space, we
227 // could just start legalizing on the root and traverse the whole graph. In
228 // practice however, this causes us to run out of stack space on large basic
229 // blocks. To avoid this problem, compute an ordering of the nodes where each
230 // node is only legalized after all of its operands are legalized.
231 DAG.AssignTopologicalOrder();
232 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
233 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
234 LegalizeOp(Op: SDValue(&*I, 0));
235
236 // Finally, it's possible the root changed. Get the new root.
237 SDValue OldRoot = DAG.getRoot();
238 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
239 DAG.setRoot(LegalizedNodes[OldRoot]);
240
241 LegalizedNodes.clear();
242
243 // Remove dead nodes now.
244 DAG.RemoveDeadNodes();
245
246 return Changed;
247}
248
249SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
250 assert(Op->getNumValues() == Result->getNumValues() &&
251 "Unexpected number of results");
252 // Generic legalization: just pass the operand through.
253 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
254 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
255 return SDValue(Result, Op.getResNo());
256}
257
258SDValue
259VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
260 MutableArrayRef<SDValue> Results) {
261 assert(Results.size() == Op->getNumValues() &&
262 "Unexpected number of results");
263 // Make sure that the generated code is itself legal.
264 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
265 Results[i] = LegalizeOp(Op: Results[i]);
266 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
267 }
268
269 return Results[Op.getResNo()];
270}
271
272SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
273 // Note that LegalizeOp may be reentered even from single-use nodes, which
274 // means that we always must cache transformed nodes.
275 auto I = LegalizedNodes.find(Val: Op);
276 if (I != LegalizedNodes.end()) return I->second;
277
278 // Legalize the operands
279 SmallVector<SDValue, 8> Ops;
280 for (const SDValue &Oper : Op->op_values())
281 Ops.push_back(Elt: LegalizeOp(Op: Oper));
282
283 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
284
285 bool HasVectorValueOrOp =
286 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
287 llvm::any_of(Range: Node->op_values(),
288 P: [](SDValue O) { return O.getValueType().isVector(); });
289 if (!HasVectorValueOrOp)
290 return TranslateLegalizeResults(Op, Result: Node);
291
292 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
293 EVT ValVT;
294 switch (Op.getOpcode()) {
295 default:
296 return TranslateLegalizeResults(Op, Result: Node);
297 case ISD::LOAD: {
298 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
299 ISD::LoadExtType ExtType = LD->getExtensionType();
300 EVT LoadedVT = LD->getMemoryVT();
301 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
302 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
303 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
304 break;
305 }
306 case ISD::STORE: {
307 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
308 EVT StVT = ST->getMemoryVT();
309 MVT ValVT = ST->getValue().getSimpleValueType();
310 if (StVT.isVector() && ST->isTruncatingStore())
311 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
312 AddrSpace: ST->getAddressSpace());
313 break;
314 }
315 case ISD::EXTRACT_VECTOR_ELT:
316 case ISD::INSERT_VECTOR_ELT: {
317 // Expanding a variable index through the stack cannot address elements
318 // narrower than a byte. Constant indices are left to LegalizeDAG.
319 EVT VecVT = Node->getOperand(Num: 0).getValueType();
320 SDValue Idx = Node->getOperand(Num: Node->getNumOperands() - 1);
321 if (!VecVT.getVectorElementType().isByteSized() &&
322 !isa<ConstantSDNode>(Val: Idx))
323 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: VecVT);
324 break;
325 }
326 case ISD::MERGE_VALUES:
327 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
328 // This operation lies about being legal: when it claims to be legal,
329 // it should actually be expanded.
330 if (Action == TargetLowering::Legal)
331 Action = TargetLowering::Expand;
332 break;
333#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
334 case ISD::STRICT_##DAGN:
335#include "llvm/IR/ConstrainedOps.def"
336 ValVT = Node->getValueType(ResNo: 0);
337 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
338 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
339 ValVT = Node->getOperand(Num: 1).getValueType();
340 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
341 Op.getOpcode() == ISD::STRICT_FSETCCS) {
342 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
343 ISD::CondCode CCCode = Node->getOperandCondCode(Num: 3);
344 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
345 if (Action == TargetLowering::Legal)
346 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
347 } else {
348 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
349 }
350 // If we're asked to expand a strict vector floating-point operation,
351 // by default we're going to simply unroll it. That is usually the
352 // best approach, except in the case where the resulting strict (scalar)
353 // operations would themselves use the fallback mutation to non-strict.
354 // In that specific case, just do the fallback on the vector op.
355 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
356 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
357 TargetLowering::Legal) {
358 EVT EltVT = ValVT.getVectorElementType();
359 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
360 == TargetLowering::Expand &&
361 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
362 == TargetLowering::Legal)
363 Action = TargetLowering::Legal;
364 }
365 break;
366 case ISD::ADD:
367 case ISD::SUB:
368 case ISD::MUL:
369 case ISD::MULHS:
370 case ISD::MULHU:
371 case ISD::SDIV:
372 case ISD::UDIV:
373 case ISD::SREM:
374 case ISD::UREM:
375 case ISD::SDIVREM:
376 case ISD::UDIVREM:
377 case ISD::FADD:
378 case ISD::FSUB:
379 case ISD::FMUL:
380 case ISD::FDIV:
381 case ISD::FREM:
382 case ISD::AND:
383 case ISD::OR:
384 case ISD::XOR:
385 case ISD::SHL:
386 case ISD::SRA:
387 case ISD::SRL:
388 case ISD::FSHL:
389 case ISD::FSHR:
390 case ISD::ROTL:
391 case ISD::ROTR:
392 case ISD::ABS:
393 case ISD::ABS_MIN_POISON:
394 case ISD::ABDS:
395 case ISD::ABDU:
396 case ISD::AVGCEILS:
397 case ISD::AVGCEILU:
398 case ISD::AVGFLOORS:
399 case ISD::AVGFLOORU:
400 case ISD::AVGSUBFLOORS:
401 case ISD::AVGSUBFLOORU:
402 case ISD::BSWAP:
403 case ISD::BITREVERSE:
404 case ISD::CTLZ:
405 case ISD::CTTZ:
406 case ISD::CTLZ_ZERO_POISON:
407 case ISD::CTTZ_ZERO_POISON:
408 case ISD::CTPOP:
409 case ISD::CLMUL:
410 case ISD::CLMULH:
411 case ISD::CLMULR:
412 case ISD::SELECT:
413 case ISD::VSELECT:
414 case ISD::SELECT_CC:
415 case ISD::ZERO_EXTEND:
416 case ISD::ANY_EXTEND:
417 case ISD::TRUNCATE:
418 case ISD::SIGN_EXTEND:
419 case ISD::FP_TO_SINT:
420 case ISD::FP_TO_UINT:
421 case ISD::FNEG:
422 case ISD::FABS:
423 case ISD::FMINNUM:
424 case ISD::FMAXNUM:
425 case ISD::FMINNUM_IEEE:
426 case ISD::FMAXNUM_IEEE:
427 case ISD::FMINIMUM:
428 case ISD::FMAXIMUM:
429 case ISD::FMINIMUMNUM:
430 case ISD::FMAXIMUMNUM:
431 case ISD::FCOPYSIGN:
432 case ISD::FSQRT:
433 case ISD::FSIN:
434 case ISD::FCOS:
435 case ISD::FTAN:
436 case ISD::FASIN:
437 case ISD::FACOS:
438 case ISD::FATAN:
439 case ISD::FATAN2:
440 case ISD::FSINH:
441 case ISD::FCOSH:
442 case ISD::FTANH:
443 case ISD::FLDEXP:
444 case ISD::FPOWI:
445 case ISD::FPOW:
446 case ISD::FCBRT:
447 case ISD::FLOG:
448 case ISD::FLOG2:
449 case ISD::FLOG10:
450 case ISD::FEXP:
451 case ISD::FEXP2:
452 case ISD::FEXP10:
453 case ISD::FCEIL:
454 case ISD::FTRUNC:
455 case ISD::FRINT:
456 case ISD::FNEARBYINT:
457 case ISD::FROUND:
458 case ISD::FROUNDEVEN:
459 case ISD::FFLOOR:
460 case ISD::FP_ROUND:
461 case ISD::FP_EXTEND:
462 case ISD::FPTRUNC_ROUND:
463 case ISD::FMA:
464 case ISD::SIGN_EXTEND_INREG:
465 case ISD::ANY_EXTEND_VECTOR_INREG:
466 case ISD::SIGN_EXTEND_VECTOR_INREG:
467 case ISD::ZERO_EXTEND_VECTOR_INREG:
468 case ISD::SMIN:
469 case ISD::SMAX:
470 case ISD::UMIN:
471 case ISD::UMAX:
472 case ISD::SMUL_LOHI:
473 case ISD::UMUL_LOHI:
474 case ISD::SADDO:
475 case ISD::UADDO:
476 case ISD::SSUBO:
477 case ISD::USUBO:
478 case ISD::SMULO:
479 case ISD::UMULO:
480 case ISD::CONVERT_FROM_ARBITRARY_FP:
481 case ISD::CONVERT_TO_ARBITRARY_FP:
482 case ISD::FCANONICALIZE:
483 case ISD::FFREXP:
484 case ISD::FMODF:
485 case ISD::FSINCOS:
486 case ISD::FSINCOSPI:
487 case ISD::SADDSAT:
488 case ISD::UADDSAT:
489 case ISD::SSUBSAT:
490 case ISD::USUBSAT:
491 case ISD::SSHLSAT:
492 case ISD::USHLSAT:
493 case ISD::FP_TO_SINT_SAT:
494 case ISD::FP_TO_UINT_SAT:
495 case ISD::MGATHER:
496 case ISD::VECTOR_COMPRESS:
497 case ISD::VECTOR_SHUFFLE_VAR:
498 case ISD::SCMP:
499 case ISD::UCMP:
500 case ISD::GET_ACTIVE_LANE_MASK:
501 case ISD::LOOP_DEPENDENCE_WAR_MASK:
502 case ISD::LOOP_DEPENDENCE_RAW_MASK:
503 case ISD::MASK_BEFOREFIRST:
504 case ISD::MASKED_UDIV:
505 case ISD::MASKED_SDIV:
506 case ISD::MASKED_UREM:
507 case ISD::MASKED_SREM:
508 case ISD::VECTOR_MATCH:
509 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
510 break;
511 case ISD::VECTOR_INTERLEAVE:
512 case ISD::VECTOR_DEINTERLEAVE: {
513 EVT VT = Node->getValueType(ResNo: 0);
514 unsigned Factor = Node->getNumOperands();
515 Action = TLI.getVectorInterleaveAction(Opc: Node->getOpcode(), Factor, VT);
516 // Early Custom Lowering causes RISC-V regressions.
517 if (Action == TargetLowering::Custom &&
518 (VT.isScalableVector() || Factor != 2)) {
519 Action = TargetLowering::Legal;
520 }
521 break;
522 }
523 case ISD::SMULFIX:
524 case ISD::SMULFIXSAT:
525 case ISD::UMULFIX:
526 case ISD::UMULFIXSAT:
527 case ISD::SDIVFIX:
528 case ISD::SDIVFIXSAT:
529 case ISD::UDIVFIX:
530 case ISD::UDIVFIXSAT: {
531 unsigned Scale = Node->getConstantOperandVal(Num: 2);
532 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
533 VT: Node->getValueType(ResNo: 0), Scale);
534 break;
535 }
536 case ISD::LROUND:
537 case ISD::LLROUND:
538 case ISD::LRINT:
539 case ISD::LLRINT:
540 case ISD::SINT_TO_FP:
541 case ISD::UINT_TO_FP:
542 case ISD::VECREDUCE_ADD:
543 case ISD::VECREDUCE_MUL:
544 case ISD::VECREDUCE_AND:
545 case ISD::VECREDUCE_OR:
546 case ISD::VECREDUCE_XOR:
547 case ISD::VECREDUCE_SMAX:
548 case ISD::VECREDUCE_SMIN:
549 case ISD::VECREDUCE_UMAX:
550 case ISD::VECREDUCE_UMIN:
551 case ISD::VECREDUCE_FADD:
552 case ISD::VECREDUCE_FMAX:
553 case ISD::VECREDUCE_FMAXIMUM:
554 case ISD::VECREDUCE_FMIN:
555 case ISD::VECREDUCE_FMINIMUM:
556 case ISD::VECREDUCE_FMAXIMUMNUM:
557 case ISD::VECREDUCE_FMINIMUMNUM:
558 case ISD::VECREDUCE_FMUL:
559 case ISD::CTTZ_ELTS:
560 case ISD::CTTZ_ELTS_ZERO_POISON:
561 case ISD::VECTOR_FIND_LAST_ACTIVE:
562 Action = TLI.getOperationAction(Op: Node->getOpcode(),
563 VT: Node->getOperand(Num: 0).getValueType());
564 break;
565 case ISD::VECREDUCE_SEQ_FADD:
566 case ISD::VECREDUCE_SEQ_FMUL:
567 Action = TLI.getOperationAction(Op: Node->getOpcode(),
568 VT: Node->getOperand(Num: 1).getValueType());
569 break;
570 case ISD::SETCC: {
571 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
572 ISD::CondCode CCCode = Node->getOperandCondCode(Num: 2);
573 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
574 if (Action == TargetLowering::Legal)
575 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
576 break;
577 }
578 case ISD::PARTIAL_REDUCE_UMLA:
579 case ISD::PARTIAL_REDUCE_SMLA:
580 case ISD::PARTIAL_REDUCE_SUMLA:
581 case ISD::PARTIAL_REDUCE_FMLA:
582 Action =
583 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
584 InputVT: Node->getOperand(Num: 1).getValueType());
585 break;
586
587#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
588 case ISD::VPID: { \
589 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
590 : Node->getOperand(LEGALPOS).getValueType(); \
591 /* Defer non-vector results to LegalizeDAG. */ \
592 if (!Node->getValueType(0).isVector() && \
593 Node->getValueType(0) != MVT::Other) { \
594 Action = TargetLowering::Legal; \
595 break; \
596 } \
597 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
598 } break;
599#include "llvm/IR/VPIntrinsics.def"
600 }
601
602 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
603
604 SmallVector<SDValue, 8> ResultVals;
605 switch (Action) {
606 default: llvm_unreachable("This action is not supported yet!");
607 case TargetLowering::Promote:
608 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
609 "This action is not supported yet!");
610 LLVM_DEBUG(dbgs() << "Promoting\n");
611 Promote(Node, Results&: ResultVals);
612 assert(!ResultVals.empty() && "No results for promotion?");
613 break;
614 case TargetLowering::Legal:
615 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
616 break;
617 case TargetLowering::Custom:
618 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
619 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
620 break;
621 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
622 [[fallthrough]];
623 case TargetLowering::Expand:
624 LLVM_DEBUG(dbgs() << "Expanding\n");
625 Expand(Node, Results&: ResultVals);
626 break;
627 }
628
629 if (ResultVals.empty())
630 return TranslateLegalizeResults(Op, Result: Node);
631
632 Changed = true;
633 return RecursivelyLegalizeResults(Op, Results: ResultVals);
634}
635
636// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
637// merge them somehow?
638bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
639 SmallVectorImpl<SDValue> &Results) {
640 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
641
642 if (!Res.getNode())
643 return false;
644
645 if (Res == SDValue(Node, 0))
646 return true;
647
648 // If the original node has one result, take the return value from
649 // LowerOperation as is. It might not be result number 0.
650 if (Node->getNumValues() == 1) {
651 Results.push_back(Elt: Res);
652 return true;
653 }
654
655 // If the original node has multiple results, then the return node should
656 // have the same number of results.
657 assert((Node->getNumValues() == Res->getNumValues()) &&
658 "Lowering returned the wrong number of results!");
659
660 // Places new result values base on N result number.
661 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
662 Results.push_back(Elt: Res.getValue(R: I));
663
664 return true;
665}
666
667void VectorLegalizer::PromoteSETCC(SDNode *Node,
668 SmallVectorImpl<SDValue> &Results) {
669 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
670 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
671
672 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
673
674 SDLoc DL(Node);
675 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
676
677 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
678 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
679 Operands[2] = Node->getOperand(Num: 2);
680
681 EVT ResVT =
682 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
683 SDValue Res =
684 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
685 if (ResVT != Node->getValueType(ResNo: 0))
686 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
687 Results.push_back(Elt: Res);
688}
689
690void VectorLegalizer::PromoteSTRICT(SDNode *Node,
691 SmallVectorImpl<SDValue> &Results) {
692 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
693 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
694
695 assert(VecVT.isFloatingPoint());
696
697 SDLoc DL(Node);
698 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
699 SmallVector<SDValue, 2> Chains;
700
701 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
702 if (Node->getOperand(Num: j).getValueType().isVector() &&
703 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
704 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
705 {
706 // promote the vector operand.
707 SDValue Ext =
708 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
709 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
710 Operands[j] = Ext.getValue(R: 0);
711 Chains.push_back(Elt: Ext.getValue(R: 1));
712 } else
713 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
714
715 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
716
717 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
718
719 SDValue Res =
720 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
721
722 SDValue Round =
723 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
724 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
725 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
726
727 Results.push_back(Elt: Round.getValue(R: 0));
728 Results.push_back(Elt: Round.getValue(R: 1));
729}
730
731void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
732 SmallVectorImpl<SDValue> &Results,
733 bool NonArithmetic) {
734 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
735 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
736 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
737
738 SDLoc DL(Node);
739 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
740 SDValue Rdx =
741 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
742 Flags: Node->getFlags());
743 SDValue Res =
744 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
745 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
746 Results.push_back(Elt: Res);
747}
748
749void VectorLegalizer::PromoteVECTOR_COMPRESS(
750 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
751 SDLoc DL(Node);
752 EVT VT = Node->getValueType(ResNo: 0);
753 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
754 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
755 "Only integer promotion or bitcasts between types is supported");
756
757 SDValue Vec = Node->getOperand(Num: 0);
758 SDValue Mask = Node->getOperand(Num: 1);
759 SDValue Passthru = Node->getOperand(Num: 2);
760 if (VT.isInteger()) {
761 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
762 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
763 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
764 } else {
765 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
766 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
767 }
768
769 SDValue Result =
770 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
771 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
772 : DAG.getBitcast(VT, V: Result);
773 Results.push_back(Elt: Result);
774}
775
776void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
777 // For a few operations there is a specific concept for promotion based on
778 // the operand's type.
779 switch (Node->getOpcode()) {
780 case ISD::SINT_TO_FP:
781 case ISD::UINT_TO_FP:
782 case ISD::STRICT_SINT_TO_FP:
783 case ISD::STRICT_UINT_TO_FP:
784 // "Promote" the operation by extending the operand.
785 PromoteINT_TO_FP(Node, Results);
786 return;
787 case ISD::FP_TO_UINT:
788 case ISD::FP_TO_SINT:
789 case ISD::STRICT_FP_TO_UINT:
790 case ISD::STRICT_FP_TO_SINT:
791 // Promote the operation by extending the operand.
792 PromoteFP_TO_INT(Node, Results);
793 return;
794 case ISD::SETCC:
795 // Promote the operation by extending the operand.
796 PromoteSETCC(Node, Results);
797 return;
798 case ISD::STRICT_FADD:
799 case ISD::STRICT_FCEIL:
800 case ISD::STRICT_FDIV:
801 case ISD::STRICT_FFLOOR:
802 case ISD::STRICT_FMA:
803 case ISD::STRICT_FMAXIMUM:
804 case ISD::STRICT_FMAXNUM:
805 case ISD::STRICT_FMINIMUM:
806 case ISD::STRICT_FMINNUM:
807 case ISD::STRICT_FMUL:
808 case ISD::STRICT_FNEARBYINT:
809 case ISD::STRICT_FRINT:
810 case ISD::STRICT_FROUND:
811 case ISD::STRICT_FROUNDEVEN:
812 case ISD::STRICT_FSQRT:
813 case ISD::STRICT_FSUB:
814 case ISD::STRICT_FTRUNC:
815 PromoteSTRICT(Node, Results);
816 return;
817 case ISD::VECREDUCE_FADD:
818 case ISD::VECREDUCE_FMUL:
819 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
820 return;
821 case ISD::VECREDUCE_FMAX:
822 case ISD::VECREDUCE_FMAXIMUM:
823 case ISD::VECREDUCE_FMIN:
824 case ISD::VECREDUCE_FMINIMUM:
825 case ISD::VECREDUCE_FMAXIMUMNUM:
826 case ISD::VECREDUCE_FMINIMUMNUM:
827 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
828 return;
829 case ISD::VECTOR_COMPRESS:
830 PromoteVECTOR_COMPRESS(Node, Results);
831 return;
832
833 case ISD::FP_ROUND:
834 case ISD::FP_EXTEND:
835 // These operations are used to do promotion so they can't be promoted
836 // themselves.
837 llvm_unreachable("Don't know how to promote this operation!");
838 }
839
840 // There are currently two cases of vector promotion:
841 // 1) Bitcasting a vector of integers to a different type to a vector of the
842 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
843 // 2) Extending a vector of floats to a vector of the same number of larger
844 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
845 assert(Node->getNumValues() == 1 &&
846 "Can't promote a vector with multiple results!");
847 MVT VT = Node->getSimpleValueType(ResNo: 0);
848 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
849 SDLoc dl(Node);
850 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
851
852 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
853 // Do not promote the mask operand of a VP OP.
854 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
855 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
856 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
857 if (Node->getOperand(Num: j)
858 .getValueType()
859 .getVectorElementType()
860 .isFloatingPoint() &&
861 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
862 Operands[j] = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
863 else
864 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
865 else
866 Operands[j] = Node->getOperand(Num: j);
867 }
868
869 SDValue Res =
870 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
871
872 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
873 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
874 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
875 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
876 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
877 else
878 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
879
880 Results.push_back(Elt: Res);
881}
882
883void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
884 SmallVectorImpl<SDValue> &Results) {
885 // INT_TO_FP operations may require the input operand be promoted even
886 // when the type is otherwise legal.
887 bool IsStrict = Node->isStrictFPOpcode();
888 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
889 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
890 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
891 "Vectors have different number of elements!");
892
893 SDLoc dl(Node);
894 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
895
896 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
897 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
898 ? ISD::ZERO_EXTEND
899 : ISD::SIGN_EXTEND;
900 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
901 if (Node->getOperand(Num: j).getValueType().isVector())
902 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
903 else
904 Operands[j] = Node->getOperand(Num: j);
905 }
906
907 if (IsStrict) {
908 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
909 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
910 Results.push_back(Elt: Res);
911 Results.push_back(Elt: Res.getValue(R: 1));
912 return;
913 }
914
915 SDValue Res =
916 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
917 Results.push_back(Elt: Res);
918}
919
920// For FP_TO_INT we promote the result type to a vector type with wider
921// elements and then truncate the result. This is different from the default
922// PromoteVector which uses bitcast to promote thus assumning that the
923// promoted vector type has the same overall size.
924void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
925 SmallVectorImpl<SDValue> &Results) {
926 MVT VT = Node->getSimpleValueType(ResNo: 0);
927 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
928 bool IsStrict = Node->isStrictFPOpcode();
929 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
930 "Vectors have different number of elements!");
931
932 unsigned NewOpc = Node->getOpcode();
933 // Change FP_TO_UINT to FP_TO_SINT if possible.
934 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
935 if (NewOpc == ISD::FP_TO_UINT &&
936 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
937 NewOpc = ISD::FP_TO_SINT;
938
939 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
940 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
941 NewOpc = ISD::STRICT_FP_TO_SINT;
942
943 SDLoc dl(Node);
944 SDValue Promoted, Chain;
945 if (IsStrict) {
946 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
947 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
948 Chain = Promoted.getValue(R: 1);
949 } else
950 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
951
952 // Assert that the converted value fits in the original type. If it doesn't
953 // (eg: because the value being converted is too big), then the result of the
954 // original operation was undefined anyway, so the assert is still correct.
955 if (Node->getOpcode() == ISD::FP_TO_UINT ||
956 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
957 NewOpc = ISD::AssertZext;
958 else
959 NewOpc = ISD::AssertSext;
960
961 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
962 N2: DAG.getValueType(VT.getScalarType()));
963 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
964 Results.push_back(Elt: Promoted);
965 if (IsStrict)
966 Results.push_back(Elt: Chain);
967}
968
969std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
970 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
971 return TLI.scalarizeVectorLoad(LD, DAG);
972}
973
974SDValue VectorLegalizer::ExpandSubByteInsertExtractVectorElt(SDNode *Node) {
975 SDLoc DL(Node);
976 LLVMContext &Ctx = *DAG.getContext();
977 SDValue Vec = Node->getOperand(Num: 0);
978 EVT VecVT = Vec.getValueType();
979 EVT EltVT = VecVT.getVectorElementType().getRoundIntegerType(Context&: Ctx);
980 EVT WideVT = VecVT.changeElementType(Context&: Ctx, EltVT);
981 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WideVT, Operand: Vec);
982
983 if (Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
984 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vec,
985 N2: Node->getOperand(Num: 1));
986 return DAG.getAnyExtOrTrunc(Op: Elt, DL, VT: Node->getValueType(ResNo: 0));
987 }
988
989 SDValue Elt = DAG.getAnyExtOrTrunc(Op: Node->getOperand(Num: 1), DL, VT: EltVT);
990 SDValue Ins = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: WideVT, N1: Vec, N2: Elt,
991 N3: Node->getOperand(Num: 2));
992 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Ins);
993}
994
995SDValue VectorLegalizer::ExpandStore(SDNode *N) {
996 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
997 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
998 return TF;
999}
1000
1001void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
1002 switch (Node->getOpcode()) {
1003 case ISD::LOAD: {
1004 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
1005 Results.push_back(Elt: Tmp.first);
1006 Results.push_back(Elt: Tmp.second);
1007 return;
1008 }
1009 case ISD::STORE:
1010 Results.push_back(Elt: ExpandStore(N: Node));
1011 return;
1012 case ISD::MERGE_VALUES:
1013 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1014 Results.push_back(Elt: Node->getOperand(Num: i));
1015 return;
1016 case ISD::EXTRACT_VECTOR_ELT:
1017 case ISD::INSERT_VECTOR_ELT:
1018 Results.push_back(Elt: ExpandSubByteInsertExtractVectorElt(Node));
1019 return;
1020 case ISD::VECTOR_INTERLEAVE:
1021 TLI.expandVectorInterleave(Node, Results, DAG);
1022 return;
1023 case ISD::VECTOR_DEINTERLEAVE:
1024 TLI.expandVectorDeinterleave(Node, Results, DAG);
1025 return;
1026 case ISD::SIGN_EXTEND_INREG:
1027 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
1028 Results.push_back(Elt: Expanded);
1029 return;
1030 }
1031 break;
1032 case ISD::ANY_EXTEND_VECTOR_INREG:
1033 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
1034 return;
1035 case ISD::SIGN_EXTEND_VECTOR_INREG:
1036 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
1037 return;
1038 case ISD::ZERO_EXTEND_VECTOR_INREG:
1039 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
1040 return;
1041 case ISD::BSWAP:
1042 if (SDValue Expanded = ExpandBSWAP(Node)) {
1043 Results.push_back(Elt: Expanded);
1044 return;
1045 }
1046 break;
1047 case ISD::VSELECT:
1048 if (SDValue Expanded = ExpandVSELECT(Node)) {
1049 Results.push_back(Elt: Expanded);
1050 return;
1051 }
1052 break;
1053 case ISD::VP_SREM:
1054 case ISD::VP_UREM:
1055 if (SDValue Expanded = ExpandVP_REM(Node)) {
1056 Results.push_back(Elt: Expanded);
1057 return;
1058 }
1059 break;
1060 case ISD::SELECT:
1061 if (SDValue Expanded = ExpandSELECT(Node)) {
1062 Results.push_back(Elt: Expanded);
1063 return;
1064 }
1065 break;
1066 case ISD::SELECT_CC: {
1067 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1068 EVT CondVT = TLI.getSetCCResultType(
1069 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
1070 SDValue SetCC =
1071 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
1072 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
1073 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1074 LHS: Node->getOperand(Num: 2),
1075 RHS: Node->getOperand(Num: 3)));
1076 return;
1077 }
1078 break;
1079 }
1080 case ISD::FP_TO_UINT:
1081 ExpandFP_TO_UINT(Node, Results);
1082 return;
1083 case ISD::UINT_TO_FP:
1084 ExpandUINT_TO_FLOAT(Node, Results);
1085 return;
1086 case ISD::FNEG:
1087 if (SDValue Expanded = ExpandFNEG(Node)) {
1088 Results.push_back(Elt: Expanded);
1089 return;
1090 }
1091 break;
1092 case ISD::FABS:
1093 if (SDValue Expanded = ExpandFABS(Node)) {
1094 Results.push_back(Elt: Expanded);
1095 return;
1096 }
1097 break;
1098 case ISD::FCOPYSIGN:
1099 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1100 Results.push_back(Elt: Expanded);
1101 return;
1102 }
1103 break;
1104 case ISD::FCANONICALIZE: {
1105 // If the scalar element type has a
1106 // Legal/Custom FCANONICALIZE, don't
1107 // mess with the vector, fall back.
1108 EVT VT = Node->getValueType(ResNo: 0);
1109 EVT EltVT = VT.getVectorElementType();
1110 if (!VT.isScalableVector() &&
1111 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1112 TargetLowering::Expand)
1113 break;
1114 // Otherwise canonicalize the whole vector.
1115 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1116 Results.push_back(Elt: Mul);
1117 return;
1118 }
1119 case ISD::FSUB:
1120 ExpandFSUB(Node, Results);
1121 return;
1122 case ISD::SETCC:
1123 ExpandSETCC(Node, Results);
1124 return;
1125 case ISD::ABS:
1126 case ISD::ABS_MIN_POISON:
1127 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1128 Results.push_back(Elt: Expanded);
1129 return;
1130 }
1131 break;
1132 case ISD::ABDS:
1133 case ISD::ABDU:
1134 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1135 Results.push_back(Elt: Expanded);
1136 return;
1137 }
1138 break;
1139 case ISD::AVGCEILS:
1140 case ISD::AVGCEILU:
1141 case ISD::AVGFLOORS:
1142 case ISD::AVGFLOORU:
1143 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1144 Results.push_back(Elt: Expanded);
1145 return;
1146 }
1147 break;
1148 case ISD::BITREVERSE:
1149 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1150 Results.push_back(Elt: Expanded);
1151 return;
1152 }
1153 break;
1154 case ISD::CTPOP:
1155 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1156 Results.push_back(Elt: Expanded);
1157 return;
1158 }
1159 break;
1160 case ISD::CTLZ:
1161 case ISD::CTLZ_ZERO_POISON:
1162 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1163 Results.push_back(Elt: Expanded);
1164 return;
1165 }
1166 break;
1167 case ISD::CTTZ:
1168 case ISD::CTTZ_ZERO_POISON:
1169 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1170 Results.push_back(Elt: Expanded);
1171 return;
1172 }
1173 break;
1174 case ISD::FSHL:
1175 case ISD::FSHR:
1176 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1177 Results.push_back(Elt: Expanded);
1178 return;
1179 }
1180 break;
1181 case ISD::CLMUL:
1182 case ISD::CLMULR:
1183 case ISD::CLMULH:
1184 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1185 Results.push_back(Elt: Expanded);
1186 return;
1187 }
1188 break;
1189 case ISD::PEXT:
1190 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1191 return;
1192 case ISD::PDEP:
1193 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1194 return;
1195 case ISD::ROTL:
1196 case ISD::ROTR:
1197 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1198 Results.push_back(Elt: Expanded);
1199 return;
1200 }
1201 break;
1202 case ISD::FMINNUM:
1203 case ISD::FMAXNUM:
1204 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1205 Results.push_back(Elt: Expanded);
1206 return;
1207 }
1208 break;
1209 case ISD::FMINIMUM:
1210 case ISD::FMAXIMUM:
1211 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1212 return;
1213 case ISD::FMINIMUMNUM:
1214 case ISD::FMAXIMUMNUM:
1215 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1216 return;
1217 case ISD::SMIN:
1218 case ISD::SMAX:
1219 case ISD::UMIN:
1220 case ISD::UMAX:
1221 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1222 Results.push_back(Elt: Expanded);
1223 return;
1224 }
1225 break;
1226 case ISD::UADDO:
1227 case ISD::USUBO:
1228 ExpandUADDSUBO(Node, Results);
1229 return;
1230 case ISD::SADDO:
1231 case ISD::SSUBO:
1232 ExpandSADDSUBO(Node, Results);
1233 return;
1234 case ISD::UMULO:
1235 case ISD::SMULO:
1236 ExpandMULO(Node, Results);
1237 return;
1238 case ISD::MULHS:
1239 case ISD::MULHU:
1240 if (SDValue Expanded = TLI.expandMULH(Node, DAG)) {
1241 Results.push_back(Elt: Expanded);
1242 return;
1243 }
1244 break;
1245 case ISD::USUBSAT:
1246 case ISD::SSUBSAT:
1247 case ISD::UADDSAT:
1248 case ISD::SADDSAT:
1249 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1250 Results.push_back(Elt: Expanded);
1251 return;
1252 }
1253 break;
1254 case ISD::USHLSAT:
1255 case ISD::SSHLSAT:
1256 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1257 Results.push_back(Elt: Expanded);
1258 return;
1259 }
1260 break;
1261 case ISD::FP_TO_SINT_SAT:
1262 case ISD::FP_TO_UINT_SAT:
1263 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1264 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1265 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1266 Results.push_back(Elt: Expanded);
1267 return;
1268 }
1269 }
1270 break;
1271 case ISD::SMULFIX:
1272 case ISD::UMULFIX:
1273 case ISD::SMULFIXSAT:
1274 case ISD::UMULFIXSAT:
1275 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1276 Results.push_back(Elt: Expanded);
1277 return;
1278 }
1279 break;
1280 case ISD::SDIVFIX:
1281 case ISD::UDIVFIX:
1282 ExpandFixedPointDiv(Node, Results);
1283 return;
1284 case ISD::SDIVFIXSAT:
1285 case ISD::UDIVFIXSAT:
1286 break;
1287#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1288 case ISD::STRICT_##DAGN:
1289#include "llvm/IR/ConstrainedOps.def"
1290 ExpandStrictFPOp(Node, Results);
1291 return;
1292 case ISD::VECREDUCE_ADD:
1293 case ISD::VECREDUCE_MUL:
1294 case ISD::VECREDUCE_AND:
1295 case ISD::VECREDUCE_OR:
1296 case ISD::VECREDUCE_XOR:
1297 case ISD::VECREDUCE_SMAX:
1298 case ISD::VECREDUCE_SMIN:
1299 case ISD::VECREDUCE_UMAX:
1300 case ISD::VECREDUCE_UMIN:
1301 case ISD::VECREDUCE_FADD:
1302 case ISD::VECREDUCE_FMUL:
1303 case ISD::VECREDUCE_FMAX:
1304 case ISD::VECREDUCE_FMIN:
1305 case ISD::VECREDUCE_FMAXIMUM:
1306 case ISD::VECREDUCE_FMINIMUM:
1307 case ISD::VECREDUCE_FMAXIMUMNUM:
1308 case ISD::VECREDUCE_FMINIMUMNUM:
1309 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1310 return;
1311 case ISD::PARTIAL_REDUCE_UMLA:
1312 case ISD::PARTIAL_REDUCE_SMLA:
1313 case ISD::PARTIAL_REDUCE_SUMLA:
1314 case ISD::PARTIAL_REDUCE_FMLA:
1315 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1316 return;
1317 case ISD::VECREDUCE_SEQ_FADD:
1318 case ISD::VECREDUCE_SEQ_FMUL:
1319 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1320 return;
1321 case ISD::VECTOR_MATCH:
1322 Results.push_back(Elt: TLI.expandVectorMatch(N: Node, DAG));
1323 return;
1324 case ISD::SREM:
1325 case ISD::UREM:
1326 ExpandREM(Node, Results);
1327 return;
1328 case ISD::VP_MERGE:
1329 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1330 Results.push_back(Elt: Expanded);
1331 return;
1332 }
1333 break;
1334 case ISD::FREM:
1335 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getREM, Results))
1336 return;
1337 break;
1338 case ISD::FSINCOS:
1339 case ISD::FSINCOSPI: {
1340 EVT VT = Node->getValueType(ResNo: 0);
1341 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1342 ? RTLIB::getSINCOS(VT)
1343 : RTLIB::getSINCOSPI(VT);
1344 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1345 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1346 return;
1347
1348 // TODO: Try to see if there's a narrower call available to use before
1349 // scalarizing.
1350 break;
1351 }
1352 case ISD::FPOW:
1353 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getPOW, Results))
1354 return;
1355
1356 // TODO: Try to see if there's a narrower call available to use before
1357 // scalarizing.
1358 break;
1359 case ISD::FCBRT:
1360 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getCBRT, Results))
1361 return;
1362
1363 // TODO: Try to see if there's a narrower call available to use before
1364 // scalarizing.
1365 break;
1366 case ISD::FMODF: {
1367 EVT VT = Node->getValueType(ResNo: 0);
1368 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1369 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1370 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1371 /*CallRetResNo=*/0))
1372 return;
1373 break;
1374 }
1375 case ISD::VECTOR_COMPRESS:
1376 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1377 return;
1378 case ISD::VECTOR_SHUFFLE_VAR:
1379 Results.push_back(Elt: TLI.expandVECTOR_SHUFFLE_VAR(Node, DAG));
1380 return;
1381 case ISD::CTTZ_ELTS:
1382 case ISD::CTTZ_ELTS_ZERO_POISON:
1383 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1384 return;
1385 case ISD::VECTOR_FIND_LAST_ACTIVE:
1386 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1387 return;
1388 case ISD::SCMP:
1389 case ISD::UCMP:
1390 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1391 return;
1392 case ISD::GET_ACTIVE_LANE_MASK:
1393 if (SDValue R = TLI.expandGetActiveLaneMask(N: Node, DAG))
1394 Results.push_back(Elt: R);
1395 return;
1396 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1397 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1398 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1399 return;
1400 case ISD::MASK_BEFOREFIRST:
1401 Results.push_back(Elt: ExpandMASK_BEFOREFIRST(N: Node));
1402 return;
1403
1404 case ISD::FADD:
1405 case ISD::FMUL:
1406 case ISD::FMA:
1407 case ISD::FDIV:
1408 case ISD::FCEIL:
1409 case ISD::FFLOOR:
1410 case ISD::FNEARBYINT:
1411 case ISD::FRINT:
1412 case ISD::FROUND:
1413 case ISD::FROUNDEVEN:
1414 case ISD::FTRUNC:
1415 case ISD::FSQRT:
1416 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1417 Results.push_back(Elt: Expanded);
1418 return;
1419 }
1420 break;
1421 case ISD::CONVERT_TO_ARBITRARY_FP:
1422 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1423 Results.push_back(Elt: Expanded);
1424 else
1425 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1426 return;
1427 case ISD::CONVERT_FROM_ARBITRARY_FP:
1428 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1429 Results.push_back(Elt: Expanded);
1430 else
1431 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1432 return;
1433 case ISD::MASKED_UDIV:
1434 case ISD::MASKED_SDIV:
1435 case ISD::MASKED_UREM:
1436 case ISD::MASKED_SREM:
1437 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1438 return;
1439 }
1440
1441 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1442 if (Node->getNumValues() == 1) {
1443 Results.push_back(Elt: Unrolled);
1444 } else {
1445 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1446 "VectorLegalizer Expand returned wrong number of results!");
1447 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1448 Results.push_back(Elt: Unrolled.getValue(R: I));
1449 }
1450}
1451
1452SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1453 // Lower a select instruction where the condition is a scalar and the
1454 // operands are vectors. Lower this select to VSELECT and implement it
1455 // using XOR AND OR. The selector bit is broadcasted.
1456 EVT VT = Node->getValueType(ResNo: 0);
1457 SDLoc DL(Node);
1458
1459 SDValue Mask = Node->getOperand(Num: 0);
1460 SDValue Op1 = Node->getOperand(Num: 1);
1461 SDValue Op2 = Node->getOperand(Num: 2);
1462
1463 assert(VT.isVector() && !Mask.getValueType().isVector()
1464 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1465
1466 // If we can't even use the basic vector operations of
1467 // AND,OR,XOR, we will have to scalarize the op.
1468 // Notice that the operation may be 'promoted' which means that it is
1469 // 'bitcasted' to another type which is handled.
1470 // Also, we need to be able to construct a splat vector using either
1471 // BUILD_VECTOR or SPLAT_VECTOR.
1472 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1473 // BUILD_VECTOR?
1474 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1475 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1476 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1477 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1478 : ISD::SPLAT_VECTOR,
1479 VT) == TargetLowering::Expand)
1480 return SDValue();
1481
1482 // Generate a mask operand.
1483 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1484
1485 // What is the size of each element in the vector mask.
1486 EVT BitTy = MaskTy.getScalarType();
1487
1488 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1489 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1490
1491 // Broadcast the mask so that the entire vector is all one or all zero.
1492 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1493
1494 // Bitcast the operands to be the same type as the mask.
1495 // This is needed when we select between FP types because
1496 // the mask is a vector of integers.
1497 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1498 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1499
1500 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1501
1502 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1503 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1504 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1505 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1506}
1507
1508SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1509 EVT VT = Node->getValueType(ResNo: 0);
1510
1511 // Make sure that the SRA and SHL instructions are available.
1512 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1513 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1514 return SDValue();
1515
1516 SDLoc DL(Node);
1517 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1518
1519 unsigned BW = VT.getScalarSizeInBits();
1520 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1521 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1522
1523 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1524 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1525}
1526
1527// Generically expand a vector anyext in register to a shuffle of the relevant
1528// lanes into the appropriate locations, with other lanes left undef.
1529SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1530 SDLoc DL(Node);
1531 EVT VT = Node->getValueType(ResNo: 0);
1532 int NumElements = VT.getVectorNumElements();
1533 SDValue Src = Node->getOperand(Num: 0);
1534 EVT SrcVT = Src.getValueType();
1535 int NumSrcElements = SrcVT.getVectorNumElements();
1536
1537 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1538 // into a larger vector type.
1539 if (SrcVT.bitsLE(VT)) {
1540 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1541 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1542 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1543 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1544 NumElements: NumSrcElements);
1545 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1546 }
1547
1548 // Build a base mask of undef shuffles.
1549 SmallVector<int, 16> ShuffleMask;
1550 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1551
1552 // Place the extended lanes into the correct locations.
1553 int ExtLaneScale = NumSrcElements / NumElements;
1554 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1555 for (int i = 0; i < NumElements; ++i)
1556 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1557
1558 return DAG.getNode(
1559 Opcode: ISD::BITCAST, DL, VT,
1560 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1561}
1562
1563SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1564 SDLoc DL(Node);
1565 EVT VT = Node->getValueType(ResNo: 0);
1566 SDValue Src = Node->getOperand(Num: 0);
1567 EVT SrcVT = Src.getValueType();
1568
1569 // First build an any-extend node which can be legalized above when we
1570 // recurse through it.
1571 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1572
1573 // Now we need sign extend. This will be exanded to shifts if it isn't
1574 // supported.
1575 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1576 NumElements: VT.getVectorNumElements());
1577 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1578 N2: DAG.getValueType(ExtVT));
1579}
1580
1581// Generically expand a vector zext in register to a shuffle of the relevant
1582// lanes into the appropriate locations, a blend of zero into the high bits,
1583// and a bitcast to the wider element type.
1584SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1585 SDLoc DL(Node);
1586 EVT VT = Node->getValueType(ResNo: 0);
1587 int NumElements = VT.getVectorNumElements();
1588 SDValue Src = Node->getOperand(Num: 0);
1589 EVT SrcVT = Src.getValueType();
1590 int NumSrcElements = SrcVT.getVectorNumElements();
1591
1592 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1593 // into a larger vector type.
1594 if (SrcVT.bitsLE(VT)) {
1595 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1596 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1597 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1598 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1599 NumElements: NumSrcElements);
1600 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1601 }
1602
1603 // Build up a zero vector to blend into this one.
1604 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1605
1606 // Shuffle the incoming lanes into the correct position, and pull all other
1607 // lanes from the zero vector.
1608 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1609
1610 int ExtLaneScale = NumSrcElements / NumElements;
1611 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1612 for (int i = 0; i < NumElements; ++i)
1613 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1614
1615 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1616 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1617}
1618
1619static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1620 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1621 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1622 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1623 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1624}
1625
1626SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1627 EVT VT = Node->getValueType(ResNo: 0);
1628
1629 // Scalable vectors can't use shuffle expansion.
1630 if (VT.isScalableVector())
1631 return TLI.expandBSWAP(N: Node, DAG);
1632
1633 // Generate a byte wise shuffle mask for the BSWAP.
1634 SmallVector<int, 16> ShuffleMask;
1635 createBSWAPShuffleMask(VT, ShuffleMask);
1636 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1637
1638 // Only emit a shuffle if the mask is legal.
1639 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1640 SDLoc DL(Node);
1641 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1642 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1643 Mask: ShuffleMask);
1644 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1645 }
1646
1647 // If we have the appropriate vector bit operations, it is better to use them
1648 // than unrolling and expanding each component.
1649 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1650 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1651 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1652 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1653 return TLI.expandBSWAP(N: Node, DAG);
1654
1655 // Otherwise let the caller unroll.
1656 return SDValue();
1657}
1658
1659SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1660 EVT VT = Node->getValueType(ResNo: 0);
1661
1662 // We can't unroll or use shuffles for scalable vectors.
1663 if (VT.isScalableVector())
1664 return TLI.expandBITREVERSE(N: Node, DAG);
1665
1666 // If we have the scalar operation, it's probably cheaper to unroll it.
1667 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1668 return SDValue();
1669
1670 // If the vector element width is a whole number of bytes, test if its legal
1671 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1672 // vector. This greatly reduces the number of bit shifts necessary.
1673 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1674 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1675 SmallVector<int, 16> BSWAPMask;
1676 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1677
1678 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1679 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1680 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1681 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1682 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1683 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1684 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1685 SDLoc DL(Node);
1686 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1687 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1688 Mask: BSWAPMask);
1689 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1690 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1691 return Op;
1692 }
1693 }
1694
1695 // If we have the appropriate vector bit operations, it is better to use them
1696 // than unrolling and expanding each component.
1697 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1698 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1699 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1700 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1701 return TLI.expandBITREVERSE(N: Node, DAG);
1702
1703 // Otherwise unroll.
1704 return SDValue();
1705}
1706
1707SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1708 // Implement VSELECT in terms of XOR, AND, OR
1709 // on platforms which do not support blend natively.
1710 SDLoc DL(Node);
1711
1712 SDValue Mask = Node->getOperand(Num: 0);
1713 SDValue Op1 = Node->getOperand(Num: 1);
1714 SDValue Op2 = Node->getOperand(Num: 2);
1715
1716 EVT VT = Mask.getValueType();
1717
1718 // If we can't even use the basic vector operations of
1719 // AND,OR,XOR, we will have to scalarize the op.
1720 // Notice that the operation may be 'promoted' which means that it is
1721 // 'bitcasted' to another type which is handled.
1722 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1723 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1724 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1725 return SDValue();
1726
1727 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1728 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1729 // vector constant to mask with.
1730 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1731 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1732 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1733 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1734 Op1.getValueType().getVectorElementType() == MVT::i1))
1735 return SDValue();
1736
1737 // If the mask and the type are different sizes, unroll the vector op. This
1738 // can occur when getSetCCResultType returns something that is different in
1739 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1740 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1741 return SDValue();
1742
1743 // Bitcast the operands to be the same type as the mask.
1744 // This is needed when we select between FP types because
1745 // the mask is a vector of integers.
1746 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1747 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1748
1749 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1750
1751 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1752 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1753 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1754 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1755}
1756
1757SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1758 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1759 // indices less than the EVL/pivot are true. Combine that with the original
1760 // mask for a full-length mask. Use a full-length VSELECT to select between
1761 // the true and false values.
1762 SDLoc DL(Node);
1763
1764 SDValue Mask = Node->getOperand(Num: 0);
1765 SDValue Op1 = Node->getOperand(Num: 1);
1766 SDValue Op2 = Node->getOperand(Num: 2);
1767 SDValue EVL = Node->getOperand(Num: 3);
1768
1769 EVT MaskVT = Mask.getValueType();
1770 bool IsFixedLen = MaskVT.isFixedLengthVector();
1771
1772 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1773 EC: MaskVT.getVectorElementCount());
1774
1775 // If we can't construct the EVL mask efficiently, it's better to unroll.
1776 if ((IsFixedLen &&
1777 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1778 (!IsFixedLen &&
1779 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1780 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1781 return SDValue();
1782
1783 // If using a SETCC would result in a different type than the mask type,
1784 // unroll.
1785 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1786 VT: EVLVecVT) != MaskVT)
1787 return SDValue();
1788
1789 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1790 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1791 SDValue EVLMask =
1792 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1793
1794 SDValue FullMask =
1795 DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: DAG.getFreeze(V: Mask), N2: EVLMask);
1796 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1797}
1798
1799SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1800 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, MUL, SUB.
1801 EVT VT = Node->getValueType(ResNo: 0);
1802
1803 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1804
1805 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1806 !TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT) ||
1807 !TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT))
1808 return SDValue();
1809
1810 SDLoc DL(Node);
1811
1812 SDValue Dividend = Node->getOperand(Num: 0);
1813 SDValue Divisor = Node->getOperand(Num: 1);
1814 SDValue Mask = Node->getOperand(Num: 2);
1815 SDValue EVL = Node->getOperand(Num: 3);
1816
1817 // X % Y -> X-X/Y*Y
1818 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1819 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Divisor, N2: Div);
1820 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Dividend, N2: Mul);
1821}
1822
1823SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1824 return TLI.expandLoopDependenceMask(N, DAG);
1825}
1826
1827SDValue VectorLegalizer::ExpandMASK_BEFOREFIRST(SDNode *N) {
1828 // Expand to (get_active_lane_mask 0, (cttz_elts x))
1829 SDLoc DL(N);
1830 EVT VT = N->getValueType(ResNo: 0);
1831 EVT VecIdxVT = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1832 SDValue CttzElts =
1833 DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: VecIdxVT, Operand: N->getOperand(Num: 0));
1834 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT,
1835 N1: DAG.getConstant(Val: 0, DL, VT: VecIdxVT), N2: CttzElts);
1836}
1837
1838SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1839 // Masked bin ops don't have undefined behaviour when dividing by zero
1840 // on disabled lanes and produce poison instead. Replace the divisor on the
1841 // disabled lanes with 1 to avoid division by zero or overflow.
1842 SDLoc dl(N);
1843 EVT VT = N->getValueType(ResNo: 0);
1844 SDValue SafeDivisor = DAG.getSelect(
1845 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1846 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1847 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1848}
1849
1850void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1851 SmallVectorImpl<SDValue> &Results) {
1852 // Attempt to expand using TargetLowering.
1853 SDValue Result, Chain;
1854 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1855 Results.push_back(Elt: Result);
1856 if (Node->isStrictFPOpcode())
1857 Results.push_back(Elt: Chain);
1858 return;
1859 }
1860
1861 // Otherwise go ahead and unroll.
1862 if (Node->isStrictFPOpcode()) {
1863 UnrollStrictFPOp(Node, Results);
1864 return;
1865 }
1866
1867 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1868}
1869
1870void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1871 SmallVectorImpl<SDValue> &Results) {
1872 bool IsStrict = Node->isStrictFPOpcode();
1873 unsigned OpNo = IsStrict ? 1 : 0;
1874 SDValue Src = Node->getOperand(Num: OpNo);
1875 EVT SrcVT = Src.getValueType();
1876 EVT DstVT = Node->getValueType(ResNo: 0);
1877 SDLoc DL(Node);
1878
1879 // Attempt to expand using TargetLowering.
1880 SDValue Result;
1881 SDValue Chain;
1882 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1883 Results.push_back(Elt: Result);
1884 if (IsStrict)
1885 Results.push_back(Elt: Chain);
1886 return;
1887 }
1888
1889 // Make sure that the SINT_TO_FP and SRL instructions are available.
1890 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1891 TargetLowering::Expand) ||
1892 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1893 TargetLowering::Expand)) ||
1894 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1895 if (IsStrict) {
1896 UnrollStrictFPOp(Node, Results);
1897 return;
1898 }
1899
1900 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1901 return;
1902 }
1903
1904 unsigned BW = SrcVT.getScalarSizeInBits();
1905 assert((BW == 64 || BW == 32) &&
1906 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1907
1908 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
1909 // UINT_TO_FP with a larger float and round to the smaller type
1910 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
1911 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
1912 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
1913 SDValue UIToFP;
1914 SDValue Result;
1915 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
1916 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
1917 if (IsStrict) {
1918 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
1919 Ops: {Node->getOperand(Num: 0), Src});
1920 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
1921 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
1922 Results.push_back(Elt: Result);
1923 Results.push_back(Elt: Result.getValue(R: 1));
1924 } else {
1925 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
1926 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
1927 Results.push_back(Elt: Result);
1928 }
1929
1930 return;
1931 }
1932
1933 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
1934
1935 // Constants to clear the upper part of the word.
1936 // Notice that we can also use SHL+SHR, but using a constant is slightly
1937 // faster on x86.
1938 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1939 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
1940
1941 // Two to the power of half-word-size.
1942 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
1943
1944 // Clear upper part of LO, lower HI
1945 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
1946 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
1947
1948 if (IsStrict) {
1949 // Convert hi and lo to floats
1950 // Convert the hi part back to the upper values
1951 // TODO: Can any fast-math-flags be set on these nodes?
1952 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1953 Ops: {Node->getOperand(Num: 0), HI});
1954 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
1955 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
1956 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1957 Ops: {Node->getOperand(Num: 0), LO});
1958
1959 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
1960 N2: fLO.getValue(R: 1));
1961
1962 // Add the two halves
1963 SDValue Result =
1964 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
1965
1966 Results.push_back(Elt: Result);
1967 Results.push_back(Elt: Result.getValue(R: 1));
1968 return;
1969 }
1970
1971 // Convert hi and lo to floats
1972 // Convert the hi part back to the upper values
1973 // TODO: Can any fast-math-flags be set on these nodes?
1974 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
1975 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
1976 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
1977
1978 // Add the two halves
1979 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
1980}
1981
1982SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1983 EVT VT = Node->getValueType(ResNo: 0);
1984 EVT IntVT = VT.changeVectorElementTypeToInteger();
1985
1986 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
1987 return SDValue();
1988
1989 // Heuristic check to determine whether vector should be expanded to integer
1990 // operations or unrolled to scalar operations.
1991 // 1. Scalable vector is never unrolled.
1992 // 2. Fixed vector is unrolled if one of followings is true:
1993 // a. Vector only has 1 element and target knows how to handle scalar
1994 // FNEG (either legal or custom expand or promote).
1995 // b. Vector has more than 1 element and target supports scalar
1996 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
1997 // FIXME: Scalar construction instruction count varies in every architecture,
1998 // here we assume 1 instruction for now.
1999 if (VT.isFixedLengthVector()) {
2000 EVT EltVT = VT.getVectorElementType();
2001 unsigned NumElts = VT.getVectorNumElements();
2002 if ((NumElts == 1 &&
2003 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
2004 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
2005 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2006 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2007 return SDValue();
2008 }
2009
2010 SDLoc DL(Node);
2011 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2012 SDValue SignMask = DAG.getConstant(
2013 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2014 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
2015 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
2016}
2017
2018SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
2019 EVT VT = Node->getValueType(ResNo: 0);
2020 EVT IntVT = VT.changeVectorElementTypeToInteger();
2021
2022 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
2023 return SDValue();
2024
2025 // Heuristic check to determine whether vector should be expanded to integer
2026 // operations or unrolled to scalar operations.
2027 // 1. Scalable vector is never unrolled.
2028 // 2. Fixed vector is unrolled if one of followings is true:
2029 // a. Vector only has 1 element and target knows how to handle scalar
2030 // FABS(either legal or custom expand or promote).
2031 // b. Vector has more than 1 element and target supports scalar
2032 // FABS natively and vector length <= 2(1 AND + 1 CONST).
2033 // FIXME: Scalar construction instruction count varies in every architecture,
2034 // here we assume 1 instruction for now.
2035 if (VT.isFixedLengthVector()) {
2036 EVT EltVT = VT.getVectorElementType();
2037 unsigned NumElts = VT.getVectorNumElements();
2038 if ((NumElts == 1 &&
2039 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
2040 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
2041 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2042 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2043 return SDValue();
2044 }
2045
2046 SDLoc DL(Node);
2047 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2048 SDValue ClearSignMask = DAG.getConstant(
2049 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2050 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
2051 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
2052}
2053
2054SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
2055 EVT VT = Node->getValueType(ResNo: 0);
2056 EVT IntVT = VT.changeVectorElementTypeToInteger();
2057
2058 if (VT != Node->getOperand(Num: 1).getValueType() ||
2059 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
2060 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
2061 return SDValue();
2062
2063 // Heuristic check to determine whether vector should be expanded to integer
2064 // operations or unrolled to scalar operations.
2065 // 1. Scalable vector is never unrolled.
2066 // 2. Fixed vector is unrolled if one of followings is true:
2067 // a. Vector only has 1 element and target knows how to handle scalar
2068 // FCOPYSIGN(either legal or custom expand or promote).
2069 // b. Vector has more than 1 element and target supports scalar
2070 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
2071 // FIXME: Scalar construction instruction count varies in every architecture,
2072 // here we assume 1 instruction for now.
2073 if (VT.isFixedLengthVector()) {
2074 EVT EltVT = VT.getVectorElementType();
2075 unsigned NumElts = VT.getVectorNumElements();
2076 if ((NumElts == 1 &&
2077 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
2078 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
2079 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2080 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2081 return SDValue();
2082 }
2083
2084 SDLoc DL(Node);
2085 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2086 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
2087
2088 SDValue SignMask = DAG.getConstant(
2089 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2090 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
2091
2092 SDValue ClearSignMask = DAG.getConstant(
2093 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2094 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
2095
2096 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2097 Flags: SDNodeFlags::Disjoint);
2098
2099 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2100}
2101
2102void VectorLegalizer::ExpandFSUB(SDNode *Node,
2103 SmallVectorImpl<SDValue> &Results) {
2104 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2105 // we can defer this to operation legalization where it will be lowered as
2106 // a+(-b).
2107 EVT VT = Node->getValueType(ResNo: 0);
2108 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2109 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2110 return; // Defer to LegalizeDAG
2111
2112 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2113 Results.push_back(Elt: Expanded);
2114 return;
2115 }
2116
2117 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2118 Results.push_back(Elt: Tmp);
2119}
2120
2121void VectorLegalizer::ExpandSETCC(SDNode *Node,
2122 SmallVectorImpl<SDValue> &Results) {
2123 bool NeedInvert = false;
2124 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2125 Node->getOpcode() == ISD::STRICT_FSETCCS;
2126 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2127 unsigned Offset = IsStrict ? 1 : 0;
2128
2129 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2130 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2131 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2132 SDValue CC = Node->getOperand(Num: 2 + Offset);
2133
2134 MVT OpVT = LHS.getSimpleValueType();
2135 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2136
2137 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2138 if (IsStrict) {
2139 UnrollStrictFPOp(Node, Results);
2140 return;
2141 }
2142 Results.push_back(Elt: UnrollVSETCC(Node));
2143 return;
2144 }
2145
2146 SDLoc dl(Node);
2147 bool Legalized =
2148 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC,
2149 NeedInvert, dl, Chain, IsSignaling);
2150
2151 if (Legalized) {
2152 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2153 // condition code, create a new SETCC node.
2154 if (CC.getNode()) {
2155 if (IsStrict) {
2156 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2157 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2158 Chain = LHS.getValue(R: 1);
2159 } else {
2160 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2161 Flags: Node->getFlags());
2162 }
2163 }
2164
2165 // If we expanded the SETCC by inverting the condition code, then wrap
2166 // the existing SETCC in a NOT to restore the intended condition.
2167 if (NeedInvert)
2168 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2169 } else {
2170 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2171
2172 // Otherwise, SETCC for the given comparison type must be completely
2173 // illegal; expand it into a SELECT_CC.
2174 EVT VT = Node->getValueType(ResNo: 0);
2175 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2176 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2177 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2178 N5: CC, Flags: Node->getFlags());
2179 }
2180
2181 Results.push_back(Elt: LHS);
2182 if (IsStrict)
2183 Results.push_back(Elt: Chain);
2184}
2185
2186void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2187 SmallVectorImpl<SDValue> &Results) {
2188 SDValue Result, Overflow;
2189 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2190 Results.push_back(Elt: Result);
2191 Results.push_back(Elt: Overflow);
2192}
2193
2194void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2195 SmallVectorImpl<SDValue> &Results) {
2196 SDValue Result, Overflow;
2197 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2198 Results.push_back(Elt: Result);
2199 Results.push_back(Elt: Overflow);
2200}
2201
2202void VectorLegalizer::ExpandMULO(SDNode *Node,
2203 SmallVectorImpl<SDValue> &Results) {
2204 SDValue Result, Overflow;
2205 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2206 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2207
2208 Results.push_back(Elt: Result);
2209 Results.push_back(Elt: Overflow);
2210}
2211
2212void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2213 SmallVectorImpl<SDValue> &Results) {
2214 SDNode *N = Node;
2215 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2216 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2217 Results.push_back(Elt: Expanded);
2218}
2219
2220void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2221 SmallVectorImpl<SDValue> &Results) {
2222 switch (Node->getOpcode()) {
2223 default:
2224 break;
2225 case ISD::STRICT_UINT_TO_FP:
2226 ExpandUINT_TO_FLOAT(Node, Results);
2227 return;
2228 case ISD::STRICT_FP_TO_UINT:
2229 ExpandFP_TO_UINT(Node, Results);
2230 return;
2231 case ISD::STRICT_FSETCC:
2232 case ISD::STRICT_FSETCCS:
2233 ExpandSETCC(Node, Results);
2234 return;
2235 case ISD::STRICT_FADD:
2236 case ISD::STRICT_FCEIL:
2237 case ISD::STRICT_FDIV:
2238 case ISD::STRICT_FFLOOR:
2239 case ISD::STRICT_FMA:
2240 case ISD::STRICT_FMUL:
2241 case ISD::STRICT_FNEARBYINT:
2242 case ISD::STRICT_FRINT:
2243 case ISD::STRICT_FROUND:
2244 case ISD::STRICT_FROUNDEVEN:
2245 case ISD::STRICT_FTRUNC:
2246 case ISD::STRICT_FSQRT:
2247 case ISD::STRICT_FSUB: {
2248 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2249 Results.push_back(Elt: Expanded.getValue(R: 0));
2250 Results.push_back(Elt: Expanded.getValue(R: 1));
2251 return;
2252 }
2253 break;
2254 }
2255 }
2256
2257 UnrollStrictFPOp(Node, Results);
2258}
2259
2260void VectorLegalizer::ExpandREM(SDNode *Node,
2261 SmallVectorImpl<SDValue> &Results) {
2262 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2263 "Expected REM node");
2264
2265 SDValue Result;
2266 if (!TLI.expandREM(Node, Result, DAG))
2267 Result = DAG.UnrollVectorOp(N: Node);
2268 Results.push_back(Elt: Result);
2269}
2270
2271// Try to expand libm nodes into vector math routine calls. Callers provide the
2272// RTLIB::get<OP>(EVT) selector of the node's libcall family, which is used to
2273// look up mappings within RuntimeLibcallsInfo. The only mappings considered are
2274// those where the result and all operands are the same vector type. While
2275// predicated nodes are not supported, we will emit calls to masked routines by
2276// passing in a mask that is true for the lanes computed by the node.
2277bool VectorLegalizer::tryExpandVecMathCall(
2278 SDNode *Node, function_ref<RTLIB::Libcall(EVT)> GetLibcall,
2279 SmallVectorImpl<SDValue> &Results) {
2280 // Chain must be propagated but currently strict fp operations are down
2281 // converted to their none strict counterpart.
2282 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2283
2284 EVT VT = Node->getValueType(ResNo: 0);
2285 LLVMContext &Ctx = *DAG.getContext();
2286 const LibcallLoweringInfo &Libcalls = DAG.getLibcalls();
2287
2288 // Try to widen the vector type when no libcall is available at that width.
2289 EVT CallVT = VT;
2290 RTLIB::LibcallImpl LCImpl =
2291 Libcalls.getVectorLibcallImplOrMasked(Call: GetLibcall(CallVT));
2292 if (LCImpl == RTLIB::Unsupported && VT.getVectorElementCount().isScalar())
2293 return false;
2294 while (LCImpl == RTLIB::Unsupported) {
2295 CallVT = CallVT.getDoubleNumVectorElementsVT(Context&: Ctx);
2296 if (!CallVT.isSimple())
2297 return false;
2298 if (TLI.isTypeLegal(VT: CallVT))
2299 LCImpl = Libcalls.getVectorLibcallImplOrMasked(Call: GetLibcall(CallVT));
2300 }
2301
2302 const RTLIB::RuntimeLibcallsInfo &RTLCI = Libcalls.getRuntimeLibcallsInfo();
2303
2304 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2305 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2306
2307 SDLoc DL(Node);
2308 TargetLowering::ArgListTy Args;
2309
2310 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2311
2312 // Sanity check just in case function has unexpected parameters.
2313 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2314 EVT::getEVT(FuncTy->getReturnType(), true) == CallVT &&
2315 "mismatch in value type and call signature type");
2316
2317 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2318 Type *ParamTy = FuncTy->getParamType(i: I);
2319
2320 if (HasMaskArg && I == E - 1) {
2321 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2322 cast<VectorType>(ParamTy)->getElementCount() ==
2323 CallVT.getVectorElementCount() &&
2324 "unexpected vector mask type");
2325 EVT MaskVT = EVT::getEVT(Ty: ParamTy, /*HandleUnknown=*/true);
2326 EVT SubMaskVT =
2327 MaskVT.changeVectorElementCount(Context&: Ctx, EC: VT.getVectorElementCount());
2328 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: SubMaskVT, OpVT: VT);
2329 // Only the lanes holding the node's elements need to be active.
2330 if (CallVT != VT)
2331 Mask = DAG.getInsertSubvector(
2332 DL, Vec: DAG.getBoolConstant(V: false, DL, VT: MaskVT, OpVT: CallVT), SubVec: Mask, Idx: 0);
2333 Args.emplace_back(args&: Mask, args&: ParamTy);
2334 } else {
2335 SDValue Op = Node->getOperand(Num: I);
2336 assert(Op.getValueType() == VT && "mismatch in vector types");
2337 if (CallVT != VT) {
2338 unsigned NumConcat =
2339 CallVT.getVectorMinNumElements() / VT.getVectorMinNumElements();
2340 SmallVector<SDValue, 4> Ops(NumConcat, Op);
2341 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: CallVT, Ops);
2342 }
2343 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2344 "mismatch in value type and call argument type");
2345 Args.emplace_back(args&: Op, args&: ParamTy);
2346 }
2347 }
2348
2349 // Emit a call to the vector function.
2350 SDValue Callee =
2351 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2352 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2353
2354 TargetLowering::CallLoweringInfo CLI(DAG);
2355 CLI.setDebugLoc(DL)
2356 .setChain(DAG.getEntryNode())
2357 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2358
2359 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2360 SDValue Result = CallResult.first;
2361 if (CallVT != VT)
2362 Result = DAG.getExtractSubvector(DL, VT, Vec: Result, Idx: 0);
2363 Results.push_back(Elt: Result);
2364 return true;
2365}
2366
2367void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2368 SmallVectorImpl<SDValue> &Results) {
2369 EVT VT = Node->getValueType(ResNo: 0);
2370
2371 // Cannot unroll a scalable vector. Delay error reporting until the final
2372 // operation legalisation phase to maximise the chances of removing the node.
2373 if (VT.isScalableVector())
2374 return;
2375
2376 EVT EltVT = VT.getVectorElementType();
2377 unsigned NumElems = VT.getVectorNumElements();
2378 unsigned NumOpers = Node->getNumOperands();
2379 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2380
2381 EVT TmpEltVT = EltVT;
2382 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2383 Node->getOpcode() == ISD::STRICT_FSETCCS)
2384 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2385 Context&: *DAG.getContext(), VT: TmpEltVT);
2386
2387 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2388 SDValue Chain = Node->getOperand(Num: 0);
2389 SDLoc dl(Node);
2390
2391 SmallVector<SDValue, 32> OpValues;
2392 SmallVector<SDValue, 32> OpChains;
2393 for (unsigned i = 0; i < NumElems; ++i) {
2394 SmallVector<SDValue, 4> Opers;
2395 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2396
2397 // The Chain is the first operand.
2398 Opers.push_back(Elt: Chain);
2399
2400 // Now process the remaining operands.
2401 for (unsigned j = 1; j < NumOpers; ++j) {
2402 SDValue Oper = Node->getOperand(Num: j);
2403 EVT OperVT = Oper.getValueType();
2404
2405 if (OperVT.isVector())
2406 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2407 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2408
2409 Opers.push_back(Elt: Oper);
2410 }
2411
2412 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2413 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2414 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2415
2416 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2417 Node->getOpcode() == ISD::STRICT_FSETCCS)
2418 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2419 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2420 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2421
2422 OpValues.push_back(Elt: ScalarResult);
2423 OpChains.push_back(Elt: ScalarChain);
2424 }
2425
2426 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2427 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2428
2429 Results.push_back(Elt: Result);
2430 Results.push_back(Elt: NewChain);
2431}
2432
2433SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2434 EVT VT = Node->getValueType(ResNo: 0);
2435 unsigned NumElems = VT.getVectorNumElements();
2436 EVT EltVT = VT.getVectorElementType();
2437 SDValue LHS = Node->getOperand(Num: 0);
2438 SDValue RHS = Node->getOperand(Num: 1);
2439 SDValue CC = Node->getOperand(Num: 2);
2440 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2441 SDLoc dl(Node);
2442 SmallVector<SDValue, 8> Ops(NumElems);
2443 for (unsigned i = 0; i < NumElems; ++i) {
2444 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2445 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2446 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2447 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2448 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2449 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2450 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2451 Context&: *DAG.getContext(), VT: TmpEltVT),
2452 N1: LHSElem, N2: RHSElem, N3: CC);
2453 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2454 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2455 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2456 }
2457 return DAG.getBuildVector(VT, DL: dl, Ops);
2458}
2459
2460bool SelectionDAG::LegalizeVectors() {
2461 return VectorLegalizer(*this).Run();
2462}
2463