1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/STLFunctionalExtras.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/Analysis/TargetLibraryInfo.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/SelectionDAG.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetLowering.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/CodeGenTypes/MachineValueType.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <iterator>
48#include <utility>
49
50using namespace llvm;
51
52#define DEBUG_TYPE "legalizevectorops"
53
54namespace {
55
56class VectorLegalizer {
57 SelectionDAG& DAG;
58 const TargetLowering &TLI;
59 bool Changed = false; // Keep track of whether anything changed
60
61 /// For nodes that are of legal width, and that have more than one use, this
62 /// map indicates what regularized operand to use. This allows us to avoid
63 /// legalizing the same thing more than once.
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65
66 /// Adds a node to the translation cache.
67 void AddLegalizedOperand(SDValue From, SDValue To) {
68 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
69 // If someone requests legalization of the new node, return itself.
70 if (From != To)
71 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
72 }
73
74 /// Legalizes the given node.
75 SDValue LegalizeOp(SDValue Op);
76
77 /// Assuming the node is legal, "legalize" the results.
78 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79
80 /// Make sure Results are legal and update the translation cache.
81 SDValue RecursivelyLegalizeResults(SDValue Op,
82 MutableArrayRef<SDValue> Results);
83
84 /// Wrapper to interface LowerOperation with a vector of Results.
85 /// Returns false if the target wants to use default expansion. Otherwise
86 /// returns true. If return is true and the Results are empty, then the
87 /// target wants to keep the input node as is.
88 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89
90 /// Implements unrolling a VSETCC.
91 SDValue UnrollVSETCC(SDNode *Node);
92
93 /// Implement expand-based legalization of vector operations.
94 ///
95 /// This is just a high-level routine to dispatch to specific code paths for
96 /// operations to legalize them.
97 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98
99 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100 /// FP_TO_SINT isn't legal.
101 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102
103 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106
107 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108 SDValue ExpandSEXTINREG(SDNode *Node);
109
110 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111 ///
112 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113 /// type. The contents of the bits in the extended part of each element are
114 /// undef.
115 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116
117 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118 ///
119 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120 /// type, then shifts left and arithmetic shifts right to introduce a sign
121 /// extension.
122 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123
124 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125 ///
126 /// Shuffles the low lanes of the operand into place and blends zeros into
127 /// the remaining lanes, finally bitcasting to the proper type.
128 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129
130 /// Expand bswap of vectors into a shuffle if legal.
131 SDValue ExpandBSWAP(SDNode *Node);
132
133 /// Implement vselect in terms of XOR, AND, OR when blend is not
134 /// supported by the target.
135 SDValue ExpandVSELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandGET_ACTIVE_LANE_MASK(SDNode *N);
139 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
140 SDValue ExpandMASK_BEFOREFIRST(SDNode *N);
141 SDValue ExpandMaskedBinOp(SDNode *N);
142 SDValue ExpandSELECT(SDNode *Node);
143 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
144 SDValue ExpandStore(SDNode *N);
145 SDValue ExpandFNEG(SDNode *Node);
146 SDValue ExpandFABS(SDNode *Node);
147 SDValue ExpandFCOPYSIGN(SDNode *Node);
148 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 SDValue ExpandBITREVERSE(SDNode *Node);
151 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158 bool tryExpandVecMathCall(SDNode *Node,
159 function_ref<RTLIB::Libcall(EVT)> GetLibcall,
160 SmallVectorImpl<SDValue> &Results);
161
162 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 /// Implements vector promotion.
165 ///
166 /// This is essentially just bitcasting the operands to a different type and
167 /// bitcasting the result back to the original type.
168 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
169
170 /// Implements [SU]INT_TO_FP vector promotion.
171 ///
172 /// This is a [zs]ext of the input operand to a larger integer type.
173 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
174
175 /// Implements FP_TO_[SU]INT vector promotion of the result type.
176 ///
177 /// It is promoted to a larger integer type. The result is then
178 /// truncated back to the original type.
179 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
180
181 /// Implements vector setcc operation promotion.
182 ///
183 /// All vector operands are promoted to a vector type with larger element
184 /// type.
185 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
188
189 /// Calculate the reduction using a type of higher precision and round the
190 /// result to match the original type. Setting NonArithmetic signifies the
191 /// rounding of the result does not affect its value.
192 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
193 bool NonArithmetic);
194
195 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
196
197public:
198 VectorLegalizer(SelectionDAG& dag) :
199 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
200
201 /// Begin legalizer the vector operations in the DAG.
202 bool Run();
203};
204
205} // end anonymous namespace
206
207bool VectorLegalizer::Run() {
208 // Before we start legalizing vector nodes, check if there are any vectors.
209 bool HasVectors = false;
210 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
211 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
212 // Check if the values of the nodes contain vectors. We don't need to check
213 // the operands because we are going to check their values at some point.
214 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
215
216 // If we found a vector node we can start the legalization.
217 if (HasVectors)
218 break;
219 }
220
221 // If this basic block has no vectors then no need to legalize vectors.
222 if (!HasVectors)
223 return false;
224
225 // The legalize process is inherently a bottom-up recursive process (users
226 // legalize their uses before themselves). Given infinite stack space, we
227 // could just start legalizing on the root and traverse the whole graph. In
228 // practice however, this causes us to run out of stack space on large basic
229 // blocks. To avoid this problem, compute an ordering of the nodes where each
230 // node is only legalized after all of its operands are legalized.
231 DAG.AssignTopologicalOrder();
232 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
233 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
234 LegalizeOp(Op: SDValue(&*I, 0));
235
236 // Finally, it's possible the root changed. Get the new root.
237 SDValue OldRoot = DAG.getRoot();
238 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
239 DAG.setRoot(LegalizedNodes[OldRoot]);
240
241 LegalizedNodes.clear();
242
243 // Remove dead nodes now.
244 DAG.RemoveDeadNodes();
245
246 return Changed;
247}
248
249SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
250 assert(Op->getNumValues() == Result->getNumValues() &&
251 "Unexpected number of results");
252 // Generic legalization: just pass the operand through.
253 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
254 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
255 return SDValue(Result, Op.getResNo());
256}
257
258SDValue
259VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
260 MutableArrayRef<SDValue> Results) {
261 assert(Results.size() == Op->getNumValues() &&
262 "Unexpected number of results");
263 // Make sure that the generated code is itself legal.
264 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
265 Results[i] = LegalizeOp(Op: Results[i]);
266 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
267 }
268
269 return Results[Op.getResNo()];
270}
271
272SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
273 // Note that LegalizeOp may be reentered even from single-use nodes, which
274 // means that we always must cache transformed nodes.
275 auto I = LegalizedNodes.find(Val: Op);
276 if (I != LegalizedNodes.end()) return I->second;
277
278 // Legalize the operands
279 SmallVector<SDValue, 8> Ops;
280 for (const SDValue &Oper : Op->op_values())
281 Ops.push_back(Elt: LegalizeOp(Op: Oper));
282
283 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
284
285 bool HasVectorValueOrOp =
286 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
287 llvm::any_of(Range: Node->op_values(),
288 P: [](SDValue O) { return O.getValueType().isVector(); });
289 if (!HasVectorValueOrOp)
290 return TranslateLegalizeResults(Op, Result: Node);
291
292 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
293 EVT ValVT;
294 switch (Op.getOpcode()) {
295 default:
296 return TranslateLegalizeResults(Op, Result: Node);
297 case ISD::LOAD: {
298 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
299 ISD::LoadExtType ExtType = LD->getExtensionType();
300 EVT LoadedVT = LD->getMemoryVT();
301 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
302 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
303 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
304 break;
305 }
306 case ISD::STORE: {
307 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
308 EVT StVT = ST->getMemoryVT();
309 MVT ValVT = ST->getValue().getSimpleValueType();
310 if (StVT.isVector() && ST->isTruncatingStore())
311 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
312 AddrSpace: ST->getAddressSpace());
313 break;
314 }
315 case ISD::MERGE_VALUES:
316 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
317 // This operation lies about being legal: when it claims to be legal,
318 // it should actually be expanded.
319 if (Action == TargetLowering::Legal)
320 Action = TargetLowering::Expand;
321 break;
322#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
323 case ISD::STRICT_##DAGN:
324#include "llvm/IR/ConstrainedOps.def"
325 ValVT = Node->getValueType(ResNo: 0);
326 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
327 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
328 ValVT = Node->getOperand(Num: 1).getValueType();
329 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
330 Op.getOpcode() == ISD::STRICT_FSETCCS) {
331 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
332 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 3))->get();
333 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
334 if (Action == TargetLowering::Legal)
335 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
336 } else {
337 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
338 }
339 // If we're asked to expand a strict vector floating-point operation,
340 // by default we're going to simply unroll it. That is usually the
341 // best approach, except in the case where the resulting strict (scalar)
342 // operations would themselves use the fallback mutation to non-strict.
343 // In that specific case, just do the fallback on the vector op.
344 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
345 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
346 TargetLowering::Legal) {
347 EVT EltVT = ValVT.getVectorElementType();
348 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
349 == TargetLowering::Expand &&
350 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
351 == TargetLowering::Legal)
352 Action = TargetLowering::Legal;
353 }
354 break;
355 case ISD::ADD:
356 case ISD::SUB:
357 case ISD::MUL:
358 case ISD::MULHS:
359 case ISD::MULHU:
360 case ISD::SDIV:
361 case ISD::UDIV:
362 case ISD::SREM:
363 case ISD::UREM:
364 case ISD::SDIVREM:
365 case ISD::UDIVREM:
366 case ISD::FADD:
367 case ISD::FSUB:
368 case ISD::FMUL:
369 case ISD::FDIV:
370 case ISD::FREM:
371 case ISD::AND:
372 case ISD::OR:
373 case ISD::XOR:
374 case ISD::SHL:
375 case ISD::SRA:
376 case ISD::SRL:
377 case ISD::FSHL:
378 case ISD::FSHR:
379 case ISD::ROTL:
380 case ISD::ROTR:
381 case ISD::ABS:
382 case ISD::ABS_MIN_POISON:
383 case ISD::ABDS:
384 case ISD::ABDU:
385 case ISD::AVGCEILS:
386 case ISD::AVGCEILU:
387 case ISD::AVGFLOORS:
388 case ISD::AVGFLOORU:
389 case ISD::BSWAP:
390 case ISD::BITREVERSE:
391 case ISD::CTLZ:
392 case ISD::CTTZ:
393 case ISD::CTLZ_ZERO_POISON:
394 case ISD::CTTZ_ZERO_POISON:
395 case ISD::CTPOP:
396 case ISD::CLMUL:
397 case ISD::CLMULH:
398 case ISD::CLMULR:
399 case ISD::SELECT:
400 case ISD::VSELECT:
401 case ISD::SELECT_CC:
402 case ISD::ZERO_EXTEND:
403 case ISD::ANY_EXTEND:
404 case ISD::TRUNCATE:
405 case ISD::SIGN_EXTEND:
406 case ISD::FP_TO_SINT:
407 case ISD::FP_TO_UINT:
408 case ISD::FNEG:
409 case ISD::FABS:
410 case ISD::FMINNUM:
411 case ISD::FMAXNUM:
412 case ISD::FMINNUM_IEEE:
413 case ISD::FMAXNUM_IEEE:
414 case ISD::FMINIMUM:
415 case ISD::FMAXIMUM:
416 case ISD::FMINIMUMNUM:
417 case ISD::FMAXIMUMNUM:
418 case ISD::FCOPYSIGN:
419 case ISD::FSQRT:
420 case ISD::FSIN:
421 case ISD::FCOS:
422 case ISD::FTAN:
423 case ISD::FASIN:
424 case ISD::FACOS:
425 case ISD::FATAN:
426 case ISD::FATAN2:
427 case ISD::FSINH:
428 case ISD::FCOSH:
429 case ISD::FTANH:
430 case ISD::FLDEXP:
431 case ISD::FPOWI:
432 case ISD::FPOW:
433 case ISD::FCBRT:
434 case ISD::FLOG:
435 case ISD::FLOG2:
436 case ISD::FLOG10:
437 case ISD::FEXP:
438 case ISD::FEXP2:
439 case ISD::FEXP10:
440 case ISD::FCEIL:
441 case ISD::FTRUNC:
442 case ISD::FRINT:
443 case ISD::FNEARBYINT:
444 case ISD::FROUND:
445 case ISD::FROUNDEVEN:
446 case ISD::FFLOOR:
447 case ISD::FP_ROUND:
448 case ISD::FP_EXTEND:
449 case ISD::FPTRUNC_ROUND:
450 case ISD::FMA:
451 case ISD::SIGN_EXTEND_INREG:
452 case ISD::ANY_EXTEND_VECTOR_INREG:
453 case ISD::SIGN_EXTEND_VECTOR_INREG:
454 case ISD::ZERO_EXTEND_VECTOR_INREG:
455 case ISD::SMIN:
456 case ISD::SMAX:
457 case ISD::UMIN:
458 case ISD::UMAX:
459 case ISD::SMUL_LOHI:
460 case ISD::UMUL_LOHI:
461 case ISD::SADDO:
462 case ISD::UADDO:
463 case ISD::SSUBO:
464 case ISD::USUBO:
465 case ISD::SMULO:
466 case ISD::UMULO:
467 case ISD::CONVERT_FROM_ARBITRARY_FP:
468 case ISD::CONVERT_TO_ARBITRARY_FP:
469 case ISD::FCANONICALIZE:
470 case ISD::FFREXP:
471 case ISD::FMODF:
472 case ISD::FSINCOS:
473 case ISD::FSINCOSPI:
474 case ISD::SADDSAT:
475 case ISD::UADDSAT:
476 case ISD::SSUBSAT:
477 case ISD::USUBSAT:
478 case ISD::SSHLSAT:
479 case ISD::USHLSAT:
480 case ISD::FP_TO_SINT_SAT:
481 case ISD::FP_TO_UINT_SAT:
482 case ISD::MGATHER:
483 case ISD::VECTOR_COMPRESS:
484 case ISD::SCMP:
485 case ISD::UCMP:
486 case ISD::GET_ACTIVE_LANE_MASK:
487 case ISD::LOOP_DEPENDENCE_WAR_MASK:
488 case ISD::LOOP_DEPENDENCE_RAW_MASK:
489 case ISD::MASK_BEFOREFIRST:
490 case ISD::MASKED_UDIV:
491 case ISD::MASKED_SDIV:
492 case ISD::MASKED_UREM:
493 case ISD::MASKED_SREM:
494 case ISD::VECTOR_MATCH:
495 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
496 break;
497 case ISD::SMULFIX:
498 case ISD::SMULFIXSAT:
499 case ISD::UMULFIX:
500 case ISD::UMULFIXSAT:
501 case ISD::SDIVFIX:
502 case ISD::SDIVFIXSAT:
503 case ISD::UDIVFIX:
504 case ISD::UDIVFIXSAT: {
505 unsigned Scale = Node->getConstantOperandVal(Num: 2);
506 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
507 VT: Node->getValueType(ResNo: 0), Scale);
508 break;
509 }
510 case ISD::LROUND:
511 case ISD::LLROUND:
512 case ISD::LRINT:
513 case ISD::LLRINT:
514 case ISD::SINT_TO_FP:
515 case ISD::UINT_TO_FP:
516 case ISD::VECREDUCE_ADD:
517 case ISD::VECREDUCE_MUL:
518 case ISD::VECREDUCE_AND:
519 case ISD::VECREDUCE_OR:
520 case ISD::VECREDUCE_XOR:
521 case ISD::VECREDUCE_SMAX:
522 case ISD::VECREDUCE_SMIN:
523 case ISD::VECREDUCE_UMAX:
524 case ISD::VECREDUCE_UMIN:
525 case ISD::VECREDUCE_FADD:
526 case ISD::VECREDUCE_FMAX:
527 case ISD::VECREDUCE_FMAXIMUM:
528 case ISD::VECREDUCE_FMIN:
529 case ISD::VECREDUCE_FMINIMUM:
530 case ISD::VECREDUCE_FMAXIMUMNUM:
531 case ISD::VECREDUCE_FMINIMUMNUM:
532 case ISD::VECREDUCE_FMUL:
533 case ISD::CTTZ_ELTS:
534 case ISD::CTTZ_ELTS_ZERO_POISON:
535 case ISD::VECTOR_FIND_LAST_ACTIVE:
536 Action = TLI.getOperationAction(Op: Node->getOpcode(),
537 VT: Node->getOperand(Num: 0).getValueType());
538 break;
539 case ISD::VECREDUCE_SEQ_FADD:
540 case ISD::VECREDUCE_SEQ_FMUL:
541 Action = TLI.getOperationAction(Op: Node->getOpcode(),
542 VT: Node->getOperand(Num: 1).getValueType());
543 break;
544 case ISD::SETCC: {
545 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
546 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
547 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
548 if (Action == TargetLowering::Legal)
549 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
550 break;
551 }
552 case ISD::PARTIAL_REDUCE_UMLA:
553 case ISD::PARTIAL_REDUCE_SMLA:
554 case ISD::PARTIAL_REDUCE_SUMLA:
555 case ISD::PARTIAL_REDUCE_FMLA:
556 Action =
557 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
558 InputVT: Node->getOperand(Num: 1).getValueType());
559 break;
560
561#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
562 case ISD::VPID: { \
563 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
564 : Node->getOperand(LEGALPOS).getValueType(); \
565 /* Defer non-vector results to LegalizeDAG. */ \
566 if (!Node->getValueType(0).isVector() && \
567 Node->getValueType(0) != MVT::Other) { \
568 Action = TargetLowering::Legal; \
569 break; \
570 } \
571 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
572 } break;
573#include "llvm/IR/VPIntrinsics.def"
574 }
575
576 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
577
578 SmallVector<SDValue, 8> ResultVals;
579 switch (Action) {
580 default: llvm_unreachable("This action is not supported yet!");
581 case TargetLowering::Promote:
582 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
583 "This action is not supported yet!");
584 LLVM_DEBUG(dbgs() << "Promoting\n");
585 Promote(Node, Results&: ResultVals);
586 assert(!ResultVals.empty() && "No results for promotion?");
587 break;
588 case TargetLowering::Legal:
589 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
590 break;
591 case TargetLowering::Custom:
592 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
593 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
594 break;
595 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
596 [[fallthrough]];
597 case TargetLowering::Expand:
598 LLVM_DEBUG(dbgs() << "Expanding\n");
599 Expand(Node, Results&: ResultVals);
600 break;
601 }
602
603 if (ResultVals.empty())
604 return TranslateLegalizeResults(Op, Result: Node);
605
606 Changed = true;
607 return RecursivelyLegalizeResults(Op, Results: ResultVals);
608}
609
610// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
611// merge them somehow?
612bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
613 SmallVectorImpl<SDValue> &Results) {
614 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
615
616 if (!Res.getNode())
617 return false;
618
619 if (Res == SDValue(Node, 0))
620 return true;
621
622 // If the original node has one result, take the return value from
623 // LowerOperation as is. It might not be result number 0.
624 if (Node->getNumValues() == 1) {
625 Results.push_back(Elt: Res);
626 return true;
627 }
628
629 // If the original node has multiple results, then the return node should
630 // have the same number of results.
631 assert((Node->getNumValues() == Res->getNumValues()) &&
632 "Lowering returned the wrong number of results!");
633
634 // Places new result values base on N result number.
635 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
636 Results.push_back(Elt: Res.getValue(R: I));
637
638 return true;
639}
640
641void VectorLegalizer::PromoteSETCC(SDNode *Node,
642 SmallVectorImpl<SDValue> &Results) {
643 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
644 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
645
646 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
647
648 SDLoc DL(Node);
649 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
650
651 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
652 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
653 Operands[2] = Node->getOperand(Num: 2);
654
655 EVT ResVT =
656 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
657 SDValue Res =
658 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
659 if (ResVT != Node->getValueType(ResNo: 0))
660 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
661 Results.push_back(Elt: Res);
662}
663
664void VectorLegalizer::PromoteSTRICT(SDNode *Node,
665 SmallVectorImpl<SDValue> &Results) {
666 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
667 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
668
669 assert(VecVT.isFloatingPoint());
670
671 SDLoc DL(Node);
672 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
673 SmallVector<SDValue, 2> Chains;
674
675 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
676 if (Node->getOperand(Num: j).getValueType().isVector() &&
677 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
678 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
679 {
680 // promote the vector operand.
681 SDValue Ext =
682 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
683 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
684 Operands[j] = Ext.getValue(R: 0);
685 Chains.push_back(Elt: Ext.getValue(R: 1));
686 } else
687 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
688
689 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
690
691 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
692
693 SDValue Res =
694 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
695
696 SDValue Round =
697 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
698 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
699 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
700
701 Results.push_back(Elt: Round.getValue(R: 0));
702 Results.push_back(Elt: Round.getValue(R: 1));
703}
704
705void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
706 SmallVectorImpl<SDValue> &Results,
707 bool NonArithmetic) {
708 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
709 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
710 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
711
712 SDLoc DL(Node);
713 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
714 SDValue Rdx =
715 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
716 Flags: Node->getFlags());
717 SDValue Res =
718 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
719 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
720 Results.push_back(Elt: Res);
721}
722
723void VectorLegalizer::PromoteVECTOR_COMPRESS(
724 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
725 SDLoc DL(Node);
726 EVT VT = Node->getValueType(ResNo: 0);
727 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
728 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
729 "Only integer promotion or bitcasts between types is supported");
730
731 SDValue Vec = Node->getOperand(Num: 0);
732 SDValue Mask = Node->getOperand(Num: 1);
733 SDValue Passthru = Node->getOperand(Num: 2);
734 if (VT.isInteger()) {
735 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
736 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
737 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
738 } else {
739 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
740 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
741 }
742
743 SDValue Result =
744 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
745 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
746 : DAG.getBitcast(VT, V: Result);
747 Results.push_back(Elt: Result);
748}
749
750void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
751 // For a few operations there is a specific concept for promotion based on
752 // the operand's type.
753 switch (Node->getOpcode()) {
754 case ISD::SINT_TO_FP:
755 case ISD::UINT_TO_FP:
756 case ISD::STRICT_SINT_TO_FP:
757 case ISD::STRICT_UINT_TO_FP:
758 // "Promote" the operation by extending the operand.
759 PromoteINT_TO_FP(Node, Results);
760 return;
761 case ISD::FP_TO_UINT:
762 case ISD::FP_TO_SINT:
763 case ISD::STRICT_FP_TO_UINT:
764 case ISD::STRICT_FP_TO_SINT:
765 // Promote the operation by extending the operand.
766 PromoteFP_TO_INT(Node, Results);
767 return;
768 case ISD::SETCC:
769 // Promote the operation by extending the operand.
770 PromoteSETCC(Node, Results);
771 return;
772 case ISD::STRICT_FADD:
773 case ISD::STRICT_FSUB:
774 case ISD::STRICT_FMUL:
775 case ISD::STRICT_FDIV:
776 case ISD::STRICT_FSQRT:
777 case ISD::STRICT_FMA:
778 PromoteSTRICT(Node, Results);
779 return;
780 case ISD::VECREDUCE_FADD:
781 case ISD::VECREDUCE_FMUL:
782 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
783 return;
784 case ISD::VECREDUCE_FMAX:
785 case ISD::VECREDUCE_FMAXIMUM:
786 case ISD::VECREDUCE_FMIN:
787 case ISD::VECREDUCE_FMINIMUM:
788 case ISD::VECREDUCE_FMAXIMUMNUM:
789 case ISD::VECREDUCE_FMINIMUMNUM:
790 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
791 return;
792 case ISD::VECTOR_COMPRESS:
793 PromoteVECTOR_COMPRESS(Node, Results);
794 return;
795
796 case ISD::FP_ROUND:
797 case ISD::FP_EXTEND:
798 // These operations are used to do promotion so they can't be promoted
799 // themselves.
800 llvm_unreachable("Don't know how to promote this operation!");
801 }
802
803 // There are currently two cases of vector promotion:
804 // 1) Bitcasting a vector of integers to a different type to a vector of the
805 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
806 // 2) Extending a vector of floats to a vector of the same number of larger
807 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
808 assert(Node->getNumValues() == 1 &&
809 "Can't promote a vector with multiple results!");
810 MVT VT = Node->getSimpleValueType(ResNo: 0);
811 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
812 SDLoc dl(Node);
813 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
814
815 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
816 // Do not promote the mask operand of a VP OP.
817 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
818 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
819 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
820 if (Node->getOperand(Num: j)
821 .getValueType()
822 .getVectorElementType()
823 .isFloatingPoint() &&
824 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
825 Operands[j] = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
826 else
827 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
828 else
829 Operands[j] = Node->getOperand(Num: j);
830 }
831
832 SDValue Res =
833 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
834
835 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
836 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
837 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
838 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
839 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
840 else
841 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
842
843 Results.push_back(Elt: Res);
844}
845
846void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
847 SmallVectorImpl<SDValue> &Results) {
848 // INT_TO_FP operations may require the input operand be promoted even
849 // when the type is otherwise legal.
850 bool IsStrict = Node->isStrictFPOpcode();
851 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
852 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
853 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
854 "Vectors have different number of elements!");
855
856 SDLoc dl(Node);
857 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
858
859 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
860 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
861 ? ISD::ZERO_EXTEND
862 : ISD::SIGN_EXTEND;
863 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
864 if (Node->getOperand(Num: j).getValueType().isVector())
865 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
866 else
867 Operands[j] = Node->getOperand(Num: j);
868 }
869
870 if (IsStrict) {
871 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
872 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
873 Results.push_back(Elt: Res);
874 Results.push_back(Elt: Res.getValue(R: 1));
875 return;
876 }
877
878 SDValue Res =
879 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
880 Results.push_back(Elt: Res);
881}
882
883// For FP_TO_INT we promote the result type to a vector type with wider
884// elements and then truncate the result. This is different from the default
885// PromoteVector which uses bitcast to promote thus assumning that the
886// promoted vector type has the same overall size.
887void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
888 SmallVectorImpl<SDValue> &Results) {
889 MVT VT = Node->getSimpleValueType(ResNo: 0);
890 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
891 bool IsStrict = Node->isStrictFPOpcode();
892 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
893 "Vectors have different number of elements!");
894
895 unsigned NewOpc = Node->getOpcode();
896 // Change FP_TO_UINT to FP_TO_SINT if possible.
897 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
898 if (NewOpc == ISD::FP_TO_UINT &&
899 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
900 NewOpc = ISD::FP_TO_SINT;
901
902 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
903 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
904 NewOpc = ISD::STRICT_FP_TO_SINT;
905
906 SDLoc dl(Node);
907 SDValue Promoted, Chain;
908 if (IsStrict) {
909 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
910 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
911 Chain = Promoted.getValue(R: 1);
912 } else
913 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
914
915 // Assert that the converted value fits in the original type. If it doesn't
916 // (eg: because the value being converted is too big), then the result of the
917 // original operation was undefined anyway, so the assert is still correct.
918 if (Node->getOpcode() == ISD::FP_TO_UINT ||
919 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
920 NewOpc = ISD::AssertZext;
921 else
922 NewOpc = ISD::AssertSext;
923
924 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
925 N2: DAG.getValueType(VT.getScalarType()));
926 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
927 Results.push_back(Elt: Promoted);
928 if (IsStrict)
929 Results.push_back(Elt: Chain);
930}
931
932std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
933 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
934 return TLI.scalarizeVectorLoad(LD, DAG);
935}
936
937SDValue VectorLegalizer::ExpandStore(SDNode *N) {
938 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
939 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
940 return TF;
941}
942
943void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
944 switch (Node->getOpcode()) {
945 case ISD::LOAD: {
946 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
947 Results.push_back(Elt: Tmp.first);
948 Results.push_back(Elt: Tmp.second);
949 return;
950 }
951 case ISD::STORE:
952 Results.push_back(Elt: ExpandStore(N: Node));
953 return;
954 case ISD::MERGE_VALUES:
955 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
956 Results.push_back(Elt: Node->getOperand(Num: i));
957 return;
958 case ISD::SIGN_EXTEND_INREG:
959 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
960 Results.push_back(Elt: Expanded);
961 return;
962 }
963 break;
964 case ISD::ANY_EXTEND_VECTOR_INREG:
965 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
966 return;
967 case ISD::SIGN_EXTEND_VECTOR_INREG:
968 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
969 return;
970 case ISD::ZERO_EXTEND_VECTOR_INREG:
971 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
972 return;
973 case ISD::BSWAP:
974 if (SDValue Expanded = ExpandBSWAP(Node)) {
975 Results.push_back(Elt: Expanded);
976 return;
977 }
978 break;
979 case ISD::VSELECT:
980 if (SDValue Expanded = ExpandVSELECT(Node)) {
981 Results.push_back(Elt: Expanded);
982 return;
983 }
984 break;
985 case ISD::VP_SREM:
986 case ISD::VP_UREM:
987 if (SDValue Expanded = ExpandVP_REM(Node)) {
988 Results.push_back(Elt: Expanded);
989 return;
990 }
991 break;
992 case ISD::SELECT:
993 if (SDValue Expanded = ExpandSELECT(Node)) {
994 Results.push_back(Elt: Expanded);
995 return;
996 }
997 break;
998 case ISD::SELECT_CC: {
999 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1000 EVT CondVT = TLI.getSetCCResultType(
1001 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
1002 SDValue SetCC =
1003 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
1004 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
1005 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1006 LHS: Node->getOperand(Num: 2),
1007 RHS: Node->getOperand(Num: 3)));
1008 return;
1009 }
1010 break;
1011 }
1012 case ISD::FP_TO_UINT:
1013 ExpandFP_TO_UINT(Node, Results);
1014 return;
1015 case ISD::UINT_TO_FP:
1016 ExpandUINT_TO_FLOAT(Node, Results);
1017 return;
1018 case ISD::FNEG:
1019 if (SDValue Expanded = ExpandFNEG(Node)) {
1020 Results.push_back(Elt: Expanded);
1021 return;
1022 }
1023 break;
1024 case ISD::FABS:
1025 if (SDValue Expanded = ExpandFABS(Node)) {
1026 Results.push_back(Elt: Expanded);
1027 return;
1028 }
1029 break;
1030 case ISD::FCOPYSIGN:
1031 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1032 Results.push_back(Elt: Expanded);
1033 return;
1034 }
1035 break;
1036 case ISD::FCANONICALIZE: {
1037 // If the scalar element type has a
1038 // Legal/Custom FCANONICALIZE, don't
1039 // mess with the vector, fall back.
1040 EVT VT = Node->getValueType(ResNo: 0);
1041 EVT EltVT = VT.getVectorElementType();
1042 if (!VT.isScalableVector() &&
1043 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1044 TargetLowering::Expand)
1045 break;
1046 // Otherwise canonicalize the whole vector.
1047 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1048 Results.push_back(Elt: Mul);
1049 return;
1050 }
1051 case ISD::FSUB:
1052 ExpandFSUB(Node, Results);
1053 return;
1054 case ISD::SETCC:
1055 ExpandSETCC(Node, Results);
1056 return;
1057 case ISD::ABS:
1058 case ISD::ABS_MIN_POISON:
1059 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1060 Results.push_back(Elt: Expanded);
1061 return;
1062 }
1063 break;
1064 case ISD::ABDS:
1065 case ISD::ABDU:
1066 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1067 Results.push_back(Elt: Expanded);
1068 return;
1069 }
1070 break;
1071 case ISD::AVGCEILS:
1072 case ISD::AVGCEILU:
1073 case ISD::AVGFLOORS:
1074 case ISD::AVGFLOORU:
1075 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1076 Results.push_back(Elt: Expanded);
1077 return;
1078 }
1079 break;
1080 case ISD::BITREVERSE:
1081 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1082 Results.push_back(Elt: Expanded);
1083 return;
1084 }
1085 break;
1086 case ISD::CTPOP:
1087 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1088 Results.push_back(Elt: Expanded);
1089 return;
1090 }
1091 break;
1092 case ISD::CTLZ:
1093 case ISD::CTLZ_ZERO_POISON:
1094 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1095 Results.push_back(Elt: Expanded);
1096 return;
1097 }
1098 break;
1099 case ISD::CTTZ:
1100 case ISD::CTTZ_ZERO_POISON:
1101 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1102 Results.push_back(Elt: Expanded);
1103 return;
1104 }
1105 break;
1106 case ISD::FSHL:
1107 case ISD::FSHR:
1108 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1109 Results.push_back(Elt: Expanded);
1110 return;
1111 }
1112 break;
1113 case ISD::CLMUL:
1114 case ISD::CLMULR:
1115 case ISD::CLMULH:
1116 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1117 Results.push_back(Elt: Expanded);
1118 return;
1119 }
1120 break;
1121 case ISD::PEXT:
1122 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1123 return;
1124 case ISD::PDEP:
1125 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1126 return;
1127 case ISD::ROTL:
1128 case ISD::ROTR:
1129 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1130 Results.push_back(Elt: Expanded);
1131 return;
1132 }
1133 break;
1134 case ISD::FMINNUM:
1135 case ISD::FMAXNUM:
1136 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1137 Results.push_back(Elt: Expanded);
1138 return;
1139 }
1140 break;
1141 case ISD::FMINIMUM:
1142 case ISD::FMAXIMUM:
1143 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1144 return;
1145 case ISD::FMINIMUMNUM:
1146 case ISD::FMAXIMUMNUM:
1147 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1148 return;
1149 case ISD::SMIN:
1150 case ISD::SMAX:
1151 case ISD::UMIN:
1152 case ISD::UMAX:
1153 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1154 Results.push_back(Elt: Expanded);
1155 return;
1156 }
1157 break;
1158 case ISD::UADDO:
1159 case ISD::USUBO:
1160 ExpandUADDSUBO(Node, Results);
1161 return;
1162 case ISD::SADDO:
1163 case ISD::SSUBO:
1164 ExpandSADDSUBO(Node, Results);
1165 return;
1166 case ISD::UMULO:
1167 case ISD::SMULO:
1168 ExpandMULO(Node, Results);
1169 return;
1170 case ISD::MULHS:
1171 case ISD::MULHU:
1172 if (SDValue Expanded = TLI.expandMULH(Node, DAG)) {
1173 Results.push_back(Elt: Expanded);
1174 return;
1175 }
1176 break;
1177 case ISD::USUBSAT:
1178 case ISD::SSUBSAT:
1179 case ISD::UADDSAT:
1180 case ISD::SADDSAT:
1181 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1182 Results.push_back(Elt: Expanded);
1183 return;
1184 }
1185 break;
1186 case ISD::USHLSAT:
1187 case ISD::SSHLSAT:
1188 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1189 Results.push_back(Elt: Expanded);
1190 return;
1191 }
1192 break;
1193 case ISD::FP_TO_SINT_SAT:
1194 case ISD::FP_TO_UINT_SAT:
1195 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1196 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1197 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1198 Results.push_back(Elt: Expanded);
1199 return;
1200 }
1201 }
1202 break;
1203 case ISD::SMULFIX:
1204 case ISD::UMULFIX:
1205 case ISD::SMULFIXSAT:
1206 case ISD::UMULFIXSAT:
1207 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1208 Results.push_back(Elt: Expanded);
1209 return;
1210 }
1211 break;
1212 case ISD::SDIVFIX:
1213 case ISD::UDIVFIX:
1214 ExpandFixedPointDiv(Node, Results);
1215 return;
1216 case ISD::SDIVFIXSAT:
1217 case ISD::UDIVFIXSAT:
1218 break;
1219#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1220 case ISD::STRICT_##DAGN:
1221#include "llvm/IR/ConstrainedOps.def"
1222 ExpandStrictFPOp(Node, Results);
1223 return;
1224 case ISD::VECREDUCE_ADD:
1225 case ISD::VECREDUCE_MUL:
1226 case ISD::VECREDUCE_AND:
1227 case ISD::VECREDUCE_OR:
1228 case ISD::VECREDUCE_XOR:
1229 case ISD::VECREDUCE_SMAX:
1230 case ISD::VECREDUCE_SMIN:
1231 case ISD::VECREDUCE_UMAX:
1232 case ISD::VECREDUCE_UMIN:
1233 case ISD::VECREDUCE_FADD:
1234 case ISD::VECREDUCE_FMUL:
1235 case ISD::VECREDUCE_FMAX:
1236 case ISD::VECREDUCE_FMIN:
1237 case ISD::VECREDUCE_FMAXIMUM:
1238 case ISD::VECREDUCE_FMINIMUM:
1239 case ISD::VECREDUCE_FMAXIMUMNUM:
1240 case ISD::VECREDUCE_FMINIMUMNUM:
1241 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1242 return;
1243 case ISD::PARTIAL_REDUCE_UMLA:
1244 case ISD::PARTIAL_REDUCE_SMLA:
1245 case ISD::PARTIAL_REDUCE_SUMLA:
1246 case ISD::PARTIAL_REDUCE_FMLA:
1247 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1248 return;
1249 case ISD::VECREDUCE_SEQ_FADD:
1250 case ISD::VECREDUCE_SEQ_FMUL:
1251 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1252 return;
1253 case ISD::VECTOR_MATCH:
1254 Results.push_back(Elt: TLI.expandVectorMatch(N: Node, DAG));
1255 return;
1256 case ISD::SREM:
1257 case ISD::UREM:
1258 ExpandREM(Node, Results);
1259 return;
1260 case ISD::VP_MERGE:
1261 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1262 Results.push_back(Elt: Expanded);
1263 return;
1264 }
1265 break;
1266 case ISD::FREM:
1267 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getREM, Results))
1268 return;
1269 break;
1270 case ISD::FSINCOS:
1271 case ISD::FSINCOSPI: {
1272 EVT VT = Node->getValueType(ResNo: 0);
1273 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1274 ? RTLIB::getSINCOS(VT)
1275 : RTLIB::getSINCOSPI(VT);
1276 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1277 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1278 return;
1279
1280 // TODO: Try to see if there's a narrower call available to use before
1281 // scalarizing.
1282 break;
1283 }
1284 case ISD::FPOW:
1285 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getPOW, Results))
1286 return;
1287
1288 // TODO: Try to see if there's a narrower call available to use before
1289 // scalarizing.
1290 break;
1291 case ISD::FCBRT:
1292 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getCBRT, Results))
1293 return;
1294
1295 // TODO: Try to see if there's a narrower call available to use before
1296 // scalarizing.
1297 break;
1298 case ISD::FMODF: {
1299 EVT VT = Node->getValueType(ResNo: 0);
1300 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1301 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1302 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1303 /*CallRetResNo=*/0))
1304 return;
1305 break;
1306 }
1307 case ISD::VECTOR_COMPRESS:
1308 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1309 return;
1310 case ISD::CTTZ_ELTS:
1311 case ISD::CTTZ_ELTS_ZERO_POISON:
1312 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1313 return;
1314 case ISD::VECTOR_FIND_LAST_ACTIVE:
1315 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1316 return;
1317 case ISD::SCMP:
1318 case ISD::UCMP:
1319 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1320 return;
1321 case ISD::GET_ACTIVE_LANE_MASK:
1322 if (SDValue R = ExpandGET_ACTIVE_LANE_MASK(N: Node))
1323 Results.push_back(Elt: R);
1324 return;
1325 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1326 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1327 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1328 return;
1329 case ISD::MASK_BEFOREFIRST:
1330 Results.push_back(Elt: ExpandMASK_BEFOREFIRST(N: Node));
1331 return;
1332
1333 case ISD::FADD:
1334 case ISD::FMUL:
1335 case ISD::FMA:
1336 case ISD::FDIV:
1337 case ISD::FCEIL:
1338 case ISD::FFLOOR:
1339 case ISD::FNEARBYINT:
1340 case ISD::FRINT:
1341 case ISD::FROUND:
1342 case ISD::FROUNDEVEN:
1343 case ISD::FTRUNC:
1344 case ISD::FSQRT:
1345 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1346 Results.push_back(Elt: Expanded);
1347 return;
1348 }
1349 break;
1350 case ISD::CONVERT_TO_ARBITRARY_FP:
1351 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1352 Results.push_back(Elt: Expanded);
1353 else
1354 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1355 return;
1356 case ISD::CONVERT_FROM_ARBITRARY_FP:
1357 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1358 Results.push_back(Elt: Expanded);
1359 else
1360 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1361 return;
1362 case ISD::MASKED_UDIV:
1363 case ISD::MASKED_SDIV:
1364 case ISD::MASKED_UREM:
1365 case ISD::MASKED_SREM:
1366 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1367 return;
1368 }
1369
1370 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1371 if (Node->getNumValues() == 1) {
1372 Results.push_back(Elt: Unrolled);
1373 } else {
1374 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1375 "VectorLegalizer Expand returned wrong number of results!");
1376 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1377 Results.push_back(Elt: Unrolled.getValue(R: I));
1378 }
1379}
1380
1381SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1382 // Lower a select instruction where the condition is a scalar and the
1383 // operands are vectors. Lower this select to VSELECT and implement it
1384 // using XOR AND OR. The selector bit is broadcasted.
1385 EVT VT = Node->getValueType(ResNo: 0);
1386 SDLoc DL(Node);
1387
1388 SDValue Mask = Node->getOperand(Num: 0);
1389 SDValue Op1 = Node->getOperand(Num: 1);
1390 SDValue Op2 = Node->getOperand(Num: 2);
1391
1392 assert(VT.isVector() && !Mask.getValueType().isVector()
1393 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1394
1395 // If we can't even use the basic vector operations of
1396 // AND,OR,XOR, we will have to scalarize the op.
1397 // Notice that the operation may be 'promoted' which means that it is
1398 // 'bitcasted' to another type which is handled.
1399 // Also, we need to be able to construct a splat vector using either
1400 // BUILD_VECTOR or SPLAT_VECTOR.
1401 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1402 // BUILD_VECTOR?
1403 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1404 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1405 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1406 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1407 : ISD::SPLAT_VECTOR,
1408 VT) == TargetLowering::Expand)
1409 return SDValue();
1410
1411 // Generate a mask operand.
1412 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1413
1414 // What is the size of each element in the vector mask.
1415 EVT BitTy = MaskTy.getScalarType();
1416
1417 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1418 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1419
1420 // Broadcast the mask so that the entire vector is all one or all zero.
1421 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1422
1423 // Bitcast the operands to be the same type as the mask.
1424 // This is needed when we select between FP types because
1425 // the mask is a vector of integers.
1426 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1427 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1428
1429 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1430
1431 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1432 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1433 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1434 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1435}
1436
1437SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1438 EVT VT = Node->getValueType(ResNo: 0);
1439
1440 // Make sure that the SRA and SHL instructions are available.
1441 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1442 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1443 return SDValue();
1444
1445 SDLoc DL(Node);
1446 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1447
1448 unsigned BW = VT.getScalarSizeInBits();
1449 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1450 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1451
1452 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1453 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1454}
1455
1456// Generically expand a vector anyext in register to a shuffle of the relevant
1457// lanes into the appropriate locations, with other lanes left undef.
1458SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1459 SDLoc DL(Node);
1460 EVT VT = Node->getValueType(ResNo: 0);
1461 int NumElements = VT.getVectorNumElements();
1462 SDValue Src = Node->getOperand(Num: 0);
1463 EVT SrcVT = Src.getValueType();
1464 int NumSrcElements = SrcVT.getVectorNumElements();
1465
1466 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1467 // into a larger vector type.
1468 if (SrcVT.bitsLE(VT)) {
1469 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1470 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1471 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1472 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1473 NumElements: NumSrcElements);
1474 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1475 }
1476
1477 // Build a base mask of undef shuffles.
1478 SmallVector<int, 16> ShuffleMask;
1479 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1480
1481 // Place the extended lanes into the correct locations.
1482 int ExtLaneScale = NumSrcElements / NumElements;
1483 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1484 for (int i = 0; i < NumElements; ++i)
1485 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1486
1487 return DAG.getNode(
1488 Opcode: ISD::BITCAST, DL, VT,
1489 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1490}
1491
1492SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1493 SDLoc DL(Node);
1494 EVT VT = Node->getValueType(ResNo: 0);
1495 SDValue Src = Node->getOperand(Num: 0);
1496 EVT SrcVT = Src.getValueType();
1497
1498 // First build an any-extend node which can be legalized above when we
1499 // recurse through it.
1500 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1501
1502 // Now we need sign extend. This will be exanded to shifts if it isn't
1503 // supported.
1504 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1505 NumElements: VT.getVectorNumElements());
1506 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1507 N2: DAG.getValueType(ExtVT));
1508}
1509
1510// Generically expand a vector zext in register to a shuffle of the relevant
1511// lanes into the appropriate locations, a blend of zero into the high bits,
1512// and a bitcast to the wider element type.
1513SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1514 SDLoc DL(Node);
1515 EVT VT = Node->getValueType(ResNo: 0);
1516 int NumElements = VT.getVectorNumElements();
1517 SDValue Src = Node->getOperand(Num: 0);
1518 EVT SrcVT = Src.getValueType();
1519 int NumSrcElements = SrcVT.getVectorNumElements();
1520
1521 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1522 // into a larger vector type.
1523 if (SrcVT.bitsLE(VT)) {
1524 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1525 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1526 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1527 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1528 NumElements: NumSrcElements);
1529 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1530 }
1531
1532 // Build up a zero vector to blend into this one.
1533 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1534
1535 // Shuffle the incoming lanes into the correct position, and pull all other
1536 // lanes from the zero vector.
1537 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1538
1539 int ExtLaneScale = NumSrcElements / NumElements;
1540 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1541 for (int i = 0; i < NumElements; ++i)
1542 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1543
1544 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1545 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1546}
1547
1548static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1549 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1550 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1551 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1552 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1553}
1554
1555SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1556 EVT VT = Node->getValueType(ResNo: 0);
1557
1558 // Scalable vectors can't use shuffle expansion.
1559 if (VT.isScalableVector())
1560 return TLI.expandBSWAP(N: Node, DAG);
1561
1562 // Generate a byte wise shuffle mask for the BSWAP.
1563 SmallVector<int, 16> ShuffleMask;
1564 createBSWAPShuffleMask(VT, ShuffleMask);
1565 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1566
1567 // Only emit a shuffle if the mask is legal.
1568 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1569 SDLoc DL(Node);
1570 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1571 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1572 Mask: ShuffleMask);
1573 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1574 }
1575
1576 // If we have the appropriate vector bit operations, it is better to use them
1577 // than unrolling and expanding each component.
1578 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1579 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1580 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1581 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1582 return TLI.expandBSWAP(N: Node, DAG);
1583
1584 // Otherwise let the caller unroll.
1585 return SDValue();
1586}
1587
1588SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1589 EVT VT = Node->getValueType(ResNo: 0);
1590
1591 // We can't unroll or use shuffles for scalable vectors.
1592 if (VT.isScalableVector())
1593 return TLI.expandBITREVERSE(N: Node, DAG);
1594
1595 // If we have the scalar operation, it's probably cheaper to unroll it.
1596 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1597 return SDValue();
1598
1599 // If the vector element width is a whole number of bytes, test if its legal
1600 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1601 // vector. This greatly reduces the number of bit shifts necessary.
1602 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1603 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1604 SmallVector<int, 16> BSWAPMask;
1605 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1606
1607 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1608 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1609 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1610 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1611 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1612 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1613 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1614 SDLoc DL(Node);
1615 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1616 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1617 Mask: BSWAPMask);
1618 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1619 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1620 return Op;
1621 }
1622 }
1623
1624 // If we have the appropriate vector bit operations, it is better to use them
1625 // than unrolling and expanding each component.
1626 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1627 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1628 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1629 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1630 return TLI.expandBITREVERSE(N: Node, DAG);
1631
1632 // Otherwise unroll.
1633 return SDValue();
1634}
1635
1636SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1637 // Implement VSELECT in terms of XOR, AND, OR
1638 // on platforms which do not support blend natively.
1639 SDLoc DL(Node);
1640
1641 SDValue Mask = Node->getOperand(Num: 0);
1642 SDValue Op1 = Node->getOperand(Num: 1);
1643 SDValue Op2 = Node->getOperand(Num: 2);
1644
1645 EVT VT = Mask.getValueType();
1646
1647 // If we can't even use the basic vector operations of
1648 // AND,OR,XOR, we will have to scalarize the op.
1649 // Notice that the operation may be 'promoted' which means that it is
1650 // 'bitcasted' to another type which is handled.
1651 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1652 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1653 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1654 return SDValue();
1655
1656 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1657 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1658 // vector constant to mask with.
1659 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1660 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1661 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1662 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1663 Op1.getValueType().getVectorElementType() == MVT::i1))
1664 return SDValue();
1665
1666 // If the mask and the type are different sizes, unroll the vector op. This
1667 // can occur when getSetCCResultType returns something that is different in
1668 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1669 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1670 return SDValue();
1671
1672 // Bitcast the operands to be the same type as the mask.
1673 // This is needed when we select between FP types because
1674 // the mask is a vector of integers.
1675 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1676 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1677
1678 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1679
1680 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1681 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1682 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1683 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1684}
1685
1686SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1687 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1688 // indices less than the EVL/pivot are true. Combine that with the original
1689 // mask for a full-length mask. Use a full-length VSELECT to select between
1690 // the true and false values.
1691 SDLoc DL(Node);
1692
1693 SDValue Mask = Node->getOperand(Num: 0);
1694 SDValue Op1 = Node->getOperand(Num: 1);
1695 SDValue Op2 = Node->getOperand(Num: 2);
1696 SDValue EVL = Node->getOperand(Num: 3);
1697
1698 EVT MaskVT = Mask.getValueType();
1699 bool IsFixedLen = MaskVT.isFixedLengthVector();
1700
1701 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1702 EC: MaskVT.getVectorElementCount());
1703
1704 // If we can't construct the EVL mask efficiently, it's better to unroll.
1705 if ((IsFixedLen &&
1706 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1707 (!IsFixedLen &&
1708 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1709 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1710 return SDValue();
1711
1712 // If using a SETCC would result in a different type than the mask type,
1713 // unroll.
1714 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1715 VT: EVLVecVT) != MaskVT)
1716 return SDValue();
1717
1718 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1719 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1720 SDValue EVLMask =
1721 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1722
1723 SDValue FullMask = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: Mask, N2: EVLMask);
1724 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1725}
1726
1727SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1728 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, MUL, SUB.
1729 EVT VT = Node->getValueType(ResNo: 0);
1730
1731 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1732
1733 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1734 !TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT) ||
1735 !TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT))
1736 return SDValue();
1737
1738 SDLoc DL(Node);
1739
1740 SDValue Dividend = Node->getOperand(Num: 0);
1741 SDValue Divisor = Node->getOperand(Num: 1);
1742 SDValue Mask = Node->getOperand(Num: 2);
1743 SDValue EVL = Node->getOperand(Num: 3);
1744
1745 // X % Y -> X-X/Y*Y
1746 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1747 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Divisor, N2: Div);
1748 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Dividend, N2: Mul);
1749}
1750
1751SDValue VectorLegalizer::ExpandGET_ACTIVE_LANE_MASK(SDNode *N) {
1752 SDLoc DL(N);
1753
1754 SDValue Start = N->getOperand(Num: 0);
1755 SDValue End = N->getOperand(Num: 1);
1756 EVT VT = N->getValueType(ResNo: 0);
1757 EVT OpVT = Start.getValueType();
1758
1759 if (VT.isScalableVector())
1760 return SDValue();
1761
1762 // Try a promoted comparison type to simplify saturation.
1763 EVT PromoteVT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: OpVT);
1764 if (TLI.isTypeLegal(VT: PromoteVT) &&
1765 isUIntN(N: OpVT.getScalarSizeInBits(), x: VT.getVectorNumElements())) {
1766 SDValue StartV = DAG.getSplat(VT: PromoteVT, DL, Op: Start);
1767 SDValue Seq = DAG.getStepVector(DL, ResVT: PromoteVT);
1768 Seq = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: PromoteVT, N1: Seq, N2: StartV);
1769
1770 EVT MaskVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1771 VT: PromoteVT);
1772 SDValue EndV = DAG.getSplat(VT: PromoteVT, DL, Op: End);
1773 SDValue Mask = DAG.getSetCC(DL, VT: MaskVT, LHS: Seq, RHS: EndV, Cond: ISD::SETULT);
1774 return DAG.getBoolExtOrTrunc(Op: Mask, SL: DL, VT, OpVT: PromoteVT);
1775 }
1776
1777 // Is VT's element type big enough to hold all rebased indices?
1778 if (!isUIntN(N: VT.getScalarSizeInBits(), x: VT.getVectorNumElements()))
1779 return SDValue();
1780
1781 // Rebase and saturate the termination value.
1782 SDValue Max = DAG.getConstant(Val: maxUIntN(N: VT.getScalarSizeInBits()), DL, VT: OpVT);
1783 End = DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: OpVT, N1: End, N2: Start);
1784 End = DAG.getNode(Opcode: ISD::UMIN, DL, VT: OpVT, N1: End, N2: Max);
1785
1786 // cmp <0, 1, 2, 3...>, End
1787 SDValue EndV = DAG.getSplat(VT, DL, Op: End);
1788 SDValue StepVector = DAG.getStepVector(DL, ResVT: VT);
1789 return DAG.getSetCC(DL, VT, LHS: StepVector, RHS: EndV, Cond: ISD::SETULT);
1790}
1791
1792SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1793 return TLI.expandLoopDependenceMask(N, DAG);
1794}
1795
1796SDValue VectorLegalizer::ExpandMASK_BEFOREFIRST(SDNode *N) {
1797 // Expand to (get_active_lane_mask 0, (cttz_elts x))
1798 SDLoc DL(N);
1799 EVT VT = N->getValueType(ResNo: 0);
1800 EVT VecIdxVT = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1801 SDValue CttzElts =
1802 DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: VecIdxVT, Operand: N->getOperand(Num: 0));
1803 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT,
1804 N1: DAG.getConstant(Val: 0, DL, VT: VecIdxVT), N2: CttzElts);
1805}
1806
1807SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1808 // Masked bin ops don't have undefined behaviour when dividing by zero
1809 // on disabled lanes and produce poison instead. Replace the divisor on the
1810 // disabled lanes with 1 to avoid division by zero or overflow.
1811 SDLoc dl(N);
1812 EVT VT = N->getValueType(ResNo: 0);
1813 SDValue SafeDivisor = DAG.getSelect(
1814 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1815 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1816 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1817}
1818
1819void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1820 SmallVectorImpl<SDValue> &Results) {
1821 // Attempt to expand using TargetLowering.
1822 SDValue Result, Chain;
1823 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1824 Results.push_back(Elt: Result);
1825 if (Node->isStrictFPOpcode())
1826 Results.push_back(Elt: Chain);
1827 return;
1828 }
1829
1830 // Otherwise go ahead and unroll.
1831 if (Node->isStrictFPOpcode()) {
1832 UnrollStrictFPOp(Node, Results);
1833 return;
1834 }
1835
1836 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1837}
1838
1839void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1840 SmallVectorImpl<SDValue> &Results) {
1841 bool IsStrict = Node->isStrictFPOpcode();
1842 unsigned OpNo = IsStrict ? 1 : 0;
1843 SDValue Src = Node->getOperand(Num: OpNo);
1844 EVT SrcVT = Src.getValueType();
1845 EVT DstVT = Node->getValueType(ResNo: 0);
1846 SDLoc DL(Node);
1847
1848 // Attempt to expand using TargetLowering.
1849 SDValue Result;
1850 SDValue Chain;
1851 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1852 Results.push_back(Elt: Result);
1853 if (IsStrict)
1854 Results.push_back(Elt: Chain);
1855 return;
1856 }
1857
1858 // Make sure that the SINT_TO_FP and SRL instructions are available.
1859 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1860 TargetLowering::Expand) ||
1861 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1862 TargetLowering::Expand)) ||
1863 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1864 if (IsStrict) {
1865 UnrollStrictFPOp(Node, Results);
1866 return;
1867 }
1868
1869 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1870 return;
1871 }
1872
1873 unsigned BW = SrcVT.getScalarSizeInBits();
1874 assert((BW == 64 || BW == 32) &&
1875 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1876
1877 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
1878 // UINT_TO_FP with a larger float and round to the smaller type
1879 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
1880 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
1881 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
1882 SDValue UIToFP;
1883 SDValue Result;
1884 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
1885 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
1886 if (IsStrict) {
1887 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
1888 Ops: {Node->getOperand(Num: 0), Src});
1889 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
1890 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
1891 Results.push_back(Elt: Result);
1892 Results.push_back(Elt: Result.getValue(R: 1));
1893 } else {
1894 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
1895 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
1896 Results.push_back(Elt: Result);
1897 }
1898
1899 return;
1900 }
1901
1902 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
1903
1904 // Constants to clear the upper part of the word.
1905 // Notice that we can also use SHL+SHR, but using a constant is slightly
1906 // faster on x86.
1907 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1908 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
1909
1910 // Two to the power of half-word-size.
1911 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
1912
1913 // Clear upper part of LO, lower HI
1914 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
1915 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
1916
1917 if (IsStrict) {
1918 // Convert hi and lo to floats
1919 // Convert the hi part back to the upper values
1920 // TODO: Can any fast-math-flags be set on these nodes?
1921 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1922 Ops: {Node->getOperand(Num: 0), HI});
1923 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
1924 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
1925 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1926 Ops: {Node->getOperand(Num: 0), LO});
1927
1928 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
1929 N2: fLO.getValue(R: 1));
1930
1931 // Add the two halves
1932 SDValue Result =
1933 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
1934
1935 Results.push_back(Elt: Result);
1936 Results.push_back(Elt: Result.getValue(R: 1));
1937 return;
1938 }
1939
1940 // Convert hi and lo to floats
1941 // Convert the hi part back to the upper values
1942 // TODO: Can any fast-math-flags be set on these nodes?
1943 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
1944 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
1945 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
1946
1947 // Add the two halves
1948 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
1949}
1950
1951SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1952 EVT VT = Node->getValueType(ResNo: 0);
1953 EVT IntVT = VT.changeVectorElementTypeToInteger();
1954
1955 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
1956 return SDValue();
1957
1958 // Heuristic check to determine whether vector should be expanded to integer
1959 // operations or unrolled to scalar operations.
1960 // 1. Scalable vector is never unrolled.
1961 // 2. Fixed vector is unrolled if one of followings is true:
1962 // a. Vector only has 1 element and target knows how to handle scalar
1963 // FNEG (either legal or custom expand or promote).
1964 // b. Vector has more than 1 element and target supports scalar
1965 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
1966 // FIXME: Scalar construction instruction count varies in every architecture,
1967 // here we assume 1 instruction for now.
1968 if (VT.isFixedLengthVector()) {
1969 EVT EltVT = VT.getVectorElementType();
1970 unsigned NumElts = VT.getVectorNumElements();
1971 if ((NumElts == 1 &&
1972 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
1973 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
1974 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1975 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1976 return SDValue();
1977 }
1978
1979 SDLoc DL(Node);
1980 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1981 SDValue SignMask = DAG.getConstant(
1982 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1983 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
1984 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
1985}
1986
1987SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
1988 EVT VT = Node->getValueType(ResNo: 0);
1989 EVT IntVT = VT.changeVectorElementTypeToInteger();
1990
1991 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
1992 return SDValue();
1993
1994 // Heuristic check to determine whether vector should be expanded to integer
1995 // operations or unrolled to scalar operations.
1996 // 1. Scalable vector is never unrolled.
1997 // 2. Fixed vector is unrolled if one of followings is true:
1998 // a. Vector only has 1 element and target knows how to handle scalar
1999 // FABS(either legal or custom expand or promote).
2000 // b. Vector has more than 1 element and target supports scalar
2001 // FABS natively and vector length <= 2(1 AND + 1 CONST).
2002 // FIXME: Scalar construction instruction count varies in every architecture,
2003 // here we assume 1 instruction for now.
2004 if (VT.isFixedLengthVector()) {
2005 EVT EltVT = VT.getVectorElementType();
2006 unsigned NumElts = VT.getVectorNumElements();
2007 if ((NumElts == 1 &&
2008 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
2009 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
2010 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2011 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2012 return SDValue();
2013 }
2014
2015 SDLoc DL(Node);
2016 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2017 SDValue ClearSignMask = DAG.getConstant(
2018 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2019 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
2020 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
2021}
2022
2023SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
2024 EVT VT = Node->getValueType(ResNo: 0);
2025 EVT IntVT = VT.changeVectorElementTypeToInteger();
2026
2027 if (VT != Node->getOperand(Num: 1).getValueType() ||
2028 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
2029 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
2030 return SDValue();
2031
2032 // Heuristic check to determine whether vector should be expanded to integer
2033 // operations or unrolled to scalar operations.
2034 // 1. Scalable vector is never unrolled.
2035 // 2. Fixed vector is unrolled if one of followings is true:
2036 // a. Vector only has 1 element and target knows how to handle scalar
2037 // FCOPYSIGN(either legal or custom expand or promote).
2038 // b. Vector has more than 1 element and target supports scalar
2039 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
2040 // FIXME: Scalar construction instruction count varies in every architecture,
2041 // here we assume 1 instruction for now.
2042 if (VT.isFixedLengthVector()) {
2043 EVT EltVT = VT.getVectorElementType();
2044 unsigned NumElts = VT.getVectorNumElements();
2045 if ((NumElts == 1 &&
2046 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
2047 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
2048 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2049 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2050 return SDValue();
2051 }
2052
2053 SDLoc DL(Node);
2054 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2055 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
2056
2057 SDValue SignMask = DAG.getConstant(
2058 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2059 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
2060
2061 SDValue ClearSignMask = DAG.getConstant(
2062 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2063 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
2064
2065 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2066 Flags: SDNodeFlags::Disjoint);
2067
2068 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2069}
2070
2071void VectorLegalizer::ExpandFSUB(SDNode *Node,
2072 SmallVectorImpl<SDValue> &Results) {
2073 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2074 // we can defer this to operation legalization where it will be lowered as
2075 // a+(-b).
2076 EVT VT = Node->getValueType(ResNo: 0);
2077 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2078 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2079 return; // Defer to LegalizeDAG
2080
2081 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2082 Results.push_back(Elt: Expanded);
2083 return;
2084 }
2085
2086 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2087 Results.push_back(Elt: Tmp);
2088}
2089
2090void VectorLegalizer::ExpandSETCC(SDNode *Node,
2091 SmallVectorImpl<SDValue> &Results) {
2092 bool NeedInvert = false;
2093 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2094 Node->getOpcode() == ISD::STRICT_FSETCCS;
2095 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2096 unsigned Offset = IsStrict ? 1 : 0;
2097
2098 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2099 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2100 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2101 SDValue CC = Node->getOperand(Num: 2 + Offset);
2102
2103 MVT OpVT = LHS.getSimpleValueType();
2104 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2105
2106 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2107 if (IsStrict) {
2108 UnrollStrictFPOp(Node, Results);
2109 return;
2110 }
2111 Results.push_back(Elt: UnrollVSETCC(Node));
2112 return;
2113 }
2114
2115 SDLoc dl(Node);
2116 bool Legalized =
2117 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC,
2118 NeedInvert, dl, Chain, IsSignaling);
2119
2120 if (Legalized) {
2121 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2122 // condition code, create a new SETCC node.
2123 if (CC.getNode()) {
2124 if (IsStrict) {
2125 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2126 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2127 Chain = LHS.getValue(R: 1);
2128 } else {
2129 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2130 Flags: Node->getFlags());
2131 }
2132 }
2133
2134 // If we expanded the SETCC by inverting the condition code, then wrap
2135 // the existing SETCC in a NOT to restore the intended condition.
2136 if (NeedInvert)
2137 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2138 } else {
2139 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2140
2141 // Otherwise, SETCC for the given comparison type must be completely
2142 // illegal; expand it into a SELECT_CC.
2143 EVT VT = Node->getValueType(ResNo: 0);
2144 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2145 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2146 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2147 N5: CC, Flags: Node->getFlags());
2148 }
2149
2150 Results.push_back(Elt: LHS);
2151 if (IsStrict)
2152 Results.push_back(Elt: Chain);
2153}
2154
2155void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2156 SmallVectorImpl<SDValue> &Results) {
2157 SDValue Result, Overflow;
2158 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2159 Results.push_back(Elt: Result);
2160 Results.push_back(Elt: Overflow);
2161}
2162
2163void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2164 SmallVectorImpl<SDValue> &Results) {
2165 SDValue Result, Overflow;
2166 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2167 Results.push_back(Elt: Result);
2168 Results.push_back(Elt: Overflow);
2169}
2170
2171void VectorLegalizer::ExpandMULO(SDNode *Node,
2172 SmallVectorImpl<SDValue> &Results) {
2173 SDValue Result, Overflow;
2174 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2175 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2176
2177 Results.push_back(Elt: Result);
2178 Results.push_back(Elt: Overflow);
2179}
2180
2181void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2182 SmallVectorImpl<SDValue> &Results) {
2183 SDNode *N = Node;
2184 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2185 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2186 Results.push_back(Elt: Expanded);
2187}
2188
2189void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2190 SmallVectorImpl<SDValue> &Results) {
2191 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
2192 ExpandUINT_TO_FLOAT(Node, Results);
2193 return;
2194 }
2195 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
2196 ExpandFP_TO_UINT(Node, Results);
2197 return;
2198 }
2199
2200 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2201 Node->getOpcode() == ISD::STRICT_FSETCCS) {
2202 ExpandSETCC(Node, Results);
2203 return;
2204 }
2205
2206 UnrollStrictFPOp(Node, Results);
2207}
2208
2209void VectorLegalizer::ExpandREM(SDNode *Node,
2210 SmallVectorImpl<SDValue> &Results) {
2211 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2212 "Expected REM node");
2213
2214 SDValue Result;
2215 if (!TLI.expandREM(Node, Result, DAG))
2216 Result = DAG.UnrollVectorOp(N: Node);
2217 Results.push_back(Elt: Result);
2218}
2219
2220// Try to expand libm nodes into vector math routine calls. Callers provide the
2221// RTLIB::get<OP>(EVT) selector of the node's libcall family, which is used to
2222// look up mappings within RuntimeLibcallsInfo. The only mappings considered are
2223// those where the result and all operands are the same vector type. While
2224// predicated nodes are not supported, we will emit calls to masked routines by
2225// passing in a mask that is true for the lanes computed by the node.
2226bool VectorLegalizer::tryExpandVecMathCall(
2227 SDNode *Node, function_ref<RTLIB::Libcall(EVT)> GetLibcall,
2228 SmallVectorImpl<SDValue> &Results) {
2229 // Chain must be propagated but currently strict fp operations are down
2230 // converted to their none strict counterpart.
2231 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2232
2233 EVT VT = Node->getValueType(ResNo: 0);
2234 LLVMContext &Ctx = *DAG.getContext();
2235 const LibcallLoweringInfo &Libcalls = DAG.getLibcalls();
2236
2237 // Try to widen the vector type when no libcall is available at that width.
2238 EVT CallVT = VT;
2239 RTLIB::LibcallImpl LCImpl = Libcalls.getLibcallImpl(Call: GetLibcall(CallVT));
2240 if (LCImpl == RTLIB::Unsupported && VT.getVectorElementCount().isScalar())
2241 return false;
2242 while (LCImpl == RTLIB::Unsupported) {
2243 CallVT = CallVT.getDoubleNumVectorElementsVT(Context&: Ctx);
2244 if (!CallVT.isSimple())
2245 return false;
2246 if (TLI.isTypeLegal(VT: CallVT))
2247 LCImpl = Libcalls.getLibcallImpl(Call: GetLibcall(CallVT));
2248 }
2249
2250 const RTLIB::RuntimeLibcallsInfo &RTLCI = TLI.getRuntimeLibcallsInfo();
2251
2252 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2253 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2254
2255 SDLoc DL(Node);
2256 TargetLowering::ArgListTy Args;
2257
2258 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2259
2260 // Sanity check just in case function has unexpected parameters.
2261 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2262 EVT::getEVT(FuncTy->getReturnType(), true) == CallVT &&
2263 "mismatch in value type and call signature type");
2264
2265 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2266 Type *ParamTy = FuncTy->getParamType(i: I);
2267
2268 if (HasMaskArg && I == E - 1) {
2269 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2270 cast<VectorType>(ParamTy)->getElementCount() ==
2271 CallVT.getVectorElementCount() &&
2272 "unexpected vector mask type");
2273 EVT MaskVT = EVT::getEVT(Ty: ParamTy, /*HandleUnknown=*/true);
2274 EVT SubMaskVT =
2275 MaskVT.changeVectorElementCount(Context&: Ctx, EC: VT.getVectorElementCount());
2276 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: SubMaskVT, OpVT: VT);
2277 // Only the lanes holding the node's elements need to be active.
2278 if (CallVT != VT)
2279 Mask = DAG.getInsertSubvector(
2280 DL, Vec: DAG.getBoolConstant(V: false, DL, VT: MaskVT, OpVT: CallVT), SubVec: Mask, Idx: 0);
2281 Args.emplace_back(args&: Mask, args&: ParamTy);
2282 } else {
2283 SDValue Op = Node->getOperand(Num: I);
2284 assert(Op.getValueType() == VT && "mismatch in vector types");
2285 if (CallVT != VT) {
2286 unsigned NumConcat =
2287 CallVT.getVectorMinNumElements() / VT.getVectorMinNumElements();
2288 SmallVector<SDValue, 4> Ops(NumConcat, Op);
2289 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: CallVT, Ops);
2290 }
2291 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2292 "mismatch in value type and call argument type");
2293 Args.emplace_back(args&: Op, args&: ParamTy);
2294 }
2295 }
2296
2297 // Emit a call to the vector function.
2298 SDValue Callee =
2299 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2300 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2301
2302 TargetLowering::CallLoweringInfo CLI(DAG);
2303 CLI.setDebugLoc(DL)
2304 .setChain(DAG.getEntryNode())
2305 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2306
2307 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2308 SDValue Result = CallResult.first;
2309 if (CallVT != VT)
2310 Result = DAG.getExtractSubvector(DL, VT, Vec: Result, Idx: 0);
2311 Results.push_back(Elt: Result);
2312 return true;
2313}
2314
2315void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2316 SmallVectorImpl<SDValue> &Results) {
2317 EVT VT = Node->getValueType(ResNo: 0);
2318
2319 // Cannot unroll a scalable vector. Delay error reporting until the final
2320 // operation legalisation phase to maximise the chances of removing the node.
2321 if (VT.isScalableVector())
2322 return;
2323
2324 EVT EltVT = VT.getVectorElementType();
2325 unsigned NumElems = VT.getVectorNumElements();
2326 unsigned NumOpers = Node->getNumOperands();
2327 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2328
2329 EVT TmpEltVT = EltVT;
2330 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2331 Node->getOpcode() == ISD::STRICT_FSETCCS)
2332 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2333 Context&: *DAG.getContext(), VT: TmpEltVT);
2334
2335 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2336 SDValue Chain = Node->getOperand(Num: 0);
2337 SDLoc dl(Node);
2338
2339 SmallVector<SDValue, 32> OpValues;
2340 SmallVector<SDValue, 32> OpChains;
2341 for (unsigned i = 0; i < NumElems; ++i) {
2342 SmallVector<SDValue, 4> Opers;
2343 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2344
2345 // The Chain is the first operand.
2346 Opers.push_back(Elt: Chain);
2347
2348 // Now process the remaining operands.
2349 for (unsigned j = 1; j < NumOpers; ++j) {
2350 SDValue Oper = Node->getOperand(Num: j);
2351 EVT OperVT = Oper.getValueType();
2352
2353 if (OperVT.isVector())
2354 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2355 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2356
2357 Opers.push_back(Elt: Oper);
2358 }
2359
2360 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2361 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2362 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2363
2364 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2365 Node->getOpcode() == ISD::STRICT_FSETCCS)
2366 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2367 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2368 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2369
2370 OpValues.push_back(Elt: ScalarResult);
2371 OpChains.push_back(Elt: ScalarChain);
2372 }
2373
2374 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2375 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2376
2377 Results.push_back(Elt: Result);
2378 Results.push_back(Elt: NewChain);
2379}
2380
2381SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2382 EVT VT = Node->getValueType(ResNo: 0);
2383 unsigned NumElems = VT.getVectorNumElements();
2384 EVT EltVT = VT.getVectorElementType();
2385 SDValue LHS = Node->getOperand(Num: 0);
2386 SDValue RHS = Node->getOperand(Num: 1);
2387 SDValue CC = Node->getOperand(Num: 2);
2388 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2389 SDLoc dl(Node);
2390 SmallVector<SDValue, 8> Ops(NumElems);
2391 for (unsigned i = 0; i < NumElems; ++i) {
2392 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2393 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2394 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2395 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2396 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2397 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2398 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2399 Context&: *DAG.getContext(), VT: TmpEltVT),
2400 N1: LHSElem, N2: RHSElem, N3: CC);
2401 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2402 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2403 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2404 }
2405 return DAG.getBuildVector(VT, DL: dl, Ops);
2406}
2407
2408bool SelectionDAG::LegalizeVectors() {
2409 return VectorLegalizer(*this).Run();
2410}
2411