1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/STLFunctionalExtras.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/Analysis/TargetLibraryInfo.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/SelectionDAG.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetLowering.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/CodeGenTypes/MachineValueType.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <iterator>
48#include <utility>
49
50using namespace llvm;
51
52#define DEBUG_TYPE "legalizevectorops"
53
54namespace {
55
56class VectorLegalizer {
57 SelectionDAG& DAG;
58 const TargetLowering &TLI;
59 bool Changed = false; // Keep track of whether anything changed
60
61 /// For nodes that are of legal width, and that have more than one use, this
62 /// map indicates what regularized operand to use. This allows us to avoid
63 /// legalizing the same thing more than once.
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65
66 /// Adds a node to the translation cache.
67 void AddLegalizedOperand(SDValue From, SDValue To) {
68 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
69 // If someone requests legalization of the new node, return itself.
70 if (From != To)
71 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
72 }
73
74 /// Legalizes the given node.
75 SDValue LegalizeOp(SDValue Op);
76
77 /// Assuming the node is legal, "legalize" the results.
78 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79
80 /// Make sure Results are legal and update the translation cache.
81 SDValue RecursivelyLegalizeResults(SDValue Op,
82 MutableArrayRef<SDValue> Results);
83
84 /// Wrapper to interface LowerOperation with a vector of Results.
85 /// Returns false if the target wants to use default expansion. Otherwise
86 /// returns true. If return is true and the Results are empty, then the
87 /// target wants to keep the input node as is.
88 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89
90 /// Implements unrolling a VSETCC.
91 SDValue UnrollVSETCC(SDNode *Node);
92
93 /// Implement expand-based legalization of vector operations.
94 ///
95 /// This is just a high-level routine to dispatch to specific code paths for
96 /// operations to legalize them.
97 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98
99 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100 /// FP_TO_SINT isn't legal.
101 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102
103 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106
107 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108 SDValue ExpandSEXTINREG(SDNode *Node);
109
110 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111 ///
112 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113 /// type. The contents of the bits in the extended part of each element are
114 /// undef.
115 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116
117 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118 ///
119 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120 /// type, then shifts left and arithmetic shifts right to introduce a sign
121 /// extension.
122 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123
124 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125 ///
126 /// Shuffles the low lanes of the operand into place and blends zeros into
127 /// the remaining lanes, finally bitcasting to the proper type.
128 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129
130 /// Expand bswap of vectors into a shuffle if legal.
131 SDValue ExpandBSWAP(SDNode *Node);
132
133 /// Implement vselect in terms of XOR, AND, OR when blend is not
134 /// supported by the target.
135 SDValue ExpandVSELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
139 SDValue ExpandMASK_BEFOREFIRST(SDNode *N);
140 SDValue ExpandMaskedBinOp(SDNode *N);
141 SDValue ExpandSELECT(SDNode *Node);
142 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
143 SDValue ExpandStore(SDNode *N);
144 SDValue ExpandFNEG(SDNode *Node);
145 SDValue ExpandFABS(SDNode *Node);
146 SDValue ExpandFCOPYSIGN(SDNode *Node);
147 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 SDValue ExpandBITREVERSE(SDNode *Node);
150 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
157 bool tryExpandVecMathCall(SDNode *Node,
158 function_ref<RTLIB::Libcall(EVT)> GetLibcall,
159 SmallVectorImpl<SDValue> &Results);
160
161 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162
163 /// Implements vector promotion.
164 ///
165 /// This is essentially just bitcasting the operands to a different type and
166 /// bitcasting the result back to the original type.
167 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
168
169 /// Implements [SU]INT_TO_FP vector promotion.
170 ///
171 /// This is a [zs]ext of the input operand to a larger integer type.
172 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
173
174 /// Implements FP_TO_[SU]INT vector promotion of the result type.
175 ///
176 /// It is promoted to a larger integer type. The result is then
177 /// truncated back to the original type.
178 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
179
180 /// Implements vector setcc operation promotion.
181 ///
182 /// All vector operands are promoted to a vector type with larger element
183 /// type.
184 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
185
186 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
187
188 /// Calculate the reduction using a type of higher precision and round the
189 /// result to match the original type. Setting NonArithmetic signifies the
190 /// rounding of the result does not affect its value.
191 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
192 bool NonArithmetic);
193
194 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
195
196public:
197 VectorLegalizer(SelectionDAG& dag) :
198 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
199
200 /// Begin legalizer the vector operations in the DAG.
201 bool Run();
202};
203
204} // end anonymous namespace
205
206bool VectorLegalizer::Run() {
207 // Before we start legalizing vector nodes, check if there are any vectors.
208 bool HasVectors = false;
209 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
210 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
211 // Check if the values of the nodes contain vectors. We don't need to check
212 // the operands because we are going to check their values at some point.
213 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
214
215 // If we found a vector node we can start the legalization.
216 if (HasVectors)
217 break;
218 }
219
220 // If this basic block has no vectors then no need to legalize vectors.
221 if (!HasVectors)
222 return false;
223
224 // The legalize process is inherently a bottom-up recursive process (users
225 // legalize their uses before themselves). Given infinite stack space, we
226 // could just start legalizing on the root and traverse the whole graph. In
227 // practice however, this causes us to run out of stack space on large basic
228 // blocks. To avoid this problem, compute an ordering of the nodes where each
229 // node is only legalized after all of its operands are legalized.
230 DAG.AssignTopologicalOrder();
231 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
232 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
233 LegalizeOp(Op: SDValue(&*I, 0));
234
235 // Finally, it's possible the root changed. Get the new root.
236 SDValue OldRoot = DAG.getRoot();
237 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
238 DAG.setRoot(LegalizedNodes[OldRoot]);
239
240 LegalizedNodes.clear();
241
242 // Remove dead nodes now.
243 DAG.RemoveDeadNodes();
244
245 return Changed;
246}
247
248SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
249 assert(Op->getNumValues() == Result->getNumValues() &&
250 "Unexpected number of results");
251 // Generic legalization: just pass the operand through.
252 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
253 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
254 return SDValue(Result, Op.getResNo());
255}
256
257SDValue
258VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
259 MutableArrayRef<SDValue> Results) {
260 assert(Results.size() == Op->getNumValues() &&
261 "Unexpected number of results");
262 // Make sure that the generated code is itself legal.
263 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
264 Results[i] = LegalizeOp(Op: Results[i]);
265 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
266 }
267
268 return Results[Op.getResNo()];
269}
270
271SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
272 // Note that LegalizeOp may be reentered even from single-use nodes, which
273 // means that we always must cache transformed nodes.
274 auto I = LegalizedNodes.find(Val: Op);
275 if (I != LegalizedNodes.end()) return I->second;
276
277 // Legalize the operands
278 SmallVector<SDValue, 8> Ops;
279 for (const SDValue &Oper : Op->op_values())
280 Ops.push_back(Elt: LegalizeOp(Op: Oper));
281
282 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
283
284 bool HasVectorValueOrOp =
285 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
286 llvm::any_of(Range: Node->op_values(),
287 P: [](SDValue O) { return O.getValueType().isVector(); });
288 if (!HasVectorValueOrOp)
289 return TranslateLegalizeResults(Op, Result: Node);
290
291 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
292 EVT ValVT;
293 switch (Op.getOpcode()) {
294 default:
295 return TranslateLegalizeResults(Op, Result: Node);
296 case ISD::LOAD: {
297 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
298 ISD::LoadExtType ExtType = LD->getExtensionType();
299 EVT LoadedVT = LD->getMemoryVT();
300 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
301 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
302 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
303 break;
304 }
305 case ISD::STORE: {
306 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
307 EVT StVT = ST->getMemoryVT();
308 MVT ValVT = ST->getValue().getSimpleValueType();
309 if (StVT.isVector() && ST->isTruncatingStore())
310 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
311 AddrSpace: ST->getAddressSpace());
312 break;
313 }
314 case ISD::MERGE_VALUES:
315 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
316 // This operation lies about being legal: when it claims to be legal,
317 // it should actually be expanded.
318 if (Action == TargetLowering::Legal)
319 Action = TargetLowering::Expand;
320 break;
321#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
322 case ISD::STRICT_##DAGN:
323#include "llvm/IR/ConstrainedOps.def"
324 ValVT = Node->getValueType(ResNo: 0);
325 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
326 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
327 ValVT = Node->getOperand(Num: 1).getValueType();
328 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
329 Op.getOpcode() == ISD::STRICT_FSETCCS) {
330 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
331 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 3))->get();
332 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
333 if (Action == TargetLowering::Legal)
334 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
335 } else {
336 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
337 }
338 // If we're asked to expand a strict vector floating-point operation,
339 // by default we're going to simply unroll it. That is usually the
340 // best approach, except in the case where the resulting strict (scalar)
341 // operations would themselves use the fallback mutation to non-strict.
342 // In that specific case, just do the fallback on the vector op.
343 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
344 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
345 TargetLowering::Legal) {
346 EVT EltVT = ValVT.getVectorElementType();
347 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
348 == TargetLowering::Expand &&
349 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
350 == TargetLowering::Legal)
351 Action = TargetLowering::Legal;
352 }
353 break;
354 case ISD::ADD:
355 case ISD::SUB:
356 case ISD::MUL:
357 case ISD::MULHS:
358 case ISD::MULHU:
359 case ISD::SDIV:
360 case ISD::UDIV:
361 case ISD::SREM:
362 case ISD::UREM:
363 case ISD::SDIVREM:
364 case ISD::UDIVREM:
365 case ISD::FADD:
366 case ISD::FSUB:
367 case ISD::FMUL:
368 case ISD::FDIV:
369 case ISD::FREM:
370 case ISD::AND:
371 case ISD::OR:
372 case ISD::XOR:
373 case ISD::SHL:
374 case ISD::SRA:
375 case ISD::SRL:
376 case ISD::FSHL:
377 case ISD::FSHR:
378 case ISD::ROTL:
379 case ISD::ROTR:
380 case ISD::ABS:
381 case ISD::ABS_MIN_POISON:
382 case ISD::ABDS:
383 case ISD::ABDU:
384 case ISD::AVGCEILS:
385 case ISD::AVGCEILU:
386 case ISD::AVGFLOORS:
387 case ISD::AVGFLOORU:
388 case ISD::AVGSUBFLOORS:
389 case ISD::AVGSUBFLOORU:
390 case ISD::BSWAP:
391 case ISD::BITREVERSE:
392 case ISD::CTLZ:
393 case ISD::CTTZ:
394 case ISD::CTLZ_ZERO_POISON:
395 case ISD::CTTZ_ZERO_POISON:
396 case ISD::CTPOP:
397 case ISD::CLMUL:
398 case ISD::CLMULH:
399 case ISD::CLMULR:
400 case ISD::SELECT:
401 case ISD::VSELECT:
402 case ISD::SELECT_CC:
403 case ISD::ZERO_EXTEND:
404 case ISD::ANY_EXTEND:
405 case ISD::TRUNCATE:
406 case ISD::SIGN_EXTEND:
407 case ISD::FP_TO_SINT:
408 case ISD::FP_TO_UINT:
409 case ISD::FNEG:
410 case ISD::FABS:
411 case ISD::FMINNUM:
412 case ISD::FMAXNUM:
413 case ISD::FMINNUM_IEEE:
414 case ISD::FMAXNUM_IEEE:
415 case ISD::FMINIMUM:
416 case ISD::FMAXIMUM:
417 case ISD::FMINIMUMNUM:
418 case ISD::FMAXIMUMNUM:
419 case ISD::FCOPYSIGN:
420 case ISD::FSQRT:
421 case ISD::FSIN:
422 case ISD::FCOS:
423 case ISD::FTAN:
424 case ISD::FASIN:
425 case ISD::FACOS:
426 case ISD::FATAN:
427 case ISD::FATAN2:
428 case ISD::FSINH:
429 case ISD::FCOSH:
430 case ISD::FTANH:
431 case ISD::FLDEXP:
432 case ISD::FPOWI:
433 case ISD::FPOW:
434 case ISD::FCBRT:
435 case ISD::FLOG:
436 case ISD::FLOG2:
437 case ISD::FLOG10:
438 case ISD::FEXP:
439 case ISD::FEXP2:
440 case ISD::FEXP10:
441 case ISD::FCEIL:
442 case ISD::FTRUNC:
443 case ISD::FRINT:
444 case ISD::FNEARBYINT:
445 case ISD::FROUND:
446 case ISD::FROUNDEVEN:
447 case ISD::FFLOOR:
448 case ISD::FP_ROUND:
449 case ISD::FP_EXTEND:
450 case ISD::FPTRUNC_ROUND:
451 case ISD::FMA:
452 case ISD::SIGN_EXTEND_INREG:
453 case ISD::ANY_EXTEND_VECTOR_INREG:
454 case ISD::SIGN_EXTEND_VECTOR_INREG:
455 case ISD::ZERO_EXTEND_VECTOR_INREG:
456 case ISD::SMIN:
457 case ISD::SMAX:
458 case ISD::UMIN:
459 case ISD::UMAX:
460 case ISD::SMUL_LOHI:
461 case ISD::UMUL_LOHI:
462 case ISD::SADDO:
463 case ISD::UADDO:
464 case ISD::SSUBO:
465 case ISD::USUBO:
466 case ISD::SMULO:
467 case ISD::UMULO:
468 case ISD::CONVERT_FROM_ARBITRARY_FP:
469 case ISD::CONVERT_TO_ARBITRARY_FP:
470 case ISD::FCANONICALIZE:
471 case ISD::FFREXP:
472 case ISD::FMODF:
473 case ISD::FSINCOS:
474 case ISD::FSINCOSPI:
475 case ISD::SADDSAT:
476 case ISD::UADDSAT:
477 case ISD::SSUBSAT:
478 case ISD::USUBSAT:
479 case ISD::SSHLSAT:
480 case ISD::USHLSAT:
481 case ISD::FP_TO_SINT_SAT:
482 case ISD::FP_TO_UINT_SAT:
483 case ISD::MGATHER:
484 case ISD::VECTOR_COMPRESS:
485 case ISD::SCMP:
486 case ISD::UCMP:
487 case ISD::GET_ACTIVE_LANE_MASK:
488 case ISD::LOOP_DEPENDENCE_WAR_MASK:
489 case ISD::LOOP_DEPENDENCE_RAW_MASK:
490 case ISD::MASK_BEFOREFIRST:
491 case ISD::MASKED_UDIV:
492 case ISD::MASKED_SDIV:
493 case ISD::MASKED_UREM:
494 case ISD::MASKED_SREM:
495 case ISD::VECTOR_MATCH:
496 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
497 break;
498 case ISD::VECTOR_INTERLEAVE:
499 case ISD::VECTOR_DEINTERLEAVE: {
500 EVT VT = Node->getValueType(ResNo: 0);
501 unsigned Factor = Node->getNumOperands();
502 Action = TLI.getVectorInterleaveAction(Opc: Node->getOpcode(), Factor, VT);
503 // Early Custom Lowering causes RISC-V regressions.
504 if (Action == TargetLowering::Custom &&
505 (VT.isScalableVector() || Factor != 2)) {
506 Action = TargetLowering::Legal;
507 }
508 break;
509 }
510 case ISD::SMULFIX:
511 case ISD::SMULFIXSAT:
512 case ISD::UMULFIX:
513 case ISD::UMULFIXSAT:
514 case ISD::SDIVFIX:
515 case ISD::SDIVFIXSAT:
516 case ISD::UDIVFIX:
517 case ISD::UDIVFIXSAT: {
518 unsigned Scale = Node->getConstantOperandVal(Num: 2);
519 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
520 VT: Node->getValueType(ResNo: 0), Scale);
521 break;
522 }
523 case ISD::LROUND:
524 case ISD::LLROUND:
525 case ISD::LRINT:
526 case ISD::LLRINT:
527 case ISD::SINT_TO_FP:
528 case ISD::UINT_TO_FP:
529 case ISD::VECREDUCE_ADD:
530 case ISD::VECREDUCE_MUL:
531 case ISD::VECREDUCE_AND:
532 case ISD::VECREDUCE_OR:
533 case ISD::VECREDUCE_XOR:
534 case ISD::VECREDUCE_SMAX:
535 case ISD::VECREDUCE_SMIN:
536 case ISD::VECREDUCE_UMAX:
537 case ISD::VECREDUCE_UMIN:
538 case ISD::VECREDUCE_FADD:
539 case ISD::VECREDUCE_FMAX:
540 case ISD::VECREDUCE_FMAXIMUM:
541 case ISD::VECREDUCE_FMIN:
542 case ISD::VECREDUCE_FMINIMUM:
543 case ISD::VECREDUCE_FMAXIMUMNUM:
544 case ISD::VECREDUCE_FMINIMUMNUM:
545 case ISD::VECREDUCE_FMUL:
546 case ISD::CTTZ_ELTS:
547 case ISD::CTTZ_ELTS_ZERO_POISON:
548 case ISD::VECTOR_FIND_LAST_ACTIVE:
549 Action = TLI.getOperationAction(Op: Node->getOpcode(),
550 VT: Node->getOperand(Num: 0).getValueType());
551 break;
552 case ISD::VECREDUCE_SEQ_FADD:
553 case ISD::VECREDUCE_SEQ_FMUL:
554 Action = TLI.getOperationAction(Op: Node->getOpcode(),
555 VT: Node->getOperand(Num: 1).getValueType());
556 break;
557 case ISD::SETCC: {
558 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
559 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
560 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
561 if (Action == TargetLowering::Legal)
562 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
563 break;
564 }
565 case ISD::PARTIAL_REDUCE_UMLA:
566 case ISD::PARTIAL_REDUCE_SMLA:
567 case ISD::PARTIAL_REDUCE_SUMLA:
568 case ISD::PARTIAL_REDUCE_FMLA:
569 Action =
570 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
571 InputVT: Node->getOperand(Num: 1).getValueType());
572 break;
573
574#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
575 case ISD::VPID: { \
576 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
577 : Node->getOperand(LEGALPOS).getValueType(); \
578 /* Defer non-vector results to LegalizeDAG. */ \
579 if (!Node->getValueType(0).isVector() && \
580 Node->getValueType(0) != MVT::Other) { \
581 Action = TargetLowering::Legal; \
582 break; \
583 } \
584 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
585 } break;
586#include "llvm/IR/VPIntrinsics.def"
587 }
588
589 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
590
591 SmallVector<SDValue, 8> ResultVals;
592 switch (Action) {
593 default: llvm_unreachable("This action is not supported yet!");
594 case TargetLowering::Promote:
595 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
596 "This action is not supported yet!");
597 LLVM_DEBUG(dbgs() << "Promoting\n");
598 Promote(Node, Results&: ResultVals);
599 assert(!ResultVals.empty() && "No results for promotion?");
600 break;
601 case TargetLowering::Legal:
602 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
603 break;
604 case TargetLowering::Custom:
605 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
606 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
607 break;
608 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
609 [[fallthrough]];
610 case TargetLowering::Expand:
611 LLVM_DEBUG(dbgs() << "Expanding\n");
612 Expand(Node, Results&: ResultVals);
613 break;
614 }
615
616 if (ResultVals.empty())
617 return TranslateLegalizeResults(Op, Result: Node);
618
619 Changed = true;
620 return RecursivelyLegalizeResults(Op, Results: ResultVals);
621}
622
623// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
624// merge them somehow?
625bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
626 SmallVectorImpl<SDValue> &Results) {
627 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
628
629 if (!Res.getNode())
630 return false;
631
632 if (Res == SDValue(Node, 0))
633 return true;
634
635 // If the original node has one result, take the return value from
636 // LowerOperation as is. It might not be result number 0.
637 if (Node->getNumValues() == 1) {
638 Results.push_back(Elt: Res);
639 return true;
640 }
641
642 // If the original node has multiple results, then the return node should
643 // have the same number of results.
644 assert((Node->getNumValues() == Res->getNumValues()) &&
645 "Lowering returned the wrong number of results!");
646
647 // Places new result values base on N result number.
648 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
649 Results.push_back(Elt: Res.getValue(R: I));
650
651 return true;
652}
653
654void VectorLegalizer::PromoteSETCC(SDNode *Node,
655 SmallVectorImpl<SDValue> &Results) {
656 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
657 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
658
659 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
660
661 SDLoc DL(Node);
662 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
663
664 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
665 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
666 Operands[2] = Node->getOperand(Num: 2);
667
668 EVT ResVT =
669 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
670 SDValue Res =
671 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
672 if (ResVT != Node->getValueType(ResNo: 0))
673 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
674 Results.push_back(Elt: Res);
675}
676
677void VectorLegalizer::PromoteSTRICT(SDNode *Node,
678 SmallVectorImpl<SDValue> &Results) {
679 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
680 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
681
682 assert(VecVT.isFloatingPoint());
683
684 SDLoc DL(Node);
685 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
686 SmallVector<SDValue, 2> Chains;
687
688 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
689 if (Node->getOperand(Num: j).getValueType().isVector() &&
690 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
691 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
692 {
693 // promote the vector operand.
694 SDValue Ext =
695 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
696 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
697 Operands[j] = Ext.getValue(R: 0);
698 Chains.push_back(Elt: Ext.getValue(R: 1));
699 } else
700 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
701
702 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
703
704 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
705
706 SDValue Res =
707 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
708
709 SDValue Round =
710 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
711 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
712 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
713
714 Results.push_back(Elt: Round.getValue(R: 0));
715 Results.push_back(Elt: Round.getValue(R: 1));
716}
717
718void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
719 SmallVectorImpl<SDValue> &Results,
720 bool NonArithmetic) {
721 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
722 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
723 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
724
725 SDLoc DL(Node);
726 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
727 SDValue Rdx =
728 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
729 Flags: Node->getFlags());
730 SDValue Res =
731 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
732 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
733 Results.push_back(Elt: Res);
734}
735
736void VectorLegalizer::PromoteVECTOR_COMPRESS(
737 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
738 SDLoc DL(Node);
739 EVT VT = Node->getValueType(ResNo: 0);
740 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
741 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
742 "Only integer promotion or bitcasts between types is supported");
743
744 SDValue Vec = Node->getOperand(Num: 0);
745 SDValue Mask = Node->getOperand(Num: 1);
746 SDValue Passthru = Node->getOperand(Num: 2);
747 if (VT.isInteger()) {
748 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
749 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
750 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
751 } else {
752 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
753 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
754 }
755
756 SDValue Result =
757 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
758 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
759 : DAG.getBitcast(VT, V: Result);
760 Results.push_back(Elt: Result);
761}
762
763void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
764 // For a few operations there is a specific concept for promotion based on
765 // the operand's type.
766 switch (Node->getOpcode()) {
767 case ISD::SINT_TO_FP:
768 case ISD::UINT_TO_FP:
769 case ISD::STRICT_SINT_TO_FP:
770 case ISD::STRICT_UINT_TO_FP:
771 // "Promote" the operation by extending the operand.
772 PromoteINT_TO_FP(Node, Results);
773 return;
774 case ISD::FP_TO_UINT:
775 case ISD::FP_TO_SINT:
776 case ISD::STRICT_FP_TO_UINT:
777 case ISD::STRICT_FP_TO_SINT:
778 // Promote the operation by extending the operand.
779 PromoteFP_TO_INT(Node, Results);
780 return;
781 case ISD::SETCC:
782 // Promote the operation by extending the operand.
783 PromoteSETCC(Node, Results);
784 return;
785 case ISD::STRICT_FADD:
786 case ISD::STRICT_FSUB:
787 case ISD::STRICT_FMUL:
788 case ISD::STRICT_FDIV:
789 case ISD::STRICT_FSQRT:
790 case ISD::STRICT_FMA:
791 PromoteSTRICT(Node, Results);
792 return;
793 case ISD::VECREDUCE_FADD:
794 case ISD::VECREDUCE_FMUL:
795 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
796 return;
797 case ISD::VECREDUCE_FMAX:
798 case ISD::VECREDUCE_FMAXIMUM:
799 case ISD::VECREDUCE_FMIN:
800 case ISD::VECREDUCE_FMINIMUM:
801 case ISD::VECREDUCE_FMAXIMUMNUM:
802 case ISD::VECREDUCE_FMINIMUMNUM:
803 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
804 return;
805 case ISD::VECTOR_COMPRESS:
806 PromoteVECTOR_COMPRESS(Node, Results);
807 return;
808
809 case ISD::FP_ROUND:
810 case ISD::FP_EXTEND:
811 // These operations are used to do promotion so they can't be promoted
812 // themselves.
813 llvm_unreachable("Don't know how to promote this operation!");
814 }
815
816 // There are currently two cases of vector promotion:
817 // 1) Bitcasting a vector of integers to a different type to a vector of the
818 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
819 // 2) Extending a vector of floats to a vector of the same number of larger
820 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
821 assert(Node->getNumValues() == 1 &&
822 "Can't promote a vector with multiple results!");
823 MVT VT = Node->getSimpleValueType(ResNo: 0);
824 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
825 SDLoc dl(Node);
826 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
827
828 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
829 // Do not promote the mask operand of a VP OP.
830 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
831 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
832 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
833 if (Node->getOperand(Num: j)
834 .getValueType()
835 .getVectorElementType()
836 .isFloatingPoint() &&
837 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
838 Operands[j] = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
839 else
840 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
841 else
842 Operands[j] = Node->getOperand(Num: j);
843 }
844
845 SDValue Res =
846 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
847
848 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
849 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
850 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
851 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
852 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
853 else
854 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
855
856 Results.push_back(Elt: Res);
857}
858
859void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
860 SmallVectorImpl<SDValue> &Results) {
861 // INT_TO_FP operations may require the input operand be promoted even
862 // when the type is otherwise legal.
863 bool IsStrict = Node->isStrictFPOpcode();
864 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
865 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
866 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
867 "Vectors have different number of elements!");
868
869 SDLoc dl(Node);
870 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
871
872 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
873 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
874 ? ISD::ZERO_EXTEND
875 : ISD::SIGN_EXTEND;
876 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
877 if (Node->getOperand(Num: j).getValueType().isVector())
878 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
879 else
880 Operands[j] = Node->getOperand(Num: j);
881 }
882
883 if (IsStrict) {
884 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
885 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
886 Results.push_back(Elt: Res);
887 Results.push_back(Elt: Res.getValue(R: 1));
888 return;
889 }
890
891 SDValue Res =
892 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
893 Results.push_back(Elt: Res);
894}
895
896// For FP_TO_INT we promote the result type to a vector type with wider
897// elements and then truncate the result. This is different from the default
898// PromoteVector which uses bitcast to promote thus assumning that the
899// promoted vector type has the same overall size.
900void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
901 SmallVectorImpl<SDValue> &Results) {
902 MVT VT = Node->getSimpleValueType(ResNo: 0);
903 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
904 bool IsStrict = Node->isStrictFPOpcode();
905 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
906 "Vectors have different number of elements!");
907
908 unsigned NewOpc = Node->getOpcode();
909 // Change FP_TO_UINT to FP_TO_SINT if possible.
910 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
911 if (NewOpc == ISD::FP_TO_UINT &&
912 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
913 NewOpc = ISD::FP_TO_SINT;
914
915 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
916 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
917 NewOpc = ISD::STRICT_FP_TO_SINT;
918
919 SDLoc dl(Node);
920 SDValue Promoted, Chain;
921 if (IsStrict) {
922 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
923 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
924 Chain = Promoted.getValue(R: 1);
925 } else
926 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
927
928 // Assert that the converted value fits in the original type. If it doesn't
929 // (eg: because the value being converted is too big), then the result of the
930 // original operation was undefined anyway, so the assert is still correct.
931 if (Node->getOpcode() == ISD::FP_TO_UINT ||
932 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
933 NewOpc = ISD::AssertZext;
934 else
935 NewOpc = ISD::AssertSext;
936
937 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
938 N2: DAG.getValueType(VT.getScalarType()));
939 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
940 Results.push_back(Elt: Promoted);
941 if (IsStrict)
942 Results.push_back(Elt: Chain);
943}
944
945std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
946 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
947 return TLI.scalarizeVectorLoad(LD, DAG);
948}
949
950SDValue VectorLegalizer::ExpandStore(SDNode *N) {
951 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
952 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
953 return TF;
954}
955
956void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
957 switch (Node->getOpcode()) {
958 case ISD::LOAD: {
959 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
960 Results.push_back(Elt: Tmp.first);
961 Results.push_back(Elt: Tmp.second);
962 return;
963 }
964 case ISD::STORE:
965 Results.push_back(Elt: ExpandStore(N: Node));
966 return;
967 case ISD::MERGE_VALUES:
968 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
969 Results.push_back(Elt: Node->getOperand(Num: i));
970 return;
971 case ISD::VECTOR_INTERLEAVE:
972 TLI.expandVectorInterleave(Node, Results, DAG);
973 return;
974 case ISD::VECTOR_DEINTERLEAVE:
975 TLI.expandVectorDeinterleave(Node, Results, DAG);
976 return;
977 case ISD::SIGN_EXTEND_INREG:
978 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
979 Results.push_back(Elt: Expanded);
980 return;
981 }
982 break;
983 case ISD::ANY_EXTEND_VECTOR_INREG:
984 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
985 return;
986 case ISD::SIGN_EXTEND_VECTOR_INREG:
987 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
988 return;
989 case ISD::ZERO_EXTEND_VECTOR_INREG:
990 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
991 return;
992 case ISD::BSWAP:
993 if (SDValue Expanded = ExpandBSWAP(Node)) {
994 Results.push_back(Elt: Expanded);
995 return;
996 }
997 break;
998 case ISD::VSELECT:
999 if (SDValue Expanded = ExpandVSELECT(Node)) {
1000 Results.push_back(Elt: Expanded);
1001 return;
1002 }
1003 break;
1004 case ISD::VP_SREM:
1005 case ISD::VP_UREM:
1006 if (SDValue Expanded = ExpandVP_REM(Node)) {
1007 Results.push_back(Elt: Expanded);
1008 return;
1009 }
1010 break;
1011 case ISD::SELECT:
1012 if (SDValue Expanded = ExpandSELECT(Node)) {
1013 Results.push_back(Elt: Expanded);
1014 return;
1015 }
1016 break;
1017 case ISD::SELECT_CC: {
1018 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1019 EVT CondVT = TLI.getSetCCResultType(
1020 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
1021 SDValue SetCC =
1022 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
1023 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
1024 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1025 LHS: Node->getOperand(Num: 2),
1026 RHS: Node->getOperand(Num: 3)));
1027 return;
1028 }
1029 break;
1030 }
1031 case ISD::FP_TO_UINT:
1032 ExpandFP_TO_UINT(Node, Results);
1033 return;
1034 case ISD::UINT_TO_FP:
1035 ExpandUINT_TO_FLOAT(Node, Results);
1036 return;
1037 case ISD::FNEG:
1038 if (SDValue Expanded = ExpandFNEG(Node)) {
1039 Results.push_back(Elt: Expanded);
1040 return;
1041 }
1042 break;
1043 case ISD::FABS:
1044 if (SDValue Expanded = ExpandFABS(Node)) {
1045 Results.push_back(Elt: Expanded);
1046 return;
1047 }
1048 break;
1049 case ISD::FCOPYSIGN:
1050 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1051 Results.push_back(Elt: Expanded);
1052 return;
1053 }
1054 break;
1055 case ISD::FCANONICALIZE: {
1056 // If the scalar element type has a
1057 // Legal/Custom FCANONICALIZE, don't
1058 // mess with the vector, fall back.
1059 EVT VT = Node->getValueType(ResNo: 0);
1060 EVT EltVT = VT.getVectorElementType();
1061 if (!VT.isScalableVector() &&
1062 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1063 TargetLowering::Expand)
1064 break;
1065 // Otherwise canonicalize the whole vector.
1066 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1067 Results.push_back(Elt: Mul);
1068 return;
1069 }
1070 case ISD::FSUB:
1071 ExpandFSUB(Node, Results);
1072 return;
1073 case ISD::SETCC:
1074 ExpandSETCC(Node, Results);
1075 return;
1076 case ISD::ABS:
1077 case ISD::ABS_MIN_POISON:
1078 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1079 Results.push_back(Elt: Expanded);
1080 return;
1081 }
1082 break;
1083 case ISD::ABDS:
1084 case ISD::ABDU:
1085 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1086 Results.push_back(Elt: Expanded);
1087 return;
1088 }
1089 break;
1090 case ISD::AVGCEILS:
1091 case ISD::AVGCEILU:
1092 case ISD::AVGFLOORS:
1093 case ISD::AVGFLOORU:
1094 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1095 Results.push_back(Elt: Expanded);
1096 return;
1097 }
1098 break;
1099 case ISD::BITREVERSE:
1100 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1101 Results.push_back(Elt: Expanded);
1102 return;
1103 }
1104 break;
1105 case ISD::CTPOP:
1106 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1107 Results.push_back(Elt: Expanded);
1108 return;
1109 }
1110 break;
1111 case ISD::CTLZ:
1112 case ISD::CTLZ_ZERO_POISON:
1113 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1114 Results.push_back(Elt: Expanded);
1115 return;
1116 }
1117 break;
1118 case ISD::CTTZ:
1119 case ISD::CTTZ_ZERO_POISON:
1120 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1121 Results.push_back(Elt: Expanded);
1122 return;
1123 }
1124 break;
1125 case ISD::FSHL:
1126 case ISD::FSHR:
1127 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1128 Results.push_back(Elt: Expanded);
1129 return;
1130 }
1131 break;
1132 case ISD::CLMUL:
1133 case ISD::CLMULR:
1134 case ISD::CLMULH:
1135 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1136 Results.push_back(Elt: Expanded);
1137 return;
1138 }
1139 break;
1140 case ISD::PEXT:
1141 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1142 return;
1143 case ISD::PDEP:
1144 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1145 return;
1146 case ISD::ROTL:
1147 case ISD::ROTR:
1148 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1149 Results.push_back(Elt: Expanded);
1150 return;
1151 }
1152 break;
1153 case ISD::FMINNUM:
1154 case ISD::FMAXNUM:
1155 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1156 Results.push_back(Elt: Expanded);
1157 return;
1158 }
1159 break;
1160 case ISD::FMINIMUM:
1161 case ISD::FMAXIMUM:
1162 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1163 return;
1164 case ISD::FMINIMUMNUM:
1165 case ISD::FMAXIMUMNUM:
1166 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1167 return;
1168 case ISD::SMIN:
1169 case ISD::SMAX:
1170 case ISD::UMIN:
1171 case ISD::UMAX:
1172 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1173 Results.push_back(Elt: Expanded);
1174 return;
1175 }
1176 break;
1177 case ISD::UADDO:
1178 case ISD::USUBO:
1179 ExpandUADDSUBO(Node, Results);
1180 return;
1181 case ISD::SADDO:
1182 case ISD::SSUBO:
1183 ExpandSADDSUBO(Node, Results);
1184 return;
1185 case ISD::UMULO:
1186 case ISD::SMULO:
1187 ExpandMULO(Node, Results);
1188 return;
1189 case ISD::MULHS:
1190 case ISD::MULHU:
1191 if (SDValue Expanded = TLI.expandMULH(Node, DAG)) {
1192 Results.push_back(Elt: Expanded);
1193 return;
1194 }
1195 break;
1196 case ISD::USUBSAT:
1197 case ISD::SSUBSAT:
1198 case ISD::UADDSAT:
1199 case ISD::SADDSAT:
1200 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1201 Results.push_back(Elt: Expanded);
1202 return;
1203 }
1204 break;
1205 case ISD::USHLSAT:
1206 case ISD::SSHLSAT:
1207 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1208 Results.push_back(Elt: Expanded);
1209 return;
1210 }
1211 break;
1212 case ISD::FP_TO_SINT_SAT:
1213 case ISD::FP_TO_UINT_SAT:
1214 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1215 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1216 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1217 Results.push_back(Elt: Expanded);
1218 return;
1219 }
1220 }
1221 break;
1222 case ISD::SMULFIX:
1223 case ISD::UMULFIX:
1224 case ISD::SMULFIXSAT:
1225 case ISD::UMULFIXSAT:
1226 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1227 Results.push_back(Elt: Expanded);
1228 return;
1229 }
1230 break;
1231 case ISD::SDIVFIX:
1232 case ISD::UDIVFIX:
1233 ExpandFixedPointDiv(Node, Results);
1234 return;
1235 case ISD::SDIVFIXSAT:
1236 case ISD::UDIVFIXSAT:
1237 break;
1238#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1239 case ISD::STRICT_##DAGN:
1240#include "llvm/IR/ConstrainedOps.def"
1241 ExpandStrictFPOp(Node, Results);
1242 return;
1243 case ISD::VECREDUCE_ADD:
1244 case ISD::VECREDUCE_MUL:
1245 case ISD::VECREDUCE_AND:
1246 case ISD::VECREDUCE_OR:
1247 case ISD::VECREDUCE_XOR:
1248 case ISD::VECREDUCE_SMAX:
1249 case ISD::VECREDUCE_SMIN:
1250 case ISD::VECREDUCE_UMAX:
1251 case ISD::VECREDUCE_UMIN:
1252 case ISD::VECREDUCE_FADD:
1253 case ISD::VECREDUCE_FMUL:
1254 case ISD::VECREDUCE_FMAX:
1255 case ISD::VECREDUCE_FMIN:
1256 case ISD::VECREDUCE_FMAXIMUM:
1257 case ISD::VECREDUCE_FMINIMUM:
1258 case ISD::VECREDUCE_FMAXIMUMNUM:
1259 case ISD::VECREDUCE_FMINIMUMNUM:
1260 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1261 return;
1262 case ISD::PARTIAL_REDUCE_UMLA:
1263 case ISD::PARTIAL_REDUCE_SMLA:
1264 case ISD::PARTIAL_REDUCE_SUMLA:
1265 case ISD::PARTIAL_REDUCE_FMLA:
1266 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1267 return;
1268 case ISD::VECREDUCE_SEQ_FADD:
1269 case ISD::VECREDUCE_SEQ_FMUL:
1270 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1271 return;
1272 case ISD::VECTOR_MATCH:
1273 Results.push_back(Elt: TLI.expandVectorMatch(N: Node, DAG));
1274 return;
1275 case ISD::SREM:
1276 case ISD::UREM:
1277 ExpandREM(Node, Results);
1278 return;
1279 case ISD::VP_MERGE:
1280 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1281 Results.push_back(Elt: Expanded);
1282 return;
1283 }
1284 break;
1285 case ISD::FREM:
1286 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getREM, Results))
1287 return;
1288 break;
1289 case ISD::FSINCOS:
1290 case ISD::FSINCOSPI: {
1291 EVT VT = Node->getValueType(ResNo: 0);
1292 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1293 ? RTLIB::getSINCOS(VT)
1294 : RTLIB::getSINCOSPI(VT);
1295 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1296 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1297 return;
1298
1299 // TODO: Try to see if there's a narrower call available to use before
1300 // scalarizing.
1301 break;
1302 }
1303 case ISD::FPOW:
1304 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getPOW, Results))
1305 return;
1306
1307 // TODO: Try to see if there's a narrower call available to use before
1308 // scalarizing.
1309 break;
1310 case ISD::FCBRT:
1311 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getCBRT, Results))
1312 return;
1313
1314 // TODO: Try to see if there's a narrower call available to use before
1315 // scalarizing.
1316 break;
1317 case ISD::FMODF: {
1318 EVT VT = Node->getValueType(ResNo: 0);
1319 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1320 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1321 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1322 /*CallRetResNo=*/0))
1323 return;
1324 break;
1325 }
1326 case ISD::VECTOR_COMPRESS:
1327 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1328 return;
1329 case ISD::CTTZ_ELTS:
1330 case ISD::CTTZ_ELTS_ZERO_POISON:
1331 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1332 return;
1333 case ISD::VECTOR_FIND_LAST_ACTIVE:
1334 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1335 return;
1336 case ISD::SCMP:
1337 case ISD::UCMP:
1338 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1339 return;
1340 case ISD::GET_ACTIVE_LANE_MASK:
1341 if (SDValue R = TLI.expandGetActiveLaneMask(N: Node, DAG))
1342 Results.push_back(Elt: R);
1343 return;
1344 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1345 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1346 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1347 return;
1348 case ISD::MASK_BEFOREFIRST:
1349 Results.push_back(Elt: ExpandMASK_BEFOREFIRST(N: Node));
1350 return;
1351
1352 case ISD::FADD:
1353 case ISD::FMUL:
1354 case ISD::FMA:
1355 case ISD::FDIV:
1356 case ISD::FCEIL:
1357 case ISD::FFLOOR:
1358 case ISD::FNEARBYINT:
1359 case ISD::FRINT:
1360 case ISD::FROUND:
1361 case ISD::FROUNDEVEN:
1362 case ISD::FTRUNC:
1363 case ISD::FSQRT:
1364 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1365 Results.push_back(Elt: Expanded);
1366 return;
1367 }
1368 break;
1369 case ISD::CONVERT_TO_ARBITRARY_FP:
1370 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1371 Results.push_back(Elt: Expanded);
1372 else
1373 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1374 return;
1375 case ISD::CONVERT_FROM_ARBITRARY_FP:
1376 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1377 Results.push_back(Elt: Expanded);
1378 else
1379 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1380 return;
1381 case ISD::MASKED_UDIV:
1382 case ISD::MASKED_SDIV:
1383 case ISD::MASKED_UREM:
1384 case ISD::MASKED_SREM:
1385 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1386 return;
1387 }
1388
1389 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1390 if (Node->getNumValues() == 1) {
1391 Results.push_back(Elt: Unrolled);
1392 } else {
1393 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1394 "VectorLegalizer Expand returned wrong number of results!");
1395 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1396 Results.push_back(Elt: Unrolled.getValue(R: I));
1397 }
1398}
1399
1400SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1401 // Lower a select instruction where the condition is a scalar and the
1402 // operands are vectors. Lower this select to VSELECT and implement it
1403 // using XOR AND OR. The selector bit is broadcasted.
1404 EVT VT = Node->getValueType(ResNo: 0);
1405 SDLoc DL(Node);
1406
1407 SDValue Mask = Node->getOperand(Num: 0);
1408 SDValue Op1 = Node->getOperand(Num: 1);
1409 SDValue Op2 = Node->getOperand(Num: 2);
1410
1411 assert(VT.isVector() && !Mask.getValueType().isVector()
1412 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1413
1414 // If we can't even use the basic vector operations of
1415 // AND,OR,XOR, we will have to scalarize the op.
1416 // Notice that the operation may be 'promoted' which means that it is
1417 // 'bitcasted' to another type which is handled.
1418 // Also, we need to be able to construct a splat vector using either
1419 // BUILD_VECTOR or SPLAT_VECTOR.
1420 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1421 // BUILD_VECTOR?
1422 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1423 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1424 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1425 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1426 : ISD::SPLAT_VECTOR,
1427 VT) == TargetLowering::Expand)
1428 return SDValue();
1429
1430 // Generate a mask operand.
1431 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1432
1433 // What is the size of each element in the vector mask.
1434 EVT BitTy = MaskTy.getScalarType();
1435
1436 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1437 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1438
1439 // Broadcast the mask so that the entire vector is all one or all zero.
1440 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1441
1442 // Bitcast the operands to be the same type as the mask.
1443 // This is needed when we select between FP types because
1444 // the mask is a vector of integers.
1445 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1446 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1447
1448 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1449
1450 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1451 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1452 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1453 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1454}
1455
1456SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1457 EVT VT = Node->getValueType(ResNo: 0);
1458
1459 // Make sure that the SRA and SHL instructions are available.
1460 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1461 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1462 return SDValue();
1463
1464 SDLoc DL(Node);
1465 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1466
1467 unsigned BW = VT.getScalarSizeInBits();
1468 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1469 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1470
1471 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1472 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1473}
1474
1475// Generically expand a vector anyext in register to a shuffle of the relevant
1476// lanes into the appropriate locations, with other lanes left undef.
1477SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1478 SDLoc DL(Node);
1479 EVT VT = Node->getValueType(ResNo: 0);
1480 int NumElements = VT.getVectorNumElements();
1481 SDValue Src = Node->getOperand(Num: 0);
1482 EVT SrcVT = Src.getValueType();
1483 int NumSrcElements = SrcVT.getVectorNumElements();
1484
1485 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1486 // into a larger vector type.
1487 if (SrcVT.bitsLE(VT)) {
1488 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1489 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1490 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1491 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1492 NumElements: NumSrcElements);
1493 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1494 }
1495
1496 // Build a base mask of undef shuffles.
1497 SmallVector<int, 16> ShuffleMask;
1498 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1499
1500 // Place the extended lanes into the correct locations.
1501 int ExtLaneScale = NumSrcElements / NumElements;
1502 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1503 for (int i = 0; i < NumElements; ++i)
1504 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1505
1506 return DAG.getNode(
1507 Opcode: ISD::BITCAST, DL, VT,
1508 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1509}
1510
1511SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1512 SDLoc DL(Node);
1513 EVT VT = Node->getValueType(ResNo: 0);
1514 SDValue Src = Node->getOperand(Num: 0);
1515 EVT SrcVT = Src.getValueType();
1516
1517 // First build an any-extend node which can be legalized above when we
1518 // recurse through it.
1519 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1520
1521 // Now we need sign extend. This will be exanded to shifts if it isn't
1522 // supported.
1523 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1524 NumElements: VT.getVectorNumElements());
1525 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1526 N2: DAG.getValueType(ExtVT));
1527}
1528
1529// Generically expand a vector zext in register to a shuffle of the relevant
1530// lanes into the appropriate locations, a blend of zero into the high bits,
1531// and a bitcast to the wider element type.
1532SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1533 SDLoc DL(Node);
1534 EVT VT = Node->getValueType(ResNo: 0);
1535 int NumElements = VT.getVectorNumElements();
1536 SDValue Src = Node->getOperand(Num: 0);
1537 EVT SrcVT = Src.getValueType();
1538 int NumSrcElements = SrcVT.getVectorNumElements();
1539
1540 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1541 // into a larger vector type.
1542 if (SrcVT.bitsLE(VT)) {
1543 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1544 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1545 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1546 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1547 NumElements: NumSrcElements);
1548 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1549 }
1550
1551 // Build up a zero vector to blend into this one.
1552 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1553
1554 // Shuffle the incoming lanes into the correct position, and pull all other
1555 // lanes from the zero vector.
1556 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1557
1558 int ExtLaneScale = NumSrcElements / NumElements;
1559 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1560 for (int i = 0; i < NumElements; ++i)
1561 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1562
1563 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1564 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1565}
1566
1567static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1568 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1569 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1570 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1571 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1572}
1573
1574SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1575 EVT VT = Node->getValueType(ResNo: 0);
1576
1577 // Scalable vectors can't use shuffle expansion.
1578 if (VT.isScalableVector())
1579 return TLI.expandBSWAP(N: Node, DAG);
1580
1581 // Generate a byte wise shuffle mask for the BSWAP.
1582 SmallVector<int, 16> ShuffleMask;
1583 createBSWAPShuffleMask(VT, ShuffleMask);
1584 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1585
1586 // Only emit a shuffle if the mask is legal.
1587 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1588 SDLoc DL(Node);
1589 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1590 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1591 Mask: ShuffleMask);
1592 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1593 }
1594
1595 // If we have the appropriate vector bit operations, it is better to use them
1596 // than unrolling and expanding each component.
1597 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1598 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1599 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1600 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1601 return TLI.expandBSWAP(N: Node, DAG);
1602
1603 // Otherwise let the caller unroll.
1604 return SDValue();
1605}
1606
1607SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1608 EVT VT = Node->getValueType(ResNo: 0);
1609
1610 // We can't unroll or use shuffles for scalable vectors.
1611 if (VT.isScalableVector())
1612 return TLI.expandBITREVERSE(N: Node, DAG);
1613
1614 // If we have the scalar operation, it's probably cheaper to unroll it.
1615 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1616 return SDValue();
1617
1618 // If the vector element width is a whole number of bytes, test if its legal
1619 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1620 // vector. This greatly reduces the number of bit shifts necessary.
1621 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1622 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1623 SmallVector<int, 16> BSWAPMask;
1624 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1625
1626 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1627 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1628 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1629 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1630 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1631 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1632 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1633 SDLoc DL(Node);
1634 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1635 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1636 Mask: BSWAPMask);
1637 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1638 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1639 return Op;
1640 }
1641 }
1642
1643 // If we have the appropriate vector bit operations, it is better to use them
1644 // than unrolling and expanding each component.
1645 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1646 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1647 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1648 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1649 return TLI.expandBITREVERSE(N: Node, DAG);
1650
1651 // Otherwise unroll.
1652 return SDValue();
1653}
1654
1655SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1656 // Implement VSELECT in terms of XOR, AND, OR
1657 // on platforms which do not support blend natively.
1658 SDLoc DL(Node);
1659
1660 SDValue Mask = Node->getOperand(Num: 0);
1661 SDValue Op1 = Node->getOperand(Num: 1);
1662 SDValue Op2 = Node->getOperand(Num: 2);
1663
1664 EVT VT = Mask.getValueType();
1665
1666 // If we can't even use the basic vector operations of
1667 // AND,OR,XOR, we will have to scalarize the op.
1668 // Notice that the operation may be 'promoted' which means that it is
1669 // 'bitcasted' to another type which is handled.
1670 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1671 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1672 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1673 return SDValue();
1674
1675 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1676 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1677 // vector constant to mask with.
1678 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1679 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1680 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1681 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1682 Op1.getValueType().getVectorElementType() == MVT::i1))
1683 return SDValue();
1684
1685 // If the mask and the type are different sizes, unroll the vector op. This
1686 // can occur when getSetCCResultType returns something that is different in
1687 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1688 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1689 return SDValue();
1690
1691 // Bitcast the operands to be the same type as the mask.
1692 // This is needed when we select between FP types because
1693 // the mask is a vector of integers.
1694 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1695 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1696
1697 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1698
1699 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1700 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1701 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1702 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1703}
1704
1705SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1706 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1707 // indices less than the EVL/pivot are true. Combine that with the original
1708 // mask for a full-length mask. Use a full-length VSELECT to select between
1709 // the true and false values.
1710 SDLoc DL(Node);
1711
1712 SDValue Mask = Node->getOperand(Num: 0);
1713 SDValue Op1 = Node->getOperand(Num: 1);
1714 SDValue Op2 = Node->getOperand(Num: 2);
1715 SDValue EVL = Node->getOperand(Num: 3);
1716
1717 EVT MaskVT = Mask.getValueType();
1718 bool IsFixedLen = MaskVT.isFixedLengthVector();
1719
1720 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1721 EC: MaskVT.getVectorElementCount());
1722
1723 // If we can't construct the EVL mask efficiently, it's better to unroll.
1724 if ((IsFixedLen &&
1725 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1726 (!IsFixedLen &&
1727 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1728 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1729 return SDValue();
1730
1731 // If using a SETCC would result in a different type than the mask type,
1732 // unroll.
1733 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1734 VT: EVLVecVT) != MaskVT)
1735 return SDValue();
1736
1737 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1738 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1739 SDValue EVLMask =
1740 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1741
1742 SDValue FullMask = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: Mask, N2: EVLMask);
1743 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1744}
1745
1746SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1747 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, MUL, SUB.
1748 EVT VT = Node->getValueType(ResNo: 0);
1749
1750 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1751
1752 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1753 !TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT) ||
1754 !TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT))
1755 return SDValue();
1756
1757 SDLoc DL(Node);
1758
1759 SDValue Dividend = Node->getOperand(Num: 0);
1760 SDValue Divisor = Node->getOperand(Num: 1);
1761 SDValue Mask = Node->getOperand(Num: 2);
1762 SDValue EVL = Node->getOperand(Num: 3);
1763
1764 // X % Y -> X-X/Y*Y
1765 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1766 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Divisor, N2: Div);
1767 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Dividend, N2: Mul);
1768}
1769
1770SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1771 return TLI.expandLoopDependenceMask(N, DAG);
1772}
1773
1774SDValue VectorLegalizer::ExpandMASK_BEFOREFIRST(SDNode *N) {
1775 // Expand to (get_active_lane_mask 0, (cttz_elts x))
1776 SDLoc DL(N);
1777 EVT VT = N->getValueType(ResNo: 0);
1778 EVT VecIdxVT = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1779 SDValue CttzElts =
1780 DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: VecIdxVT, Operand: N->getOperand(Num: 0));
1781 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT,
1782 N1: DAG.getConstant(Val: 0, DL, VT: VecIdxVT), N2: CttzElts);
1783}
1784
1785SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1786 // Masked bin ops don't have undefined behaviour when dividing by zero
1787 // on disabled lanes and produce poison instead. Replace the divisor on the
1788 // disabled lanes with 1 to avoid division by zero or overflow.
1789 SDLoc dl(N);
1790 EVT VT = N->getValueType(ResNo: 0);
1791 SDValue SafeDivisor = DAG.getSelect(
1792 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1793 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1794 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1795}
1796
1797void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1798 SmallVectorImpl<SDValue> &Results) {
1799 // Attempt to expand using TargetLowering.
1800 SDValue Result, Chain;
1801 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1802 Results.push_back(Elt: Result);
1803 if (Node->isStrictFPOpcode())
1804 Results.push_back(Elt: Chain);
1805 return;
1806 }
1807
1808 // Otherwise go ahead and unroll.
1809 if (Node->isStrictFPOpcode()) {
1810 UnrollStrictFPOp(Node, Results);
1811 return;
1812 }
1813
1814 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1815}
1816
1817void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1818 SmallVectorImpl<SDValue> &Results) {
1819 bool IsStrict = Node->isStrictFPOpcode();
1820 unsigned OpNo = IsStrict ? 1 : 0;
1821 SDValue Src = Node->getOperand(Num: OpNo);
1822 EVT SrcVT = Src.getValueType();
1823 EVT DstVT = Node->getValueType(ResNo: 0);
1824 SDLoc DL(Node);
1825
1826 // Attempt to expand using TargetLowering.
1827 SDValue Result;
1828 SDValue Chain;
1829 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1830 Results.push_back(Elt: Result);
1831 if (IsStrict)
1832 Results.push_back(Elt: Chain);
1833 return;
1834 }
1835
1836 // Make sure that the SINT_TO_FP and SRL instructions are available.
1837 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1838 TargetLowering::Expand) ||
1839 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1840 TargetLowering::Expand)) ||
1841 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1842 if (IsStrict) {
1843 UnrollStrictFPOp(Node, Results);
1844 return;
1845 }
1846
1847 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1848 return;
1849 }
1850
1851 unsigned BW = SrcVT.getScalarSizeInBits();
1852 assert((BW == 64 || BW == 32) &&
1853 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1854
1855 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
1856 // UINT_TO_FP with a larger float and round to the smaller type
1857 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
1858 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
1859 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
1860 SDValue UIToFP;
1861 SDValue Result;
1862 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
1863 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
1864 if (IsStrict) {
1865 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
1866 Ops: {Node->getOperand(Num: 0), Src});
1867 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
1868 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
1869 Results.push_back(Elt: Result);
1870 Results.push_back(Elt: Result.getValue(R: 1));
1871 } else {
1872 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
1873 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
1874 Results.push_back(Elt: Result);
1875 }
1876
1877 return;
1878 }
1879
1880 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
1881
1882 // Constants to clear the upper part of the word.
1883 // Notice that we can also use SHL+SHR, but using a constant is slightly
1884 // faster on x86.
1885 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1886 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
1887
1888 // Two to the power of half-word-size.
1889 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
1890
1891 // Clear upper part of LO, lower HI
1892 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
1893 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
1894
1895 if (IsStrict) {
1896 // Convert hi and lo to floats
1897 // Convert the hi part back to the upper values
1898 // TODO: Can any fast-math-flags be set on these nodes?
1899 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1900 Ops: {Node->getOperand(Num: 0), HI});
1901 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
1902 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
1903 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1904 Ops: {Node->getOperand(Num: 0), LO});
1905
1906 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
1907 N2: fLO.getValue(R: 1));
1908
1909 // Add the two halves
1910 SDValue Result =
1911 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
1912
1913 Results.push_back(Elt: Result);
1914 Results.push_back(Elt: Result.getValue(R: 1));
1915 return;
1916 }
1917
1918 // Convert hi and lo to floats
1919 // Convert the hi part back to the upper values
1920 // TODO: Can any fast-math-flags be set on these nodes?
1921 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
1922 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
1923 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
1924
1925 // Add the two halves
1926 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
1927}
1928
1929SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1930 EVT VT = Node->getValueType(ResNo: 0);
1931 EVT IntVT = VT.changeVectorElementTypeToInteger();
1932
1933 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
1934 return SDValue();
1935
1936 // Heuristic check to determine whether vector should be expanded to integer
1937 // operations or unrolled to scalar operations.
1938 // 1. Scalable vector is never unrolled.
1939 // 2. Fixed vector is unrolled if one of followings is true:
1940 // a. Vector only has 1 element and target knows how to handle scalar
1941 // FNEG (either legal or custom expand or promote).
1942 // b. Vector has more than 1 element and target supports scalar
1943 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
1944 // FIXME: Scalar construction instruction count varies in every architecture,
1945 // here we assume 1 instruction for now.
1946 if (VT.isFixedLengthVector()) {
1947 EVT EltVT = VT.getVectorElementType();
1948 unsigned NumElts = VT.getVectorNumElements();
1949 if ((NumElts == 1 &&
1950 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
1951 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
1952 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1953 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1954 return SDValue();
1955 }
1956
1957 SDLoc DL(Node);
1958 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1959 SDValue SignMask = DAG.getConstant(
1960 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1961 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
1962 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
1963}
1964
1965SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
1966 EVT VT = Node->getValueType(ResNo: 0);
1967 EVT IntVT = VT.changeVectorElementTypeToInteger();
1968
1969 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
1970 return SDValue();
1971
1972 // Heuristic check to determine whether vector should be expanded to integer
1973 // operations or unrolled to scalar operations.
1974 // 1. Scalable vector is never unrolled.
1975 // 2. Fixed vector is unrolled if one of followings is true:
1976 // a. Vector only has 1 element and target knows how to handle scalar
1977 // FABS(either legal or custom expand or promote).
1978 // b. Vector has more than 1 element and target supports scalar
1979 // FABS natively and vector length <= 2(1 AND + 1 CONST).
1980 // FIXME: Scalar construction instruction count varies in every architecture,
1981 // here we assume 1 instruction for now.
1982 if (VT.isFixedLengthVector()) {
1983 EVT EltVT = VT.getVectorElementType();
1984 unsigned NumElts = VT.getVectorNumElements();
1985 if ((NumElts == 1 &&
1986 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
1987 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
1988 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1989 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1990 return SDValue();
1991 }
1992
1993 SDLoc DL(Node);
1994 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1995 SDValue ClearSignMask = DAG.getConstant(
1996 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1997 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
1998 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
1999}
2000
2001SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
2002 EVT VT = Node->getValueType(ResNo: 0);
2003 EVT IntVT = VT.changeVectorElementTypeToInteger();
2004
2005 if (VT != Node->getOperand(Num: 1).getValueType() ||
2006 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
2007 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
2008 return SDValue();
2009
2010 // Heuristic check to determine whether vector should be expanded to integer
2011 // operations or unrolled to scalar operations.
2012 // 1. Scalable vector is never unrolled.
2013 // 2. Fixed vector is unrolled if one of followings is true:
2014 // a. Vector only has 1 element and target knows how to handle scalar
2015 // FCOPYSIGN(either legal or custom expand or promote).
2016 // b. Vector has more than 1 element and target supports scalar
2017 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
2018 // FIXME: Scalar construction instruction count varies in every architecture,
2019 // here we assume 1 instruction for now.
2020 if (VT.isFixedLengthVector()) {
2021 EVT EltVT = VT.getVectorElementType();
2022 unsigned NumElts = VT.getVectorNumElements();
2023 if ((NumElts == 1 &&
2024 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
2025 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
2026 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2027 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2028 return SDValue();
2029 }
2030
2031 SDLoc DL(Node);
2032 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2033 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
2034
2035 SDValue SignMask = DAG.getConstant(
2036 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2037 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
2038
2039 SDValue ClearSignMask = DAG.getConstant(
2040 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2041 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
2042
2043 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2044 Flags: SDNodeFlags::Disjoint);
2045
2046 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2047}
2048
2049void VectorLegalizer::ExpandFSUB(SDNode *Node,
2050 SmallVectorImpl<SDValue> &Results) {
2051 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2052 // we can defer this to operation legalization where it will be lowered as
2053 // a+(-b).
2054 EVT VT = Node->getValueType(ResNo: 0);
2055 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2056 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2057 return; // Defer to LegalizeDAG
2058
2059 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2060 Results.push_back(Elt: Expanded);
2061 return;
2062 }
2063
2064 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2065 Results.push_back(Elt: Tmp);
2066}
2067
2068void VectorLegalizer::ExpandSETCC(SDNode *Node,
2069 SmallVectorImpl<SDValue> &Results) {
2070 bool NeedInvert = false;
2071 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2072 Node->getOpcode() == ISD::STRICT_FSETCCS;
2073 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2074 unsigned Offset = IsStrict ? 1 : 0;
2075
2076 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2077 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2078 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2079 SDValue CC = Node->getOperand(Num: 2 + Offset);
2080
2081 MVT OpVT = LHS.getSimpleValueType();
2082 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2083
2084 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2085 if (IsStrict) {
2086 UnrollStrictFPOp(Node, Results);
2087 return;
2088 }
2089 Results.push_back(Elt: UnrollVSETCC(Node));
2090 return;
2091 }
2092
2093 SDLoc dl(Node);
2094 bool Legalized =
2095 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC,
2096 NeedInvert, dl, Chain, IsSignaling);
2097
2098 if (Legalized) {
2099 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2100 // condition code, create a new SETCC node.
2101 if (CC.getNode()) {
2102 if (IsStrict) {
2103 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2104 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2105 Chain = LHS.getValue(R: 1);
2106 } else {
2107 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2108 Flags: Node->getFlags());
2109 }
2110 }
2111
2112 // If we expanded the SETCC by inverting the condition code, then wrap
2113 // the existing SETCC in a NOT to restore the intended condition.
2114 if (NeedInvert)
2115 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2116 } else {
2117 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2118
2119 // Otherwise, SETCC for the given comparison type must be completely
2120 // illegal; expand it into a SELECT_CC.
2121 EVT VT = Node->getValueType(ResNo: 0);
2122 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2123 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2124 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2125 N5: CC, Flags: Node->getFlags());
2126 }
2127
2128 Results.push_back(Elt: LHS);
2129 if (IsStrict)
2130 Results.push_back(Elt: Chain);
2131}
2132
2133void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2134 SmallVectorImpl<SDValue> &Results) {
2135 SDValue Result, Overflow;
2136 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2137 Results.push_back(Elt: Result);
2138 Results.push_back(Elt: Overflow);
2139}
2140
2141void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2142 SmallVectorImpl<SDValue> &Results) {
2143 SDValue Result, Overflow;
2144 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2145 Results.push_back(Elt: Result);
2146 Results.push_back(Elt: Overflow);
2147}
2148
2149void VectorLegalizer::ExpandMULO(SDNode *Node,
2150 SmallVectorImpl<SDValue> &Results) {
2151 SDValue Result, Overflow;
2152 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2153 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2154
2155 Results.push_back(Elt: Result);
2156 Results.push_back(Elt: Overflow);
2157}
2158
2159void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2160 SmallVectorImpl<SDValue> &Results) {
2161 SDNode *N = Node;
2162 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2163 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2164 Results.push_back(Elt: Expanded);
2165}
2166
2167void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2168 SmallVectorImpl<SDValue> &Results) {
2169 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
2170 ExpandUINT_TO_FLOAT(Node, Results);
2171 return;
2172 }
2173 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
2174 ExpandFP_TO_UINT(Node, Results);
2175 return;
2176 }
2177
2178 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2179 Node->getOpcode() == ISD::STRICT_FSETCCS) {
2180 ExpandSETCC(Node, Results);
2181 return;
2182 }
2183
2184 UnrollStrictFPOp(Node, Results);
2185}
2186
2187void VectorLegalizer::ExpandREM(SDNode *Node,
2188 SmallVectorImpl<SDValue> &Results) {
2189 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2190 "Expected REM node");
2191
2192 SDValue Result;
2193 if (!TLI.expandREM(Node, Result, DAG))
2194 Result = DAG.UnrollVectorOp(N: Node);
2195 Results.push_back(Elt: Result);
2196}
2197
2198// Try to expand libm nodes into vector math routine calls. Callers provide the
2199// RTLIB::get<OP>(EVT) selector of the node's libcall family, which is used to
2200// look up mappings within RuntimeLibcallsInfo. The only mappings considered are
2201// those where the result and all operands are the same vector type. While
2202// predicated nodes are not supported, we will emit calls to masked routines by
2203// passing in a mask that is true for the lanes computed by the node.
2204bool VectorLegalizer::tryExpandVecMathCall(
2205 SDNode *Node, function_ref<RTLIB::Libcall(EVT)> GetLibcall,
2206 SmallVectorImpl<SDValue> &Results) {
2207 // Chain must be propagated but currently strict fp operations are down
2208 // converted to their none strict counterpart.
2209 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2210
2211 EVT VT = Node->getValueType(ResNo: 0);
2212 LLVMContext &Ctx = *DAG.getContext();
2213 const LibcallLoweringInfo &Libcalls = DAG.getLibcalls();
2214
2215 // Try to widen the vector type when no libcall is available at that width.
2216 EVT CallVT = VT;
2217 RTLIB::LibcallImpl LCImpl =
2218 Libcalls.getVectorLibcallImplOrMasked(Call: GetLibcall(CallVT));
2219 if (LCImpl == RTLIB::Unsupported && VT.getVectorElementCount().isScalar())
2220 return false;
2221 while (LCImpl == RTLIB::Unsupported) {
2222 CallVT = CallVT.getDoubleNumVectorElementsVT(Context&: Ctx);
2223 if (!CallVT.isSimple())
2224 return false;
2225 if (TLI.isTypeLegal(VT: CallVT))
2226 LCImpl = Libcalls.getVectorLibcallImplOrMasked(Call: GetLibcall(CallVT));
2227 }
2228
2229 const RTLIB::RuntimeLibcallsInfo &RTLCI = TLI.getRuntimeLibcallsInfo();
2230
2231 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2232 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2233
2234 SDLoc DL(Node);
2235 TargetLowering::ArgListTy Args;
2236
2237 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2238
2239 // Sanity check just in case function has unexpected parameters.
2240 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2241 EVT::getEVT(FuncTy->getReturnType(), true) == CallVT &&
2242 "mismatch in value type and call signature type");
2243
2244 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2245 Type *ParamTy = FuncTy->getParamType(i: I);
2246
2247 if (HasMaskArg && I == E - 1) {
2248 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2249 cast<VectorType>(ParamTy)->getElementCount() ==
2250 CallVT.getVectorElementCount() &&
2251 "unexpected vector mask type");
2252 EVT MaskVT = EVT::getEVT(Ty: ParamTy, /*HandleUnknown=*/true);
2253 EVT SubMaskVT =
2254 MaskVT.changeVectorElementCount(Context&: Ctx, EC: VT.getVectorElementCount());
2255 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: SubMaskVT, OpVT: VT);
2256 // Only the lanes holding the node's elements need to be active.
2257 if (CallVT != VT)
2258 Mask = DAG.getInsertSubvector(
2259 DL, Vec: DAG.getBoolConstant(V: false, DL, VT: MaskVT, OpVT: CallVT), SubVec: Mask, Idx: 0);
2260 Args.emplace_back(args&: Mask, args&: ParamTy);
2261 } else {
2262 SDValue Op = Node->getOperand(Num: I);
2263 assert(Op.getValueType() == VT && "mismatch in vector types");
2264 if (CallVT != VT) {
2265 unsigned NumConcat =
2266 CallVT.getVectorMinNumElements() / VT.getVectorMinNumElements();
2267 SmallVector<SDValue, 4> Ops(NumConcat, Op);
2268 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: CallVT, Ops);
2269 }
2270 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2271 "mismatch in value type and call argument type");
2272 Args.emplace_back(args&: Op, args&: ParamTy);
2273 }
2274 }
2275
2276 // Emit a call to the vector function.
2277 SDValue Callee =
2278 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2279 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2280
2281 TargetLowering::CallLoweringInfo CLI(DAG);
2282 CLI.setDebugLoc(DL)
2283 .setChain(DAG.getEntryNode())
2284 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2285
2286 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2287 SDValue Result = CallResult.first;
2288 if (CallVT != VT)
2289 Result = DAG.getExtractSubvector(DL, VT, Vec: Result, Idx: 0);
2290 Results.push_back(Elt: Result);
2291 return true;
2292}
2293
2294void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2295 SmallVectorImpl<SDValue> &Results) {
2296 EVT VT = Node->getValueType(ResNo: 0);
2297
2298 // Cannot unroll a scalable vector. Delay error reporting until the final
2299 // operation legalisation phase to maximise the chances of removing the node.
2300 if (VT.isScalableVector())
2301 return;
2302
2303 EVT EltVT = VT.getVectorElementType();
2304 unsigned NumElems = VT.getVectorNumElements();
2305 unsigned NumOpers = Node->getNumOperands();
2306 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2307
2308 EVT TmpEltVT = EltVT;
2309 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2310 Node->getOpcode() == ISD::STRICT_FSETCCS)
2311 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2312 Context&: *DAG.getContext(), VT: TmpEltVT);
2313
2314 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2315 SDValue Chain = Node->getOperand(Num: 0);
2316 SDLoc dl(Node);
2317
2318 SmallVector<SDValue, 32> OpValues;
2319 SmallVector<SDValue, 32> OpChains;
2320 for (unsigned i = 0; i < NumElems; ++i) {
2321 SmallVector<SDValue, 4> Opers;
2322 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2323
2324 // The Chain is the first operand.
2325 Opers.push_back(Elt: Chain);
2326
2327 // Now process the remaining operands.
2328 for (unsigned j = 1; j < NumOpers; ++j) {
2329 SDValue Oper = Node->getOperand(Num: j);
2330 EVT OperVT = Oper.getValueType();
2331
2332 if (OperVT.isVector())
2333 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2334 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2335
2336 Opers.push_back(Elt: Oper);
2337 }
2338
2339 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2340 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2341 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2342
2343 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2344 Node->getOpcode() == ISD::STRICT_FSETCCS)
2345 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2346 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2347 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2348
2349 OpValues.push_back(Elt: ScalarResult);
2350 OpChains.push_back(Elt: ScalarChain);
2351 }
2352
2353 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2354 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2355
2356 Results.push_back(Elt: Result);
2357 Results.push_back(Elt: NewChain);
2358}
2359
2360SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2361 EVT VT = Node->getValueType(ResNo: 0);
2362 unsigned NumElems = VT.getVectorNumElements();
2363 EVT EltVT = VT.getVectorElementType();
2364 SDValue LHS = Node->getOperand(Num: 0);
2365 SDValue RHS = Node->getOperand(Num: 1);
2366 SDValue CC = Node->getOperand(Num: 2);
2367 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2368 SDLoc dl(Node);
2369 SmallVector<SDValue, 8> Ops(NumElems);
2370 for (unsigned i = 0; i < NumElems; ++i) {
2371 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2372 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2373 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2374 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2375 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2376 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2377 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2378 Context&: *DAG.getContext(), VT: TmpEltVT),
2379 N1: LHSElem, N2: RHSElem, N3: CC);
2380 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2381 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2382 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2383 }
2384 return DAG.getBuildVector(VT, DL: dl, Ops);
2385}
2386
2387bool SelectionDAG::LegalizeVectors() {
2388 return VectorLegalizer(*this).Run();
2389}
2390