1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/STLFunctionalExtras.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/Analysis/TargetLibraryInfo.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/SelectionDAG.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetLowering.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/CodeGenTypes/MachineValueType.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Compiler.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <iterator>
48#include <utility>
49
50using namespace llvm;
51
52#define DEBUG_TYPE "legalizevectorops"
53
54namespace {
55
56class VectorLegalizer {
57 SelectionDAG& DAG;
58 const TargetLowering &TLI;
59 bool Changed = false; // Keep track of whether anything changed
60
61 /// For nodes that are of legal width, and that have more than one use, this
62 /// map indicates what regularized operand to use. This allows us to avoid
63 /// legalizing the same thing more than once.
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65
66 /// Adds a node to the translation cache.
67 void AddLegalizedOperand(SDValue From, SDValue To) {
68 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
69 // If someone requests legalization of the new node, return itself.
70 if (From != To)
71 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
72 }
73
74 /// Legalizes the given node.
75 SDValue LegalizeOp(SDValue Op);
76
77 /// Assuming the node is legal, "legalize" the results.
78 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79
80 /// Make sure Results are legal and update the translation cache.
81 SDValue RecursivelyLegalizeResults(SDValue Op,
82 MutableArrayRef<SDValue> Results);
83
84 /// Wrapper to interface LowerOperation with a vector of Results.
85 /// Returns false if the target wants to use default expansion. Otherwise
86 /// returns true. If return is true and the Results are empty, then the
87 /// target wants to keep the input node as is.
88 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89
90 /// Implements unrolling a VSETCC.
91 SDValue UnrollVSETCC(SDNode *Node);
92
93 /// Implement expand-based legalization of vector operations.
94 ///
95 /// This is just a high-level routine to dispatch to specific code paths for
96 /// operations to legalize them.
97 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98
99 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100 /// FP_TO_SINT isn't legal.
101 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102
103 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106
107 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108 SDValue ExpandSEXTINREG(SDNode *Node);
109
110 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111 ///
112 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113 /// type. The contents of the bits in the extended part of each element are
114 /// undef.
115 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116
117 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118 ///
119 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120 /// type, then shifts left and arithmetic shifts right to introduce a sign
121 /// extension.
122 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123
124 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125 ///
126 /// Shuffles the low lanes of the operand into place and blends zeros into
127 /// the remaining lanes, finally bitcasting to the proper type.
128 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129
130 /// Expand bswap of vectors into a shuffle if legal.
131 SDValue ExpandBSWAP(SDNode *Node);
132
133 /// Implement vselect in terms of XOR, AND, OR when blend is not
134 /// supported by the target.
135 SDValue ExpandVSELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
139 SDValue ExpandMaskedBinOp(SDNode *N);
140 SDValue ExpandSELECT(SDNode *Node);
141 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
142 SDValue ExpandStore(SDNode *N);
143 SDValue ExpandFNEG(SDNode *Node);
144 SDValue ExpandFABS(SDNode *Node);
145 SDValue ExpandFCOPYSIGN(SDNode *Node);
146 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 SDValue ExpandBITREVERSE(SDNode *Node);
149 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155
156 bool tryExpandVecMathCall(SDNode *Node,
157 function_ref<RTLIB::Libcall(EVT)> GetLibcall,
158 SmallVectorImpl<SDValue> &Results);
159
160 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 /// Implements vector promotion.
163 ///
164 /// This is essentially just bitcasting the operands to a different type and
165 /// bitcasting the result back to the original type.
166 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
167
168 /// Implements [SU]INT_TO_FP vector promotion.
169 ///
170 /// This is a [zs]ext of the input operand to a larger integer type.
171 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
172
173 /// Implements FP_TO_[SU]INT vector promotion of the result type.
174 ///
175 /// It is promoted to a larger integer type. The result is then
176 /// truncated back to the original type.
177 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
178
179 /// Implements vector setcc operation promotion.
180 ///
181 /// All vector operands are promoted to a vector type with larger element
182 /// type.
183 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
184
185 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187 /// Calculate the reduction using a type of higher precision and round the
188 /// result to match the original type. Setting NonArithmetic signifies the
189 /// rounding of the result does not affect its value.
190 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
191 bool NonArithmetic);
192
193 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
194
195public:
196 VectorLegalizer(SelectionDAG& dag) :
197 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
198
199 /// Begin legalizer the vector operations in the DAG.
200 bool Run();
201};
202
203} // end anonymous namespace
204
205bool VectorLegalizer::Run() {
206 // Before we start legalizing vector nodes, check if there are any vectors.
207 bool HasVectors = false;
208 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
209 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
210 // Check if the values of the nodes contain vectors. We don't need to check
211 // the operands because we are going to check their values at some point.
212 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
213
214 // If we found a vector node we can start the legalization.
215 if (HasVectors)
216 break;
217 }
218
219 // If this basic block has no vectors then no need to legalize vectors.
220 if (!HasVectors)
221 return false;
222
223 // The legalize process is inherently a bottom-up recursive process (users
224 // legalize their uses before themselves). Given infinite stack space, we
225 // could just start legalizing on the root and traverse the whole graph. In
226 // practice however, this causes us to run out of stack space on large basic
227 // blocks. To avoid this problem, compute an ordering of the nodes where each
228 // node is only legalized after all of its operands are legalized.
229 DAG.AssignTopologicalOrder();
230 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
231 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
232 LegalizeOp(Op: SDValue(&*I, 0));
233
234 // Finally, it's possible the root changed. Get the new root.
235 SDValue OldRoot = DAG.getRoot();
236 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
237 DAG.setRoot(LegalizedNodes[OldRoot]);
238
239 LegalizedNodes.clear();
240
241 // Remove dead nodes now.
242 DAG.RemoveDeadNodes();
243
244 return Changed;
245}
246
247SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
248 assert(Op->getNumValues() == Result->getNumValues() &&
249 "Unexpected number of results");
250 // Generic legalization: just pass the operand through.
251 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
252 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
253 return SDValue(Result, Op.getResNo());
254}
255
256SDValue
257VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
258 MutableArrayRef<SDValue> Results) {
259 assert(Results.size() == Op->getNumValues() &&
260 "Unexpected number of results");
261 // Make sure that the generated code is itself legal.
262 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
263 Results[i] = LegalizeOp(Op: Results[i]);
264 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
265 }
266
267 return Results[Op.getResNo()];
268}
269
270SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
271 // Note that LegalizeOp may be reentered even from single-use nodes, which
272 // means that we always must cache transformed nodes.
273 auto I = LegalizedNodes.find(Val: Op);
274 if (I != LegalizedNodes.end()) return I->second;
275
276 // Legalize the operands
277 SmallVector<SDValue, 8> Ops;
278 for (const SDValue &Oper : Op->op_values())
279 Ops.push_back(Elt: LegalizeOp(Op: Oper));
280
281 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
282
283 bool HasVectorValueOrOp =
284 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
285 llvm::any_of(Range: Node->op_values(),
286 P: [](SDValue O) { return O.getValueType().isVector(); });
287 if (!HasVectorValueOrOp)
288 return TranslateLegalizeResults(Op, Result: Node);
289
290 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
291 EVT ValVT;
292 switch (Op.getOpcode()) {
293 default:
294 return TranslateLegalizeResults(Op, Result: Node);
295 case ISD::LOAD: {
296 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
297 ISD::LoadExtType ExtType = LD->getExtensionType();
298 EVT LoadedVT = LD->getMemoryVT();
299 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
300 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
301 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
302 break;
303 }
304 case ISD::STORE: {
305 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
306 EVT StVT = ST->getMemoryVT();
307 MVT ValVT = ST->getValue().getSimpleValueType();
308 if (StVT.isVector() && ST->isTruncatingStore())
309 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
310 AddrSpace: ST->getAddressSpace());
311 break;
312 }
313 case ISD::MERGE_VALUES:
314 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
315 // This operation lies about being legal: when it claims to be legal,
316 // it should actually be expanded.
317 if (Action == TargetLowering::Legal)
318 Action = TargetLowering::Expand;
319 break;
320#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
321 case ISD::STRICT_##DAGN:
322#include "llvm/IR/ConstrainedOps.def"
323 ValVT = Node->getValueType(ResNo: 0);
324 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
325 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
326 ValVT = Node->getOperand(Num: 1).getValueType();
327 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
328 Op.getOpcode() == ISD::STRICT_FSETCCS) {
329 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
330 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 3))->get();
331 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
332 if (Action == TargetLowering::Legal)
333 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
334 } else {
335 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
336 }
337 // If we're asked to expand a strict vector floating-point operation,
338 // by default we're going to simply unroll it. That is usually the
339 // best approach, except in the case where the resulting strict (scalar)
340 // operations would themselves use the fallback mutation to non-strict.
341 // In that specific case, just do the fallback on the vector op.
342 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
343 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
344 TargetLowering::Legal) {
345 EVT EltVT = ValVT.getVectorElementType();
346 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
347 == TargetLowering::Expand &&
348 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
349 == TargetLowering::Legal)
350 Action = TargetLowering::Legal;
351 }
352 break;
353 case ISD::ADD:
354 case ISD::SUB:
355 case ISD::MUL:
356 case ISD::MULHS:
357 case ISD::MULHU:
358 case ISD::SDIV:
359 case ISD::UDIV:
360 case ISD::SREM:
361 case ISD::UREM:
362 case ISD::SDIVREM:
363 case ISD::UDIVREM:
364 case ISD::FADD:
365 case ISD::FSUB:
366 case ISD::FMUL:
367 case ISD::FDIV:
368 case ISD::FREM:
369 case ISD::AND:
370 case ISD::OR:
371 case ISD::XOR:
372 case ISD::SHL:
373 case ISD::SRA:
374 case ISD::SRL:
375 case ISD::FSHL:
376 case ISD::FSHR:
377 case ISD::ROTL:
378 case ISD::ROTR:
379 case ISD::ABS:
380 case ISD::ABS_MIN_POISON:
381 case ISD::ABDS:
382 case ISD::ABDU:
383 case ISD::AVGCEILS:
384 case ISD::AVGCEILU:
385 case ISD::AVGFLOORS:
386 case ISD::AVGFLOORU:
387 case ISD::BSWAP:
388 case ISD::BITREVERSE:
389 case ISD::CTLZ:
390 case ISD::CTTZ:
391 case ISD::CTLZ_ZERO_POISON:
392 case ISD::CTTZ_ZERO_POISON:
393 case ISD::CTPOP:
394 case ISD::CLMUL:
395 case ISD::CLMULH:
396 case ISD::CLMULR:
397 case ISD::SELECT:
398 case ISD::VSELECT:
399 case ISD::SELECT_CC:
400 case ISD::ZERO_EXTEND:
401 case ISD::ANY_EXTEND:
402 case ISD::TRUNCATE:
403 case ISD::SIGN_EXTEND:
404 case ISD::FP_TO_SINT:
405 case ISD::FP_TO_UINT:
406 case ISD::FNEG:
407 case ISD::FABS:
408 case ISD::FMINNUM:
409 case ISD::FMAXNUM:
410 case ISD::FMINNUM_IEEE:
411 case ISD::FMAXNUM_IEEE:
412 case ISD::FMINIMUM:
413 case ISD::FMAXIMUM:
414 case ISD::FMINIMUMNUM:
415 case ISD::FMAXIMUMNUM:
416 case ISD::FCOPYSIGN:
417 case ISD::FSQRT:
418 case ISD::FSIN:
419 case ISD::FCOS:
420 case ISD::FTAN:
421 case ISD::FASIN:
422 case ISD::FACOS:
423 case ISD::FATAN:
424 case ISD::FATAN2:
425 case ISD::FSINH:
426 case ISD::FCOSH:
427 case ISD::FTANH:
428 case ISD::FLDEXP:
429 case ISD::FPOWI:
430 case ISD::FPOW:
431 case ISD::FCBRT:
432 case ISD::FLOG:
433 case ISD::FLOG2:
434 case ISD::FLOG10:
435 case ISD::FEXP:
436 case ISD::FEXP2:
437 case ISD::FEXP10:
438 case ISD::FCEIL:
439 case ISD::FTRUNC:
440 case ISD::FRINT:
441 case ISD::FNEARBYINT:
442 case ISD::FROUND:
443 case ISD::FROUNDEVEN:
444 case ISD::FFLOOR:
445 case ISD::FP_ROUND:
446 case ISD::FP_EXTEND:
447 case ISD::FPTRUNC_ROUND:
448 case ISD::FMA:
449 case ISD::SIGN_EXTEND_INREG:
450 case ISD::ANY_EXTEND_VECTOR_INREG:
451 case ISD::SIGN_EXTEND_VECTOR_INREG:
452 case ISD::ZERO_EXTEND_VECTOR_INREG:
453 case ISD::SMIN:
454 case ISD::SMAX:
455 case ISD::UMIN:
456 case ISD::UMAX:
457 case ISD::SMUL_LOHI:
458 case ISD::UMUL_LOHI:
459 case ISD::SADDO:
460 case ISD::UADDO:
461 case ISD::SSUBO:
462 case ISD::USUBO:
463 case ISD::SMULO:
464 case ISD::UMULO:
465 case ISD::CONVERT_FROM_ARBITRARY_FP:
466 case ISD::CONVERT_TO_ARBITRARY_FP:
467 case ISD::FCANONICALIZE:
468 case ISD::FFREXP:
469 case ISD::FMODF:
470 case ISD::FSINCOS:
471 case ISD::FSINCOSPI:
472 case ISD::SADDSAT:
473 case ISD::UADDSAT:
474 case ISD::SSUBSAT:
475 case ISD::USUBSAT:
476 case ISD::SSHLSAT:
477 case ISD::USHLSAT:
478 case ISD::FP_TO_SINT_SAT:
479 case ISD::FP_TO_UINT_SAT:
480 case ISD::MGATHER:
481 case ISD::VECTOR_COMPRESS:
482 case ISD::SCMP:
483 case ISD::UCMP:
484 case ISD::LOOP_DEPENDENCE_WAR_MASK:
485 case ISD::LOOP_DEPENDENCE_RAW_MASK:
486 case ISD::MASKED_UDIV:
487 case ISD::MASKED_SDIV:
488 case ISD::MASKED_UREM:
489 case ISD::MASKED_SREM:
490 case ISD::VECTOR_MATCH:
491 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
492 break;
493 case ISD::SMULFIX:
494 case ISD::SMULFIXSAT:
495 case ISD::UMULFIX:
496 case ISD::UMULFIXSAT:
497 case ISD::SDIVFIX:
498 case ISD::SDIVFIXSAT:
499 case ISD::UDIVFIX:
500 case ISD::UDIVFIXSAT: {
501 unsigned Scale = Node->getConstantOperandVal(Num: 2);
502 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
503 VT: Node->getValueType(ResNo: 0), Scale);
504 break;
505 }
506 case ISD::LROUND:
507 case ISD::LLROUND:
508 case ISD::LRINT:
509 case ISD::LLRINT:
510 case ISD::SINT_TO_FP:
511 case ISD::UINT_TO_FP:
512 case ISD::VECREDUCE_ADD:
513 case ISD::VECREDUCE_MUL:
514 case ISD::VECREDUCE_AND:
515 case ISD::VECREDUCE_OR:
516 case ISD::VECREDUCE_XOR:
517 case ISD::VECREDUCE_SMAX:
518 case ISD::VECREDUCE_SMIN:
519 case ISD::VECREDUCE_UMAX:
520 case ISD::VECREDUCE_UMIN:
521 case ISD::VECREDUCE_FADD:
522 case ISD::VECREDUCE_FMAX:
523 case ISD::VECREDUCE_FMAXIMUM:
524 case ISD::VECREDUCE_FMIN:
525 case ISD::VECREDUCE_FMINIMUM:
526 case ISD::VECREDUCE_FMAXIMUMNUM:
527 case ISD::VECREDUCE_FMINIMUMNUM:
528 case ISD::VECREDUCE_FMUL:
529 case ISD::CTTZ_ELTS:
530 case ISD::CTTZ_ELTS_ZERO_POISON:
531 case ISD::VECTOR_FIND_LAST_ACTIVE:
532 Action = TLI.getOperationAction(Op: Node->getOpcode(),
533 VT: Node->getOperand(Num: 0).getValueType());
534 break;
535 case ISD::VECREDUCE_SEQ_FADD:
536 case ISD::VECREDUCE_SEQ_FMUL:
537 Action = TLI.getOperationAction(Op: Node->getOpcode(),
538 VT: Node->getOperand(Num: 1).getValueType());
539 break;
540 case ISD::SETCC: {
541 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
542 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
543 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
544 if (Action == TargetLowering::Legal)
545 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
546 break;
547 }
548 case ISD::PARTIAL_REDUCE_UMLA:
549 case ISD::PARTIAL_REDUCE_SMLA:
550 case ISD::PARTIAL_REDUCE_SUMLA:
551 case ISD::PARTIAL_REDUCE_FMLA:
552 Action =
553 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
554 InputVT: Node->getOperand(Num: 1).getValueType());
555 break;
556
557#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
558 case ISD::VPID: { \
559 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
560 : Node->getOperand(LEGALPOS).getValueType(); \
561 /* Defer non-vector results to LegalizeDAG. */ \
562 if (!Node->getValueType(0).isVector() && \
563 Node->getValueType(0) != MVT::Other) { \
564 Action = TargetLowering::Legal; \
565 break; \
566 } \
567 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
568 } break;
569#include "llvm/IR/VPIntrinsics.def"
570 }
571
572 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
573
574 SmallVector<SDValue, 8> ResultVals;
575 switch (Action) {
576 default: llvm_unreachable("This action is not supported yet!");
577 case TargetLowering::Promote:
578 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
579 "This action is not supported yet!");
580 LLVM_DEBUG(dbgs() << "Promoting\n");
581 Promote(Node, Results&: ResultVals);
582 assert(!ResultVals.empty() && "No results for promotion?");
583 break;
584 case TargetLowering::Legal:
585 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
586 break;
587 case TargetLowering::Custom:
588 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
589 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
590 break;
591 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
592 [[fallthrough]];
593 case TargetLowering::Expand:
594 LLVM_DEBUG(dbgs() << "Expanding\n");
595 Expand(Node, Results&: ResultVals);
596 break;
597 }
598
599 if (ResultVals.empty())
600 return TranslateLegalizeResults(Op, Result: Node);
601
602 Changed = true;
603 return RecursivelyLegalizeResults(Op, Results: ResultVals);
604}
605
606// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
607// merge them somehow?
608bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
609 SmallVectorImpl<SDValue> &Results) {
610 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
611
612 if (!Res.getNode())
613 return false;
614
615 if (Res == SDValue(Node, 0))
616 return true;
617
618 // If the original node has one result, take the return value from
619 // LowerOperation as is. It might not be result number 0.
620 if (Node->getNumValues() == 1) {
621 Results.push_back(Elt: Res);
622 return true;
623 }
624
625 // If the original node has multiple results, then the return node should
626 // have the same number of results.
627 assert((Node->getNumValues() == Res->getNumValues()) &&
628 "Lowering returned the wrong number of results!");
629
630 // Places new result values base on N result number.
631 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
632 Results.push_back(Elt: Res.getValue(R: I));
633
634 return true;
635}
636
637void VectorLegalizer::PromoteSETCC(SDNode *Node,
638 SmallVectorImpl<SDValue> &Results) {
639 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
640 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
641
642 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
643
644 SDLoc DL(Node);
645 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
646
647 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
648 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
649 Operands[2] = Node->getOperand(Num: 2);
650
651 EVT ResVT =
652 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
653 SDValue Res =
654 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
655 if (ResVT != Node->getValueType(ResNo: 0))
656 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
657 Results.push_back(Elt: Res);
658}
659
660void VectorLegalizer::PromoteSTRICT(SDNode *Node,
661 SmallVectorImpl<SDValue> &Results) {
662 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
663 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
664
665 assert(VecVT.isFloatingPoint());
666
667 SDLoc DL(Node);
668 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
669 SmallVector<SDValue, 2> Chains;
670
671 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
672 if (Node->getOperand(Num: j).getValueType().isVector() &&
673 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
674 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
675 {
676 // promote the vector operand.
677 SDValue Ext =
678 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
679 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
680 Operands[j] = Ext.getValue(R: 0);
681 Chains.push_back(Elt: Ext.getValue(R: 1));
682 } else
683 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
684
685 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
686
687 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
688
689 SDValue Res =
690 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
691
692 SDValue Round =
693 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
694 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
695 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
696
697 Results.push_back(Elt: Round.getValue(R: 0));
698 Results.push_back(Elt: Round.getValue(R: 1));
699}
700
701void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
702 SmallVectorImpl<SDValue> &Results,
703 bool NonArithmetic) {
704 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
705 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
706 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
707
708 SDLoc DL(Node);
709 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
710 SDValue Rdx =
711 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
712 Flags: Node->getFlags());
713 SDValue Res =
714 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
715 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
716 Results.push_back(Elt: Res);
717}
718
719void VectorLegalizer::PromoteVECTOR_COMPRESS(
720 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
721 SDLoc DL(Node);
722 EVT VT = Node->getValueType(ResNo: 0);
723 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
724 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
725 "Only integer promotion or bitcasts between types is supported");
726
727 SDValue Vec = Node->getOperand(Num: 0);
728 SDValue Mask = Node->getOperand(Num: 1);
729 SDValue Passthru = Node->getOperand(Num: 2);
730 if (VT.isInteger()) {
731 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
732 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
733 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
734 } else {
735 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
736 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
737 }
738
739 SDValue Result =
740 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
741 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
742 : DAG.getBitcast(VT, V: Result);
743 Results.push_back(Elt: Result);
744}
745
746void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
747 // For a few operations there is a specific concept for promotion based on
748 // the operand's type.
749 switch (Node->getOpcode()) {
750 case ISD::SINT_TO_FP:
751 case ISD::UINT_TO_FP:
752 case ISD::STRICT_SINT_TO_FP:
753 case ISD::STRICT_UINT_TO_FP:
754 // "Promote" the operation by extending the operand.
755 PromoteINT_TO_FP(Node, Results);
756 return;
757 case ISD::FP_TO_UINT:
758 case ISD::FP_TO_SINT:
759 case ISD::STRICT_FP_TO_UINT:
760 case ISD::STRICT_FP_TO_SINT:
761 // Promote the operation by extending the operand.
762 PromoteFP_TO_INT(Node, Results);
763 return;
764 case ISD::SETCC:
765 // Promote the operation by extending the operand.
766 PromoteSETCC(Node, Results);
767 return;
768 case ISD::STRICT_FADD:
769 case ISD::STRICT_FSUB:
770 case ISD::STRICT_FMUL:
771 case ISD::STRICT_FDIV:
772 case ISD::STRICT_FSQRT:
773 case ISD::STRICT_FMA:
774 PromoteSTRICT(Node, Results);
775 return;
776 case ISD::VECREDUCE_FADD:
777 case ISD::VECREDUCE_FMUL:
778 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
779 return;
780 case ISD::VECREDUCE_FMAX:
781 case ISD::VECREDUCE_FMAXIMUM:
782 case ISD::VECREDUCE_FMIN:
783 case ISD::VECREDUCE_FMINIMUM:
784 case ISD::VECREDUCE_FMAXIMUMNUM:
785 case ISD::VECREDUCE_FMINIMUMNUM:
786 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
787 return;
788 case ISD::VECTOR_COMPRESS:
789 PromoteVECTOR_COMPRESS(Node, Results);
790 return;
791
792 case ISD::FP_ROUND:
793 case ISD::FP_EXTEND:
794 // These operations are used to do promotion so they can't be promoted
795 // themselves.
796 llvm_unreachable("Don't know how to promote this operation!");
797 }
798
799 // There are currently two cases of vector promotion:
800 // 1) Bitcasting a vector of integers to a different type to a vector of the
801 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
802 // 2) Extending a vector of floats to a vector of the same number of larger
803 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
804 assert(Node->getNumValues() == 1 &&
805 "Can't promote a vector with multiple results!");
806 MVT VT = Node->getSimpleValueType(ResNo: 0);
807 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
808 SDLoc dl(Node);
809 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
810
811 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
812 // Do not promote the mask operand of a VP OP.
813 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
814 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
815 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
816 if (Node->getOperand(Num: j)
817 .getValueType()
818 .getVectorElementType()
819 .isFloatingPoint() &&
820 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
821 Operands[j] = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
822 else
823 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
824 else
825 Operands[j] = Node->getOperand(Num: j);
826 }
827
828 SDValue Res =
829 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
830
831 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
832 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
833 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
834 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
835 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
836 else
837 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
838
839 Results.push_back(Elt: Res);
840}
841
842void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
843 SmallVectorImpl<SDValue> &Results) {
844 // INT_TO_FP operations may require the input operand be promoted even
845 // when the type is otherwise legal.
846 bool IsStrict = Node->isStrictFPOpcode();
847 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
848 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
849 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
850 "Vectors have different number of elements!");
851
852 SDLoc dl(Node);
853 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
854
855 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
856 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
857 ? ISD::ZERO_EXTEND
858 : ISD::SIGN_EXTEND;
859 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
860 if (Node->getOperand(Num: j).getValueType().isVector())
861 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
862 else
863 Operands[j] = Node->getOperand(Num: j);
864 }
865
866 if (IsStrict) {
867 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
868 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
869 Results.push_back(Elt: Res);
870 Results.push_back(Elt: Res.getValue(R: 1));
871 return;
872 }
873
874 SDValue Res =
875 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
876 Results.push_back(Elt: Res);
877}
878
879// For FP_TO_INT we promote the result type to a vector type with wider
880// elements and then truncate the result. This is different from the default
881// PromoteVector which uses bitcast to promote thus assumning that the
882// promoted vector type has the same overall size.
883void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
884 SmallVectorImpl<SDValue> &Results) {
885 MVT VT = Node->getSimpleValueType(ResNo: 0);
886 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
887 bool IsStrict = Node->isStrictFPOpcode();
888 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
889 "Vectors have different number of elements!");
890
891 unsigned NewOpc = Node->getOpcode();
892 // Change FP_TO_UINT to FP_TO_SINT if possible.
893 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
894 if (NewOpc == ISD::FP_TO_UINT &&
895 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
896 NewOpc = ISD::FP_TO_SINT;
897
898 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
899 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
900 NewOpc = ISD::STRICT_FP_TO_SINT;
901
902 SDLoc dl(Node);
903 SDValue Promoted, Chain;
904 if (IsStrict) {
905 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
906 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
907 Chain = Promoted.getValue(R: 1);
908 } else
909 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
910
911 // Assert that the converted value fits in the original type. If it doesn't
912 // (eg: because the value being converted is too big), then the result of the
913 // original operation was undefined anyway, so the assert is still correct.
914 if (Node->getOpcode() == ISD::FP_TO_UINT ||
915 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
916 NewOpc = ISD::AssertZext;
917 else
918 NewOpc = ISD::AssertSext;
919
920 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
921 N2: DAG.getValueType(VT.getScalarType()));
922 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
923 Results.push_back(Elt: Promoted);
924 if (IsStrict)
925 Results.push_back(Elt: Chain);
926}
927
928std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
929 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
930 return TLI.scalarizeVectorLoad(LD, DAG);
931}
932
933SDValue VectorLegalizer::ExpandStore(SDNode *N) {
934 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
935 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
936 return TF;
937}
938
939void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
940 switch (Node->getOpcode()) {
941 case ISD::LOAD: {
942 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
943 Results.push_back(Elt: Tmp.first);
944 Results.push_back(Elt: Tmp.second);
945 return;
946 }
947 case ISD::STORE:
948 Results.push_back(Elt: ExpandStore(N: Node));
949 return;
950 case ISD::MERGE_VALUES:
951 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
952 Results.push_back(Elt: Node->getOperand(Num: i));
953 return;
954 case ISD::SIGN_EXTEND_INREG:
955 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
956 Results.push_back(Elt: Expanded);
957 return;
958 }
959 break;
960 case ISD::ANY_EXTEND_VECTOR_INREG:
961 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
962 return;
963 case ISD::SIGN_EXTEND_VECTOR_INREG:
964 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
965 return;
966 case ISD::ZERO_EXTEND_VECTOR_INREG:
967 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
968 return;
969 case ISD::BSWAP:
970 if (SDValue Expanded = ExpandBSWAP(Node)) {
971 Results.push_back(Elt: Expanded);
972 return;
973 }
974 break;
975 case ISD::VSELECT:
976 if (SDValue Expanded = ExpandVSELECT(Node)) {
977 Results.push_back(Elt: Expanded);
978 return;
979 }
980 break;
981 case ISD::VP_SREM:
982 case ISD::VP_UREM:
983 if (SDValue Expanded = ExpandVP_REM(Node)) {
984 Results.push_back(Elt: Expanded);
985 return;
986 }
987 break;
988 case ISD::SELECT:
989 if (SDValue Expanded = ExpandSELECT(Node)) {
990 Results.push_back(Elt: Expanded);
991 return;
992 }
993 break;
994 case ISD::SELECT_CC: {
995 if (Node->getValueType(ResNo: 0).isScalableVector()) {
996 EVT CondVT = TLI.getSetCCResultType(
997 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
998 SDValue SetCC =
999 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
1000 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
1001 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1002 LHS: Node->getOperand(Num: 2),
1003 RHS: Node->getOperand(Num: 3)));
1004 return;
1005 }
1006 break;
1007 }
1008 case ISD::FP_TO_UINT:
1009 ExpandFP_TO_UINT(Node, Results);
1010 return;
1011 case ISD::UINT_TO_FP:
1012 ExpandUINT_TO_FLOAT(Node, Results);
1013 return;
1014 case ISD::FNEG:
1015 if (SDValue Expanded = ExpandFNEG(Node)) {
1016 Results.push_back(Elt: Expanded);
1017 return;
1018 }
1019 break;
1020 case ISD::FABS:
1021 if (SDValue Expanded = ExpandFABS(Node)) {
1022 Results.push_back(Elt: Expanded);
1023 return;
1024 }
1025 break;
1026 case ISD::FCOPYSIGN:
1027 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1028 Results.push_back(Elt: Expanded);
1029 return;
1030 }
1031 break;
1032 case ISD::FCANONICALIZE: {
1033 // If the scalar element type has a
1034 // Legal/Custom FCANONICALIZE, don't
1035 // mess with the vector, fall back.
1036 EVT VT = Node->getValueType(ResNo: 0);
1037 EVT EltVT = VT.getVectorElementType();
1038 if (!VT.isScalableVector() &&
1039 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1040 TargetLowering::Expand)
1041 break;
1042 // Otherwise canonicalize the whole vector.
1043 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1044 Results.push_back(Elt: Mul);
1045 return;
1046 }
1047 case ISD::FSUB:
1048 ExpandFSUB(Node, Results);
1049 return;
1050 case ISD::SETCC:
1051 ExpandSETCC(Node, Results);
1052 return;
1053 case ISD::ABS:
1054 case ISD::ABS_MIN_POISON:
1055 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1056 Results.push_back(Elt: Expanded);
1057 return;
1058 }
1059 break;
1060 case ISD::ABDS:
1061 case ISD::ABDU:
1062 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1063 Results.push_back(Elt: Expanded);
1064 return;
1065 }
1066 break;
1067 case ISD::AVGCEILS:
1068 case ISD::AVGCEILU:
1069 case ISD::AVGFLOORS:
1070 case ISD::AVGFLOORU:
1071 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1072 Results.push_back(Elt: Expanded);
1073 return;
1074 }
1075 break;
1076 case ISD::BITREVERSE:
1077 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1078 Results.push_back(Elt: Expanded);
1079 return;
1080 }
1081 break;
1082 case ISD::CTPOP:
1083 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1084 Results.push_back(Elt: Expanded);
1085 return;
1086 }
1087 break;
1088 case ISD::CTLZ:
1089 case ISD::CTLZ_ZERO_POISON:
1090 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1091 Results.push_back(Elt: Expanded);
1092 return;
1093 }
1094 break;
1095 case ISD::CTTZ:
1096 case ISD::CTTZ_ZERO_POISON:
1097 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1098 Results.push_back(Elt: Expanded);
1099 return;
1100 }
1101 break;
1102 case ISD::FSHL:
1103 case ISD::FSHR:
1104 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1105 Results.push_back(Elt: Expanded);
1106 return;
1107 }
1108 break;
1109 case ISD::CLMUL:
1110 case ISD::CLMULR:
1111 case ISD::CLMULH:
1112 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1113 Results.push_back(Elt: Expanded);
1114 return;
1115 }
1116 break;
1117 case ISD::PEXT:
1118 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1119 return;
1120 case ISD::PDEP:
1121 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1122 return;
1123 case ISD::ROTL:
1124 case ISD::ROTR:
1125 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1126 Results.push_back(Elt: Expanded);
1127 return;
1128 }
1129 break;
1130 case ISD::FMINNUM:
1131 case ISD::FMAXNUM:
1132 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1133 Results.push_back(Elt: Expanded);
1134 return;
1135 }
1136 break;
1137 case ISD::FMINIMUM:
1138 case ISD::FMAXIMUM:
1139 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1140 return;
1141 case ISD::FMINIMUMNUM:
1142 case ISD::FMAXIMUMNUM:
1143 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1144 return;
1145 case ISD::SMIN:
1146 case ISD::SMAX:
1147 case ISD::UMIN:
1148 case ISD::UMAX:
1149 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1150 Results.push_back(Elt: Expanded);
1151 return;
1152 }
1153 break;
1154 case ISD::UADDO:
1155 case ISD::USUBO:
1156 ExpandUADDSUBO(Node, Results);
1157 return;
1158 case ISD::SADDO:
1159 case ISD::SSUBO:
1160 ExpandSADDSUBO(Node, Results);
1161 return;
1162 case ISD::UMULO:
1163 case ISD::SMULO:
1164 ExpandMULO(Node, Results);
1165 return;
1166 case ISD::MULHS:
1167 case ISD::MULHU:
1168 if (SDValue Expanded = TLI.expandMULH(Node, DAG)) {
1169 Results.push_back(Elt: Expanded);
1170 return;
1171 }
1172 break;
1173 case ISD::USUBSAT:
1174 case ISD::SSUBSAT:
1175 case ISD::UADDSAT:
1176 case ISD::SADDSAT:
1177 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1178 Results.push_back(Elt: Expanded);
1179 return;
1180 }
1181 break;
1182 case ISD::USHLSAT:
1183 case ISD::SSHLSAT:
1184 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1185 Results.push_back(Elt: Expanded);
1186 return;
1187 }
1188 break;
1189 case ISD::FP_TO_SINT_SAT:
1190 case ISD::FP_TO_UINT_SAT:
1191 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1192 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1193 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1194 Results.push_back(Elt: Expanded);
1195 return;
1196 }
1197 }
1198 break;
1199 case ISD::SMULFIX:
1200 case ISD::UMULFIX:
1201 case ISD::SMULFIXSAT:
1202 case ISD::UMULFIXSAT:
1203 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1204 Results.push_back(Elt: Expanded);
1205 return;
1206 }
1207 break;
1208 case ISD::SDIVFIX:
1209 case ISD::UDIVFIX:
1210 ExpandFixedPointDiv(Node, Results);
1211 return;
1212 case ISD::SDIVFIXSAT:
1213 case ISD::UDIVFIXSAT:
1214 break;
1215#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1216 case ISD::STRICT_##DAGN:
1217#include "llvm/IR/ConstrainedOps.def"
1218 ExpandStrictFPOp(Node, Results);
1219 return;
1220 case ISD::VECREDUCE_ADD:
1221 case ISD::VECREDUCE_MUL:
1222 case ISD::VECREDUCE_AND:
1223 case ISD::VECREDUCE_OR:
1224 case ISD::VECREDUCE_XOR:
1225 case ISD::VECREDUCE_SMAX:
1226 case ISD::VECREDUCE_SMIN:
1227 case ISD::VECREDUCE_UMAX:
1228 case ISD::VECREDUCE_UMIN:
1229 case ISD::VECREDUCE_FADD:
1230 case ISD::VECREDUCE_FMUL:
1231 case ISD::VECREDUCE_FMAX:
1232 case ISD::VECREDUCE_FMIN:
1233 case ISD::VECREDUCE_FMAXIMUM:
1234 case ISD::VECREDUCE_FMINIMUM:
1235 case ISD::VECREDUCE_FMAXIMUMNUM:
1236 case ISD::VECREDUCE_FMINIMUMNUM:
1237 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1238 return;
1239 case ISD::PARTIAL_REDUCE_UMLA:
1240 case ISD::PARTIAL_REDUCE_SMLA:
1241 case ISD::PARTIAL_REDUCE_SUMLA:
1242 case ISD::PARTIAL_REDUCE_FMLA:
1243 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1244 return;
1245 case ISD::VECREDUCE_SEQ_FADD:
1246 case ISD::VECREDUCE_SEQ_FMUL:
1247 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1248 return;
1249 case ISD::VECTOR_MATCH:
1250 Results.push_back(Elt: TLI.expandVectorMatch(N: Node, DAG));
1251 return;
1252 case ISD::SREM:
1253 case ISD::UREM:
1254 ExpandREM(Node, Results);
1255 return;
1256 case ISD::VP_MERGE:
1257 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1258 Results.push_back(Elt: Expanded);
1259 return;
1260 }
1261 break;
1262 case ISD::FREM:
1263 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getREM, Results))
1264 return;
1265 break;
1266 case ISD::FSINCOS:
1267 case ISD::FSINCOSPI: {
1268 EVT VT = Node->getValueType(ResNo: 0);
1269 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1270 ? RTLIB::getSINCOS(VT)
1271 : RTLIB::getSINCOSPI(VT);
1272 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1273 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1274 return;
1275
1276 // TODO: Try to see if there's a narrower call available to use before
1277 // scalarizing.
1278 break;
1279 }
1280 case ISD::FPOW:
1281 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getPOW, Results))
1282 return;
1283
1284 // TODO: Try to see if there's a narrower call available to use before
1285 // scalarizing.
1286 break;
1287 case ISD::FCBRT:
1288 if (tryExpandVecMathCall(Node, GetLibcall: RTLIB::getCBRT, Results))
1289 return;
1290
1291 // TODO: Try to see if there's a narrower call available to use before
1292 // scalarizing.
1293 break;
1294 case ISD::FMODF: {
1295 EVT VT = Node->getValueType(ResNo: 0);
1296 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1297 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1298 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1299 /*CallRetResNo=*/0))
1300 return;
1301 break;
1302 }
1303 case ISD::VECTOR_COMPRESS:
1304 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1305 return;
1306 case ISD::CTTZ_ELTS:
1307 case ISD::CTTZ_ELTS_ZERO_POISON:
1308 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1309 return;
1310 case ISD::VECTOR_FIND_LAST_ACTIVE:
1311 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1312 return;
1313 case ISD::SCMP:
1314 case ISD::UCMP:
1315 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1316 return;
1317 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1318 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1319 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1320 return;
1321
1322 case ISD::FADD:
1323 case ISD::FMUL:
1324 case ISD::FMA:
1325 case ISD::FDIV:
1326 case ISD::FCEIL:
1327 case ISD::FFLOOR:
1328 case ISD::FNEARBYINT:
1329 case ISD::FRINT:
1330 case ISD::FROUND:
1331 case ISD::FROUNDEVEN:
1332 case ISD::FTRUNC:
1333 case ISD::FSQRT:
1334 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1335 Results.push_back(Elt: Expanded);
1336 return;
1337 }
1338 break;
1339 case ISD::CONVERT_TO_ARBITRARY_FP:
1340 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1341 Results.push_back(Elt: Expanded);
1342 else
1343 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1344 return;
1345 case ISD::CONVERT_FROM_ARBITRARY_FP:
1346 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1347 Results.push_back(Elt: Expanded);
1348 else
1349 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1350 return;
1351 case ISD::MASKED_UDIV:
1352 case ISD::MASKED_SDIV:
1353 case ISD::MASKED_UREM:
1354 case ISD::MASKED_SREM:
1355 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1356 return;
1357 }
1358
1359 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1360 if (Node->getNumValues() == 1) {
1361 Results.push_back(Elt: Unrolled);
1362 } else {
1363 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1364 "VectorLegalizer Expand returned wrong number of results!");
1365 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1366 Results.push_back(Elt: Unrolled.getValue(R: I));
1367 }
1368}
1369
1370SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1371 // Lower a select instruction where the condition is a scalar and the
1372 // operands are vectors. Lower this select to VSELECT and implement it
1373 // using XOR AND OR. The selector bit is broadcasted.
1374 EVT VT = Node->getValueType(ResNo: 0);
1375 SDLoc DL(Node);
1376
1377 SDValue Mask = Node->getOperand(Num: 0);
1378 SDValue Op1 = Node->getOperand(Num: 1);
1379 SDValue Op2 = Node->getOperand(Num: 2);
1380
1381 assert(VT.isVector() && !Mask.getValueType().isVector()
1382 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1383
1384 // If we can't even use the basic vector operations of
1385 // AND,OR,XOR, we will have to scalarize the op.
1386 // Notice that the operation may be 'promoted' which means that it is
1387 // 'bitcasted' to another type which is handled.
1388 // Also, we need to be able to construct a splat vector using either
1389 // BUILD_VECTOR or SPLAT_VECTOR.
1390 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1391 // BUILD_VECTOR?
1392 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1393 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1394 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1395 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1396 : ISD::SPLAT_VECTOR,
1397 VT) == TargetLowering::Expand)
1398 return SDValue();
1399
1400 // Generate a mask operand.
1401 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1402
1403 // What is the size of each element in the vector mask.
1404 EVT BitTy = MaskTy.getScalarType();
1405
1406 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1407 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1408
1409 // Broadcast the mask so that the entire vector is all one or all zero.
1410 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1411
1412 // Bitcast the operands to be the same type as the mask.
1413 // This is needed when we select between FP types because
1414 // the mask is a vector of integers.
1415 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1416 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1417
1418 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1419
1420 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1421 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1422 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1423 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1424}
1425
1426SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1427 EVT VT = Node->getValueType(ResNo: 0);
1428
1429 // Make sure that the SRA and SHL instructions are available.
1430 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1431 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1432 return SDValue();
1433
1434 SDLoc DL(Node);
1435 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1436
1437 unsigned BW = VT.getScalarSizeInBits();
1438 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1439 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1440
1441 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1442 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1443}
1444
1445// Generically expand a vector anyext in register to a shuffle of the relevant
1446// lanes into the appropriate locations, with other lanes left undef.
1447SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1448 SDLoc DL(Node);
1449 EVT VT = Node->getValueType(ResNo: 0);
1450 int NumElements = VT.getVectorNumElements();
1451 SDValue Src = Node->getOperand(Num: 0);
1452 EVT SrcVT = Src.getValueType();
1453 int NumSrcElements = SrcVT.getVectorNumElements();
1454
1455 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1456 // into a larger vector type.
1457 if (SrcVT.bitsLE(VT)) {
1458 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1459 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1460 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1461 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1462 NumElements: NumSrcElements);
1463 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1464 }
1465
1466 // Build a base mask of undef shuffles.
1467 SmallVector<int, 16> ShuffleMask;
1468 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1469
1470 // Place the extended lanes into the correct locations.
1471 int ExtLaneScale = NumSrcElements / NumElements;
1472 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1473 for (int i = 0; i < NumElements; ++i)
1474 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1475
1476 return DAG.getNode(
1477 Opcode: ISD::BITCAST, DL, VT,
1478 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1479}
1480
1481SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1482 SDLoc DL(Node);
1483 EVT VT = Node->getValueType(ResNo: 0);
1484 SDValue Src = Node->getOperand(Num: 0);
1485 EVT SrcVT = Src.getValueType();
1486
1487 // First build an any-extend node which can be legalized above when we
1488 // recurse through it.
1489 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1490
1491 // Now we need sign extend. This will be exanded to shifts if it isn't
1492 // supported.
1493 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1494 NumElements: VT.getVectorNumElements());
1495 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1496 N2: DAG.getValueType(ExtVT));
1497}
1498
1499// Generically expand a vector zext in register to a shuffle of the relevant
1500// lanes into the appropriate locations, a blend of zero into the high bits,
1501// and a bitcast to the wider element type.
1502SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1503 SDLoc DL(Node);
1504 EVT VT = Node->getValueType(ResNo: 0);
1505 int NumElements = VT.getVectorNumElements();
1506 SDValue Src = Node->getOperand(Num: 0);
1507 EVT SrcVT = Src.getValueType();
1508 int NumSrcElements = SrcVT.getVectorNumElements();
1509
1510 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1511 // into a larger vector type.
1512 if (SrcVT.bitsLE(VT)) {
1513 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1514 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1515 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1516 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1517 NumElements: NumSrcElements);
1518 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1519 }
1520
1521 // Build up a zero vector to blend into this one.
1522 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1523
1524 // Shuffle the incoming lanes into the correct position, and pull all other
1525 // lanes from the zero vector.
1526 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1527
1528 int ExtLaneScale = NumSrcElements / NumElements;
1529 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1530 for (int i = 0; i < NumElements; ++i)
1531 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1532
1533 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1534 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1535}
1536
1537static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1538 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1539 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1540 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1541 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1542}
1543
1544SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1545 EVT VT = Node->getValueType(ResNo: 0);
1546
1547 // Scalable vectors can't use shuffle expansion.
1548 if (VT.isScalableVector())
1549 return TLI.expandBSWAP(N: Node, DAG);
1550
1551 // Generate a byte wise shuffle mask for the BSWAP.
1552 SmallVector<int, 16> ShuffleMask;
1553 createBSWAPShuffleMask(VT, ShuffleMask);
1554 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1555
1556 // Only emit a shuffle if the mask is legal.
1557 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1558 SDLoc DL(Node);
1559 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1560 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1561 Mask: ShuffleMask);
1562 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1563 }
1564
1565 // If we have the appropriate vector bit operations, it is better to use them
1566 // than unrolling and expanding each component.
1567 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1568 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1569 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1570 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1571 return TLI.expandBSWAP(N: Node, DAG);
1572
1573 // Otherwise let the caller unroll.
1574 return SDValue();
1575}
1576
1577SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1578 EVT VT = Node->getValueType(ResNo: 0);
1579
1580 // We can't unroll or use shuffles for scalable vectors.
1581 if (VT.isScalableVector())
1582 return TLI.expandBITREVERSE(N: Node, DAG);
1583
1584 // If we have the scalar operation, it's probably cheaper to unroll it.
1585 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1586 return SDValue();
1587
1588 // If the vector element width is a whole number of bytes, test if its legal
1589 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1590 // vector. This greatly reduces the number of bit shifts necessary.
1591 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1592 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1593 SmallVector<int, 16> BSWAPMask;
1594 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1595
1596 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1597 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1598 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1599 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1600 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1601 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1602 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1603 SDLoc DL(Node);
1604 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1605 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1606 Mask: BSWAPMask);
1607 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1608 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1609 return Op;
1610 }
1611 }
1612
1613 // If we have the appropriate vector bit operations, it is better to use them
1614 // than unrolling and expanding each component.
1615 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1616 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1617 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1618 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1619 return TLI.expandBITREVERSE(N: Node, DAG);
1620
1621 // Otherwise unroll.
1622 return SDValue();
1623}
1624
1625SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1626 // Implement VSELECT in terms of XOR, AND, OR
1627 // on platforms which do not support blend natively.
1628 SDLoc DL(Node);
1629
1630 SDValue Mask = Node->getOperand(Num: 0);
1631 SDValue Op1 = Node->getOperand(Num: 1);
1632 SDValue Op2 = Node->getOperand(Num: 2);
1633
1634 EVT VT = Mask.getValueType();
1635
1636 // If we can't even use the basic vector operations of
1637 // AND,OR,XOR, we will have to scalarize the op.
1638 // Notice that the operation may be 'promoted' which means that it is
1639 // 'bitcasted' to another type which is handled.
1640 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1641 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1642 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1643 return SDValue();
1644
1645 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1646 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1647 // vector constant to mask with.
1648 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1649 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1650 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1651 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1652 Op1.getValueType().getVectorElementType() == MVT::i1))
1653 return SDValue();
1654
1655 // If the mask and the type are different sizes, unroll the vector op. This
1656 // can occur when getSetCCResultType returns something that is different in
1657 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1658 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1659 return SDValue();
1660
1661 // Bitcast the operands to be the same type as the mask.
1662 // This is needed when we select between FP types because
1663 // the mask is a vector of integers.
1664 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1665 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1666
1667 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1668
1669 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1670 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1671 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1672 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1673}
1674
1675SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1676 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1677 // indices less than the EVL/pivot are true. Combine that with the original
1678 // mask for a full-length mask. Use a full-length VSELECT to select between
1679 // the true and false values.
1680 SDLoc DL(Node);
1681
1682 SDValue Mask = Node->getOperand(Num: 0);
1683 SDValue Op1 = Node->getOperand(Num: 1);
1684 SDValue Op2 = Node->getOperand(Num: 2);
1685 SDValue EVL = Node->getOperand(Num: 3);
1686
1687 EVT MaskVT = Mask.getValueType();
1688 bool IsFixedLen = MaskVT.isFixedLengthVector();
1689
1690 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1691 EC: MaskVT.getVectorElementCount());
1692
1693 // If we can't construct the EVL mask efficiently, it's better to unroll.
1694 if ((IsFixedLen &&
1695 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1696 (!IsFixedLen &&
1697 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1698 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1699 return SDValue();
1700
1701 // If using a SETCC would result in a different type than the mask type,
1702 // unroll.
1703 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1704 VT: EVLVecVT) != MaskVT)
1705 return SDValue();
1706
1707 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1708 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1709 SDValue EVLMask =
1710 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1711
1712 SDValue FullMask = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: Mask, N2: EVLMask);
1713 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1714}
1715
1716SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1717 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, MUL, SUB.
1718 EVT VT = Node->getValueType(ResNo: 0);
1719
1720 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1721
1722 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1723 !TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT) ||
1724 !TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT))
1725 return SDValue();
1726
1727 SDLoc DL(Node);
1728
1729 SDValue Dividend = Node->getOperand(Num: 0);
1730 SDValue Divisor = Node->getOperand(Num: 1);
1731 SDValue Mask = Node->getOperand(Num: 2);
1732 SDValue EVL = Node->getOperand(Num: 3);
1733
1734 // X % Y -> X-X/Y*Y
1735 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1736 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Divisor, N2: Div);
1737 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Dividend, N2: Mul);
1738}
1739
1740SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1741 return TLI.expandLoopDependenceMask(N, DAG);
1742}
1743
1744SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1745 // Masked bin ops don't have undefined behaviour when dividing by zero
1746 // on disabled lanes and produce poison instead. Replace the divisor on the
1747 // disabled lanes with 1 to avoid division by zero or overflow.
1748 SDLoc dl(N);
1749 EVT VT = N->getValueType(ResNo: 0);
1750 SDValue SafeDivisor = DAG.getSelect(
1751 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1752 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1753 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1754}
1755
1756void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1757 SmallVectorImpl<SDValue> &Results) {
1758 // Attempt to expand using TargetLowering.
1759 SDValue Result, Chain;
1760 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1761 Results.push_back(Elt: Result);
1762 if (Node->isStrictFPOpcode())
1763 Results.push_back(Elt: Chain);
1764 return;
1765 }
1766
1767 // Otherwise go ahead and unroll.
1768 if (Node->isStrictFPOpcode()) {
1769 UnrollStrictFPOp(Node, Results);
1770 return;
1771 }
1772
1773 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1774}
1775
1776void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1777 SmallVectorImpl<SDValue> &Results) {
1778 bool IsStrict = Node->isStrictFPOpcode();
1779 unsigned OpNo = IsStrict ? 1 : 0;
1780 SDValue Src = Node->getOperand(Num: OpNo);
1781 EVT SrcVT = Src.getValueType();
1782 EVT DstVT = Node->getValueType(ResNo: 0);
1783 SDLoc DL(Node);
1784
1785 // Attempt to expand using TargetLowering.
1786 SDValue Result;
1787 SDValue Chain;
1788 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1789 Results.push_back(Elt: Result);
1790 if (IsStrict)
1791 Results.push_back(Elt: Chain);
1792 return;
1793 }
1794
1795 // Make sure that the SINT_TO_FP and SRL instructions are available.
1796 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1797 TargetLowering::Expand) ||
1798 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1799 TargetLowering::Expand)) ||
1800 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1801 if (IsStrict) {
1802 UnrollStrictFPOp(Node, Results);
1803 return;
1804 }
1805
1806 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1807 return;
1808 }
1809
1810 unsigned BW = SrcVT.getScalarSizeInBits();
1811 assert((BW == 64 || BW == 32) &&
1812 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1813
1814 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
1815 // UINT_TO_FP with a larger float and round to the smaller type
1816 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
1817 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
1818 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
1819 SDValue UIToFP;
1820 SDValue Result;
1821 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
1822 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
1823 if (IsStrict) {
1824 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
1825 Ops: {Node->getOperand(Num: 0), Src});
1826 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
1827 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
1828 Results.push_back(Elt: Result);
1829 Results.push_back(Elt: Result.getValue(R: 1));
1830 } else {
1831 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
1832 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
1833 Results.push_back(Elt: Result);
1834 }
1835
1836 return;
1837 }
1838
1839 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
1840
1841 // Constants to clear the upper part of the word.
1842 // Notice that we can also use SHL+SHR, but using a constant is slightly
1843 // faster on x86.
1844 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1845 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
1846
1847 // Two to the power of half-word-size.
1848 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
1849
1850 // Clear upper part of LO, lower HI
1851 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
1852 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
1853
1854 if (IsStrict) {
1855 // Convert hi and lo to floats
1856 // Convert the hi part back to the upper values
1857 // TODO: Can any fast-math-flags be set on these nodes?
1858 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1859 Ops: {Node->getOperand(Num: 0), HI});
1860 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
1861 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
1862 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1863 Ops: {Node->getOperand(Num: 0), LO});
1864
1865 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
1866 N2: fLO.getValue(R: 1));
1867
1868 // Add the two halves
1869 SDValue Result =
1870 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
1871
1872 Results.push_back(Elt: Result);
1873 Results.push_back(Elt: Result.getValue(R: 1));
1874 return;
1875 }
1876
1877 // Convert hi and lo to floats
1878 // Convert the hi part back to the upper values
1879 // TODO: Can any fast-math-flags be set on these nodes?
1880 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
1881 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
1882 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
1883
1884 // Add the two halves
1885 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
1886}
1887
1888SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1889 EVT VT = Node->getValueType(ResNo: 0);
1890 EVT IntVT = VT.changeVectorElementTypeToInteger();
1891
1892 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
1893 return SDValue();
1894
1895 // Heuristic check to determine whether vector should be expanded to integer
1896 // operations or unrolled to scalar operations.
1897 // 1. Scalable vector is never unrolled.
1898 // 2. Fixed vector is unrolled if one of followings is true:
1899 // a. Vector only has 1 element and target knows how to handle scalar
1900 // FNEG (either legal or custom expand or promote).
1901 // b. Vector has more than 1 element and target supports scalar
1902 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
1903 // FIXME: Scalar construction instruction count varies in every architecture,
1904 // here we assume 1 instruction for now.
1905 if (VT.isFixedLengthVector()) {
1906 EVT EltVT = VT.getVectorElementType();
1907 unsigned NumElts = VT.getVectorNumElements();
1908 if ((NumElts == 1 &&
1909 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
1910 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
1911 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1912 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1913 return SDValue();
1914 }
1915
1916 SDLoc DL(Node);
1917 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1918 SDValue SignMask = DAG.getConstant(
1919 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1920 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
1921 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
1922}
1923
1924SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
1925 EVT VT = Node->getValueType(ResNo: 0);
1926 EVT IntVT = VT.changeVectorElementTypeToInteger();
1927
1928 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
1929 return SDValue();
1930
1931 // Heuristic check to determine whether vector should be expanded to integer
1932 // operations or unrolled to scalar operations.
1933 // 1. Scalable vector is never unrolled.
1934 // 2. Fixed vector is unrolled if one of followings is true:
1935 // a. Vector only has 1 element and target knows how to handle scalar
1936 // FABS(either legal or custom expand or promote).
1937 // b. Vector has more than 1 element and target supports scalar
1938 // FABS natively and vector length <= 2(1 AND + 1 CONST).
1939 // FIXME: Scalar construction instruction count varies in every architecture,
1940 // here we assume 1 instruction for now.
1941 if (VT.isFixedLengthVector()) {
1942 EVT EltVT = VT.getVectorElementType();
1943 unsigned NumElts = VT.getVectorNumElements();
1944 if ((NumElts == 1 &&
1945 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
1946 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
1947 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1948 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1949 return SDValue();
1950 }
1951
1952 SDLoc DL(Node);
1953 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1954 SDValue ClearSignMask = DAG.getConstant(
1955 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1956 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
1957 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
1958}
1959
1960SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
1961 EVT VT = Node->getValueType(ResNo: 0);
1962 EVT IntVT = VT.changeVectorElementTypeToInteger();
1963
1964 if (VT != Node->getOperand(Num: 1).getValueType() ||
1965 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
1966 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
1967 return SDValue();
1968
1969 // Heuristic check to determine whether vector should be expanded to integer
1970 // operations or unrolled to scalar operations.
1971 // 1. Scalable vector is never unrolled.
1972 // 2. Fixed vector is unrolled if one of followings is true:
1973 // a. Vector only has 1 element and target knows how to handle scalar
1974 // FCOPYSIGN(either legal or custom expand or promote).
1975 // b. Vector has more than 1 element and target supports scalar
1976 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
1977 // FIXME: Scalar construction instruction count varies in every architecture,
1978 // here we assume 1 instruction for now.
1979 if (VT.isFixedLengthVector()) {
1980 EVT EltVT = VT.getVectorElementType();
1981 unsigned NumElts = VT.getVectorNumElements();
1982 if ((NumElts == 1 &&
1983 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
1984 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
1985 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1986 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1987 return SDValue();
1988 }
1989
1990 SDLoc DL(Node);
1991 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1992 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
1993
1994 SDValue SignMask = DAG.getConstant(
1995 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1996 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
1997
1998 SDValue ClearSignMask = DAG.getConstant(
1999 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2000 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
2001
2002 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2003 Flags: SDNodeFlags::Disjoint);
2004
2005 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2006}
2007
2008void VectorLegalizer::ExpandFSUB(SDNode *Node,
2009 SmallVectorImpl<SDValue> &Results) {
2010 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2011 // we can defer this to operation legalization where it will be lowered as
2012 // a+(-b).
2013 EVT VT = Node->getValueType(ResNo: 0);
2014 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2015 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2016 return; // Defer to LegalizeDAG
2017
2018 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2019 Results.push_back(Elt: Expanded);
2020 return;
2021 }
2022
2023 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2024 Results.push_back(Elt: Tmp);
2025}
2026
2027void VectorLegalizer::ExpandSETCC(SDNode *Node,
2028 SmallVectorImpl<SDValue> &Results) {
2029 bool NeedInvert = false;
2030 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2031 Node->getOpcode() == ISD::STRICT_FSETCCS;
2032 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2033 unsigned Offset = IsStrict ? 1 : 0;
2034
2035 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2036 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2037 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2038 SDValue CC = Node->getOperand(Num: 2 + Offset);
2039
2040 MVT OpVT = LHS.getSimpleValueType();
2041 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2042
2043 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2044 if (IsStrict) {
2045 UnrollStrictFPOp(Node, Results);
2046 return;
2047 }
2048 Results.push_back(Elt: UnrollVSETCC(Node));
2049 return;
2050 }
2051
2052 SDLoc dl(Node);
2053 bool Legalized =
2054 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC,
2055 NeedInvert, dl, Chain, IsSignaling);
2056
2057 if (Legalized) {
2058 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2059 // condition code, create a new SETCC node.
2060 if (CC.getNode()) {
2061 if (IsStrict) {
2062 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2063 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2064 Chain = LHS.getValue(R: 1);
2065 } else {
2066 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2067 Flags: Node->getFlags());
2068 }
2069 }
2070
2071 // If we expanded the SETCC by inverting the condition code, then wrap
2072 // the existing SETCC in a NOT to restore the intended condition.
2073 if (NeedInvert)
2074 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2075 } else {
2076 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2077
2078 // Otherwise, SETCC for the given comparison type must be completely
2079 // illegal; expand it into a SELECT_CC.
2080 EVT VT = Node->getValueType(ResNo: 0);
2081 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2082 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2083 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2084 N5: CC, Flags: Node->getFlags());
2085 }
2086
2087 Results.push_back(Elt: LHS);
2088 if (IsStrict)
2089 Results.push_back(Elt: Chain);
2090}
2091
2092void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2093 SmallVectorImpl<SDValue> &Results) {
2094 SDValue Result, Overflow;
2095 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2096 Results.push_back(Elt: Result);
2097 Results.push_back(Elt: Overflow);
2098}
2099
2100void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2101 SmallVectorImpl<SDValue> &Results) {
2102 SDValue Result, Overflow;
2103 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2104 Results.push_back(Elt: Result);
2105 Results.push_back(Elt: Overflow);
2106}
2107
2108void VectorLegalizer::ExpandMULO(SDNode *Node,
2109 SmallVectorImpl<SDValue> &Results) {
2110 SDValue Result, Overflow;
2111 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2112 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2113
2114 Results.push_back(Elt: Result);
2115 Results.push_back(Elt: Overflow);
2116}
2117
2118void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2119 SmallVectorImpl<SDValue> &Results) {
2120 SDNode *N = Node;
2121 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2122 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2123 Results.push_back(Elt: Expanded);
2124}
2125
2126void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2127 SmallVectorImpl<SDValue> &Results) {
2128 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
2129 ExpandUINT_TO_FLOAT(Node, Results);
2130 return;
2131 }
2132 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
2133 ExpandFP_TO_UINT(Node, Results);
2134 return;
2135 }
2136
2137 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2138 Node->getOpcode() == ISD::STRICT_FSETCCS) {
2139 ExpandSETCC(Node, Results);
2140 return;
2141 }
2142
2143 UnrollStrictFPOp(Node, Results);
2144}
2145
2146void VectorLegalizer::ExpandREM(SDNode *Node,
2147 SmallVectorImpl<SDValue> &Results) {
2148 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2149 "Expected REM node");
2150
2151 SDValue Result;
2152 if (!TLI.expandREM(Node, Result, DAG))
2153 Result = DAG.UnrollVectorOp(N: Node);
2154 Results.push_back(Elt: Result);
2155}
2156
2157// Try to expand libm nodes into vector math routine calls. Callers provide the
2158// RTLIB::get<OP>(EVT) selector of the node's libcall family, which is used to
2159// look up mappings within RuntimeLibcallsInfo. The only mappings considered are
2160// those where the result and all operands are the same vector type. While
2161// predicated nodes are not supported, we will emit calls to masked routines by
2162// passing in a mask that is true for the lanes computed by the node.
2163bool VectorLegalizer::tryExpandVecMathCall(
2164 SDNode *Node, function_ref<RTLIB::Libcall(EVT)> GetLibcall,
2165 SmallVectorImpl<SDValue> &Results) {
2166 // Chain must be propagated but currently strict fp operations are down
2167 // converted to their none strict counterpart.
2168 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2169
2170 EVT VT = Node->getValueType(ResNo: 0);
2171 LLVMContext &Ctx = *DAG.getContext();
2172 const LibcallLoweringInfo &Libcalls = DAG.getLibcalls();
2173
2174 // Try to widen the vector type when no libcall is available at that width.
2175 EVT CallVT = VT;
2176 RTLIB::LibcallImpl LCImpl = Libcalls.getLibcallImpl(Call: GetLibcall(CallVT));
2177 if (LCImpl == RTLIB::Unsupported && VT.getVectorElementCount().isScalar())
2178 return false;
2179 while (LCImpl == RTLIB::Unsupported) {
2180 CallVT = CallVT.getDoubleNumVectorElementsVT(Context&: Ctx);
2181 if (!CallVT.isSimple())
2182 return false;
2183 if (TLI.isTypeLegal(VT: CallVT))
2184 LCImpl = Libcalls.getLibcallImpl(Call: GetLibcall(CallVT));
2185 }
2186
2187 const RTLIB::RuntimeLibcallsInfo &RTLCI = TLI.getRuntimeLibcallsInfo();
2188
2189 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2190 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2191
2192 SDLoc DL(Node);
2193 TargetLowering::ArgListTy Args;
2194
2195 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2196
2197 // Sanity check just in case function has unexpected parameters.
2198 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2199 EVT::getEVT(FuncTy->getReturnType(), true) == CallVT &&
2200 "mismatch in value type and call signature type");
2201
2202 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2203 Type *ParamTy = FuncTy->getParamType(i: I);
2204
2205 if (HasMaskArg && I == E - 1) {
2206 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2207 cast<VectorType>(ParamTy)->getElementCount() ==
2208 CallVT.getVectorElementCount() &&
2209 "unexpected vector mask type");
2210 EVT MaskVT = EVT::getEVT(Ty: ParamTy, /*HandleUnknown=*/true);
2211 EVT SubMaskVT =
2212 MaskVT.changeVectorElementCount(Context&: Ctx, EC: VT.getVectorElementCount());
2213 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: SubMaskVT, OpVT: VT);
2214 // Only the lanes holding the node's elements need to be active.
2215 if (CallVT != VT)
2216 Mask = DAG.getInsertSubvector(
2217 DL, Vec: DAG.getBoolConstant(V: false, DL, VT: MaskVT, OpVT: CallVT), SubVec: Mask, Idx: 0);
2218 Args.emplace_back(args&: Mask, args&: ParamTy);
2219 } else {
2220 SDValue Op = Node->getOperand(Num: I);
2221 assert(Op.getValueType() == VT && "mismatch in vector types");
2222 if (CallVT != VT) {
2223 unsigned NumConcat =
2224 CallVT.getVectorMinNumElements() / VT.getVectorMinNumElements();
2225 SmallVector<SDValue, 4> Ops(NumConcat, Op);
2226 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: CallVT, Ops);
2227 }
2228 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2229 "mismatch in value type and call argument type");
2230 Args.emplace_back(args&: Op, args&: ParamTy);
2231 }
2232 }
2233
2234 // Emit a call to the vector function.
2235 SDValue Callee =
2236 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2237 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2238
2239 TargetLowering::CallLoweringInfo CLI(DAG);
2240 CLI.setDebugLoc(DL)
2241 .setChain(DAG.getEntryNode())
2242 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2243
2244 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2245 SDValue Result = CallResult.first;
2246 if (CallVT != VT)
2247 Result = DAG.getExtractSubvector(DL, VT, Vec: Result, Idx: 0);
2248 Results.push_back(Elt: Result);
2249 return true;
2250}
2251
2252void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2253 SmallVectorImpl<SDValue> &Results) {
2254 EVT VT = Node->getValueType(ResNo: 0);
2255 EVT EltVT = VT.getVectorElementType();
2256 unsigned NumElems = VT.getVectorNumElements();
2257 unsigned NumOpers = Node->getNumOperands();
2258 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2259
2260 EVT TmpEltVT = EltVT;
2261 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2262 Node->getOpcode() == ISD::STRICT_FSETCCS)
2263 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2264 Context&: *DAG.getContext(), VT: TmpEltVT);
2265
2266 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2267 SDValue Chain = Node->getOperand(Num: 0);
2268 SDLoc dl(Node);
2269
2270 SmallVector<SDValue, 32> OpValues;
2271 SmallVector<SDValue, 32> OpChains;
2272 for (unsigned i = 0; i < NumElems; ++i) {
2273 SmallVector<SDValue, 4> Opers;
2274 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2275
2276 // The Chain is the first operand.
2277 Opers.push_back(Elt: Chain);
2278
2279 // Now process the remaining operands.
2280 for (unsigned j = 1; j < NumOpers; ++j) {
2281 SDValue Oper = Node->getOperand(Num: j);
2282 EVT OperVT = Oper.getValueType();
2283
2284 if (OperVT.isVector())
2285 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2286 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2287
2288 Opers.push_back(Elt: Oper);
2289 }
2290
2291 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2292 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2293 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2294
2295 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2296 Node->getOpcode() == ISD::STRICT_FSETCCS)
2297 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2298 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2299 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2300
2301 OpValues.push_back(Elt: ScalarResult);
2302 OpChains.push_back(Elt: ScalarChain);
2303 }
2304
2305 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2306 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2307
2308 Results.push_back(Elt: Result);
2309 Results.push_back(Elt: NewChain);
2310}
2311
2312SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2313 EVT VT = Node->getValueType(ResNo: 0);
2314 unsigned NumElems = VT.getVectorNumElements();
2315 EVT EltVT = VT.getVectorElementType();
2316 SDValue LHS = Node->getOperand(Num: 0);
2317 SDValue RHS = Node->getOperand(Num: 1);
2318 SDValue CC = Node->getOperand(Num: 2);
2319 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2320 SDLoc dl(Node);
2321 SmallVector<SDValue, 8> Ops(NumElems);
2322 for (unsigned i = 0; i < NumElems; ++i) {
2323 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2324 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2325 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2326 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2327 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2328 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2329 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2330 Context&: *DAG.getContext(), VT: TmpEltVT),
2331 N1: LHSElem, N2: RHSElem, N3: CC);
2332 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2333 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2334 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2335 }
2336 return DAG.getBuildVector(VT, DL: dl, Ops);
2337}
2338
2339bool SelectionDAG::LegalizeVectors() {
2340 return VectorLegalizer(*this).Run();
2341}
2342