1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/Analysis/TargetLibraryInfo.h"
32#include "llvm/Analysis/VectorUtils.h"
33#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/SelectionDAG.h"
35#include "llvm/CodeGen/SelectionDAGNodes.h"
36#include "llvm/CodeGen/TargetLowering.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/CodeGenTypes/MachineValueType.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/Compiler.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include <cassert>
45#include <cstdint>
46#include <iterator>
47#include <utility>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "legalizevectorops"
52
53namespace {
54
55class VectorLegalizer {
56 SelectionDAG& DAG;
57 const TargetLowering &TLI;
58 bool Changed = false; // Keep track of whether anything changed
59
60 /// For nodes that are of legal width, and that have more than one use, this
61 /// map indicates what regularized operand to use. This allows us to avoid
62 /// legalizing the same thing more than once.
63 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
64
65 /// Adds a node to the translation cache.
66 void AddLegalizedOperand(SDValue From, SDValue To) {
67 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
68 // If someone requests legalization of the new node, return itself.
69 if (From != To)
70 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
71 }
72
73 /// Legalizes the given node.
74 SDValue LegalizeOp(SDValue Op);
75
76 /// Assuming the node is legal, "legalize" the results.
77 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
78
79 /// Make sure Results are legal and update the translation cache.
80 SDValue RecursivelyLegalizeResults(SDValue Op,
81 MutableArrayRef<SDValue> Results);
82
83 /// Wrapper to interface LowerOperation with a vector of Results.
84 /// Returns false if the target wants to use default expansion. Otherwise
85 /// returns true. If return is true and the Results are empty, then the
86 /// target wants to keep the input node as is.
87 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
88
89 /// Implements unrolling a VSETCC.
90 SDValue UnrollVSETCC(SDNode *Node);
91
92 /// Implement expand-based legalization of vector operations.
93 ///
94 /// This is just a high-level routine to dispatch to specific code paths for
95 /// operations to legalize them.
96 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
97
98 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
99 /// FP_TO_SINT isn't legal.
100 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
101
102 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
103 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
104 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
105
106 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
107 SDValue ExpandSEXTINREG(SDNode *Node);
108
109 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
110 ///
111 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
112 /// type. The contents of the bits in the extended part of each element are
113 /// undef.
114 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
115
116 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
117 ///
118 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
119 /// type, then shifts left and arithmetic shifts right to introduce a sign
120 /// extension.
121 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
122
123 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
124 ///
125 /// Shuffles the low lanes of the operand into place and blends zeros into
126 /// the remaining lanes, finally bitcasting to the proper type.
127 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
128
129 /// Expand bswap of vectors into a shuffle if legal.
130 SDValue ExpandBSWAP(SDNode *Node);
131
132 /// Implement vselect in terms of XOR, AND, OR when blend is not
133 /// supported by the target.
134 SDValue ExpandVSELECT(SDNode *Node);
135 SDValue ExpandVP_MERGE(SDNode *Node);
136 SDValue ExpandVP_REM(SDNode *Node);
137 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
138 SDValue ExpandMaskedBinOp(SDNode *N);
139 SDValue ExpandSELECT(SDNode *Node);
140 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
141 SDValue ExpandStore(SDNode *N);
142 SDValue ExpandFNEG(SDNode *Node);
143 SDValue ExpandFABS(SDNode *Node);
144 SDValue ExpandFCOPYSIGN(SDNode *Node);
145 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 SDValue ExpandBITREVERSE(SDNode *Node);
148 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154
155 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
156 SmallVectorImpl<SDValue> &Results);
157
158 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
159
160 /// Implements vector promotion.
161 ///
162 /// This is essentially just bitcasting the operands to a different type and
163 /// bitcasting the result back to the original type.
164 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
165
166 /// Implements [SU]INT_TO_FP vector promotion.
167 ///
168 /// This is a [zs]ext of the input operand to a larger integer type.
169 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
170
171 /// Implements FP_TO_[SU]INT vector promotion of the result type.
172 ///
173 /// It is promoted to a larger integer type. The result is then
174 /// truncated back to the original type.
175 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
176
177 /// Implements vector setcc operation promotion.
178 ///
179 /// All vector operands are promoted to a vector type with larger element
180 /// type.
181 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
182
183 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
184
185 /// Calculate the reduction using a type of higher precision and round the
186 /// result to match the original type. Setting NonArithmetic signifies the
187 /// rounding of the result does not affect its value.
188 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
189 bool NonArithmetic);
190
191 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
192
193public:
194 VectorLegalizer(SelectionDAG& dag) :
195 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
196
197 /// Begin legalizer the vector operations in the DAG.
198 bool Run();
199};
200
201} // end anonymous namespace
202
203bool VectorLegalizer::Run() {
204 // Before we start legalizing vector nodes, check if there are any vectors.
205 bool HasVectors = false;
206 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
207 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
208 // Check if the values of the nodes contain vectors. We don't need to check
209 // the operands because we are going to check their values at some point.
210 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
211
212 // If we found a vector node we can start the legalization.
213 if (HasVectors)
214 break;
215 }
216
217 // If this basic block has no vectors then no need to legalize vectors.
218 if (!HasVectors)
219 return false;
220
221 // The legalize process is inherently a bottom-up recursive process (users
222 // legalize their uses before themselves). Given infinite stack space, we
223 // could just start legalizing on the root and traverse the whole graph. In
224 // practice however, this causes us to run out of stack space on large basic
225 // blocks. To avoid this problem, compute an ordering of the nodes where each
226 // node is only legalized after all of its operands are legalized.
227 DAG.AssignTopologicalOrder();
228 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
229 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
230 LegalizeOp(Op: SDValue(&*I, 0));
231
232 // Finally, it's possible the root changed. Get the new root.
233 SDValue OldRoot = DAG.getRoot();
234 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
235 DAG.setRoot(LegalizedNodes[OldRoot]);
236
237 LegalizedNodes.clear();
238
239 // Remove dead nodes now.
240 DAG.RemoveDeadNodes();
241
242 return Changed;
243}
244
245SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
246 assert(Op->getNumValues() == Result->getNumValues() &&
247 "Unexpected number of results");
248 // Generic legalization: just pass the operand through.
249 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
250 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
251 return SDValue(Result, Op.getResNo());
252}
253
254SDValue
255VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
256 MutableArrayRef<SDValue> Results) {
257 assert(Results.size() == Op->getNumValues() &&
258 "Unexpected number of results");
259 // Make sure that the generated code is itself legal.
260 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
261 Results[i] = LegalizeOp(Op: Results[i]);
262 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
263 }
264
265 return Results[Op.getResNo()];
266}
267
268SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
269 // Note that LegalizeOp may be reentered even from single-use nodes, which
270 // means that we always must cache transformed nodes.
271 auto I = LegalizedNodes.find(Val: Op);
272 if (I != LegalizedNodes.end()) return I->second;
273
274 // Legalize the operands
275 SmallVector<SDValue, 8> Ops;
276 for (const SDValue &Oper : Op->op_values())
277 Ops.push_back(Elt: LegalizeOp(Op: Oper));
278
279 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
280
281 bool HasVectorValueOrOp =
282 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
283 llvm::any_of(Range: Node->op_values(),
284 P: [](SDValue O) { return O.getValueType().isVector(); });
285 if (!HasVectorValueOrOp)
286 return TranslateLegalizeResults(Op, Result: Node);
287
288 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
289 EVT ValVT;
290 switch (Op.getOpcode()) {
291 default:
292 return TranslateLegalizeResults(Op, Result: Node);
293 case ISD::LOAD: {
294 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
295 ISD::LoadExtType ExtType = LD->getExtensionType();
296 EVT LoadedVT = LD->getMemoryVT();
297 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
298 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
299 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
300 break;
301 }
302 case ISD::STORE: {
303 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
304 EVT StVT = ST->getMemoryVT();
305 MVT ValVT = ST->getValue().getSimpleValueType();
306 if (StVT.isVector() && ST->isTruncatingStore())
307 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
308 AddrSpace: ST->getAddressSpace());
309 break;
310 }
311 case ISD::MERGE_VALUES:
312 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
313 // This operation lies about being legal: when it claims to be legal,
314 // it should actually be expanded.
315 if (Action == TargetLowering::Legal)
316 Action = TargetLowering::Expand;
317 break;
318#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
319 case ISD::STRICT_##DAGN:
320#include "llvm/IR/ConstrainedOps.def"
321 ValVT = Node->getValueType(ResNo: 0);
322 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
323 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
324 ValVT = Node->getOperand(Num: 1).getValueType();
325 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
326 Op.getOpcode() == ISD::STRICT_FSETCCS) {
327 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
328 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 3))->get();
329 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
330 if (Action == TargetLowering::Legal)
331 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
332 } else {
333 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
334 }
335 // If we're asked to expand a strict vector floating-point operation,
336 // by default we're going to simply unroll it. That is usually the
337 // best approach, except in the case where the resulting strict (scalar)
338 // operations would themselves use the fallback mutation to non-strict.
339 // In that specific case, just do the fallback on the vector op.
340 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
341 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
342 TargetLowering::Legal) {
343 EVT EltVT = ValVT.getVectorElementType();
344 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
345 == TargetLowering::Expand &&
346 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
347 == TargetLowering::Legal)
348 Action = TargetLowering::Legal;
349 }
350 break;
351 case ISD::ADD:
352 case ISD::SUB:
353 case ISD::MUL:
354 case ISD::MULHS:
355 case ISD::MULHU:
356 case ISD::SDIV:
357 case ISD::UDIV:
358 case ISD::SREM:
359 case ISD::UREM:
360 case ISD::SDIVREM:
361 case ISD::UDIVREM:
362 case ISD::FADD:
363 case ISD::FSUB:
364 case ISD::FMUL:
365 case ISD::FDIV:
366 case ISD::FREM:
367 case ISD::AND:
368 case ISD::OR:
369 case ISD::XOR:
370 case ISD::SHL:
371 case ISD::SRA:
372 case ISD::SRL:
373 case ISD::FSHL:
374 case ISD::FSHR:
375 case ISD::ROTL:
376 case ISD::ROTR:
377 case ISD::ABS:
378 case ISD::ABS_MIN_POISON:
379 case ISD::ABDS:
380 case ISD::ABDU:
381 case ISD::AVGCEILS:
382 case ISD::AVGCEILU:
383 case ISD::AVGFLOORS:
384 case ISD::AVGFLOORU:
385 case ISD::BSWAP:
386 case ISD::BITREVERSE:
387 case ISD::CTLZ:
388 case ISD::CTTZ:
389 case ISD::CTLZ_ZERO_POISON:
390 case ISD::CTTZ_ZERO_POISON:
391 case ISD::CTPOP:
392 case ISD::CLMUL:
393 case ISD::CLMULH:
394 case ISD::CLMULR:
395 case ISD::SELECT:
396 case ISD::VSELECT:
397 case ISD::SELECT_CC:
398 case ISD::ZERO_EXTEND:
399 case ISD::ANY_EXTEND:
400 case ISD::TRUNCATE:
401 case ISD::SIGN_EXTEND:
402 case ISD::FP_TO_SINT:
403 case ISD::FP_TO_UINT:
404 case ISD::FNEG:
405 case ISD::FABS:
406 case ISD::FMINNUM:
407 case ISD::FMAXNUM:
408 case ISD::FMINNUM_IEEE:
409 case ISD::FMAXNUM_IEEE:
410 case ISD::FMINIMUM:
411 case ISD::FMAXIMUM:
412 case ISD::FMINIMUMNUM:
413 case ISD::FMAXIMUMNUM:
414 case ISD::FCOPYSIGN:
415 case ISD::FSQRT:
416 case ISD::FSIN:
417 case ISD::FCOS:
418 case ISD::FTAN:
419 case ISD::FASIN:
420 case ISD::FACOS:
421 case ISD::FATAN:
422 case ISD::FATAN2:
423 case ISD::FSINH:
424 case ISD::FCOSH:
425 case ISD::FTANH:
426 case ISD::FLDEXP:
427 case ISD::FPOWI:
428 case ISD::FPOW:
429 case ISD::FCBRT:
430 case ISD::FLOG:
431 case ISD::FLOG2:
432 case ISD::FLOG10:
433 case ISD::FEXP:
434 case ISD::FEXP2:
435 case ISD::FEXP10:
436 case ISD::FCEIL:
437 case ISD::FTRUNC:
438 case ISD::FRINT:
439 case ISD::FNEARBYINT:
440 case ISD::FROUND:
441 case ISD::FROUNDEVEN:
442 case ISD::FFLOOR:
443 case ISD::FP_ROUND:
444 case ISD::FP_EXTEND:
445 case ISD::FPTRUNC_ROUND:
446 case ISD::FMA:
447 case ISD::SIGN_EXTEND_INREG:
448 case ISD::ANY_EXTEND_VECTOR_INREG:
449 case ISD::SIGN_EXTEND_VECTOR_INREG:
450 case ISD::ZERO_EXTEND_VECTOR_INREG:
451 case ISD::SMIN:
452 case ISD::SMAX:
453 case ISD::UMIN:
454 case ISD::UMAX:
455 case ISD::SMUL_LOHI:
456 case ISD::UMUL_LOHI:
457 case ISD::SADDO:
458 case ISD::UADDO:
459 case ISD::SSUBO:
460 case ISD::USUBO:
461 case ISD::SMULO:
462 case ISD::UMULO:
463 case ISD::CONVERT_FROM_ARBITRARY_FP:
464 case ISD::CONVERT_TO_ARBITRARY_FP:
465 case ISD::FCANONICALIZE:
466 case ISD::FFREXP:
467 case ISD::FMODF:
468 case ISD::FSINCOS:
469 case ISD::FSINCOSPI:
470 case ISD::SADDSAT:
471 case ISD::UADDSAT:
472 case ISD::SSUBSAT:
473 case ISD::USUBSAT:
474 case ISD::SSHLSAT:
475 case ISD::USHLSAT:
476 case ISD::FP_TO_SINT_SAT:
477 case ISD::FP_TO_UINT_SAT:
478 case ISD::MGATHER:
479 case ISD::VECTOR_COMPRESS:
480 case ISD::SCMP:
481 case ISD::UCMP:
482 case ISD::LOOP_DEPENDENCE_WAR_MASK:
483 case ISD::LOOP_DEPENDENCE_RAW_MASK:
484 case ISD::MASKED_UDIV:
485 case ISD::MASKED_SDIV:
486 case ISD::MASKED_UREM:
487 case ISD::MASKED_SREM:
488 case ISD::VECTOR_MATCH:
489 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
490 break;
491 case ISD::SMULFIX:
492 case ISD::SMULFIXSAT:
493 case ISD::UMULFIX:
494 case ISD::UMULFIXSAT:
495 case ISD::SDIVFIX:
496 case ISD::SDIVFIXSAT:
497 case ISD::UDIVFIX:
498 case ISD::UDIVFIXSAT: {
499 unsigned Scale = Node->getConstantOperandVal(Num: 2);
500 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
501 VT: Node->getValueType(ResNo: 0), Scale);
502 break;
503 }
504 case ISD::LROUND:
505 case ISD::LLROUND:
506 case ISD::LRINT:
507 case ISD::LLRINT:
508 case ISD::SINT_TO_FP:
509 case ISD::UINT_TO_FP:
510 case ISD::VECREDUCE_ADD:
511 case ISD::VECREDUCE_MUL:
512 case ISD::VECREDUCE_AND:
513 case ISD::VECREDUCE_OR:
514 case ISD::VECREDUCE_XOR:
515 case ISD::VECREDUCE_SMAX:
516 case ISD::VECREDUCE_SMIN:
517 case ISD::VECREDUCE_UMAX:
518 case ISD::VECREDUCE_UMIN:
519 case ISD::VECREDUCE_FADD:
520 case ISD::VECREDUCE_FMAX:
521 case ISD::VECREDUCE_FMAXIMUM:
522 case ISD::VECREDUCE_FMIN:
523 case ISD::VECREDUCE_FMINIMUM:
524 case ISD::VECREDUCE_FMAXIMUMNUM:
525 case ISD::VECREDUCE_FMINIMUMNUM:
526 case ISD::VECREDUCE_FMUL:
527 case ISD::CTTZ_ELTS:
528 case ISD::CTTZ_ELTS_ZERO_POISON:
529 case ISD::VECTOR_FIND_LAST_ACTIVE:
530 Action = TLI.getOperationAction(Op: Node->getOpcode(),
531 VT: Node->getOperand(Num: 0).getValueType());
532 break;
533 case ISD::VECREDUCE_SEQ_FADD:
534 case ISD::VECREDUCE_SEQ_FMUL:
535 Action = TLI.getOperationAction(Op: Node->getOpcode(),
536 VT: Node->getOperand(Num: 1).getValueType());
537 break;
538 case ISD::SETCC: {
539 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
540 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
541 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
542 if (Action == TargetLowering::Legal)
543 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
544 break;
545 }
546 case ISD::PARTIAL_REDUCE_UMLA:
547 case ISD::PARTIAL_REDUCE_SMLA:
548 case ISD::PARTIAL_REDUCE_SUMLA:
549 case ISD::PARTIAL_REDUCE_FMLA:
550 Action =
551 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
552 InputVT: Node->getOperand(Num: 1).getValueType());
553 break;
554
555#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
556 case ISD::VPID: { \
557 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
558 : Node->getOperand(LEGALPOS).getValueType(); \
559 /* Defer non-vector results to LegalizeDAG. */ \
560 if (!Node->getValueType(0).isVector() && \
561 Node->getValueType(0) != MVT::Other) { \
562 Action = TargetLowering::Legal; \
563 break; \
564 } \
565 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
566 } break;
567#include "llvm/IR/VPIntrinsics.def"
568 }
569
570 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
571
572 SmallVector<SDValue, 8> ResultVals;
573 switch (Action) {
574 default: llvm_unreachable("This action is not supported yet!");
575 case TargetLowering::Promote:
576 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
577 "This action is not supported yet!");
578 LLVM_DEBUG(dbgs() << "Promoting\n");
579 Promote(Node, Results&: ResultVals);
580 assert(!ResultVals.empty() && "No results for promotion?");
581 break;
582 case TargetLowering::Legal:
583 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
584 break;
585 case TargetLowering::Custom:
586 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
587 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
588 break;
589 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
590 [[fallthrough]];
591 case TargetLowering::Expand:
592 LLVM_DEBUG(dbgs() << "Expanding\n");
593 Expand(Node, Results&: ResultVals);
594 break;
595 }
596
597 if (ResultVals.empty())
598 return TranslateLegalizeResults(Op, Result: Node);
599
600 Changed = true;
601 return RecursivelyLegalizeResults(Op, Results: ResultVals);
602}
603
604// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
605// merge them somehow?
606bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
607 SmallVectorImpl<SDValue> &Results) {
608 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
609
610 if (!Res.getNode())
611 return false;
612
613 if (Res == SDValue(Node, 0))
614 return true;
615
616 // If the original node has one result, take the return value from
617 // LowerOperation as is. It might not be result number 0.
618 if (Node->getNumValues() == 1) {
619 Results.push_back(Elt: Res);
620 return true;
621 }
622
623 // If the original node has multiple results, then the return node should
624 // have the same number of results.
625 assert((Node->getNumValues() == Res->getNumValues()) &&
626 "Lowering returned the wrong number of results!");
627
628 // Places new result values base on N result number.
629 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
630 Results.push_back(Elt: Res.getValue(R: I));
631
632 return true;
633}
634
635void VectorLegalizer::PromoteSETCC(SDNode *Node,
636 SmallVectorImpl<SDValue> &Results) {
637 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
638 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
639
640 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
641
642 SDLoc DL(Node);
643 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
644
645 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
646 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
647 Operands[2] = Node->getOperand(Num: 2);
648
649 EVT ResVT =
650 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
651 SDValue Res =
652 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
653 if (ResVT != Node->getValueType(ResNo: 0))
654 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
655 Results.push_back(Elt: Res);
656}
657
658void VectorLegalizer::PromoteSTRICT(SDNode *Node,
659 SmallVectorImpl<SDValue> &Results) {
660 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
661 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
662
663 assert(VecVT.isFloatingPoint());
664
665 SDLoc DL(Node);
666 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
667 SmallVector<SDValue, 2> Chains;
668
669 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
670 if (Node->getOperand(Num: j).getValueType().isVector() &&
671 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
672 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
673 {
674 // promote the vector operand.
675 SDValue Ext =
676 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
677 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
678 Operands[j] = Ext.getValue(R: 0);
679 Chains.push_back(Elt: Ext.getValue(R: 1));
680 } else
681 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
682
683 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
684
685 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
686
687 SDValue Res =
688 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
689
690 SDValue Round =
691 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
692 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
693 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
694
695 Results.push_back(Elt: Round.getValue(R: 0));
696 Results.push_back(Elt: Round.getValue(R: 1));
697}
698
699void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
700 SmallVectorImpl<SDValue> &Results,
701 bool NonArithmetic) {
702 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
703 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
704 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
705
706 SDLoc DL(Node);
707 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
708 SDValue Rdx =
709 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
710 Flags: Node->getFlags());
711 SDValue Res =
712 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
713 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
714 Results.push_back(Elt: Res);
715}
716
717void VectorLegalizer::PromoteVECTOR_COMPRESS(
718 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
719 SDLoc DL(Node);
720 EVT VT = Node->getValueType(ResNo: 0);
721 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
722 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
723 "Only integer promotion or bitcasts between types is supported");
724
725 SDValue Vec = Node->getOperand(Num: 0);
726 SDValue Mask = Node->getOperand(Num: 1);
727 SDValue Passthru = Node->getOperand(Num: 2);
728 if (VT.isInteger()) {
729 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
730 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
731 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
732 } else {
733 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
734 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
735 }
736
737 SDValue Result =
738 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
739 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
740 : DAG.getBitcast(VT, V: Result);
741 Results.push_back(Elt: Result);
742}
743
744void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
745 // For a few operations there is a specific concept for promotion based on
746 // the operand's type.
747 switch (Node->getOpcode()) {
748 case ISD::SINT_TO_FP:
749 case ISD::UINT_TO_FP:
750 case ISD::STRICT_SINT_TO_FP:
751 case ISD::STRICT_UINT_TO_FP:
752 // "Promote" the operation by extending the operand.
753 PromoteINT_TO_FP(Node, Results);
754 return;
755 case ISD::FP_TO_UINT:
756 case ISD::FP_TO_SINT:
757 case ISD::STRICT_FP_TO_UINT:
758 case ISD::STRICT_FP_TO_SINT:
759 // Promote the operation by extending the operand.
760 PromoteFP_TO_INT(Node, Results);
761 return;
762 case ISD::SETCC:
763 // Promote the operation by extending the operand.
764 PromoteSETCC(Node, Results);
765 return;
766 case ISD::STRICT_FADD:
767 case ISD::STRICT_FSUB:
768 case ISD::STRICT_FMUL:
769 case ISD::STRICT_FDIV:
770 case ISD::STRICT_FSQRT:
771 case ISD::STRICT_FMA:
772 PromoteSTRICT(Node, Results);
773 return;
774 case ISD::VECREDUCE_FADD:
775 case ISD::VECREDUCE_FMUL:
776 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
777 return;
778 case ISD::VECREDUCE_FMAX:
779 case ISD::VECREDUCE_FMAXIMUM:
780 case ISD::VECREDUCE_FMIN:
781 case ISD::VECREDUCE_FMINIMUM:
782 case ISD::VECREDUCE_FMAXIMUMNUM:
783 case ISD::VECREDUCE_FMINIMUMNUM:
784 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
785 return;
786 case ISD::VECTOR_COMPRESS:
787 PromoteVECTOR_COMPRESS(Node, Results);
788 return;
789
790 case ISD::FP_ROUND:
791 case ISD::FP_EXTEND:
792 // These operations are used to do promotion so they can't be promoted
793 // themselves.
794 llvm_unreachable("Don't know how to promote this operation!");
795 }
796
797 // There are currently two cases of vector promotion:
798 // 1) Bitcasting a vector of integers to a different type to a vector of the
799 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
800 // 2) Extending a vector of floats to a vector of the same number of larger
801 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
802 assert(Node->getNumValues() == 1 &&
803 "Can't promote a vector with multiple results!");
804 MVT VT = Node->getSimpleValueType(ResNo: 0);
805 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
806 SDLoc dl(Node);
807 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
808
809 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
810 // Do not promote the mask operand of a VP OP.
811 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
812 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
813 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
814 if (Node->getOperand(Num: j)
815 .getValueType()
816 .getVectorElementType()
817 .isFloatingPoint() &&
818 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
819 Operands[j] = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
820 else
821 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
822 else
823 Operands[j] = Node->getOperand(Num: j);
824 }
825
826 SDValue Res =
827 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
828
829 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
830 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
831 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
832 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
833 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
834 else
835 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
836
837 Results.push_back(Elt: Res);
838}
839
840void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
841 SmallVectorImpl<SDValue> &Results) {
842 // INT_TO_FP operations may require the input operand be promoted even
843 // when the type is otherwise legal.
844 bool IsStrict = Node->isStrictFPOpcode();
845 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
846 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
847 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
848 "Vectors have different number of elements!");
849
850 SDLoc dl(Node);
851 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
852
853 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
854 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
855 ? ISD::ZERO_EXTEND
856 : ISD::SIGN_EXTEND;
857 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
858 if (Node->getOperand(Num: j).getValueType().isVector())
859 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
860 else
861 Operands[j] = Node->getOperand(Num: j);
862 }
863
864 if (IsStrict) {
865 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
866 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
867 Results.push_back(Elt: Res);
868 Results.push_back(Elt: Res.getValue(R: 1));
869 return;
870 }
871
872 SDValue Res =
873 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
874 Results.push_back(Elt: Res);
875}
876
877// For FP_TO_INT we promote the result type to a vector type with wider
878// elements and then truncate the result. This is different from the default
879// PromoteVector which uses bitcast to promote thus assumning that the
880// promoted vector type has the same overall size.
881void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
882 SmallVectorImpl<SDValue> &Results) {
883 MVT VT = Node->getSimpleValueType(ResNo: 0);
884 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
885 bool IsStrict = Node->isStrictFPOpcode();
886 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
887 "Vectors have different number of elements!");
888
889 unsigned NewOpc = Node->getOpcode();
890 // Change FP_TO_UINT to FP_TO_SINT if possible.
891 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
892 if (NewOpc == ISD::FP_TO_UINT &&
893 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
894 NewOpc = ISD::FP_TO_SINT;
895
896 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
897 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
898 NewOpc = ISD::STRICT_FP_TO_SINT;
899
900 SDLoc dl(Node);
901 SDValue Promoted, Chain;
902 if (IsStrict) {
903 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
904 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
905 Chain = Promoted.getValue(R: 1);
906 } else
907 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
908
909 // Assert that the converted value fits in the original type. If it doesn't
910 // (eg: because the value being converted is too big), then the result of the
911 // original operation was undefined anyway, so the assert is still correct.
912 if (Node->getOpcode() == ISD::FP_TO_UINT ||
913 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
914 NewOpc = ISD::AssertZext;
915 else
916 NewOpc = ISD::AssertSext;
917
918 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
919 N2: DAG.getValueType(VT.getScalarType()));
920 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
921 Results.push_back(Elt: Promoted);
922 if (IsStrict)
923 Results.push_back(Elt: Chain);
924}
925
926std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
927 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
928 return TLI.scalarizeVectorLoad(LD, DAG);
929}
930
931SDValue VectorLegalizer::ExpandStore(SDNode *N) {
932 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
933 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
934 return TF;
935}
936
937void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
938 switch (Node->getOpcode()) {
939 case ISD::LOAD: {
940 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
941 Results.push_back(Elt: Tmp.first);
942 Results.push_back(Elt: Tmp.second);
943 return;
944 }
945 case ISD::STORE:
946 Results.push_back(Elt: ExpandStore(N: Node));
947 return;
948 case ISD::MERGE_VALUES:
949 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
950 Results.push_back(Elt: Node->getOperand(Num: i));
951 return;
952 case ISD::SIGN_EXTEND_INREG:
953 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
954 Results.push_back(Elt: Expanded);
955 return;
956 }
957 break;
958 case ISD::ANY_EXTEND_VECTOR_INREG:
959 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
960 return;
961 case ISD::SIGN_EXTEND_VECTOR_INREG:
962 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
963 return;
964 case ISD::ZERO_EXTEND_VECTOR_INREG:
965 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
966 return;
967 case ISD::BSWAP:
968 if (SDValue Expanded = ExpandBSWAP(Node)) {
969 Results.push_back(Elt: Expanded);
970 return;
971 }
972 break;
973 case ISD::VSELECT:
974 if (SDValue Expanded = ExpandVSELECT(Node)) {
975 Results.push_back(Elt: Expanded);
976 return;
977 }
978 break;
979 case ISD::VP_SREM:
980 case ISD::VP_UREM:
981 if (SDValue Expanded = ExpandVP_REM(Node)) {
982 Results.push_back(Elt: Expanded);
983 return;
984 }
985 break;
986 case ISD::SELECT:
987 if (SDValue Expanded = ExpandSELECT(Node)) {
988 Results.push_back(Elt: Expanded);
989 return;
990 }
991 break;
992 case ISD::SELECT_CC: {
993 if (Node->getValueType(ResNo: 0).isScalableVector()) {
994 EVT CondVT = TLI.getSetCCResultType(
995 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
996 SDValue SetCC =
997 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
998 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
999 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1000 LHS: Node->getOperand(Num: 2),
1001 RHS: Node->getOperand(Num: 3)));
1002 return;
1003 }
1004 break;
1005 }
1006 case ISD::FP_TO_UINT:
1007 ExpandFP_TO_UINT(Node, Results);
1008 return;
1009 case ISD::UINT_TO_FP:
1010 ExpandUINT_TO_FLOAT(Node, Results);
1011 return;
1012 case ISD::FNEG:
1013 if (SDValue Expanded = ExpandFNEG(Node)) {
1014 Results.push_back(Elt: Expanded);
1015 return;
1016 }
1017 break;
1018 case ISD::FABS:
1019 if (SDValue Expanded = ExpandFABS(Node)) {
1020 Results.push_back(Elt: Expanded);
1021 return;
1022 }
1023 break;
1024 case ISD::FCOPYSIGN:
1025 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1026 Results.push_back(Elt: Expanded);
1027 return;
1028 }
1029 break;
1030 case ISD::FCANONICALIZE: {
1031 // If the scalar element type has a
1032 // Legal/Custom FCANONICALIZE, don't
1033 // mess with the vector, fall back.
1034 EVT VT = Node->getValueType(ResNo: 0);
1035 EVT EltVT = VT.getVectorElementType();
1036 if (!VT.isScalableVector() &&
1037 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1038 TargetLowering::Expand)
1039 break;
1040 // Otherwise canonicalize the whole vector.
1041 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1042 Results.push_back(Elt: Mul);
1043 return;
1044 }
1045 case ISD::FSUB:
1046 ExpandFSUB(Node, Results);
1047 return;
1048 case ISD::SETCC:
1049 ExpandSETCC(Node, Results);
1050 return;
1051 case ISD::ABS:
1052 case ISD::ABS_MIN_POISON:
1053 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1054 Results.push_back(Elt: Expanded);
1055 return;
1056 }
1057 break;
1058 case ISD::ABDS:
1059 case ISD::ABDU:
1060 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1061 Results.push_back(Elt: Expanded);
1062 return;
1063 }
1064 break;
1065 case ISD::AVGCEILS:
1066 case ISD::AVGCEILU:
1067 case ISD::AVGFLOORS:
1068 case ISD::AVGFLOORU:
1069 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1070 Results.push_back(Elt: Expanded);
1071 return;
1072 }
1073 break;
1074 case ISD::BITREVERSE:
1075 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1076 Results.push_back(Elt: Expanded);
1077 return;
1078 }
1079 break;
1080 case ISD::CTPOP:
1081 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1082 Results.push_back(Elt: Expanded);
1083 return;
1084 }
1085 break;
1086 case ISD::CTLZ:
1087 case ISD::CTLZ_ZERO_POISON:
1088 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1089 Results.push_back(Elt: Expanded);
1090 return;
1091 }
1092 break;
1093 case ISD::CTTZ:
1094 case ISD::CTTZ_ZERO_POISON:
1095 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1096 Results.push_back(Elt: Expanded);
1097 return;
1098 }
1099 break;
1100 case ISD::FSHL:
1101 case ISD::FSHR:
1102 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1103 Results.push_back(Elt: Expanded);
1104 return;
1105 }
1106 break;
1107 case ISD::CLMUL:
1108 case ISD::CLMULR:
1109 case ISD::CLMULH:
1110 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1111 Results.push_back(Elt: Expanded);
1112 return;
1113 }
1114 break;
1115 case ISD::PEXT:
1116 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1117 return;
1118 case ISD::PDEP:
1119 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1120 return;
1121 case ISD::ROTL:
1122 case ISD::ROTR:
1123 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1124 Results.push_back(Elt: Expanded);
1125 return;
1126 }
1127 break;
1128 case ISD::FMINNUM:
1129 case ISD::FMAXNUM:
1130 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1131 Results.push_back(Elt: Expanded);
1132 return;
1133 }
1134 break;
1135 case ISD::FMINIMUM:
1136 case ISD::FMAXIMUM:
1137 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1138 return;
1139 case ISD::FMINIMUMNUM:
1140 case ISD::FMAXIMUMNUM:
1141 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1142 return;
1143 case ISD::SMIN:
1144 case ISD::SMAX:
1145 case ISD::UMIN:
1146 case ISD::UMAX:
1147 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1148 Results.push_back(Elt: Expanded);
1149 return;
1150 }
1151 break;
1152 case ISD::UADDO:
1153 case ISD::USUBO:
1154 ExpandUADDSUBO(Node, Results);
1155 return;
1156 case ISD::SADDO:
1157 case ISD::SSUBO:
1158 ExpandSADDSUBO(Node, Results);
1159 return;
1160 case ISD::UMULO:
1161 case ISD::SMULO:
1162 ExpandMULO(Node, Results);
1163 return;
1164 case ISD::USUBSAT:
1165 case ISD::SSUBSAT:
1166 case ISD::UADDSAT:
1167 case ISD::SADDSAT:
1168 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1169 Results.push_back(Elt: Expanded);
1170 return;
1171 }
1172 break;
1173 case ISD::USHLSAT:
1174 case ISD::SSHLSAT:
1175 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1176 Results.push_back(Elt: Expanded);
1177 return;
1178 }
1179 break;
1180 case ISD::FP_TO_SINT_SAT:
1181 case ISD::FP_TO_UINT_SAT:
1182 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1183 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1184 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1185 Results.push_back(Elt: Expanded);
1186 return;
1187 }
1188 }
1189 break;
1190 case ISD::SMULFIX:
1191 case ISD::UMULFIX:
1192 case ISD::SMULFIXSAT:
1193 case ISD::UMULFIXSAT:
1194 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1195 Results.push_back(Elt: Expanded);
1196 return;
1197 }
1198 break;
1199 case ISD::SDIVFIX:
1200 case ISD::UDIVFIX:
1201 ExpandFixedPointDiv(Node, Results);
1202 return;
1203 case ISD::SDIVFIXSAT:
1204 case ISD::UDIVFIXSAT:
1205 break;
1206#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1207 case ISD::STRICT_##DAGN:
1208#include "llvm/IR/ConstrainedOps.def"
1209 ExpandStrictFPOp(Node, Results);
1210 return;
1211 case ISD::VECREDUCE_ADD:
1212 case ISD::VECREDUCE_MUL:
1213 case ISD::VECREDUCE_AND:
1214 case ISD::VECREDUCE_OR:
1215 case ISD::VECREDUCE_XOR:
1216 case ISD::VECREDUCE_SMAX:
1217 case ISD::VECREDUCE_SMIN:
1218 case ISD::VECREDUCE_UMAX:
1219 case ISD::VECREDUCE_UMIN:
1220 case ISD::VECREDUCE_FADD:
1221 case ISD::VECREDUCE_FMUL:
1222 case ISD::VECREDUCE_FMAX:
1223 case ISD::VECREDUCE_FMIN:
1224 case ISD::VECREDUCE_FMAXIMUM:
1225 case ISD::VECREDUCE_FMINIMUM:
1226 case ISD::VECREDUCE_FMAXIMUMNUM:
1227 case ISD::VECREDUCE_FMINIMUMNUM:
1228 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1229 return;
1230 case ISD::PARTIAL_REDUCE_UMLA:
1231 case ISD::PARTIAL_REDUCE_SMLA:
1232 case ISD::PARTIAL_REDUCE_SUMLA:
1233 case ISD::PARTIAL_REDUCE_FMLA:
1234 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1235 return;
1236 case ISD::VECREDUCE_SEQ_FADD:
1237 case ISD::VECREDUCE_SEQ_FMUL:
1238 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1239 return;
1240 case ISD::VECTOR_MATCH:
1241 Results.push_back(Elt: TLI.expandVectorMatch(N: Node, DAG));
1242 return;
1243 case ISD::SREM:
1244 case ISD::UREM:
1245 ExpandREM(Node, Results);
1246 return;
1247 case ISD::VP_MERGE:
1248 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1249 Results.push_back(Elt: Expanded);
1250 return;
1251 }
1252 break;
1253 case ISD::FREM: {
1254 RTLIB::Libcall LC = RTLIB::getREM(VT: Node->getValueType(ResNo: 0));
1255 if (tryExpandVecMathCall(Node, LC, Results))
1256 return;
1257
1258 break;
1259 }
1260 case ISD::FSINCOS:
1261 case ISD::FSINCOSPI: {
1262 EVT VT = Node->getValueType(ResNo: 0);
1263 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1264 ? RTLIB::getSINCOS(VT)
1265 : RTLIB::getSINCOSPI(VT);
1266 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1267 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1268 return;
1269
1270 // TODO: Try to see if there's a narrower call available to use before
1271 // scalarizing.
1272 break;
1273 }
1274 case ISD::FPOW: {
1275 RTLIB::Libcall LC = RTLIB::getPOW(VT: Node->getValueType(ResNo: 0));
1276 if (tryExpandVecMathCall(Node, LC, Results))
1277 return;
1278
1279 // TODO: Try to see if there's a narrower call available to use before
1280 // scalarizing.
1281 break;
1282 }
1283 case ISD::FCBRT: {
1284 RTLIB::Libcall LC = RTLIB::getCBRT(VT: Node->getValueType(ResNo: 0));
1285 if (tryExpandVecMathCall(Node, LC, Results))
1286 return;
1287
1288 // TODO: Try to see if there's a narrower call available to use before
1289 // scalarizing.
1290 break;
1291 }
1292 case ISD::FMODF: {
1293 EVT VT = Node->getValueType(ResNo: 0);
1294 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1295 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1296 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1297 /*CallRetResNo=*/0))
1298 return;
1299 break;
1300 }
1301 case ISD::VECTOR_COMPRESS:
1302 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1303 return;
1304 case ISD::CTTZ_ELTS:
1305 case ISD::CTTZ_ELTS_ZERO_POISON:
1306 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1307 return;
1308 case ISD::VECTOR_FIND_LAST_ACTIVE:
1309 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1310 return;
1311 case ISD::SCMP:
1312 case ISD::UCMP:
1313 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1314 return;
1315 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1316 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1317 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1318 return;
1319
1320 case ISD::FADD:
1321 case ISD::FMUL:
1322 case ISD::FMA:
1323 case ISD::FDIV:
1324 case ISD::FCEIL:
1325 case ISD::FFLOOR:
1326 case ISD::FNEARBYINT:
1327 case ISD::FRINT:
1328 case ISD::FROUND:
1329 case ISD::FROUNDEVEN:
1330 case ISD::FTRUNC:
1331 case ISD::FSQRT:
1332 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1333 Results.push_back(Elt: Expanded);
1334 return;
1335 }
1336 break;
1337 case ISD::CONVERT_TO_ARBITRARY_FP:
1338 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1339 Results.push_back(Elt: Expanded);
1340 else
1341 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1342 return;
1343 case ISD::CONVERT_FROM_ARBITRARY_FP:
1344 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1345 Results.push_back(Elt: Expanded);
1346 else
1347 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1348 return;
1349 case ISD::MASKED_UDIV:
1350 case ISD::MASKED_SDIV:
1351 case ISD::MASKED_UREM:
1352 case ISD::MASKED_SREM:
1353 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1354 return;
1355 }
1356
1357 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1358 if (Node->getNumValues() == 1) {
1359 Results.push_back(Elt: Unrolled);
1360 } else {
1361 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1362 "VectorLegalizer Expand returned wrong number of results!");
1363 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1364 Results.push_back(Elt: Unrolled.getValue(R: I));
1365 }
1366}
1367
1368SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1369 // Lower a select instruction where the condition is a scalar and the
1370 // operands are vectors. Lower this select to VSELECT and implement it
1371 // using XOR AND OR. The selector bit is broadcasted.
1372 EVT VT = Node->getValueType(ResNo: 0);
1373 SDLoc DL(Node);
1374
1375 SDValue Mask = Node->getOperand(Num: 0);
1376 SDValue Op1 = Node->getOperand(Num: 1);
1377 SDValue Op2 = Node->getOperand(Num: 2);
1378
1379 assert(VT.isVector() && !Mask.getValueType().isVector()
1380 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1381
1382 // If we can't even use the basic vector operations of
1383 // AND,OR,XOR, we will have to scalarize the op.
1384 // Notice that the operation may be 'promoted' which means that it is
1385 // 'bitcasted' to another type which is handled.
1386 // Also, we need to be able to construct a splat vector using either
1387 // BUILD_VECTOR or SPLAT_VECTOR.
1388 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1389 // BUILD_VECTOR?
1390 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1391 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1392 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1393 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1394 : ISD::SPLAT_VECTOR,
1395 VT) == TargetLowering::Expand)
1396 return SDValue();
1397
1398 // Generate a mask operand.
1399 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1400
1401 // What is the size of each element in the vector mask.
1402 EVT BitTy = MaskTy.getScalarType();
1403
1404 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1405 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1406
1407 // Broadcast the mask so that the entire vector is all one or all zero.
1408 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1409
1410 // Bitcast the operands to be the same type as the mask.
1411 // This is needed when we select between FP types because
1412 // the mask is a vector of integers.
1413 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1414 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1415
1416 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1417
1418 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1419 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1420 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1421 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1422}
1423
1424SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1425 EVT VT = Node->getValueType(ResNo: 0);
1426
1427 // Make sure that the SRA and SHL instructions are available.
1428 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1429 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1430 return SDValue();
1431
1432 SDLoc DL(Node);
1433 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1434
1435 unsigned BW = VT.getScalarSizeInBits();
1436 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1437 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1438
1439 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1440 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1441}
1442
1443// Generically expand a vector anyext in register to a shuffle of the relevant
1444// lanes into the appropriate locations, with other lanes left undef.
1445SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1446 SDLoc DL(Node);
1447 EVT VT = Node->getValueType(ResNo: 0);
1448 int NumElements = VT.getVectorNumElements();
1449 SDValue Src = Node->getOperand(Num: 0);
1450 EVT SrcVT = Src.getValueType();
1451 int NumSrcElements = SrcVT.getVectorNumElements();
1452
1453 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1454 // into a larger vector type.
1455 if (SrcVT.bitsLE(VT)) {
1456 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1457 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1458 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1459 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1460 NumElements: NumSrcElements);
1461 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1462 }
1463
1464 // Build a base mask of undef shuffles.
1465 SmallVector<int, 16> ShuffleMask;
1466 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1467
1468 // Place the extended lanes into the correct locations.
1469 int ExtLaneScale = NumSrcElements / NumElements;
1470 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1471 for (int i = 0; i < NumElements; ++i)
1472 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1473
1474 return DAG.getNode(
1475 Opcode: ISD::BITCAST, DL, VT,
1476 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1477}
1478
1479SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1480 SDLoc DL(Node);
1481 EVT VT = Node->getValueType(ResNo: 0);
1482 SDValue Src = Node->getOperand(Num: 0);
1483 EVT SrcVT = Src.getValueType();
1484
1485 // First build an any-extend node which can be legalized above when we
1486 // recurse through it.
1487 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1488
1489 // Now we need sign extend. This will be exanded to shifts if it isn't
1490 // supported.
1491 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1492 NumElements: VT.getVectorNumElements());
1493 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1494 N2: DAG.getValueType(ExtVT));
1495}
1496
1497// Generically expand a vector zext in register to a shuffle of the relevant
1498// lanes into the appropriate locations, a blend of zero into the high bits,
1499// and a bitcast to the wider element type.
1500SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1501 SDLoc DL(Node);
1502 EVT VT = Node->getValueType(ResNo: 0);
1503 int NumElements = VT.getVectorNumElements();
1504 SDValue Src = Node->getOperand(Num: 0);
1505 EVT SrcVT = Src.getValueType();
1506 int NumSrcElements = SrcVT.getVectorNumElements();
1507
1508 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1509 // into a larger vector type.
1510 if (SrcVT.bitsLE(VT)) {
1511 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1512 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1513 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1514 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1515 NumElements: NumSrcElements);
1516 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1517 }
1518
1519 // Build up a zero vector to blend into this one.
1520 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1521
1522 // Shuffle the incoming lanes into the correct position, and pull all other
1523 // lanes from the zero vector.
1524 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1525
1526 int ExtLaneScale = NumSrcElements / NumElements;
1527 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1528 for (int i = 0; i < NumElements; ++i)
1529 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1530
1531 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1532 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1533}
1534
1535static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1536 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1537 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1538 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1539 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1540}
1541
1542SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1543 EVT VT = Node->getValueType(ResNo: 0);
1544
1545 // Scalable vectors can't use shuffle expansion.
1546 if (VT.isScalableVector())
1547 return TLI.expandBSWAP(N: Node, DAG);
1548
1549 // Generate a byte wise shuffle mask for the BSWAP.
1550 SmallVector<int, 16> ShuffleMask;
1551 createBSWAPShuffleMask(VT, ShuffleMask);
1552 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1553
1554 // Only emit a shuffle if the mask is legal.
1555 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1556 SDLoc DL(Node);
1557 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1558 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1559 Mask: ShuffleMask);
1560 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1561 }
1562
1563 // If we have the appropriate vector bit operations, it is better to use them
1564 // than unrolling and expanding each component.
1565 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1566 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1567 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1568 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1569 return TLI.expandBSWAP(N: Node, DAG);
1570
1571 // Otherwise let the caller unroll.
1572 return SDValue();
1573}
1574
1575SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1576 EVT VT = Node->getValueType(ResNo: 0);
1577
1578 // We can't unroll or use shuffles for scalable vectors.
1579 if (VT.isScalableVector())
1580 return TLI.expandBITREVERSE(N: Node, DAG);
1581
1582 // If we have the scalar operation, it's probably cheaper to unroll it.
1583 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1584 return SDValue();
1585
1586 // If the vector element width is a whole number of bytes, test if its legal
1587 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1588 // vector. This greatly reduces the number of bit shifts necessary.
1589 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1590 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1591 SmallVector<int, 16> BSWAPMask;
1592 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1593
1594 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1595 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1596 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1597 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1598 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1599 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1600 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1601 SDLoc DL(Node);
1602 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1603 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1604 Mask: BSWAPMask);
1605 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1606 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1607 return Op;
1608 }
1609 }
1610
1611 // If we have the appropriate vector bit operations, it is better to use them
1612 // than unrolling and expanding each component.
1613 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1614 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1615 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1616 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1617 return TLI.expandBITREVERSE(N: Node, DAG);
1618
1619 // Otherwise unroll.
1620 return SDValue();
1621}
1622
1623SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1624 // Implement VSELECT in terms of XOR, AND, OR
1625 // on platforms which do not support blend natively.
1626 SDLoc DL(Node);
1627
1628 SDValue Mask = Node->getOperand(Num: 0);
1629 SDValue Op1 = Node->getOperand(Num: 1);
1630 SDValue Op2 = Node->getOperand(Num: 2);
1631
1632 EVT VT = Mask.getValueType();
1633
1634 // If we can't even use the basic vector operations of
1635 // AND,OR,XOR, we will have to scalarize the op.
1636 // Notice that the operation may be 'promoted' which means that it is
1637 // 'bitcasted' to another type which is handled.
1638 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1639 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1640 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1641 return SDValue();
1642
1643 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1644 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1645 // vector constant to mask with.
1646 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1647 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1648 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1649 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1650 Op1.getValueType().getVectorElementType() == MVT::i1))
1651 return SDValue();
1652
1653 // If the mask and the type are different sizes, unroll the vector op. This
1654 // can occur when getSetCCResultType returns something that is different in
1655 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1656 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1657 return SDValue();
1658
1659 // Bitcast the operands to be the same type as the mask.
1660 // This is needed when we select between FP types because
1661 // the mask is a vector of integers.
1662 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1663 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1664
1665 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1666
1667 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1668 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1669 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1670 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1671}
1672
1673SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1674 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1675 // indices less than the EVL/pivot are true. Combine that with the original
1676 // mask for a full-length mask. Use a full-length VSELECT to select between
1677 // the true and false values.
1678 SDLoc DL(Node);
1679
1680 SDValue Mask = Node->getOperand(Num: 0);
1681 SDValue Op1 = Node->getOperand(Num: 1);
1682 SDValue Op2 = Node->getOperand(Num: 2);
1683 SDValue EVL = Node->getOperand(Num: 3);
1684
1685 EVT MaskVT = Mask.getValueType();
1686 bool IsFixedLen = MaskVT.isFixedLengthVector();
1687
1688 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1689 EC: MaskVT.getVectorElementCount());
1690
1691 // If we can't construct the EVL mask efficiently, it's better to unroll.
1692 if ((IsFixedLen &&
1693 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1694 (!IsFixedLen &&
1695 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1696 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1697 return SDValue();
1698
1699 // If using a SETCC would result in a different type than the mask type,
1700 // unroll.
1701 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1702 VT: EVLVecVT) != MaskVT)
1703 return SDValue();
1704
1705 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1706 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1707 SDValue EVLMask =
1708 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1709
1710 SDValue FullMask = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: Mask, N2: EVLMask);
1711 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1712}
1713
1714SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1715 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, MUL, SUB.
1716 EVT VT = Node->getValueType(ResNo: 0);
1717
1718 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1719
1720 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1721 !TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT) ||
1722 !TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT))
1723 return SDValue();
1724
1725 SDLoc DL(Node);
1726
1727 SDValue Dividend = Node->getOperand(Num: 0);
1728 SDValue Divisor = Node->getOperand(Num: 1);
1729 SDValue Mask = Node->getOperand(Num: 2);
1730 SDValue EVL = Node->getOperand(Num: 3);
1731
1732 // X % Y -> X-X/Y*Y
1733 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1734 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Divisor, N2: Div);
1735 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Dividend, N2: Mul);
1736}
1737
1738SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1739 return TLI.expandLoopDependenceMask(N, DAG);
1740}
1741
1742SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1743 // Masked bin ops don't have undefined behaviour when dividing by zero
1744 // on disabled lanes and produce poison instead. Replace the divisor on the
1745 // disabled lanes with 1 to avoid division by zero or overflow.
1746 SDLoc dl(N);
1747 EVT VT = N->getValueType(ResNo: 0);
1748 SDValue SafeDivisor = DAG.getSelect(
1749 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1750 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1751 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1752}
1753
1754void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1755 SmallVectorImpl<SDValue> &Results) {
1756 // Attempt to expand using TargetLowering.
1757 SDValue Result, Chain;
1758 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1759 Results.push_back(Elt: Result);
1760 if (Node->isStrictFPOpcode())
1761 Results.push_back(Elt: Chain);
1762 return;
1763 }
1764
1765 // Otherwise go ahead and unroll.
1766 if (Node->isStrictFPOpcode()) {
1767 UnrollStrictFPOp(Node, Results);
1768 return;
1769 }
1770
1771 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1772}
1773
1774void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1775 SmallVectorImpl<SDValue> &Results) {
1776 bool IsStrict = Node->isStrictFPOpcode();
1777 unsigned OpNo = IsStrict ? 1 : 0;
1778 SDValue Src = Node->getOperand(Num: OpNo);
1779 EVT SrcVT = Src.getValueType();
1780 EVT DstVT = Node->getValueType(ResNo: 0);
1781 SDLoc DL(Node);
1782
1783 // Attempt to expand using TargetLowering.
1784 SDValue Result;
1785 SDValue Chain;
1786 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1787 Results.push_back(Elt: Result);
1788 if (IsStrict)
1789 Results.push_back(Elt: Chain);
1790 return;
1791 }
1792
1793 // Make sure that the SINT_TO_FP and SRL instructions are available.
1794 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1795 TargetLowering::Expand) ||
1796 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1797 TargetLowering::Expand)) ||
1798 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1799 if (IsStrict) {
1800 UnrollStrictFPOp(Node, Results);
1801 return;
1802 }
1803
1804 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1805 return;
1806 }
1807
1808 unsigned BW = SrcVT.getScalarSizeInBits();
1809 assert((BW == 64 || BW == 32) &&
1810 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1811
1812 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
1813 // UINT_TO_FP with a larger float and round to the smaller type
1814 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
1815 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
1816 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
1817 SDValue UIToFP;
1818 SDValue Result;
1819 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
1820 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
1821 if (IsStrict) {
1822 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
1823 Ops: {Node->getOperand(Num: 0), Src});
1824 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
1825 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
1826 Results.push_back(Elt: Result);
1827 Results.push_back(Elt: Result.getValue(R: 1));
1828 } else {
1829 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
1830 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
1831 Results.push_back(Elt: Result);
1832 }
1833
1834 return;
1835 }
1836
1837 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
1838
1839 // Constants to clear the upper part of the word.
1840 // Notice that we can also use SHL+SHR, but using a constant is slightly
1841 // faster on x86.
1842 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1843 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
1844
1845 // Two to the power of half-word-size.
1846 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
1847
1848 // Clear upper part of LO, lower HI
1849 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
1850 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
1851
1852 if (IsStrict) {
1853 // Convert hi and lo to floats
1854 // Convert the hi part back to the upper values
1855 // TODO: Can any fast-math-flags be set on these nodes?
1856 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1857 Ops: {Node->getOperand(Num: 0), HI});
1858 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
1859 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
1860 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
1861 Ops: {Node->getOperand(Num: 0), LO});
1862
1863 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
1864 N2: fLO.getValue(R: 1));
1865
1866 // Add the two halves
1867 SDValue Result =
1868 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
1869
1870 Results.push_back(Elt: Result);
1871 Results.push_back(Elt: Result.getValue(R: 1));
1872 return;
1873 }
1874
1875 // Convert hi and lo to floats
1876 // Convert the hi part back to the upper values
1877 // TODO: Can any fast-math-flags be set on these nodes?
1878 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
1879 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
1880 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
1881
1882 // Add the two halves
1883 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
1884}
1885
1886SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1887 EVT VT = Node->getValueType(ResNo: 0);
1888 EVT IntVT = VT.changeVectorElementTypeToInteger();
1889
1890 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
1891 return SDValue();
1892
1893 // Heuristic check to determine whether vector should be expanded to integer
1894 // operations or unrolled to scalar operations.
1895 // 1. Scalable vector is never unrolled.
1896 // 2. Fixed vector is unrolled if one of followings is true:
1897 // a. Vector only has 1 element and target knows how to handle scalar
1898 // FNEG (either legal or custom expand or promote).
1899 // b. Vector has more than 1 element and target supports scalar
1900 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
1901 // FIXME: Scalar construction instruction count varies in every architecture,
1902 // here we assume 1 instruction for now.
1903 if (VT.isFixedLengthVector()) {
1904 EVT EltVT = VT.getVectorElementType();
1905 unsigned NumElts = VT.getVectorNumElements();
1906 if ((NumElts == 1 &&
1907 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
1908 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
1909 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1910 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1911 return SDValue();
1912 }
1913
1914 SDLoc DL(Node);
1915 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1916 SDValue SignMask = DAG.getConstant(
1917 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1918 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
1919 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
1920}
1921
1922SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
1923 EVT VT = Node->getValueType(ResNo: 0);
1924 EVT IntVT = VT.changeVectorElementTypeToInteger();
1925
1926 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
1927 return SDValue();
1928
1929 // Heuristic check to determine whether vector should be expanded to integer
1930 // operations or unrolled to scalar operations.
1931 // 1. Scalable vector is never unrolled.
1932 // 2. Fixed vector is unrolled if one of followings is true:
1933 // a. Vector only has 1 element and target knows how to handle scalar
1934 // FABS(either legal or custom expand or promote).
1935 // b. Vector has more than 1 element and target supports scalar
1936 // FABS natively and vector length <= 2(1 AND + 1 CONST).
1937 // FIXME: Scalar construction instruction count varies in every architecture,
1938 // here we assume 1 instruction for now.
1939 if (VT.isFixedLengthVector()) {
1940 EVT EltVT = VT.getVectorElementType();
1941 unsigned NumElts = VT.getVectorNumElements();
1942 if ((NumElts == 1 &&
1943 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
1944 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
1945 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1946 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1947 return SDValue();
1948 }
1949
1950 SDLoc DL(Node);
1951 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1952 SDValue ClearSignMask = DAG.getConstant(
1953 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1954 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
1955 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
1956}
1957
1958SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
1959 EVT VT = Node->getValueType(ResNo: 0);
1960 EVT IntVT = VT.changeVectorElementTypeToInteger();
1961
1962 if (VT != Node->getOperand(Num: 1).getValueType() ||
1963 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
1964 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
1965 return SDValue();
1966
1967 // Heuristic check to determine whether vector should be expanded to integer
1968 // operations or unrolled to scalar operations.
1969 // 1. Scalable vector is never unrolled.
1970 // 2. Fixed vector is unrolled if one of followings is true:
1971 // a. Vector only has 1 element and target knows how to handle scalar
1972 // FCOPYSIGN(either legal or custom expand or promote).
1973 // b. Vector has more than 1 element and target supports scalar
1974 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
1975 // FIXME: Scalar construction instruction count varies in every architecture,
1976 // here we assume 1 instruction for now.
1977 if (VT.isFixedLengthVector()) {
1978 EVT EltVT = VT.getVectorElementType();
1979 unsigned NumElts = VT.getVectorNumElements();
1980 if ((NumElts == 1 &&
1981 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
1982 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
1983 TLI.isExtractVecEltCheap(VT, Index: 0) &&
1984 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
1985 return SDValue();
1986 }
1987
1988 SDLoc DL(Node);
1989 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1990 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
1991
1992 SDValue SignMask = DAG.getConstant(
1993 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1994 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
1995
1996 SDValue ClearSignMask = DAG.getConstant(
1997 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1998 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
1999
2000 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2001 Flags: SDNodeFlags::Disjoint);
2002
2003 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2004}
2005
2006void VectorLegalizer::ExpandFSUB(SDNode *Node,
2007 SmallVectorImpl<SDValue> &Results) {
2008 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2009 // we can defer this to operation legalization where it will be lowered as
2010 // a+(-b).
2011 EVT VT = Node->getValueType(ResNo: 0);
2012 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2013 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2014 return; // Defer to LegalizeDAG
2015
2016 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2017 Results.push_back(Elt: Expanded);
2018 return;
2019 }
2020
2021 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2022 Results.push_back(Elt: Tmp);
2023}
2024
2025void VectorLegalizer::ExpandSETCC(SDNode *Node,
2026 SmallVectorImpl<SDValue> &Results) {
2027 bool NeedInvert = false;
2028 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2029 Node->getOpcode() == ISD::STRICT_FSETCCS;
2030 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2031 unsigned Offset = IsStrict ? 1 : 0;
2032
2033 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2034 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2035 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2036 SDValue CC = Node->getOperand(Num: 2 + Offset);
2037
2038 MVT OpVT = LHS.getSimpleValueType();
2039 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2040
2041 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2042 if (IsStrict) {
2043 UnrollStrictFPOp(Node, Results);
2044 return;
2045 }
2046 Results.push_back(Elt: UnrollVSETCC(Node));
2047 return;
2048 }
2049
2050 SDLoc dl(Node);
2051 bool Legalized =
2052 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC,
2053 NeedInvert, dl, Chain, IsSignaling);
2054
2055 if (Legalized) {
2056 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2057 // condition code, create a new SETCC node.
2058 if (CC.getNode()) {
2059 if (IsStrict) {
2060 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2061 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2062 Chain = LHS.getValue(R: 1);
2063 } else {
2064 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2065 Flags: Node->getFlags());
2066 }
2067 }
2068
2069 // If we expanded the SETCC by inverting the condition code, then wrap
2070 // the existing SETCC in a NOT to restore the intended condition.
2071 if (NeedInvert)
2072 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2073 } else {
2074 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2075
2076 // Otherwise, SETCC for the given comparison type must be completely
2077 // illegal; expand it into a SELECT_CC.
2078 EVT VT = Node->getValueType(ResNo: 0);
2079 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2080 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2081 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2082 N5: CC, Flags: Node->getFlags());
2083 }
2084
2085 Results.push_back(Elt: LHS);
2086 if (IsStrict)
2087 Results.push_back(Elt: Chain);
2088}
2089
2090void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2091 SmallVectorImpl<SDValue> &Results) {
2092 SDValue Result, Overflow;
2093 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2094 Results.push_back(Elt: Result);
2095 Results.push_back(Elt: Overflow);
2096}
2097
2098void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2099 SmallVectorImpl<SDValue> &Results) {
2100 SDValue Result, Overflow;
2101 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2102 Results.push_back(Elt: Result);
2103 Results.push_back(Elt: Overflow);
2104}
2105
2106void VectorLegalizer::ExpandMULO(SDNode *Node,
2107 SmallVectorImpl<SDValue> &Results) {
2108 SDValue Result, Overflow;
2109 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2110 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2111
2112 Results.push_back(Elt: Result);
2113 Results.push_back(Elt: Overflow);
2114}
2115
2116void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2117 SmallVectorImpl<SDValue> &Results) {
2118 SDNode *N = Node;
2119 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2120 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2121 Results.push_back(Elt: Expanded);
2122}
2123
2124void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2125 SmallVectorImpl<SDValue> &Results) {
2126 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
2127 ExpandUINT_TO_FLOAT(Node, Results);
2128 return;
2129 }
2130 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
2131 ExpandFP_TO_UINT(Node, Results);
2132 return;
2133 }
2134
2135 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2136 Node->getOpcode() == ISD::STRICT_FSETCCS) {
2137 ExpandSETCC(Node, Results);
2138 return;
2139 }
2140
2141 UnrollStrictFPOp(Node, Results);
2142}
2143
2144void VectorLegalizer::ExpandREM(SDNode *Node,
2145 SmallVectorImpl<SDValue> &Results) {
2146 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2147 "Expected REM node");
2148
2149 SDValue Result;
2150 if (!TLI.expandREM(Node, Result, DAG))
2151 Result = DAG.UnrollVectorOp(N: Node);
2152 Results.push_back(Elt: Result);
2153}
2154
2155// Try to expand libm nodes into vector math routine calls. Callers provide the
2156// LibFunc equivalent of the passed in Node, which is used to lookup mappings
2157// within TargetLibraryInfo. The only mappings considered are those where the
2158// result and all operands are the same vector type. While predicated nodes are
2159// not supported, we will emit calls to masked routines by passing in an all
2160// true mask.
2161bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
2162 SmallVectorImpl<SDValue> &Results) {
2163 // Chain must be propagated but currently strict fp operations are down
2164 // converted to their none strict counterpart.
2165 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2166
2167 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2168 if (LCImpl == RTLIB::Unsupported)
2169 return false;
2170
2171 EVT VT = Node->getValueType(ResNo: 0);
2172 const RTLIB::RuntimeLibcallsInfo &RTLCI = TLI.getRuntimeLibcallsInfo();
2173 LLVMContext &Ctx = *DAG.getContext();
2174
2175 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2176 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2177
2178 SDLoc DL(Node);
2179 TargetLowering::ArgListTy Args;
2180
2181 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2182
2183 // Sanity check just in case function has unexpected parameters.
2184 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2185 EVT::getEVT(FuncTy->getReturnType(), true) == VT &&
2186 "mismatch in value type and call signature type");
2187
2188 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2189 Type *ParamTy = FuncTy->getParamType(i: I);
2190
2191 if (HasMaskArg && I == E - 1) {
2192 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2193 "unexpected vector mask type");
2194 EVT MaskVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
2195 Args.emplace_back(args: DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT),
2196 args: MaskVT.getTypeForEVT(Context&: Ctx));
2197
2198 } else {
2199 SDValue Op = Node->getOperand(Num: I);
2200 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2201 "mismatch in value type and call argument type");
2202 Args.emplace_back(args&: Op, args&: ParamTy);
2203 }
2204 }
2205
2206 // Emit a call to the vector function.
2207 SDValue Callee =
2208 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2209 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2210
2211 TargetLowering::CallLoweringInfo CLI(DAG);
2212 CLI.setDebugLoc(DL)
2213 .setChain(DAG.getEntryNode())
2214 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2215
2216 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2217 Results.push_back(Elt: CallResult.first);
2218 return true;
2219}
2220
2221void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2222 SmallVectorImpl<SDValue> &Results) {
2223 EVT VT = Node->getValueType(ResNo: 0);
2224 EVT EltVT = VT.getVectorElementType();
2225 unsigned NumElems = VT.getVectorNumElements();
2226 unsigned NumOpers = Node->getNumOperands();
2227 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2228
2229 EVT TmpEltVT = EltVT;
2230 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2231 Node->getOpcode() == ISD::STRICT_FSETCCS)
2232 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2233 Context&: *DAG.getContext(), VT: TmpEltVT);
2234
2235 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2236 SDValue Chain = Node->getOperand(Num: 0);
2237 SDLoc dl(Node);
2238
2239 SmallVector<SDValue, 32> OpValues;
2240 SmallVector<SDValue, 32> OpChains;
2241 for (unsigned i = 0; i < NumElems; ++i) {
2242 SmallVector<SDValue, 4> Opers;
2243 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2244
2245 // The Chain is the first operand.
2246 Opers.push_back(Elt: Chain);
2247
2248 // Now process the remaining operands.
2249 for (unsigned j = 1; j < NumOpers; ++j) {
2250 SDValue Oper = Node->getOperand(Num: j);
2251 EVT OperVT = Oper.getValueType();
2252
2253 if (OperVT.isVector())
2254 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2255 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2256
2257 Opers.push_back(Elt: Oper);
2258 }
2259
2260 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2261 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2262 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2263
2264 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2265 Node->getOpcode() == ISD::STRICT_FSETCCS)
2266 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2267 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2268 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2269
2270 OpValues.push_back(Elt: ScalarResult);
2271 OpChains.push_back(Elt: ScalarChain);
2272 }
2273
2274 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2275 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2276
2277 Results.push_back(Elt: Result);
2278 Results.push_back(Elt: NewChain);
2279}
2280
2281SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2282 EVT VT = Node->getValueType(ResNo: 0);
2283 unsigned NumElems = VT.getVectorNumElements();
2284 EVT EltVT = VT.getVectorElementType();
2285 SDValue LHS = Node->getOperand(Num: 0);
2286 SDValue RHS = Node->getOperand(Num: 1);
2287 SDValue CC = Node->getOperand(Num: 2);
2288 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2289 SDLoc dl(Node);
2290 SmallVector<SDValue, 8> Ops(NumElems);
2291 for (unsigned i = 0; i < NumElems; ++i) {
2292 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2293 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2294 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2295 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2296 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2297 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2298 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2299 Context&: *DAG.getContext(), VT: TmpEltVT),
2300 N1: LHSElem, N2: RHSElem, N3: CC);
2301 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2302 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2303 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2304 }
2305 return DAG.getBuildVector(VT, DL: dl, Ops);
2306}
2307
2308bool SelectionDAG::LegalizeVectors() {
2309 return VectorLegalizer(*this).Run();
2310}
2311