1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::LegalizeVectors method.
10//
11// The vector legalizer looks for vector operations which might need to be
12// scalarized and legalizes them. This is a separate step from Legalize because
13// scalarizing can introduce illegal types. For example, suppose we have an
14// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16// operation, which introduces nodes with the illegal type i64 which must be
17// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18// the operation must be unrolled, which introduces nodes with the illegal
19// type i8 which must be promoted.
20//
21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22// or operations that happen to take a vector which are custom-lowered;
23// the legalization for such operations never produces nodes
24// with illegal types, so it's okay to put off legalizing them until
25// SelectionDAG::Legalize runs.
26//
27//===----------------------------------------------------------------------===//
28
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/Analysis/TargetLibraryInfo.h"
32#include "llvm/Analysis/VectorUtils.h"
33#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/SelectionDAG.h"
35#include "llvm/CodeGen/SelectionDAGNodes.h"
36#include "llvm/CodeGen/TargetLowering.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/CodeGenTypes/MachineValueType.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/Compiler.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include <cassert>
45#include <cstdint>
46#include <iterator>
47#include <utility>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "legalizevectorops"
52
53namespace {
54
55class VectorLegalizer {
56 SelectionDAG& DAG;
57 const TargetLowering &TLI;
58 bool Changed = false; // Keep track of whether anything changed
59
60 /// For nodes that are of legal width, and that have more than one use, this
61 /// map indicates what regularized operand to use. This allows us to avoid
62 /// legalizing the same thing more than once.
63 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
64
65 /// Adds a node to the translation cache.
66 void AddLegalizedOperand(SDValue From, SDValue To) {
67 LegalizedNodes.insert(KV: std::make_pair(x&: From, y&: To));
68 // If someone requests legalization of the new node, return itself.
69 if (From != To)
70 LegalizedNodes.insert(KV: std::make_pair(x&: To, y&: To));
71 }
72
73 /// Legalizes the given node.
74 SDValue LegalizeOp(SDValue Op);
75
76 /// Assuming the node is legal, "legalize" the results.
77 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
78
79 /// Make sure Results are legal and update the translation cache.
80 SDValue RecursivelyLegalizeResults(SDValue Op,
81 MutableArrayRef<SDValue> Results);
82
83 /// Wrapper to interface LowerOperation with a vector of Results.
84 /// Returns false if the target wants to use default expansion. Otherwise
85 /// returns true. If return is true and the Results are empty, then the
86 /// target wants to keep the input node as is.
87 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
88
89 /// Implements unrolling a VSETCC.
90 SDValue UnrollVSETCC(SDNode *Node);
91
92 /// Implement expand-based legalization of vector operations.
93 ///
94 /// This is just a high-level routine to dispatch to specific code paths for
95 /// operations to legalize them.
96 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
97
98 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
99 /// FP_TO_SINT isn't legal.
100 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
101
102 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
103 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
104 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
105
106 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
107 SDValue ExpandSEXTINREG(SDNode *Node);
108
109 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
110 ///
111 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
112 /// type. The contents of the bits in the extended part of each element are
113 /// undef.
114 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
115
116 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
117 ///
118 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
119 /// type, then shifts left and arithmetic shifts right to introduce a sign
120 /// extension.
121 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
122
123 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
124 ///
125 /// Shuffles the low lanes of the operand into place and blends zeros into
126 /// the remaining lanes, finally bitcasting to the proper type.
127 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
128
129 /// Expand bswap of vectors into a shuffle if legal.
130 SDValue ExpandBSWAP(SDNode *Node);
131
132 /// Implement vselect in terms of XOR, AND, OR when blend is not
133 /// supported by the target.
134 SDValue ExpandVSELECT(SDNode *Node);
135 SDValue ExpandVP_SELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandVP_FNEG(SDNode *Node);
139 SDValue ExpandVP_FABS(SDNode *Node);
140 SDValue ExpandVP_FCOPYSIGN(SDNode *Node);
141 SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
142 SDValue ExpandMaskedBinOp(SDNode *N);
143 SDValue ExpandSELECT(SDNode *Node);
144 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
145 SDValue ExpandStore(SDNode *N);
146 SDValue ExpandFNEG(SDNode *Node);
147 SDValue ExpandFABS(SDNode *Node);
148 SDValue ExpandFCOPYSIGN(SDNode *Node);
149 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 SDValue ExpandBITREVERSE(SDNode *Node);
152 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158
159 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
160 SmallVectorImpl<SDValue> &Results);
161
162 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 /// Implements vector promotion.
165 ///
166 /// This is essentially just bitcasting the operands to a different type and
167 /// bitcasting the result back to the original type.
168 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
169
170 /// Implements [SU]INT_TO_FP vector promotion.
171 ///
172 /// This is a [zs]ext of the input operand to a larger integer type.
173 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
174
175 /// Implements FP_TO_[SU]INT vector promotion of the result type.
176 ///
177 /// It is promoted to a larger integer type. The result is then
178 /// truncated back to the original type.
179 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
180
181 /// Implements vector setcc operation promotion.
182 ///
183 /// All vector operands are promoted to a vector type with larger element
184 /// type.
185 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
186
187 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
188
189 /// Calculate the reduction using a type of higher precision and round the
190 /// result to match the original type. Setting NonArithmetic signifies the
191 /// rounding of the result does not affect its value.
192 void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
193 bool NonArithmetic);
194
195 void PromoteVECTOR_COMPRESS(SDNode *Node, SmallVectorImpl<SDValue> &Results);
196
197public:
198 VectorLegalizer(SelectionDAG& dag) :
199 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
200
201 /// Begin legalizer the vector operations in the DAG.
202 bool Run();
203};
204
205} // end anonymous namespace
206
207bool VectorLegalizer::Run() {
208 // Before we start legalizing vector nodes, check if there are any vectors.
209 bool HasVectors = false;
210 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
211 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I) {
212 // Check if the values of the nodes contain vectors. We don't need to check
213 // the operands because we are going to check their values at some point.
214 HasVectors = llvm::any_of(Range: I->values(), P: [](EVT T) { return T.isVector(); });
215
216 // If we found a vector node we can start the legalization.
217 if (HasVectors)
218 break;
219 }
220
221 // If this basic block has no vectors then no need to legalize vectors.
222 if (!HasVectors)
223 return false;
224
225 // The legalize process is inherently a bottom-up recursive process (users
226 // legalize their uses before themselves). Given infinite stack space, we
227 // could just start legalizing on the root and traverse the whole graph. In
228 // practice however, this causes us to run out of stack space on large basic
229 // blocks. To avoid this problem, compute an ordering of the nodes where each
230 // node is only legalized after all of its operands are legalized.
231 DAG.AssignTopologicalOrder();
232 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
233 E = std::prev(x: DAG.allnodes_end()); I != std::next(x: E); ++I)
234 LegalizeOp(Op: SDValue(&*I, 0));
235
236 // Finally, it's possible the root changed. Get the new root.
237 SDValue OldRoot = DAG.getRoot();
238 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
239 DAG.setRoot(LegalizedNodes[OldRoot]);
240
241 LegalizedNodes.clear();
242
243 // Remove dead nodes now.
244 DAG.RemoveDeadNodes();
245
246 return Changed;
247}
248
249SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
250 assert(Op->getNumValues() == Result->getNumValues() &&
251 "Unexpected number of results");
252 // Generic legalization: just pass the operand through.
253 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
254 AddLegalizedOperand(From: Op.getValue(R: i), To: SDValue(Result, i));
255 return SDValue(Result, Op.getResNo());
256}
257
258SDValue
259VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
260 MutableArrayRef<SDValue> Results) {
261 assert(Results.size() == Op->getNumValues() &&
262 "Unexpected number of results");
263 // Make sure that the generated code is itself legal.
264 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
265 Results[i] = LegalizeOp(Op: Results[i]);
266 AddLegalizedOperand(From: Op.getValue(R: i), To: Results[i]);
267 }
268
269 return Results[Op.getResNo()];
270}
271
272SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
273 // Note that LegalizeOp may be reentered even from single-use nodes, which
274 // means that we always must cache transformed nodes.
275 auto I = LegalizedNodes.find(Val: Op);
276 if (I != LegalizedNodes.end()) return I->second;
277
278 // Legalize the operands
279 SmallVector<SDValue, 8> Ops;
280 for (const SDValue &Oper : Op->op_values())
281 Ops.push_back(Elt: LegalizeOp(Op: Oper));
282
283 SDNode *Node = DAG.UpdateNodeOperands(N: Op.getNode(), Ops);
284
285 bool HasVectorValueOrOp =
286 llvm::any_of(Range: Node->values(), P: [](EVT T) { return T.isVector(); }) ||
287 llvm::any_of(Range: Node->op_values(),
288 P: [](SDValue O) { return O.getValueType().isVector(); });
289 if (!HasVectorValueOrOp)
290 return TranslateLegalizeResults(Op, Result: Node);
291
292 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
293 EVT ValVT;
294 switch (Op.getOpcode()) {
295 default:
296 return TranslateLegalizeResults(Op, Result: Node);
297 case ISD::LOAD: {
298 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
299 ISD::LoadExtType ExtType = LD->getExtensionType();
300 EVT LoadedVT = LD->getMemoryVT();
301 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
302 Action = TLI.getLoadAction(ValVT: LD->getValueType(ResNo: 0), MemVT: LoadedVT, Alignment: LD->getAlign(),
303 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false);
304 break;
305 }
306 case ISD::STORE: {
307 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
308 EVT StVT = ST->getMemoryVT();
309 MVT ValVT = ST->getValue().getSimpleValueType();
310 if (StVT.isVector() && ST->isTruncatingStore())
311 Action = TLI.getTruncStoreAction(ValVT, MemVT: StVT, Alignment: ST->getAlign(),
312 AddrSpace: ST->getAddressSpace());
313 break;
314 }
315 case ISD::MERGE_VALUES:
316 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
317 // This operation lies about being legal: when it claims to be legal,
318 // it should actually be expanded.
319 if (Action == TargetLowering::Legal)
320 Action = TargetLowering::Expand;
321 break;
322#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
323 case ISD::STRICT_##DAGN:
324#include "llvm/IR/ConstrainedOps.def"
325 ValVT = Node->getValueType(ResNo: 0);
326 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
327 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
328 ValVT = Node->getOperand(Num: 1).getValueType();
329 if (Op.getOpcode() == ISD::STRICT_FSETCC ||
330 Op.getOpcode() == ISD::STRICT_FSETCCS) {
331 MVT OpVT = Node->getOperand(Num: 1).getSimpleValueType();
332 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 3))->get();
333 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
334 if (Action == TargetLowering::Legal)
335 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
336 } else {
337 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: ValVT);
338 }
339 // If we're asked to expand a strict vector floating-point operation,
340 // by default we're going to simply unroll it. That is usually the
341 // best approach, except in the case where the resulting strict (scalar)
342 // operations would themselves use the fallback mutation to non-strict.
343 // In that specific case, just do the fallback on the vector op.
344 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
345 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: ValVT) ==
346 TargetLowering::Legal) {
347 EVT EltVT = ValVT.getVectorElementType();
348 if (TLI.getOperationAction(Op: Node->getOpcode(), VT: EltVT)
349 == TargetLowering::Expand &&
350 TLI.getStrictFPOperationAction(Op: Node->getOpcode(), VT: EltVT)
351 == TargetLowering::Legal)
352 Action = TargetLowering::Legal;
353 }
354 break;
355 case ISD::ADD:
356 case ISD::SUB:
357 case ISD::MUL:
358 case ISD::MULHS:
359 case ISD::MULHU:
360 case ISD::SDIV:
361 case ISD::UDIV:
362 case ISD::SREM:
363 case ISD::UREM:
364 case ISD::SDIVREM:
365 case ISD::UDIVREM:
366 case ISD::FADD:
367 case ISD::FSUB:
368 case ISD::FMUL:
369 case ISD::FDIV:
370 case ISD::FREM:
371 case ISD::AND:
372 case ISD::OR:
373 case ISD::XOR:
374 case ISD::SHL:
375 case ISD::SRA:
376 case ISD::SRL:
377 case ISD::FSHL:
378 case ISD::FSHR:
379 case ISD::ROTL:
380 case ISD::ROTR:
381 case ISD::ABS:
382 case ISD::ABS_MIN_POISON:
383 case ISD::ABDS:
384 case ISD::ABDU:
385 case ISD::AVGCEILS:
386 case ISD::AVGCEILU:
387 case ISD::AVGFLOORS:
388 case ISD::AVGFLOORU:
389 case ISD::BSWAP:
390 case ISD::BITREVERSE:
391 case ISD::CTLZ:
392 case ISD::CTTZ:
393 case ISD::CTLZ_ZERO_POISON:
394 case ISD::CTTZ_ZERO_POISON:
395 case ISD::CTPOP:
396 case ISD::CLMUL:
397 case ISD::CLMULH:
398 case ISD::CLMULR:
399 case ISD::SELECT:
400 case ISD::VSELECT:
401 case ISD::SELECT_CC:
402 case ISD::ZERO_EXTEND:
403 case ISD::ANY_EXTEND:
404 case ISD::TRUNCATE:
405 case ISD::SIGN_EXTEND:
406 case ISD::FP_TO_SINT:
407 case ISD::FP_TO_UINT:
408 case ISD::FNEG:
409 case ISD::FABS:
410 case ISD::FMINNUM:
411 case ISD::FMAXNUM:
412 case ISD::FMINNUM_IEEE:
413 case ISD::FMAXNUM_IEEE:
414 case ISD::FMINIMUM:
415 case ISD::FMAXIMUM:
416 case ISD::FMINIMUMNUM:
417 case ISD::FMAXIMUMNUM:
418 case ISD::FCOPYSIGN:
419 case ISD::FSQRT:
420 case ISD::FSIN:
421 case ISD::FCOS:
422 case ISD::FTAN:
423 case ISD::FASIN:
424 case ISD::FACOS:
425 case ISD::FATAN:
426 case ISD::FATAN2:
427 case ISD::FSINH:
428 case ISD::FCOSH:
429 case ISD::FTANH:
430 case ISD::FLDEXP:
431 case ISD::FPOWI:
432 case ISD::FPOW:
433 case ISD::FCBRT:
434 case ISD::FLOG:
435 case ISD::FLOG2:
436 case ISD::FLOG10:
437 case ISD::FEXP:
438 case ISD::FEXP2:
439 case ISD::FEXP10:
440 case ISD::FCEIL:
441 case ISD::FTRUNC:
442 case ISD::FRINT:
443 case ISD::FNEARBYINT:
444 case ISD::FROUND:
445 case ISD::FROUNDEVEN:
446 case ISD::FFLOOR:
447 case ISD::FP_ROUND:
448 case ISD::FP_EXTEND:
449 case ISD::FPTRUNC_ROUND:
450 case ISD::FMA:
451 case ISD::SIGN_EXTEND_INREG:
452 case ISD::ANY_EXTEND_VECTOR_INREG:
453 case ISD::SIGN_EXTEND_VECTOR_INREG:
454 case ISD::ZERO_EXTEND_VECTOR_INREG:
455 case ISD::SMIN:
456 case ISD::SMAX:
457 case ISD::UMIN:
458 case ISD::UMAX:
459 case ISD::SMUL_LOHI:
460 case ISD::UMUL_LOHI:
461 case ISD::SADDO:
462 case ISD::UADDO:
463 case ISD::SSUBO:
464 case ISD::USUBO:
465 case ISD::SMULO:
466 case ISD::UMULO:
467 case ISD::CONVERT_FROM_ARBITRARY_FP:
468 case ISD::CONVERT_TO_ARBITRARY_FP:
469 case ISD::FCANONICALIZE:
470 case ISD::FFREXP:
471 case ISD::FMODF:
472 case ISD::FSINCOS:
473 case ISD::FSINCOSPI:
474 case ISD::SADDSAT:
475 case ISD::UADDSAT:
476 case ISD::SSUBSAT:
477 case ISD::USUBSAT:
478 case ISD::SSHLSAT:
479 case ISD::USHLSAT:
480 case ISD::FP_TO_SINT_SAT:
481 case ISD::FP_TO_UINT_SAT:
482 case ISD::MGATHER:
483 case ISD::VECTOR_COMPRESS:
484 case ISD::SCMP:
485 case ISD::UCMP:
486 case ISD::LOOP_DEPENDENCE_WAR_MASK:
487 case ISD::LOOP_DEPENDENCE_RAW_MASK:
488 case ISD::MASKED_UDIV:
489 case ISD::MASKED_SDIV:
490 case ISD::MASKED_UREM:
491 case ISD::MASKED_SREM:
492 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
493 break;
494 case ISD::SMULFIX:
495 case ISD::SMULFIXSAT:
496 case ISD::UMULFIX:
497 case ISD::UMULFIXSAT:
498 case ISD::SDIVFIX:
499 case ISD::SDIVFIXSAT:
500 case ISD::UDIVFIX:
501 case ISD::UDIVFIXSAT: {
502 unsigned Scale = Node->getConstantOperandVal(Num: 2);
503 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
504 VT: Node->getValueType(ResNo: 0), Scale);
505 break;
506 }
507 case ISD::LROUND:
508 case ISD::LLROUND:
509 case ISD::LRINT:
510 case ISD::LLRINT:
511 case ISD::SINT_TO_FP:
512 case ISD::UINT_TO_FP:
513 case ISD::VECREDUCE_ADD:
514 case ISD::VECREDUCE_MUL:
515 case ISD::VECREDUCE_AND:
516 case ISD::VECREDUCE_OR:
517 case ISD::VECREDUCE_XOR:
518 case ISD::VECREDUCE_SMAX:
519 case ISD::VECREDUCE_SMIN:
520 case ISD::VECREDUCE_UMAX:
521 case ISD::VECREDUCE_UMIN:
522 case ISD::VECREDUCE_FADD:
523 case ISD::VECREDUCE_FMAX:
524 case ISD::VECREDUCE_FMAXIMUM:
525 case ISD::VECREDUCE_FMIN:
526 case ISD::VECREDUCE_FMINIMUM:
527 case ISD::VECREDUCE_FMUL:
528 case ISD::CTTZ_ELTS:
529 case ISD::CTTZ_ELTS_ZERO_POISON:
530 case ISD::VECTOR_FIND_LAST_ACTIVE:
531 Action = TLI.getOperationAction(Op: Node->getOpcode(),
532 VT: Node->getOperand(Num: 0).getValueType());
533 break;
534 case ISD::VECREDUCE_SEQ_FADD:
535 case ISD::VECREDUCE_SEQ_FMUL:
536 Action = TLI.getOperationAction(Op: Node->getOpcode(),
537 VT: Node->getOperand(Num: 1).getValueType());
538 break;
539 case ISD::SETCC: {
540 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
541 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
542 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
543 if (Action == TargetLowering::Legal)
544 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
545 break;
546 }
547 case ISD::PARTIAL_REDUCE_UMLA:
548 case ISD::PARTIAL_REDUCE_SMLA:
549 case ISD::PARTIAL_REDUCE_SUMLA:
550 case ISD::PARTIAL_REDUCE_FMLA:
551 Action =
552 TLI.getPartialReduceMLAAction(Opc: Op.getOpcode(), AccVT: Node->getValueType(ResNo: 0),
553 InputVT: Node->getOperand(Num: 1).getValueType());
554 break;
555
556#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
557 case ISD::VPID: { \
558 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
559 : Node->getOperand(LEGALPOS).getValueType(); \
560 if (ISD::VPID == ISD::VP_SETCC) { \
561 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
562 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
563 if (Action != TargetLowering::Legal) \
564 break; \
565 } \
566 /* Defer non-vector results to LegalizeDAG. */ \
567 if (!Node->getValueType(0).isVector() && \
568 Node->getValueType(0) != MVT::Other) { \
569 Action = TargetLowering::Legal; \
570 break; \
571 } \
572 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
573 } break;
574#include "llvm/IR/VPIntrinsics.def"
575 }
576
577 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
578
579 SmallVector<SDValue, 8> ResultVals;
580 switch (Action) {
581 default: llvm_unreachable("This action is not supported yet!");
582 case TargetLowering::Promote:
583 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
584 "This action is not supported yet!");
585 LLVM_DEBUG(dbgs() << "Promoting\n");
586 Promote(Node, Results&: ResultVals);
587 assert(!ResultVals.empty() && "No results for promotion?");
588 break;
589 case TargetLowering::Legal:
590 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
591 break;
592 case TargetLowering::Custom:
593 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
594 if (LowerOperationWrapper(N: Node, Results&: ResultVals))
595 break;
596 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
597 [[fallthrough]];
598 case TargetLowering::Expand:
599 LLVM_DEBUG(dbgs() << "Expanding\n");
600 Expand(Node, Results&: ResultVals);
601 break;
602 }
603
604 if (ResultVals.empty())
605 return TranslateLegalizeResults(Op, Result: Node);
606
607 Changed = true;
608 return RecursivelyLegalizeResults(Op, Results: ResultVals);
609}
610
611// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
612// merge them somehow?
613bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
614 SmallVectorImpl<SDValue> &Results) {
615 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
616
617 if (!Res.getNode())
618 return false;
619
620 if (Res == SDValue(Node, 0))
621 return true;
622
623 // If the original node has one result, take the return value from
624 // LowerOperation as is. It might not be result number 0.
625 if (Node->getNumValues() == 1) {
626 Results.push_back(Elt: Res);
627 return true;
628 }
629
630 // If the original node has multiple results, then the return node should
631 // have the same number of results.
632 assert((Node->getNumValues() == Res->getNumValues()) &&
633 "Lowering returned the wrong number of results!");
634
635 // Places new result values base on N result number.
636 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
637 Results.push_back(Elt: Res.getValue(R: I));
638
639 return true;
640}
641
642void VectorLegalizer::PromoteSETCC(SDNode *Node,
643 SmallVectorImpl<SDValue> &Results) {
644 MVT VecVT = Node->getOperand(Num: 0).getSimpleValueType();
645 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
646
647 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
648
649 SDLoc DL(Node);
650 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
651
652 Operands[0] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 0));
653 Operands[1] = DAG.getNode(Opcode: ExtOp, DL, VT: NewVecVT, Operand: Node->getOperand(Num: 1));
654 Operands[2] = Node->getOperand(Num: 2);
655
656 if (Node->getOpcode() == ISD::VP_SETCC) {
657 Operands[3] = Node->getOperand(Num: 3); // mask
658 Operands[4] = Node->getOperand(Num: 4); // evl
659 }
660
661 EVT ResVT =
662 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: NewVecVT);
663 SDValue Res =
664 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Ops: Operands, Flags: Node->getFlags());
665 if (ResVT != Node->getValueType(ResNo: 0))
666 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: Node->getValueType(ResNo: 0), OpVT: NewVecVT);
667 Results.push_back(Elt: Res);
668}
669
670void VectorLegalizer::PromoteSTRICT(SDNode *Node,
671 SmallVectorImpl<SDValue> &Results) {
672 MVT VecVT = Node->getOperand(Num: 1).getSimpleValueType();
673 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
674
675 assert(VecVT.isFloatingPoint());
676
677 SDLoc DL(Node);
678 SmallVector<SDValue, 5> Operands(Node->getNumOperands());
679 SmallVector<SDValue, 2> Chains;
680
681 for (unsigned j = 1; j != Node->getNumOperands(); ++j)
682 if (Node->getOperand(Num: j).getValueType().isVector() &&
683 !(ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
684 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) // Skip mask operand.
685 {
686 // promote the vector operand.
687 SDValue Ext =
688 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {NewVecVT, MVT::Other},
689 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: j)});
690 Operands[j] = Ext.getValue(R: 0);
691 Chains.push_back(Elt: Ext.getValue(R: 1));
692 } else
693 Operands[j] = Node->getOperand(Num: j); // Skip no vector operand.
694
695 SDVTList VTs = DAG.getVTList(VT1: NewVecVT, VT2: Node->getValueType(ResNo: 1));
696
697 Operands[0] = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
698
699 SDValue Res =
700 DAG.getNode(Opcode: Node->getOpcode(), DL, VTList: VTs, Ops: Operands, Flags: Node->getFlags());
701
702 SDValue Round =
703 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VecVT, MVT::Other},
704 Ops: {Res.getValue(R: 1), Res.getValue(R: 0),
705 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
706
707 Results.push_back(Elt: Round.getValue(R: 0));
708 Results.push_back(Elt: Round.getValue(R: 1));
709}
710
711void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
712 SmallVectorImpl<SDValue> &Results,
713 bool NonArithmetic) {
714 MVT OpVT = Node->getOperand(Num: 0).getSimpleValueType();
715 assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
716 MVT NewOpVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OpVT);
717
718 SDLoc DL(Node);
719 SDValue NewOp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewOpVT, Operand: Node->getOperand(Num: 0));
720 SDValue Rdx =
721 DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewOpVT.getVectorElementType(), Operand: NewOp,
722 Flags: Node->getFlags());
723 SDValue Res =
724 DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Node->getValueType(ResNo: 0), N1: Rdx,
725 N2: DAG.getIntPtrConstant(Val: NonArithmetic, DL, /*isTarget=*/true));
726 Results.push_back(Elt: Res);
727}
728
729void VectorLegalizer::PromoteVECTOR_COMPRESS(
730 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
731 SDLoc DL(Node);
732 EVT VT = Node->getValueType(ResNo: 0);
733 MVT PromotedVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VT.getSimpleVT());
734 assert((VT.isInteger() || VT.getSizeInBits() == PromotedVT.getSizeInBits()) &&
735 "Only integer promotion or bitcasts between types is supported");
736
737 SDValue Vec = Node->getOperand(Num: 0);
738 SDValue Mask = Node->getOperand(Num: 1);
739 SDValue Passthru = Node->getOperand(Num: 2);
740 if (VT.isInteger()) {
741 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Vec);
742 Mask = TLI.promoteTargetBoolean(DAG, Bool: Mask, ValVT: PromotedVT);
743 Passthru = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PromotedVT, Operand: Passthru);
744 } else {
745 Vec = DAG.getBitcast(VT: PromotedVT, V: Vec);
746 Passthru = DAG.getBitcast(VT: PromotedVT, V: Passthru);
747 }
748
749 SDValue Result =
750 DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: PromotedVT, N1: Vec, N2: Mask, N3: Passthru);
751 Result = VT.isInteger() ? DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Result)
752 : DAG.getBitcast(VT, V: Result);
753 Results.push_back(Elt: Result);
754}
755
756void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
757 // For a few operations there is a specific concept for promotion based on
758 // the operand's type.
759 switch (Node->getOpcode()) {
760 case ISD::SINT_TO_FP:
761 case ISD::UINT_TO_FP:
762 case ISD::STRICT_SINT_TO_FP:
763 case ISD::STRICT_UINT_TO_FP:
764 // "Promote" the operation by extending the operand.
765 PromoteINT_TO_FP(Node, Results);
766 return;
767 case ISD::FP_TO_UINT:
768 case ISD::FP_TO_SINT:
769 case ISD::STRICT_FP_TO_UINT:
770 case ISD::STRICT_FP_TO_SINT:
771 // Promote the operation by extending the operand.
772 PromoteFP_TO_INT(Node, Results);
773 return;
774 case ISD::VP_SETCC:
775 case ISD::SETCC:
776 // Promote the operation by extending the operand.
777 PromoteSETCC(Node, Results);
778 return;
779 case ISD::STRICT_FADD:
780 case ISD::STRICT_FSUB:
781 case ISD::STRICT_FMUL:
782 case ISD::STRICT_FDIV:
783 case ISD::STRICT_FSQRT:
784 case ISD::STRICT_FMA:
785 PromoteSTRICT(Node, Results);
786 return;
787 case ISD::VECREDUCE_FADD:
788 case ISD::VECREDUCE_FMUL:
789 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
790 return;
791 case ISD::VECREDUCE_FMAX:
792 case ISD::VECREDUCE_FMAXIMUM:
793 case ISD::VECREDUCE_FMIN:
794 case ISD::VECREDUCE_FMINIMUM:
795 PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
796 return;
797 case ISD::VECTOR_COMPRESS:
798 PromoteVECTOR_COMPRESS(Node, Results);
799 return;
800
801 case ISD::FP_ROUND:
802 case ISD::FP_EXTEND:
803 // These operations are used to do promotion so they can't be promoted
804 // themselves.
805 llvm_unreachable("Don't know how to promote this operation!");
806 case ISD::VP_FABS:
807 case ISD::VP_FCOPYSIGN:
808 case ISD::VP_FNEG:
809 // Promoting fabs, fneg, and fcopysign changes their semantics.
810 llvm_unreachable("These operations should not be promoted");
811 }
812
813 // There are currently two cases of vector promotion:
814 // 1) Bitcasting a vector of integers to a different type to a vector of the
815 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
816 // 2) Extending a vector of floats to a vector of the same number of larger
817 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
818 assert(Node->getNumValues() == 1 &&
819 "Can't promote a vector with multiple results!");
820 MVT VT = Node->getSimpleValueType(ResNo: 0);
821 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
822 SDLoc dl(Node);
823 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
824
825 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
826 // Do not promote the mask operand of a VP OP.
827 bool SkipPromote = ISD::isVPOpcode(Opcode: Node->getOpcode()) &&
828 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j;
829 if (Node->getOperand(Num: j).getValueType().isVector() && !SkipPromote)
830 if (Node->getOperand(Num: j)
831 .getValueType()
832 .getVectorElementType()
833 .isFloatingPoint() &&
834 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
835 if (ISD::isVPOpcode(Opcode: Node->getOpcode())) {
836 unsigned EVLIdx =
837 *ISD::getVPExplicitVectorLengthIdx(Opcode: Node->getOpcode());
838 unsigned MaskIdx = *ISD::getVPMaskIdx(Opcode: Node->getOpcode());
839 Operands[j] =
840 DAG.getNode(Opcode: ISD::VP_FP_EXTEND, DL: dl, VT: NVT, N1: Node->getOperand(Num: j),
841 N2: Node->getOperand(Num: MaskIdx), N3: Node->getOperand(Num: EVLIdx));
842 } else {
843 Operands[j] =
844 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
845 }
846 else
847 Operands[j] = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
848 else
849 Operands[j] = Node->getOperand(Num: j);
850 }
851
852 SDValue Res =
853 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Ops: Operands, Flags: Node->getFlags());
854
855 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
856 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
857 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
858 if (ISD::isVPOpcode(Opcode: Node->getOpcode())) {
859 unsigned EVLIdx = *ISD::getVPExplicitVectorLengthIdx(Opcode: Node->getOpcode());
860 unsigned MaskIdx = *ISD::getVPMaskIdx(Opcode: Node->getOpcode());
861 Res = DAG.getNode(Opcode: ISD::VP_FP_ROUND, DL: dl, VT, N1: Res,
862 N2: Node->getOperand(Num: MaskIdx), N3: Node->getOperand(Num: EVLIdx));
863 } else {
864 Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Res,
865 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
866 }
867 else
868 Res = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
869
870 Results.push_back(Elt: Res);
871}
872
873void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
874 SmallVectorImpl<SDValue> &Results) {
875 // INT_TO_FP operations may require the input operand be promoted even
876 // when the type is otherwise legal.
877 bool IsStrict = Node->isStrictFPOpcode();
878 MVT VT = Node->getOperand(Num: IsStrict ? 1 : 0).getSimpleValueType();
879 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
880 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
881 "Vectors have different number of elements!");
882
883 SDLoc dl(Node);
884 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
885
886 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
887 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
888 ? ISD::ZERO_EXTEND
889 : ISD::SIGN_EXTEND;
890 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
891 if (Node->getOperand(Num: j).getValueType().isVector())
892 Operands[j] = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: j));
893 else
894 Operands[j] = Node->getOperand(Num: j);
895 }
896
897 if (IsStrict) {
898 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
899 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other}, Ops: Operands);
900 Results.push_back(Elt: Res);
901 Results.push_back(Elt: Res.getValue(R: 1));
902 return;
903 }
904
905 SDValue Res =
906 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Ops: Operands);
907 Results.push_back(Elt: Res);
908}
909
910// For FP_TO_INT we promote the result type to a vector type with wider
911// elements and then truncate the result. This is different from the default
912// PromoteVector which uses bitcast to promote thus assumning that the
913// promoted vector type has the same overall size.
914void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
915 SmallVectorImpl<SDValue> &Results) {
916 MVT VT = Node->getSimpleValueType(ResNo: 0);
917 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
918 bool IsStrict = Node->isStrictFPOpcode();
919 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
920 "Vectors have different number of elements!");
921
922 unsigned NewOpc = Node->getOpcode();
923 // Change FP_TO_UINT to FP_TO_SINT if possible.
924 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
925 if (NewOpc == ISD::FP_TO_UINT &&
926 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT))
927 NewOpc = ISD::FP_TO_SINT;
928
929 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
930 TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT))
931 NewOpc = ISD::STRICT_FP_TO_SINT;
932
933 SDLoc dl(Node);
934 SDValue Promoted, Chain;
935 if (IsStrict) {
936 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
937 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
938 Chain = Promoted.getValue(R: 1);
939 } else
940 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
941
942 // Assert that the converted value fits in the original type. If it doesn't
943 // (eg: because the value being converted is too big), then the result of the
944 // original operation was undefined anyway, so the assert is still correct.
945 if (Node->getOpcode() == ISD::FP_TO_UINT ||
946 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
947 NewOpc = ISD::AssertZext;
948 else
949 NewOpc = ISD::AssertSext;
950
951 Promoted = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Promoted,
952 N2: DAG.getValueType(VT.getScalarType()));
953 Promoted = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Promoted);
954 Results.push_back(Elt: Promoted);
955 if (IsStrict)
956 Results.push_back(Elt: Chain);
957}
958
959std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
960 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
961 return TLI.scalarizeVectorLoad(LD, DAG);
962}
963
964SDValue VectorLegalizer::ExpandStore(SDNode *N) {
965 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
966 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
967 return TF;
968}
969
970void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
971 switch (Node->getOpcode()) {
972 case ISD::LOAD: {
973 std::pair<SDValue, SDValue> Tmp = ExpandLoad(N: Node);
974 Results.push_back(Elt: Tmp.first);
975 Results.push_back(Elt: Tmp.second);
976 return;
977 }
978 case ISD::STORE:
979 Results.push_back(Elt: ExpandStore(N: Node));
980 return;
981 case ISD::MERGE_VALUES:
982 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
983 Results.push_back(Elt: Node->getOperand(Num: i));
984 return;
985 case ISD::SIGN_EXTEND_INREG:
986 if (SDValue Expanded = ExpandSEXTINREG(Node)) {
987 Results.push_back(Elt: Expanded);
988 return;
989 }
990 break;
991 case ISD::ANY_EXTEND_VECTOR_INREG:
992 Results.push_back(Elt: ExpandANY_EXTEND_VECTOR_INREG(Node));
993 return;
994 case ISD::SIGN_EXTEND_VECTOR_INREG:
995 Results.push_back(Elt: ExpandSIGN_EXTEND_VECTOR_INREG(Node));
996 return;
997 case ISD::ZERO_EXTEND_VECTOR_INREG:
998 Results.push_back(Elt: ExpandZERO_EXTEND_VECTOR_INREG(Node));
999 return;
1000 case ISD::BSWAP:
1001 if (SDValue Expanded = ExpandBSWAP(Node)) {
1002 Results.push_back(Elt: Expanded);
1003 return;
1004 }
1005 break;
1006 case ISD::VP_BSWAP:
1007 Results.push_back(Elt: TLI.expandVPBSWAP(N: Node, DAG));
1008 return;
1009 case ISD::VSELECT:
1010 if (SDValue Expanded = ExpandVSELECT(Node)) {
1011 Results.push_back(Elt: Expanded);
1012 return;
1013 }
1014 break;
1015 case ISD::VP_SELECT:
1016 if (SDValue Expanded = ExpandVP_SELECT(Node)) {
1017 Results.push_back(Elt: Expanded);
1018 return;
1019 }
1020 break;
1021 case ISD::VP_SREM:
1022 case ISD::VP_UREM:
1023 if (SDValue Expanded = ExpandVP_REM(Node)) {
1024 Results.push_back(Elt: Expanded);
1025 return;
1026 }
1027 break;
1028 case ISD::VP_FNEG:
1029 if (SDValue Expanded = ExpandVP_FNEG(Node)) {
1030 Results.push_back(Elt: Expanded);
1031 return;
1032 }
1033 break;
1034 case ISD::VP_FABS:
1035 if (SDValue Expanded = ExpandVP_FABS(Node)) {
1036 Results.push_back(Elt: Expanded);
1037 return;
1038 }
1039 break;
1040 case ISD::VP_FCOPYSIGN:
1041 if (SDValue Expanded = ExpandVP_FCOPYSIGN(Node)) {
1042 Results.push_back(Elt: Expanded);
1043 return;
1044 }
1045 break;
1046 case ISD::SELECT:
1047 if (SDValue Expanded = ExpandSELECT(Node)) {
1048 Results.push_back(Elt: Expanded);
1049 return;
1050 }
1051 break;
1052 case ISD::SELECT_CC: {
1053 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1054 EVT CondVT = TLI.getSetCCResultType(
1055 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
1056 SDValue SetCC =
1057 DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(Node), VT: CondVT, N1: Node->getOperand(Num: 0),
1058 N2: Node->getOperand(Num: 1), N3: Node->getOperand(Num: 4));
1059 Results.push_back(Elt: DAG.getSelect(DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0), Cond: SetCC,
1060 LHS: Node->getOperand(Num: 2),
1061 RHS: Node->getOperand(Num: 3)));
1062 return;
1063 }
1064 break;
1065 }
1066 case ISD::FP_TO_UINT:
1067 ExpandFP_TO_UINT(Node, Results);
1068 return;
1069 case ISD::UINT_TO_FP:
1070 ExpandUINT_TO_FLOAT(Node, Results);
1071 return;
1072 case ISD::FNEG:
1073 if (SDValue Expanded = ExpandFNEG(Node)) {
1074 Results.push_back(Elt: Expanded);
1075 return;
1076 }
1077 break;
1078 case ISD::FABS:
1079 if (SDValue Expanded = ExpandFABS(Node)) {
1080 Results.push_back(Elt: Expanded);
1081 return;
1082 }
1083 break;
1084 case ISD::FCOPYSIGN:
1085 if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {
1086 Results.push_back(Elt: Expanded);
1087 return;
1088 }
1089 break;
1090 case ISD::FCANONICALIZE: {
1091 // If the scalar element type has a
1092 // Legal/Custom FCANONICALIZE, don't
1093 // mess with the vector, fall back.
1094 EVT VT = Node->getValueType(ResNo: 0);
1095 EVT EltVT = VT.getVectorElementType();
1096 if (!VT.isScalableVector() &&
1097 TLI.getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT.getSimpleVT()) !=
1098 TargetLowering::Expand)
1099 break;
1100 // Otherwise canonicalize the whole vector.
1101 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
1102 Results.push_back(Elt: Mul);
1103 return;
1104 }
1105 case ISD::FSUB:
1106 ExpandFSUB(Node, Results);
1107 return;
1108 case ISD::SETCC:
1109 case ISD::VP_SETCC:
1110 ExpandSETCC(Node, Results);
1111 return;
1112 case ISD::ABS:
1113 case ISD::ABS_MIN_POISON:
1114 if (SDValue Expanded = TLI.expandABS(N: Node, DAG)) {
1115 Results.push_back(Elt: Expanded);
1116 return;
1117 }
1118 break;
1119 case ISD::ABDS:
1120 case ISD::ABDU:
1121 if (SDValue Expanded = TLI.expandABD(N: Node, DAG)) {
1122 Results.push_back(Elt: Expanded);
1123 return;
1124 }
1125 break;
1126 case ISD::AVGCEILS:
1127 case ISD::AVGCEILU:
1128 case ISD::AVGFLOORS:
1129 case ISD::AVGFLOORU:
1130 if (SDValue Expanded = TLI.expandAVG(N: Node, DAG)) {
1131 Results.push_back(Elt: Expanded);
1132 return;
1133 }
1134 break;
1135 case ISD::BITREVERSE:
1136 if (SDValue Expanded = ExpandBITREVERSE(Node)) {
1137 Results.push_back(Elt: Expanded);
1138 return;
1139 }
1140 break;
1141 case ISD::VP_BITREVERSE:
1142 if (SDValue Expanded = TLI.expandVPBITREVERSE(N: Node, DAG)) {
1143 Results.push_back(Elt: Expanded);
1144 return;
1145 }
1146 break;
1147 case ISD::CTPOP:
1148 if (SDValue Expanded = TLI.expandCTPOP(N: Node, DAG)) {
1149 Results.push_back(Elt: Expanded);
1150 return;
1151 }
1152 break;
1153 case ISD::VP_CTPOP:
1154 if (SDValue Expanded = TLI.expandVPCTPOP(N: Node, DAG)) {
1155 Results.push_back(Elt: Expanded);
1156 return;
1157 }
1158 break;
1159 case ISD::CTLZ:
1160 case ISD::CTLZ_ZERO_POISON:
1161 if (SDValue Expanded = TLI.expandCTLZ(N: Node, DAG)) {
1162 Results.push_back(Elt: Expanded);
1163 return;
1164 }
1165 break;
1166 case ISD::VP_CTLZ:
1167 case ISD::VP_CTLZ_ZERO_POISON:
1168 if (SDValue Expanded = TLI.expandVPCTLZ(N: Node, DAG)) {
1169 Results.push_back(Elt: Expanded);
1170 return;
1171 }
1172 break;
1173 case ISD::CTTZ:
1174 case ISD::CTTZ_ZERO_POISON:
1175 if (SDValue Expanded = TLI.expandCTTZ(N: Node, DAG)) {
1176 Results.push_back(Elt: Expanded);
1177 return;
1178 }
1179 break;
1180 case ISD::VP_CTTZ:
1181 case ISD::VP_CTTZ_ZERO_POISON:
1182 if (SDValue Expanded = TLI.expandVPCTTZ(N: Node, DAG)) {
1183 Results.push_back(Elt: Expanded);
1184 return;
1185 }
1186 break;
1187 case ISD::FSHL:
1188 case ISD::VP_FSHL:
1189 case ISD::FSHR:
1190 case ISD::VP_FSHR:
1191 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG)) {
1192 Results.push_back(Elt: Expanded);
1193 return;
1194 }
1195 break;
1196 case ISD::CLMUL:
1197 case ISD::CLMULR:
1198 case ISD::CLMULH:
1199 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG)) {
1200 Results.push_back(Elt: Expanded);
1201 return;
1202 }
1203 break;
1204 case ISD::PEXT:
1205 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
1206 return;
1207 case ISD::PDEP:
1208 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
1209 return;
1210 case ISD::ROTL:
1211 case ISD::ROTR:
1212 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: false /*AllowVectorOps*/, DAG)) {
1213 Results.push_back(Elt: Expanded);
1214 return;
1215 }
1216 break;
1217 case ISD::FMINNUM:
1218 case ISD::FMAXNUM:
1219 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG)) {
1220 Results.push_back(Elt: Expanded);
1221 return;
1222 }
1223 break;
1224 case ISD::FMINIMUM:
1225 case ISD::FMAXIMUM:
1226 Results.push_back(Elt: TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG));
1227 return;
1228 case ISD::FMINIMUMNUM:
1229 case ISD::FMAXIMUMNUM:
1230 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
1231 return;
1232 case ISD::SMIN:
1233 case ISD::SMAX:
1234 case ISD::UMIN:
1235 case ISD::UMAX:
1236 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
1237 Results.push_back(Elt: Expanded);
1238 return;
1239 }
1240 break;
1241 case ISD::UADDO:
1242 case ISD::USUBO:
1243 ExpandUADDSUBO(Node, Results);
1244 return;
1245 case ISD::SADDO:
1246 case ISD::SSUBO:
1247 ExpandSADDSUBO(Node, Results);
1248 return;
1249 case ISD::UMULO:
1250 case ISD::SMULO:
1251 ExpandMULO(Node, Results);
1252 return;
1253 case ISD::USUBSAT:
1254 case ISD::SSUBSAT:
1255 case ISD::UADDSAT:
1256 case ISD::SADDSAT:
1257 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
1258 Results.push_back(Elt: Expanded);
1259 return;
1260 }
1261 break;
1262 case ISD::USHLSAT:
1263 case ISD::SSHLSAT:
1264 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {
1265 Results.push_back(Elt: Expanded);
1266 return;
1267 }
1268 break;
1269 case ISD::FP_TO_SINT_SAT:
1270 case ISD::FP_TO_UINT_SAT:
1271 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
1272 if (Node->getValueType(ResNo: 0).isScalableVector()) {
1273 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(N: Node, DAG)) {
1274 Results.push_back(Elt: Expanded);
1275 return;
1276 }
1277 }
1278 break;
1279 case ISD::SMULFIX:
1280 case ISD::UMULFIX:
1281 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
1282 Results.push_back(Elt: Expanded);
1283 return;
1284 }
1285 break;
1286 case ISD::SMULFIXSAT:
1287 case ISD::UMULFIXSAT:
1288 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
1289 // why. Maybe it results in worse codegen compared to the unroll for some
1290 // targets? This should probably be investigated. And if we still prefer to
1291 // unroll an explanation could be helpful.
1292 break;
1293 case ISD::SDIVFIX:
1294 case ISD::UDIVFIX:
1295 ExpandFixedPointDiv(Node, Results);
1296 return;
1297 case ISD::SDIVFIXSAT:
1298 case ISD::UDIVFIXSAT:
1299 break;
1300#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1301 case ISD::STRICT_##DAGN:
1302#include "llvm/IR/ConstrainedOps.def"
1303 ExpandStrictFPOp(Node, Results);
1304 return;
1305 case ISD::VECREDUCE_ADD:
1306 case ISD::VECREDUCE_MUL:
1307 case ISD::VECREDUCE_AND:
1308 case ISD::VECREDUCE_OR:
1309 case ISD::VECREDUCE_XOR:
1310 case ISD::VECREDUCE_SMAX:
1311 case ISD::VECREDUCE_SMIN:
1312 case ISD::VECREDUCE_UMAX:
1313 case ISD::VECREDUCE_UMIN:
1314 case ISD::VECREDUCE_FADD:
1315 case ISD::VECREDUCE_FMUL:
1316 case ISD::VECREDUCE_FMAX:
1317 case ISD::VECREDUCE_FMIN:
1318 case ISD::VECREDUCE_FMAXIMUM:
1319 case ISD::VECREDUCE_FMINIMUM:
1320 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
1321 return;
1322 case ISD::PARTIAL_REDUCE_UMLA:
1323 case ISD::PARTIAL_REDUCE_SMLA:
1324 case ISD::PARTIAL_REDUCE_SUMLA:
1325 case ISD::PARTIAL_REDUCE_FMLA:
1326 Results.push_back(Elt: TLI.expandPartialReduceMLA(Node, DAG));
1327 return;
1328 case ISD::VECREDUCE_SEQ_FADD:
1329 case ISD::VECREDUCE_SEQ_FMUL:
1330 Results.push_back(Elt: TLI.expandVecReduceSeq(Node, DAG));
1331 return;
1332 case ISD::SREM:
1333 case ISD::UREM:
1334 ExpandREM(Node, Results);
1335 return;
1336 case ISD::VP_MERGE:
1337 if (SDValue Expanded = ExpandVP_MERGE(Node)) {
1338 Results.push_back(Elt: Expanded);
1339 return;
1340 }
1341 break;
1342 case ISD::FREM: {
1343 RTLIB::Libcall LC = RTLIB::getREM(VT: Node->getValueType(ResNo: 0));
1344 if (tryExpandVecMathCall(Node, LC, Results))
1345 return;
1346
1347 break;
1348 }
1349 case ISD::FSINCOS:
1350 case ISD::FSINCOSPI: {
1351 EVT VT = Node->getValueType(ResNo: 0);
1352 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1353 ? RTLIB::getSINCOS(RetVT: VT)
1354 : RTLIB::getSINCOSPI(RetVT: VT);
1355 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1356 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))
1357 return;
1358
1359 // TODO: Try to see if there's a narrower call available to use before
1360 // scalarizing.
1361 break;
1362 }
1363 case ISD::FPOW: {
1364 RTLIB::Libcall LC = RTLIB::getPOW(RetVT: Node->getValueType(ResNo: 0));
1365 if (tryExpandVecMathCall(Node, LC, Results))
1366 return;
1367
1368 // TODO: Try to see if there's a narrower call available to use before
1369 // scalarizing.
1370 break;
1371 }
1372 case ISD::FCBRT: {
1373 RTLIB::Libcall LC = RTLIB::getCBRT(RetVT: Node->getValueType(ResNo: 0));
1374 if (tryExpandVecMathCall(Node, LC, Results))
1375 return;
1376
1377 // TODO: Try to see if there's a narrower call available to use before
1378 // scalarizing.
1379 break;
1380 }
1381 case ISD::FMODF: {
1382 EVT VT = Node->getValueType(ResNo: 0);
1383 RTLIB::Libcall LC = RTLIB::getMODF(VT);
1384 if (LC != RTLIB::UNKNOWN_LIBCALL &&
1385 TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
1386 /*CallRetResNo=*/0))
1387 return;
1388 break;
1389 }
1390 case ISD::VECTOR_COMPRESS:
1391 Results.push_back(Elt: TLI.expandVECTOR_COMPRESS(Node, DAG));
1392 return;
1393 case ISD::CTTZ_ELTS:
1394 case ISD::CTTZ_ELTS_ZERO_POISON:
1395 Results.push_back(Elt: TLI.expandCttzElts(Node, DAG));
1396 return;
1397 case ISD::VECTOR_FIND_LAST_ACTIVE:
1398 Results.push_back(Elt: TLI.expandVectorFindLastActive(N: Node, DAG));
1399 return;
1400 case ISD::SCMP:
1401 case ISD::UCMP:
1402 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
1403 return;
1404 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1405 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1406 Results.push_back(Elt: ExpandLOOP_DEPENDENCE_MASK(N: Node));
1407 return;
1408
1409 case ISD::FADD:
1410 case ISD::FMUL:
1411 case ISD::FMA:
1412 case ISD::FDIV:
1413 case ISD::FCEIL:
1414 case ISD::FFLOOR:
1415 case ISD::FNEARBYINT:
1416 case ISD::FRINT:
1417 case ISD::FROUND:
1418 case ISD::FROUNDEVEN:
1419 case ISD::FTRUNC:
1420 case ISD::FSQRT:
1421 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
1422 Results.push_back(Elt: Expanded);
1423 return;
1424 }
1425 break;
1426 case ISD::CONVERT_TO_ARBITRARY_FP:
1427 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
1428 Results.push_back(Elt: Expanded);
1429 else
1430 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1431 return;
1432 case ISD::CONVERT_FROM_ARBITRARY_FP:
1433 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
1434 Results.push_back(Elt: Expanded);
1435 else
1436 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
1437 return;
1438 case ISD::MASKED_UDIV:
1439 case ISD::MASKED_SDIV:
1440 case ISD::MASKED_UREM:
1441 case ISD::MASKED_SREM:
1442 Results.push_back(Elt: ExpandMaskedBinOp(N: Node));
1443 return;
1444 }
1445
1446 SDValue Unrolled = DAG.UnrollVectorOp(N: Node);
1447 if (Node->getNumValues() == 1) {
1448 Results.push_back(Elt: Unrolled);
1449 } else {
1450 assert(Node->getNumValues() == Unrolled->getNumValues() &&
1451 "VectorLegalizer Expand returned wrong number of results!");
1452 for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
1453 Results.push_back(Elt: Unrolled.getValue(R: I));
1454 }
1455}
1456
1457SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
1458 // Lower a select instruction where the condition is a scalar and the
1459 // operands are vectors. Lower this select to VSELECT and implement it
1460 // using XOR AND OR. The selector bit is broadcasted.
1461 EVT VT = Node->getValueType(ResNo: 0);
1462 SDLoc DL(Node);
1463
1464 SDValue Mask = Node->getOperand(Num: 0);
1465 SDValue Op1 = Node->getOperand(Num: 1);
1466 SDValue Op2 = Node->getOperand(Num: 2);
1467
1468 assert(VT.isVector() && !Mask.getValueType().isVector()
1469 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1470
1471 // If we can't even use the basic vector operations of
1472 // AND,OR,XOR, we will have to scalarize the op.
1473 // Notice that the operation may be 'promoted' which means that it is
1474 // 'bitcasted' to another type which is handled.
1475 // Also, we need to be able to construct a splat vector using either
1476 // BUILD_VECTOR or SPLAT_VECTOR.
1477 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
1478 // BUILD_VECTOR?
1479 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1480 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1481 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand ||
1482 TLI.getOperationAction(Op: VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
1483 : ISD::SPLAT_VECTOR,
1484 VT) == TargetLowering::Expand)
1485 return SDValue();
1486
1487 // Generate a mask operand.
1488 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1489
1490 // What is the size of each element in the vector mask.
1491 EVT BitTy = MaskTy.getScalarType();
1492
1493 Mask = DAG.getSelect(DL, VT: BitTy, Cond: Mask, LHS: DAG.getAllOnesConstant(DL, VT: BitTy),
1494 RHS: DAG.getConstant(Val: 0, DL, VT: BitTy));
1495
1496 // Broadcast the mask so that the entire vector is all one or all zero.
1497 Mask = DAG.getSplat(VT: MaskTy, DL, Op: Mask);
1498
1499 // Bitcast the operands to be the same type as the mask.
1500 // This is needed when we select between FP types because
1501 // the mask is a vector of integers.
1502 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op1);
1503 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MaskTy, Operand: Op2);
1504
1505 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT: MaskTy);
1506
1507 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op1, N2: Mask);
1508 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskTy, N1: Op2, N2: NotMask);
1509 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MaskTy, N1: Op1, N2: Op2);
1510 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1511}
1512
1513SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1514 EVT VT = Node->getValueType(ResNo: 0);
1515
1516 // Make sure that the SRA and SHL instructions are available.
1517 if (TLI.getOperationAction(Op: ISD::SRA, VT) == TargetLowering::Expand ||
1518 TLI.getOperationAction(Op: ISD::SHL, VT) == TargetLowering::Expand)
1519 return SDValue();
1520
1521 SDLoc DL(Node);
1522 EVT OrigTy = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1523
1524 unsigned BW = VT.getScalarSizeInBits();
1525 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1526 SDValue ShiftSz = DAG.getConstant(Val: BW - OrigBW, DL, VT);
1527
1528 SDValue Op = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Node->getOperand(Num: 0), N2: ShiftSz);
1529 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Op, N2: ShiftSz);
1530}
1531
1532// Generically expand a vector anyext in register to a shuffle of the relevant
1533// lanes into the appropriate locations, with other lanes left undef.
1534SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1535 SDLoc DL(Node);
1536 EVT VT = Node->getValueType(ResNo: 0);
1537 int NumElements = VT.getVectorNumElements();
1538 SDValue Src = Node->getOperand(Num: 0);
1539 EVT SrcVT = Src.getValueType();
1540 int NumSrcElements = SrcVT.getVectorNumElements();
1541
1542 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1543 // into a larger vector type.
1544 if (SrcVT.bitsLE(VT)) {
1545 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1546 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1547 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1548 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1549 NumElements: NumSrcElements);
1550 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1551 }
1552
1553 // Build a base mask of undef shuffles.
1554 SmallVector<int, 16> ShuffleMask;
1555 ShuffleMask.resize(N: NumSrcElements, NV: -1);
1556
1557 // Place the extended lanes into the correct locations.
1558 int ExtLaneScale = NumSrcElements / NumElements;
1559 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1560 for (int i = 0; i < NumElements; ++i)
1561 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1562
1563 return DAG.getNode(
1564 Opcode: ISD::BITCAST, DL, VT,
1565 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Src, N2: DAG.getPOISON(VT: SrcVT), Mask: ShuffleMask));
1566}
1567
1568SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1569 SDLoc DL(Node);
1570 EVT VT = Node->getValueType(ResNo: 0);
1571 SDValue Src = Node->getOperand(Num: 0);
1572 EVT SrcVT = Src.getValueType();
1573
1574 // First build an any-extend node which can be legalized above when we
1575 // recurse through it.
1576 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: Src);
1577
1578 // Now we need sign extend. This will be exanded to shifts if it isn't
1579 // supported.
1580 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getVectorElementType(),
1581 NumElements: VT.getVectorNumElements());
1582 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
1583 N2: DAG.getValueType(ExtVT));
1584}
1585
1586// Generically expand a vector zext in register to a shuffle of the relevant
1587// lanes into the appropriate locations, a blend of zero into the high bits,
1588// and a bitcast to the wider element type.
1589SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1590 SDLoc DL(Node);
1591 EVT VT = Node->getValueType(ResNo: 0);
1592 int NumElements = VT.getVectorNumElements();
1593 SDValue Src = Node->getOperand(Num: 0);
1594 EVT SrcVT = Src.getValueType();
1595 int NumSrcElements = SrcVT.getVectorNumElements();
1596
1597 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1598 // into a larger vector type.
1599 if (SrcVT.bitsLE(VT)) {
1600 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1601 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1602 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1603 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(),
1604 NumElements: NumSrcElements);
1605 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SrcVT), SubVec: Src, Idx: 0);
1606 }
1607
1608 // Build up a zero vector to blend into this one.
1609 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1610
1611 // Shuffle the incoming lanes into the correct position, and pull all other
1612 // lanes from the zero vector.
1613 auto ShuffleMask = llvm::to_vector<16>(Range: llvm::seq<int>(Begin: 0, End: NumSrcElements));
1614
1615 int ExtLaneScale = NumSrcElements / NumElements;
1616 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1617 for (int i = 0; i < NumElements; ++i)
1618 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1619
1620 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
1621 Operand: DAG.getVectorShuffle(VT: SrcVT, dl: DL, N1: Zero, N2: Src, Mask: ShuffleMask));
1622}
1623
1624static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1625 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1626 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1627 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1628 ShuffleMask.push_back(Elt: (I * ScalarSizeInBytes) + J);
1629}
1630
1631SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1632 EVT VT = Node->getValueType(ResNo: 0);
1633
1634 // Scalable vectors can't use shuffle expansion.
1635 if (VT.isScalableVector())
1636 return TLI.expandBSWAP(N: Node, DAG);
1637
1638 // Generate a byte wise shuffle mask for the BSWAP.
1639 SmallVector<int, 16> ShuffleMask;
1640 createBSWAPShuffleMask(VT, ShuffleMask);
1641 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: ShuffleMask.size());
1642
1643 // Only emit a shuffle if the mask is legal.
1644 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1645 SDLoc DL(Node);
1646 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1647 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1648 Mask: ShuffleMask);
1649 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1650 }
1651
1652 // If we have the appropriate vector bit operations, it is better to use them
1653 // than unrolling and expanding each component.
1654 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1655 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1656 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1657 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1658 return TLI.expandBSWAP(N: Node, DAG);
1659
1660 // Otherwise let the caller unroll.
1661 return SDValue();
1662}
1663
1664SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {
1665 EVT VT = Node->getValueType(ResNo: 0);
1666
1667 // We can't unroll or use shuffles for scalable vectors.
1668 if (VT.isScalableVector())
1669 return TLI.expandBITREVERSE(N: Node, DAG);
1670
1671 // If we have the scalar operation, it's probably cheaper to unroll it.
1672 if (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: VT.getScalarType()))
1673 return SDValue();
1674
1675 // If the vector element width is a whole number of bytes, test if its legal
1676 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1677 // vector. This greatly reduces the number of bit shifts necessary.
1678 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1679 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1680 SmallVector<int, 16> BSWAPMask;
1681 createBSWAPShuffleMask(VT, ShuffleMask&: BSWAPMask);
1682
1683 EVT ByteVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: BSWAPMask.size());
1684 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1685 (TLI.isOperationLegalOrCustom(Op: ISD::BITREVERSE, VT: ByteVT) ||
1686 (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT: ByteVT) &&
1687 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT: ByteVT) &&
1688 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: ByteVT) &&
1689 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: ByteVT)))) {
1690 SDLoc DL(Node);
1691 SDValue Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ByteVT, Operand: Node->getOperand(Num: 0));
1692 Op = DAG.getVectorShuffle(VT: ByteVT, dl: DL, N1: Op, N2: DAG.getPOISON(VT: ByteVT),
1693 Mask: BSWAPMask);
1694 Op = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ByteVT, Operand: Op);
1695 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1696 return Op;
1697 }
1698 }
1699
1700 // If we have the appropriate vector bit operations, it is better to use them
1701 // than unrolling and expanding each component.
1702 if (TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
1703 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
1704 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) &&
1705 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT))
1706 return TLI.expandBITREVERSE(N: Node, DAG);
1707
1708 // Otherwise unroll.
1709 return SDValue();
1710}
1711
1712SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1713 // Implement VSELECT in terms of XOR, AND, OR
1714 // on platforms which do not support blend natively.
1715 SDLoc DL(Node);
1716
1717 SDValue Mask = Node->getOperand(Num: 0);
1718 SDValue Op1 = Node->getOperand(Num: 1);
1719 SDValue Op2 = Node->getOperand(Num: 2);
1720
1721 EVT VT = Mask.getValueType();
1722
1723 // If we can't even use the basic vector operations of
1724 // AND,OR,XOR, we will have to scalarize the op.
1725 // Notice that the operation may be 'promoted' which means that it is
1726 // 'bitcasted' to another type which is handled.
1727 if (TLI.getOperationAction(Op: ISD::AND, VT) == TargetLowering::Expand ||
1728 TLI.getOperationAction(Op: ISD::XOR, VT) == TargetLowering::Expand ||
1729 TLI.getOperationAction(Op: ISD::OR, VT) == TargetLowering::Expand)
1730 return SDValue();
1731
1732 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1733 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1734 // vector constant to mask with.
1735 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1736 auto BoolContents = TLI.getBooleanContents(Type: Op1.getValueType());
1737 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1738 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1739 Op1.getValueType().getVectorElementType() == MVT::i1))
1740 return SDValue();
1741
1742 // If the mask and the type are different sizes, unroll the vector op. This
1743 // can occur when getSetCCResultType returns something that is different in
1744 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1745 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1746 return SDValue();
1747
1748 // Bitcast the operands to be the same type as the mask.
1749 // This is needed when we select between FP types because
1750 // the mask is a vector of integers.
1751 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op1);
1752 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op2);
1753
1754 SDValue NotMask = DAG.getNOT(DL, Val: Mask, VT);
1755
1756 Op1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op1, N2: Mask);
1757 Op2 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op2, N2: NotMask);
1758 SDValue Val = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Op1, N2: Op2);
1759 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Node->getValueType(ResNo: 0), Operand: Val);
1760}
1761
1762SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1763 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1764 // do not support it natively.
1765 SDLoc DL(Node);
1766
1767 SDValue Mask = Node->getOperand(Num: 0);
1768 SDValue Op1 = Node->getOperand(Num: 1);
1769 SDValue Op2 = Node->getOperand(Num: 2);
1770 SDValue EVL = Node->getOperand(Num: 3);
1771
1772 EVT VT = Mask.getValueType();
1773
1774 // If we can't even use the basic vector operations of
1775 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1776 if (TLI.getOperationAction(Op: ISD::VP_AND, VT) == TargetLowering::Expand ||
1777 TLI.getOperationAction(Op: ISD::VP_XOR, VT) == TargetLowering::Expand ||
1778 TLI.getOperationAction(Op: ISD::VP_OR, VT) == TargetLowering::Expand)
1779 return SDValue();
1780
1781 // This operation also isn't safe when the operands aren't also booleans.
1782 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1783 return SDValue();
1784
1785 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1786 SDValue NotMask = DAG.getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Mask, N2: Ones, N3: Ones, N4: EVL);
1787
1788 Op1 = DAG.getNode(Opcode: ISD::VP_AND, DL, VT, N1: Op1, N2: Mask, N3: Ones, N4: EVL);
1789 Op2 = DAG.getNode(Opcode: ISD::VP_AND, DL, VT, N1: Op2, N2: NotMask, N3: Ones, N4: EVL);
1790 return DAG.getNode(Opcode: ISD::VP_OR, DL, VT, N1: Op1, N2: Op2, N3: Ones, N4: EVL);
1791}
1792
1793SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1794 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1795 // indices less than the EVL/pivot are true. Combine that with the original
1796 // mask for a full-length mask. Use a full-length VSELECT to select between
1797 // the true and false values.
1798 SDLoc DL(Node);
1799
1800 SDValue Mask = Node->getOperand(Num: 0);
1801 SDValue Op1 = Node->getOperand(Num: 1);
1802 SDValue Op2 = Node->getOperand(Num: 2);
1803 SDValue EVL = Node->getOperand(Num: 3);
1804
1805 EVT MaskVT = Mask.getValueType();
1806 bool IsFixedLen = MaskVT.isFixedLengthVector();
1807
1808 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EVL.getValueType(),
1809 EC: MaskVT.getVectorElementCount());
1810
1811 // If we can't construct the EVL mask efficiently, it's better to unroll.
1812 if ((IsFixedLen &&
1813 !TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: EVLVecVT)) ||
1814 (!IsFixedLen &&
1815 (!TLI.isOperationLegalOrCustom(Op: ISD::STEP_VECTOR, VT: EVLVecVT) ||
1816 !TLI.isOperationLegalOrCustom(Op: ISD::SPLAT_VECTOR, VT: EVLVecVT))))
1817 return SDValue();
1818
1819 // If using a SETCC would result in a different type than the mask type,
1820 // unroll.
1821 if (TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
1822 VT: EVLVecVT) != MaskVT)
1823 return SDValue();
1824
1825 SDValue StepVec = DAG.getStepVector(DL, ResVT: EVLVecVT);
1826 SDValue SplatEVL = DAG.getSplat(VT: EVLVecVT, DL, Op: EVL);
1827 SDValue EVLMask =
1828 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVec, RHS: SplatEVL, Cond: ISD::CondCode::SETULT);
1829
1830 SDValue FullMask = DAG.getNode(Opcode: ISD::AND, DL, VT: MaskVT, N1: Mask, N2: EVLMask);
1831 return DAG.getSelect(DL, VT: Node->getValueType(ResNo: 0), Cond: FullMask, LHS: Op1, RHS: Op2);
1832}
1833
1834SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
1835 // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.
1836 EVT VT = Node->getValueType(ResNo: 0);
1837
1838 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1839
1840 if (!TLI.isOperationLegalOrCustom(Op: DivOpc, VT) ||
1841 !TLI.isOperationLegalOrCustom(Op: ISD::VP_MUL, VT) ||
1842 !TLI.isOperationLegalOrCustom(Op: ISD::VP_SUB, VT))
1843 return SDValue();
1844
1845 SDLoc DL(Node);
1846
1847 SDValue Dividend = Node->getOperand(Num: 0);
1848 SDValue Divisor = Node->getOperand(Num: 1);
1849 SDValue Mask = Node->getOperand(Num: 2);
1850 SDValue EVL = Node->getOperand(Num: 3);
1851
1852 // X % Y -> X-X/Y*Y
1853 SDValue Div = DAG.getNode(Opcode: DivOpc, DL, VT, N1: Dividend, N2: Divisor, N3: Mask, N4: EVL);
1854 SDValue Mul = DAG.getNode(Opcode: ISD::VP_MUL, DL, VT, N1: Divisor, N2: Div, N3: Mask, N4: EVL);
1855 return DAG.getNode(Opcode: ISD::VP_SUB, DL, VT, N1: Dividend, N2: Mul, N3: Mask, N4: EVL);
1856}
1857
1858SDValue VectorLegalizer::ExpandVP_FNEG(SDNode *Node) {
1859 EVT VT = Node->getValueType(ResNo: 0);
1860 EVT IntVT = VT.changeVectorElementTypeToInteger();
1861
1862 if (!TLI.isOperationLegalOrCustom(Op: ISD::VP_XOR, VT: IntVT))
1863 return SDValue();
1864
1865 SDValue Mask = Node->getOperand(Num: 1);
1866 SDValue EVL = Node->getOperand(Num: 2);
1867
1868 SDLoc DL(Node);
1869 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1870 SDValue SignMask = DAG.getConstant(
1871 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1872 SDValue Xor = DAG.getNode(Opcode: ISD::VP_XOR, DL, VT: IntVT, N1: Cast, N2: SignMask, N3: Mask, N4: EVL);
1873 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
1874}
1875
1876SDValue VectorLegalizer::ExpandVP_FABS(SDNode *Node) {
1877 EVT VT = Node->getValueType(ResNo: 0);
1878 EVT IntVT = VT.changeVectorElementTypeToInteger();
1879
1880 if (!TLI.isOperationLegalOrCustom(Op: ISD::VP_AND, VT: IntVT))
1881 return SDValue();
1882
1883 SDValue Mask = Node->getOperand(Num: 1);
1884 SDValue EVL = Node->getOperand(Num: 2);
1885
1886 SDLoc DL(Node);
1887 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1888 SDValue ClearSignMask = DAG.getConstant(
1889 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1890 SDValue ClearSign =
1891 DAG.getNode(Opcode: ISD::VP_AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask, N3: Mask, N4: EVL);
1892 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearSign);
1893}
1894
1895SDValue VectorLegalizer::ExpandVP_FCOPYSIGN(SDNode *Node) {
1896 EVT VT = Node->getValueType(ResNo: 0);
1897
1898 if (VT != Node->getOperand(Num: 1).getValueType())
1899 return SDValue();
1900
1901 EVT IntVT = VT.changeVectorElementTypeToInteger();
1902 if (!TLI.isOperationLegalOrCustom(Op: ISD::VP_AND, VT: IntVT) ||
1903 !TLI.isOperationLegalOrCustom(Op: ISD::VP_XOR, VT: IntVT))
1904 return SDValue();
1905
1906 SDValue Mask = Node->getOperand(Num: 2);
1907 SDValue EVL = Node->getOperand(Num: 3);
1908
1909 SDLoc DL(Node);
1910 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
1911 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
1912
1913 SDValue SignMask = DAG.getConstant(
1914 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1915 SDValue SignBit =
1916 DAG.getNode(Opcode: ISD::VP_AND, DL, VT: IntVT, N1: Sign, N2: SignMask, N3: Mask, N4: EVL);
1917
1918 SDValue ClearSignMask = DAG.getConstant(
1919 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
1920 SDValue ClearedSign =
1921 DAG.getNode(Opcode: ISD::VP_AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask, N3: Mask, N4: EVL);
1922
1923 SDValue CopiedSign = DAG.getNode(Opcode: ISD::VP_OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
1924 N3: Mask, N4: EVL, Flags: SDNodeFlags::Disjoint);
1925
1926 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
1927}
1928
1929SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
1930 return TLI.expandLoopDependenceMask(N, DAG);
1931}
1932
1933SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
1934 // Masked bin ops don't have undefined behaviour when dividing by zero
1935 // on disabled lanes and produce poison instead. Replace the divisor on the
1936 // disabled lanes with 1 to avoid division by zero or overflow.
1937 SDLoc dl(N);
1938 EVT VT = N->getValueType(ResNo: 0);
1939 SDValue SafeDivisor = DAG.getSelect(
1940 DL: dl, VT, Cond: N->getOperand(Num: 2), LHS: N->getOperand(Num: 1), RHS: DAG.getConstant(Val: 1, DL: dl, VT));
1941 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL: dl, VT,
1942 N1: N->getOperand(Num: 0), N2: SafeDivisor);
1943}
1944
1945void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1946 SmallVectorImpl<SDValue> &Results) {
1947 // Attempt to expand using TargetLowering.
1948 SDValue Result, Chain;
1949 if (TLI.expandFP_TO_UINT(N: Node, Result, Chain, DAG)) {
1950 Results.push_back(Elt: Result);
1951 if (Node->isStrictFPOpcode())
1952 Results.push_back(Elt: Chain);
1953 return;
1954 }
1955
1956 // Otherwise go ahead and unroll.
1957 if (Node->isStrictFPOpcode()) {
1958 UnrollStrictFPOp(Node, Results);
1959 return;
1960 }
1961
1962 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1963}
1964
1965void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1966 SmallVectorImpl<SDValue> &Results) {
1967 bool IsStrict = Node->isStrictFPOpcode();
1968 unsigned OpNo = IsStrict ? 1 : 0;
1969 SDValue Src = Node->getOperand(Num: OpNo);
1970 EVT SrcVT = Src.getValueType();
1971 EVT DstVT = Node->getValueType(ResNo: 0);
1972 SDLoc DL(Node);
1973
1974 // Attempt to expand using TargetLowering.
1975 SDValue Result;
1976 SDValue Chain;
1977 if (TLI.expandUINT_TO_FP(N: Node, Result, Chain, DAG)) {
1978 Results.push_back(Elt: Result);
1979 if (IsStrict)
1980 Results.push_back(Elt: Chain);
1981 return;
1982 }
1983
1984 // Make sure that the SINT_TO_FP and SRL instructions are available.
1985 if (((!IsStrict && TLI.getOperationAction(Op: ISD::SINT_TO_FP, VT: SrcVT) ==
1986 TargetLowering::Expand) ||
1987 (IsStrict && TLI.getOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: SrcVT) ==
1988 TargetLowering::Expand)) ||
1989 TLI.getOperationAction(Op: ISD::SRL, VT: SrcVT) == TargetLowering::Expand) {
1990 if (IsStrict) {
1991 UnrollStrictFPOp(Node, Results);
1992 return;
1993 }
1994
1995 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
1996 return;
1997 }
1998
1999 unsigned BW = SrcVT.getScalarSizeInBits();
2000 assert((BW == 64 || BW == 32) &&
2001 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
2002
2003 // If STRICT_/FMUL is not supported by the target (in case of f16) replace the
2004 // UINT_TO_FP with a larger float and round to the smaller type
2005 if ((!IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT: DstVT)) ||
2006 (IsStrict && !TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FMUL, VT: DstVT))) {
2007 EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;
2008 SDValue UIToFP;
2009 SDValue Result;
2010 SDValue TargetZero = DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true);
2011 EVT FloatVecVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: FPVT);
2012 if (IsStrict) {
2013 UIToFP = DAG.getNode(Opcode: ISD::STRICT_UINT_TO_FP, DL, ResultTys: {FloatVecVT, MVT::Other},
2014 Ops: {Node->getOperand(Num: 0), Src});
2015 Result = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
2016 Ops: {Node->getOperand(Num: 0), UIToFP, TargetZero});
2017 Results.push_back(Elt: Result);
2018 Results.push_back(Elt: Result.getValue(R: 1));
2019 } else {
2020 UIToFP = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVecVT, Operand: Src);
2021 Result = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: UIToFP, N2: TargetZero);
2022 Results.push_back(Elt: Result);
2023 }
2024
2025 return;
2026 }
2027
2028 SDValue HalfWord = DAG.getConstant(Val: BW / 2, DL, VT: SrcVT);
2029
2030 // Constants to clear the upper part of the word.
2031 // Notice that we can also use SHL+SHR, but using a constant is slightly
2032 // faster on x86.
2033 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
2034 SDValue HalfWordMask = DAG.getConstant(Val: HWMask, DL, VT: SrcVT);
2035
2036 // Two to the power of half-word-size.
2037 SDValue TWOHW = DAG.getConstantFP(Val: 1ULL << (BW / 2), DL, VT: DstVT);
2038
2039 // Clear upper part of LO, lower HI
2040 SDValue HI = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: HalfWord);
2041 SDValue LO = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: HalfWordMask);
2042
2043 if (IsStrict) {
2044 // Convert hi and lo to floats
2045 // Convert the hi part back to the upper values
2046 // TODO: Can any fast-math-flags be set on these nodes?
2047 SDValue fHI = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
2048 Ops: {Node->getOperand(Num: 0), HI});
2049 fHI = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {DstVT, MVT::Other},
2050 Ops: {fHI.getValue(R: 1), fHI, TWOHW});
2051 SDValue fLO = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL, ResultTys: {DstVT, MVT::Other},
2052 Ops: {Node->getOperand(Num: 0), LO});
2053
2054 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: fHI.getValue(R: 1),
2055 N2: fLO.getValue(R: 1));
2056
2057 // Add the two halves
2058 SDValue Result =
2059 DAG.getNode(Opcode: ISD::STRICT_FADD, DL, ResultTys: {DstVT, MVT::Other}, Ops: {TF, fHI, fLO});
2060
2061 Results.push_back(Elt: Result);
2062 Results.push_back(Elt: Result.getValue(R: 1));
2063 return;
2064 }
2065
2066 // Convert hi and lo to floats
2067 // Convert the hi part back to the upper values
2068 // TODO: Can any fast-math-flags be set on these nodes?
2069 SDValue fHI = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: HI);
2070 fHI = DAG.getNode(Opcode: ISD::FMUL, DL, VT: DstVT, N1: fHI, N2: TWOHW);
2071 SDValue fLO = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: LO);
2072
2073 // Add the two halves
2074 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: fHI, N2: fLO));
2075}
2076
2077SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
2078 EVT VT = Node->getValueType(ResNo: 0);
2079 EVT IntVT = VT.changeVectorElementTypeToInteger();
2080
2081 if (!TLI.isOperationLegalOrCustom(Op: ISD::XOR, VT: IntVT))
2082 return SDValue();
2083
2084 // Heuristic check to determine whether vector should be expanded to integer
2085 // operations or unrolled to scalar operations.
2086 // 1. Scalable vector is never unrolled.
2087 // 2. Fixed vector is unrolled if one of followings is true:
2088 // a. Vector only has 1 element and target knows how to handle scalar
2089 // FNEG (either legal or custom expand or promote).
2090 // b. Vector has more than 1 element and target supports scalar
2091 // FNEG natively and vector length <= 2(1 XOR + 1 CONST).
2092 // FIXME: Scalar construction instruction count varies in every architecture,
2093 // here we assume 1 instruction for now.
2094 if (VT.isFixedLengthVector()) {
2095 EVT EltVT = VT.getVectorElementType();
2096 unsigned NumElts = VT.getVectorNumElements();
2097 if ((NumElts == 1 &&
2098 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FNEG, VT: EltVT)) ||
2099 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FNEG, VT: EltVT) &&
2100 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2101 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2102 return SDValue();
2103 }
2104
2105 SDLoc DL(Node);
2106 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2107 SDValue SignMask = DAG.getConstant(
2108 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2109 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: Cast, N2: SignMask);
2110 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Xor);
2111}
2112
2113SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {
2114 EVT VT = Node->getValueType(ResNo: 0);
2115 EVT IntVT = VT.changeVectorElementTypeToInteger();
2116
2117 if (!TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
2118 return SDValue();
2119
2120 // Heuristic check to determine whether vector should be expanded to integer
2121 // operations or unrolled to scalar operations.
2122 // 1. Scalable vector is never unrolled.
2123 // 2. Fixed vector is unrolled if one of followings is true:
2124 // a. Vector only has 1 element and target knows how to handle scalar
2125 // FABS(either legal or custom expand or promote).
2126 // b. Vector has more than 1 element and target supports scalar
2127 // FABS natively and vector length <= 2(1 AND + 1 CONST).
2128 // FIXME: Scalar construction instruction count varies in every architecture,
2129 // here we assume 1 instruction for now.
2130 if (VT.isFixedLengthVector()) {
2131 EVT EltVT = VT.getVectorElementType();
2132 unsigned NumElts = VT.getVectorNumElements();
2133 if ((NumElts == 1 &&
2134 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FABS, VT: EltVT)) ||
2135 (NumElts < 3 && TLI.isOperationLegal(Op: ISD::FABS, VT: EltVT) &&
2136 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2137 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2138 return SDValue();
2139 }
2140
2141 SDLoc DL(Node);
2142 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2143 SDValue ClearSignMask = DAG.getConstant(
2144 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2145 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Cast, N2: ClearSignMask);
2146 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ClearedSign);
2147}
2148
2149SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {
2150 EVT VT = Node->getValueType(ResNo: 0);
2151 EVT IntVT = VT.changeVectorElementTypeToInteger();
2152
2153 if (VT != Node->getOperand(Num: 1).getValueType() ||
2154 !TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT) ||
2155 !TLI.isOperationLegalOrCustom(Op: ISD::OR, VT: IntVT))
2156 return SDValue();
2157
2158 // Heuristic check to determine whether vector should be expanded to integer
2159 // operations or unrolled to scalar operations.
2160 // 1. Scalable vector is never unrolled.
2161 // 2. Fixed vector is unrolled if one of followings is true:
2162 // a. Vector only has 1 element and target knows how to handle scalar
2163 // FCOPYSIGN(either legal or custom expand or promote).
2164 // b. Vector has more than 1 element and target supports scalar
2165 // FCOPYSIGN natively and vector length <= 5(2 AND + 1 OR + 2 CONST).
2166 // FIXME: Scalar construction instruction count varies in every architecture,
2167 // here we assume 1 instruction for now.
2168 if (VT.isFixedLengthVector()) {
2169 EVT EltVT = VT.getVectorElementType();
2170 unsigned NumElts = VT.getVectorNumElements();
2171 if ((NumElts == 1 &&
2172 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::FCOPYSIGN, VT: EltVT)) ||
2173 (NumElts < 6 && TLI.isOperationLegal(Op: ISD::FCOPYSIGN, VT: EltVT) &&
2174 TLI.isExtractVecEltCheap(VT, Index: 0) &&
2175 (NumElts == 1 || TLI.isExtractVecEltCheap(VT, Index: 1))))
2176 return SDValue();
2177 }
2178
2179 SDLoc DL(Node);
2180 SDValue Mag = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 0));
2181 SDValue Sign = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Node->getOperand(Num: 1));
2182
2183 SDValue SignMask = DAG.getConstant(
2184 Val: APInt::getSignMask(BitWidth: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2185 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Sign, N2: SignMask);
2186
2187 SDValue ClearSignMask = DAG.getConstant(
2188 Val: APInt::getSignedMaxValue(numBits: IntVT.getScalarSizeInBits()), DL, VT: IntVT);
2189 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: Mag, N2: ClearSignMask);
2190
2191 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: ClearedSign, N2: SignBit,
2192 Flags: SDNodeFlags::Disjoint);
2193
2194 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: CopiedSign);
2195}
2196
2197void VectorLegalizer::ExpandFSUB(SDNode *Node,
2198 SmallVectorImpl<SDValue> &Results) {
2199 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
2200 // we can defer this to operation legalization where it will be lowered as
2201 // a+(-b).
2202 EVT VT = Node->getValueType(ResNo: 0);
2203 if (TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT) &&
2204 TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT))
2205 return; // Defer to LegalizeDAG
2206
2207 if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
2208 Results.push_back(Elt: Expanded);
2209 return;
2210 }
2211
2212 SDValue Tmp = DAG.UnrollVectorOp(N: Node);
2213 Results.push_back(Elt: Tmp);
2214}
2215
2216void VectorLegalizer::ExpandSETCC(SDNode *Node,
2217 SmallVectorImpl<SDValue> &Results) {
2218 bool NeedInvert = false;
2219 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
2220 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
2221 Node->getOpcode() == ISD::STRICT_FSETCCS;
2222 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
2223 unsigned Offset = IsStrict ? 1 : 0;
2224
2225 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
2226 SDValue LHS = Node->getOperand(Num: 0 + Offset);
2227 SDValue RHS = Node->getOperand(Num: 1 + Offset);
2228 SDValue CC = Node->getOperand(Num: 2 + Offset);
2229
2230 MVT OpVT = LHS.getSimpleValueType();
2231 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
2232
2233 if (TLI.getCondCodeAction(CC: CCCode, VT: OpVT) != TargetLowering::Expand) {
2234 if (IsStrict) {
2235 UnrollStrictFPOp(Node, Results);
2236 return;
2237 }
2238 Results.push_back(Elt: UnrollVSETCC(Node));
2239 return;
2240 }
2241
2242 SDValue Mask, EVL;
2243 if (IsVP) {
2244 Mask = Node->getOperand(Num: 3 + Offset);
2245 EVL = Node->getOperand(Num: 4 + Offset);
2246 }
2247
2248 SDLoc dl(Node);
2249 bool Legalized =
2250 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS, RHS, CC, Mask,
2251 EVL, NeedInvert, dl, Chain, IsSignaling);
2252
2253 if (Legalized) {
2254 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
2255 // condition code, create a new SETCC node.
2256 if (CC.getNode()) {
2257 if (IsStrict) {
2258 LHS = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
2259 Ops: {Chain, LHS, RHS, CC}, Flags: Node->getFlags());
2260 Chain = LHS.getValue(R: 1);
2261 } else if (IsVP) {
2262 LHS = DAG.getNode(Opcode: ISD::VP_SETCC, DL: dl, VT: Node->getValueType(ResNo: 0),
2263 Ops: {LHS, RHS, CC, Mask, EVL}, Flags: Node->getFlags());
2264 } else {
2265 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: LHS, N2: RHS, N3: CC,
2266 Flags: Node->getFlags());
2267 }
2268 }
2269
2270 // If we expanded the SETCC by inverting the condition code, then wrap
2271 // the existing SETCC in a NOT to restore the intended condition.
2272 if (NeedInvert) {
2273 if (!IsVP)
2274 LHS = DAG.getLogicalNOT(DL: dl, Val: LHS, VT: LHS->getValueType(ResNo: 0));
2275 else
2276 LHS = DAG.getVPLogicalNOT(DL: dl, Val: LHS, Mask, EVL, VT: LHS->getValueType(ResNo: 0));
2277 }
2278 } else {
2279 assert(!IsStrict && "Don't know how to expand for strict nodes.");
2280
2281 // Otherwise, SETCC for the given comparison type must be completely
2282 // illegal; expand it into a SELECT_CC.
2283 EVT VT = Node->getValueType(ResNo: 0);
2284 LHS = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: LHS, N2: RHS,
2285 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: LHS.getValueType()),
2286 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: LHS.getValueType()),
2287 N5: CC, Flags: Node->getFlags());
2288 }
2289
2290 Results.push_back(Elt: LHS);
2291 if (IsStrict)
2292 Results.push_back(Elt: Chain);
2293}
2294
2295void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
2296 SmallVectorImpl<SDValue> &Results) {
2297 SDValue Result, Overflow;
2298 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
2299 Results.push_back(Elt: Result);
2300 Results.push_back(Elt: Overflow);
2301}
2302
2303void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
2304 SmallVectorImpl<SDValue> &Results) {
2305 SDValue Result, Overflow;
2306 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
2307 Results.push_back(Elt: Result);
2308 Results.push_back(Elt: Overflow);
2309}
2310
2311void VectorLegalizer::ExpandMULO(SDNode *Node,
2312 SmallVectorImpl<SDValue> &Results) {
2313 SDValue Result, Overflow;
2314 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
2315 std::tie(args&: Result, args&: Overflow) = DAG.UnrollVectorOverflowOp(N: Node);
2316
2317 Results.push_back(Elt: Result);
2318 Results.push_back(Elt: Overflow);
2319}
2320
2321void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
2322 SmallVectorImpl<SDValue> &Results) {
2323 SDNode *N = Node;
2324 if (SDValue Expanded = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl: SDLoc(N),
2325 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Scale: N->getConstantOperandVal(Num: 2), DAG))
2326 Results.push_back(Elt: Expanded);
2327}
2328
2329void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
2330 SmallVectorImpl<SDValue> &Results) {
2331 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
2332 ExpandUINT_TO_FLOAT(Node, Results);
2333 return;
2334 }
2335 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
2336 ExpandFP_TO_UINT(Node, Results);
2337 return;
2338 }
2339
2340 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2341 Node->getOpcode() == ISD::STRICT_FSETCCS) {
2342 ExpandSETCC(Node, Results);
2343 return;
2344 }
2345
2346 UnrollStrictFPOp(Node, Results);
2347}
2348
2349void VectorLegalizer::ExpandREM(SDNode *Node,
2350 SmallVectorImpl<SDValue> &Results) {
2351 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
2352 "Expected REM node");
2353
2354 SDValue Result;
2355 if (!TLI.expandREM(Node, Result, DAG))
2356 Result = DAG.UnrollVectorOp(N: Node);
2357 Results.push_back(Elt: Result);
2358}
2359
2360// Try to expand libm nodes into vector math routine calls. Callers provide the
2361// LibFunc equivalent of the passed in Node, which is used to lookup mappings
2362// within TargetLibraryInfo. The only mappings considered are those where the
2363// result and all operands are the same vector type. While predicated nodes are
2364// not supported, we will emit calls to masked routines by passing in an all
2365// true mask.
2366bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
2367 SmallVectorImpl<SDValue> &Results) {
2368 // Chain must be propagated but currently strict fp operations are down
2369 // converted to their none strict counterpart.
2370 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");
2371
2372 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2373 if (LCImpl == RTLIB::Unsupported)
2374 return false;
2375
2376 EVT VT = Node->getValueType(ResNo: 0);
2377 const RTLIB::RuntimeLibcallsInfo &RTLCI = TLI.getRuntimeLibcallsInfo();
2378 LLVMContext &Ctx = *DAG.getContext();
2379
2380 auto [FuncTy, FuncAttrs] = RTLCI.getFunctionTy(
2381 Ctx, TT: DAG.getSubtarget().getTargetTriple(), DL: DAG.getDataLayout(), LibcallImpl: LCImpl);
2382
2383 SDLoc DL(Node);
2384 TargetLowering::ArgListTy Args;
2385
2386 bool HasMaskArg = RTLCI.hasVectorMaskArgument(Impl: LCImpl);
2387
2388 // Sanity check just in case function has unexpected parameters.
2389 assert(FuncTy->getNumParams() == Node->getNumOperands() + HasMaskArg &&
2390 EVT::getEVT(FuncTy->getReturnType(), true) == VT &&
2391 "mismatch in value type and call signature type");
2392
2393 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
2394 Type *ParamTy = FuncTy->getParamType(i: I);
2395
2396 if (HasMaskArg && I == E - 1) {
2397 assert(cast<VectorType>(ParamTy)->getElementType()->isIntegerTy(1) &&
2398 "unexpected vector mask type");
2399 EVT MaskVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
2400 Args.emplace_back(args: DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT),
2401 args: MaskVT.getTypeForEVT(Context&: Ctx));
2402
2403 } else {
2404 SDValue Op = Node->getOperand(Num: I);
2405 assert(Op.getValueType() == EVT::getEVT(ParamTy, true) &&
2406 "mismatch in value type and call argument type");
2407 Args.emplace_back(args&: Op, args&: ParamTy);
2408 }
2409 }
2410
2411 // Emit a call to the vector function.
2412 SDValue Callee =
2413 DAG.getExternalSymbol(LCImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2414 CallingConv::ID CC = RTLCI.getLibcallImplCallingConv(Call: LCImpl);
2415
2416 TargetLowering::CallLoweringInfo CLI(DAG);
2417 CLI.setDebugLoc(DL)
2418 .setChain(DAG.getEntryNode())
2419 .setLibCallee(CC, ResultType: FuncTy->getReturnType(), Target: Callee, ArgsList: std::move(Args));
2420
2421 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2422 Results.push_back(Elt: CallResult.first);
2423 return true;
2424}
2425
2426void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
2427 SmallVectorImpl<SDValue> &Results) {
2428 EVT VT = Node->getValueType(ResNo: 0);
2429 EVT EltVT = VT.getVectorElementType();
2430 unsigned NumElems = VT.getVectorNumElements();
2431 unsigned NumOpers = Node->getNumOperands();
2432 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2433
2434 EVT TmpEltVT = EltVT;
2435 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2436 Node->getOpcode() == ISD::STRICT_FSETCCS)
2437 TmpEltVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2438 Context&: *DAG.getContext(), VT: TmpEltVT);
2439
2440 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2441 SDValue Chain = Node->getOperand(Num: 0);
2442 SDLoc dl(Node);
2443
2444 SmallVector<SDValue, 32> OpValues;
2445 SmallVector<SDValue, 32> OpChains;
2446 for (unsigned i = 0; i < NumElems; ++i) {
2447 SmallVector<SDValue, 4> Opers;
2448 SDValue Idx = DAG.getVectorIdxConstant(Val: i, DL: dl);
2449
2450 // The Chain is the first operand.
2451 Opers.push_back(Elt: Chain);
2452
2453 // Now process the remaining operands.
2454 for (unsigned j = 1; j < NumOpers; ++j) {
2455 SDValue Oper = Node->getOperand(Num: j);
2456 EVT OperVT = Oper.getValueType();
2457
2458 if (OperVT.isVector())
2459 Oper = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
2460 VT: OperVT.getVectorElementType(), N1: Oper, N2: Idx);
2461
2462 Opers.push_back(Elt: Oper);
2463 }
2464
2465 SDValue ScalarOp = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: ValueVTs, Ops: Opers);
2466 SDValue ScalarResult = ScalarOp.getValue(R: 0);
2467 SDValue ScalarChain = ScalarOp.getValue(R: 1);
2468
2469 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
2470 Node->getOpcode() == ISD::STRICT_FSETCCS)
2471 ScalarResult = DAG.getSelect(DL: dl, VT: EltVT, Cond: ScalarResult,
2472 LHS: DAG.getAllOnesConstant(DL: dl, VT: EltVT),
2473 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2474
2475 OpValues.push_back(Elt: ScalarResult);
2476 OpChains.push_back(Elt: ScalarChain);
2477 }
2478
2479 SDValue Result = DAG.getBuildVector(VT, DL: dl, Ops: OpValues);
2480 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
2481
2482 Results.push_back(Elt: Result);
2483 Results.push_back(Elt: NewChain);
2484}
2485
2486SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
2487 EVT VT = Node->getValueType(ResNo: 0);
2488 unsigned NumElems = VT.getVectorNumElements();
2489 EVT EltVT = VT.getVectorElementType();
2490 SDValue LHS = Node->getOperand(Num: 0);
2491 SDValue RHS = Node->getOperand(Num: 1);
2492 SDValue CC = Node->getOperand(Num: 2);
2493 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
2494 SDLoc dl(Node);
2495 SmallVector<SDValue, 8> Ops(NumElems);
2496 for (unsigned i = 0; i < NumElems; ++i) {
2497 SDValue LHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: LHS,
2498 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2499 SDValue RHSElem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: TmpEltVT, N1: RHS,
2500 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
2501 // FIXME: We should use i1 setcc + boolext here, but it causes regressions.
2502 Ops[i] = DAG.getNode(Opcode: ISD::SETCC, DL: dl,
2503 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
2504 Context&: *DAG.getContext(), VT: TmpEltVT),
2505 N1: LHSElem, N2: RHSElem, N3: CC);
2506 Ops[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Ops[i],
2507 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
2508 RHS: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
2509 }
2510 return DAG.getBuildVector(VT, DL: dl, Ops);
2511}
2512
2513bool SelectionDAG::LegalizeVectors() {
2514 return VectorLegalizer(*this).Run();
2515}
2516