1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/CodeGen/StackMaps.h"
23#include "llvm/CodeGen/TargetLowering.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/Support/ErrorHandling.h"
27#include "llvm/Support/KnownBits.h"
28#include "llvm/Support/raw_ostream.h"
29#include <algorithm>
30using namespace llvm;
31
32#define DEBUG_TYPE "legalize-types"
33
34//===----------------------------------------------------------------------===//
35// Integer Result Promotion
36//===----------------------------------------------------------------------===//
37
38/// PromoteIntegerResult - This method is called when a result of a node is
39/// found to be in need of promotion to a larger type. At this point, the node
40/// may also have invalid operands or may have other results that need
41/// expansion, we just know that (at least) one result needs promotion.
42void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
43 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
44 SDValue Res = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
48 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
49 return;
50 }
51
52 switch (N->getOpcode()) {
53 default:
54#ifndef NDEBUG
55 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
56 N->dump(&DAG); dbgs() << "\n";
57#endif
58 report_fatal_error(reason: "Do not know how to promote this operator!");
59 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
60 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
61 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
62 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
63 case ISD::VP_BITREVERSE:
64 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
65 case ISD::VP_BSWAP:
66 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
67 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
68 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
69 case ISD::VP_CTLZ_ZERO_UNDEF:
70 case ISD::VP_CTLZ:
71 case ISD::CTLZ_ZERO_UNDEF:
72 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
73 case ISD::CTLS: Res = PromoteIntRes_CTLS(N); break;
74 case ISD::PARITY:
75 case ISD::VP_CTPOP:
76 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
77 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::VP_CTTZ:
79 case ISD::CTTZ_ZERO_UNDEF:
80 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
81 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
82 case ISD::VP_CTTZ_ELTS:
83 Res = PromoteIntRes_VP_CttzElements(N);
84 break;
85 case ISD::EXTRACT_VECTOR_ELT:
86 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
87 case ISD::LOAD: Res = PromoteIntRes_LOAD(N: cast<LoadSDNode>(Val: N)); break;
88 case ISD::VP_LOAD:
89 Res = PromoteIntRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
90 break;
91 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
92 break;
93 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
94 break;
95 case ISD::VECTOR_COMPRESS:
96 Res = PromoteIntRes_VECTOR_COMPRESS(N);
97 break;
98 case ISD::SELECT:
99 case ISD::VSELECT:
100 case ISD::VP_SELECT:
101 case ISD::VP_MERGE:
102 Res = PromoteIntRes_Select(N);
103 break;
104 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
105 case ISD::STRICT_FSETCC:
106 case ISD::STRICT_FSETCCS:
107 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
108 case ISD::SMIN:
109 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
110 case ISD::UMIN:
111 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
112
113 case ISD::SHL:
114 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
115 case ISD::SIGN_EXTEND_INREG:
116 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
117 case ISD::SRA:
118 case ISD::VP_SRA: Res = PromoteIntRes_SRA(N); break;
119 case ISD::SRL:
120 case ISD::VP_SRL: Res = PromoteIntRes_SRL(N); break;
121 case ISD::VP_TRUNCATE:
122 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
123 case ISD::POISON:
124 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
125 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
126 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
127
128 case ISD::EXTRACT_SUBVECTOR:
129 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
130 case ISD::INSERT_SUBVECTOR:
131 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
132 case ISD::VECTOR_REVERSE:
133 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
134 case ISD::VECTOR_SHUFFLE:
135 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
136 case ISD::VECTOR_SPLICE_LEFT:
137 case ISD::VECTOR_SPLICE_RIGHT:
138 Res = PromoteIntRes_VECTOR_SPLICE(N);
139 break;
140 case ISD::VECTOR_INTERLEAVE:
141 case ISD::VECTOR_DEINTERLEAVE:
142 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
143 return;
144 case ISD::INSERT_VECTOR_ELT:
145 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
146 case ISD::BUILD_VECTOR:
147 Res = PromoteIntRes_BUILD_VECTOR(N);
148 break;
149 case ISD::SPLAT_VECTOR:
150 case ISD::SCALAR_TO_VECTOR:
151 Res = PromoteIntRes_ScalarOp(N);
152 break;
153 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
154 case ISD::CONCAT_VECTORS:
155 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
156
157 case ISD::ANY_EXTEND_VECTOR_INREG:
158 case ISD::SIGN_EXTEND_VECTOR_INREG:
159 case ISD::ZERO_EXTEND_VECTOR_INREG:
160 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
161
162 case ISD::VECTOR_FIND_LAST_ACTIVE:
163 Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
164 break;
165
166 case ISD::GET_ACTIVE_LANE_MASK:
167 Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
168 break;
169
170 case ISD::PARTIAL_REDUCE_UMLA:
171 case ISD::PARTIAL_REDUCE_SMLA:
172 case ISD::PARTIAL_REDUCE_SUMLA:
173 Res = PromoteIntRes_PARTIAL_REDUCE_MLA(N);
174 break;
175
176 case ISD::SIGN_EXTEND:
177 case ISD::VP_SIGN_EXTEND:
178 case ISD::ZERO_EXTEND:
179 case ISD::VP_ZERO_EXTEND:
180 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
181
182 case ISD::VP_FP_TO_SINT:
183 case ISD::VP_FP_TO_UINT:
184 case ISD::STRICT_FP_TO_SINT:
185 case ISD::STRICT_FP_TO_UINT:
186 case ISD::FP_TO_SINT:
187 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
188
189 case ISD::FP_TO_SINT_SAT:
190 case ISD::FP_TO_UINT_SAT:
191 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
192
193 case ISD::FP_TO_BF16:
194 case ISD::FP_TO_FP16:
195 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
196 break;
197 case ISD::STRICT_FP_TO_BF16:
198 case ISD::STRICT_FP_TO_FP16:
199 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
200 break;
201 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
202
203 case ISD::AND:
204 case ISD::OR:
205 case ISD::XOR:
206 case ISD::ADD:
207 case ISD::SUB:
208 case ISD::MUL:
209 case ISD::VP_AND:
210 case ISD::VP_OR:
211 case ISD::VP_XOR:
212 case ISD::VP_ADD:
213 case ISD::VP_SUB:
214 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
215
216 case ISD::ABDS:
217 case ISD::AVGCEILS:
218 case ISD::AVGFLOORS:
219 case ISD::VP_SMIN:
220 case ISD::VP_SMAX:
221 case ISD::SDIV:
222 case ISD::SREM:
223 case ISD::VP_SDIV:
224 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
225
226 case ISD::ABDU:
227 case ISD::AVGCEILU:
228 case ISD::AVGFLOORU:
229 case ISD::VP_UMIN:
230 case ISD::VP_UMAX:
231 case ISD::UDIV:
232 case ISD::UREM:
233 case ISD::VP_UDIV:
234 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
235
236 case ISD::SADDO:
237 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
238 case ISD::UADDO:
239 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
240 case ISD::SMULO:
241 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
242
243 case ISD::ADDE:
244 case ISD::SUBE:
245 case ISD::UADDO_CARRY:
246 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
247
248 case ISD::SADDO_CARRY:
249 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
250
251 case ISD::SADDSAT:
252 case ISD::UADDSAT:
253 case ISD::SSUBSAT:
254 case ISD::USUBSAT:
255 case ISD::SSHLSAT:
256 case ISD::USHLSAT:
257 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N);
258 break;
259 case ISD::VP_SADDSAT:
260 case ISD::VP_UADDSAT:
261 case ISD::VP_SSUBSAT:
262 case ISD::VP_USUBSAT:
263 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N);
264 break;
265
266 case ISD::SCMP:
267 case ISD::UCMP:
268 Res = PromoteIntRes_CMP(N);
269 break;
270
271 case ISD::SMULFIX:
272 case ISD::SMULFIXSAT:
273 case ISD::UMULFIX:
274 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
275
276 case ISD::SDIVFIX:
277 case ISD::SDIVFIXSAT:
278 case ISD::UDIVFIX:
279 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
280
281 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
282
283 case ISD::ATOMIC_LOAD:
284 Res = PromoteIntRes_Atomic0(N: cast<AtomicSDNode>(Val: N)); break;
285
286 case ISD::ATOMIC_LOAD_ADD:
287 case ISD::ATOMIC_LOAD_SUB:
288 case ISD::ATOMIC_LOAD_AND:
289 case ISD::ATOMIC_LOAD_CLR:
290 case ISD::ATOMIC_LOAD_OR:
291 case ISD::ATOMIC_LOAD_XOR:
292 case ISD::ATOMIC_LOAD_NAND:
293 case ISD::ATOMIC_LOAD_MIN:
294 case ISD::ATOMIC_LOAD_MAX:
295 case ISD::ATOMIC_LOAD_UMIN:
296 case ISD::ATOMIC_LOAD_UMAX:
297 case ISD::ATOMIC_SWAP:
298 Res = PromoteIntRes_Atomic1(N: cast<AtomicSDNode>(Val: N)); break;
299
300 case ISD::ATOMIC_CMP_SWAP:
301 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
302 Res = PromoteIntRes_AtomicCmpSwap(N: cast<AtomicSDNode>(Val: N), ResNo);
303 break;
304
305 case ISD::VECREDUCE_ADD:
306 case ISD::VECREDUCE_MUL:
307 case ISD::VECREDUCE_AND:
308 case ISD::VECREDUCE_OR:
309 case ISD::VECREDUCE_XOR:
310 case ISD::VECREDUCE_SMAX:
311 case ISD::VECREDUCE_SMIN:
312 case ISD::VECREDUCE_UMAX:
313 case ISD::VECREDUCE_UMIN:
314 Res = PromoteIntRes_VECREDUCE(N);
315 break;
316
317 case ISD::VP_REDUCE_ADD:
318 case ISD::VP_REDUCE_MUL:
319 case ISD::VP_REDUCE_AND:
320 case ISD::VP_REDUCE_OR:
321 case ISD::VP_REDUCE_XOR:
322 case ISD::VP_REDUCE_SMAX:
323 case ISD::VP_REDUCE_SMIN:
324 case ISD::VP_REDUCE_UMAX:
325 case ISD::VP_REDUCE_UMIN:
326 Res = PromoteIntRes_VP_REDUCE(N);
327 break;
328
329 case ISD::LOOP_DEPENDENCE_WAR_MASK:
330 case ISD::LOOP_DEPENDENCE_RAW_MASK:
331 Res = PromoteIntRes_LOOP_DEPENDENCE_MASK(N);
332 break;
333
334 case ISD::FREEZE:
335 Res = PromoteIntRes_FREEZE(N);
336 break;
337
338 case ISD::ROTL:
339 case ISD::ROTR:
340 Res = PromoteIntRes_Rotate(N);
341 break;
342
343 case ISD::FSHL:
344 case ISD::FSHR:
345 Res = PromoteIntRes_FunnelShift(N);
346 break;
347
348 case ISD::VP_FSHL:
349 case ISD::VP_FSHR:
350 Res = PromoteIntRes_VPFunnelShift(N);
351 break;
352
353 case ISD::CLMUL:
354 case ISD::CLMULH:
355 case ISD::CLMULR:
356 Res = PromoteIntRes_CLMUL(N);
357 break;
358
359 case ISD::IS_FPCLASS:
360 Res = PromoteIntRes_IS_FPCLASS(N);
361 break;
362 case ISD::FFREXP:
363 Res = PromoteIntRes_FFREXP(N);
364 break;
365
366 case ISD::LRINT:
367 case ISD::LLRINT:
368 Res = PromoteIntRes_XRINT(N);
369 break;
370
371 case ISD::PATCHPOINT:
372 Res = PromoteIntRes_PATCHPOINT(N);
373 break;
374 case ISD::READ_REGISTER:
375 Res = PromoteIntRes_READ_REGISTER(N);
376 break;
377 }
378
379 // If the result is null then the sub-method took care of registering it.
380 if (Res.getNode())
381 SetPromotedInteger(Op: SDValue(N, ResNo), Result: Res);
382}
383
384SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
385 unsigned ResNo) {
386 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
387 return GetPromotedInteger(Op);
388}
389
390SDValue DAGTypeLegalizer::PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
391 EVT VT = N->getValueType(ResNo: 0);
392 EVT NewVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
393 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NewVT, Ops: N->ops());
394}
395
396SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
397 // Sign-extend the new bits, and continue the assertion.
398 SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 0));
399 return DAG.getNode(Opcode: ISD::AssertSext, DL: SDLoc(N),
400 VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1));
401}
402
403SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
404 // Zero the new bits, and continue the assertion.
405 SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
406 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N),
407 VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1));
408}
409
410SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
411 EVT ResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
412 ISD::LoadExtType ExtType = N->getExtensionType();
413 if (ExtType == ISD::NON_EXTLOAD) {
414 switch (TLI.getExtendForAtomicOps()) {
415 case ISD::SIGN_EXTEND:
416 ExtType = ISD::SEXTLOAD;
417 break;
418 case ISD::ZERO_EXTEND:
419 ExtType = ISD::ZEXTLOAD;
420 break;
421 case ISD::ANY_EXTEND:
422 ExtType = ISD::EXTLOAD;
423 break;
424 default:
425 llvm_unreachable("Invalid atomic op extension");
426 }
427 }
428
429 SDValue Res =
430 DAG.getAtomicLoad(ExtType, dl: SDLoc(N), MemVT: N->getMemoryVT(), VT: ResVT,
431 Chain: N->getChain(), Ptr: N->getBasePtr(), MMO: N->getMemOperand());
432
433 // Legalize the chain result - switch anything that used the old chain to
434 // use the new one.
435 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
436 return Res;
437}
438
439SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
440 SDValue Op2 = N->getOperand(Num: 2);
441 switch (TLI.getExtendForAtomicRMWArg(Op: N->getOpcode())) {
442 case ISD::SIGN_EXTEND:
443 Op2 = SExtPromotedInteger(Op: Op2);
444 break;
445 case ISD::ZERO_EXTEND:
446 Op2 = ZExtPromotedInteger(Op: Op2);
447 break;
448 case ISD::ANY_EXTEND:
449 Op2 = GetPromotedInteger(Op: Op2);
450 break;
451 default:
452 llvm_unreachable("Invalid atomic op extension");
453 }
454 SDValue Res = DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N),
455 MemVT: N->getMemoryVT(),
456 Chain: N->getChain(), Ptr: N->getBasePtr(),
457 Val: Op2, MMO: N->getMemOperand());
458 // Legalize the chain result - switch anything that used the old chain to
459 // use the new one.
460 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
461 return Res;
462}
463
464SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
465 unsigned ResNo) {
466 if (ResNo == 1) {
467 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
468 EVT SVT = getSetCCResultType(VT: N->getOperand(Num: 2).getValueType());
469 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1));
470
471 // Only use the result of getSetCCResultType if it is legal,
472 // otherwise just use the promoted result type (NVT).
473 if (!TLI.isTypeLegal(VT: SVT))
474 SVT = NVT;
475
476 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: SVT, VT3: MVT::Other);
477 SDValue Res = DAG.getAtomicCmpSwap(
478 Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs,
479 Chain: N->getChain(), Ptr: N->getBasePtr(), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3),
480 MMO: N->getMemOperand());
481 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
482 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
483 return DAG.getSExtOrTrunc(Op: Res.getValue(R: 1), DL: SDLoc(N), VT: NVT);
484 }
485
486 // Op2 is used for the comparison and thus must be extended according to the
487 // target's atomic operations. Op3 is merely stored and so can be left alone.
488 SDValue Op2 = N->getOperand(Num: 2);
489 SDValue Op3 = GetPromotedInteger(Op: N->getOperand(Num: 3));
490 switch (TLI.getExtendForAtomicCmpSwapArg()) {
491 case ISD::SIGN_EXTEND:
492 Op2 = SExtPromotedInteger(Op: Op2);
493 break;
494 case ISD::ZERO_EXTEND:
495 Op2 = ZExtPromotedInteger(Op: Op2);
496 break;
497 case ISD::ANY_EXTEND:
498 Op2 = GetPromotedInteger(Op: Op2);
499 break;
500 default:
501 llvm_unreachable("Invalid atomic op extension");
502 }
503
504 SDVTList VTs =
505 DAG.getVTList(VT1: Op2.getValueType(), VT2: N->getValueType(ResNo: 1), VT3: MVT::Other);
506 SDValue Res = DAG.getAtomicCmpSwap(
507 Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs, Chain: N->getChain(),
508 Ptr: N->getBasePtr(), Cmp: Op2, Swp: Op3, MMO: N->getMemOperand());
509 // Update the use to N with the newly created Res.
510 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
511 ReplaceValueWith(From: SDValue(N, i), To: Res.getValue(R: i));
512 return Res;
513}
514
515SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
516 SDValue InOp = N->getOperand(Num: 0);
517 EVT InVT = InOp.getValueType();
518 EVT NInVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT);
519 EVT OutVT = N->getValueType(ResNo: 0);
520 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
521 SDLoc dl(N);
522
523 switch (getTypeAction(VT: InVT)) {
524 case TargetLowering::TypeLegal:
525 break;
526 case TargetLowering::TypePromoteInteger:
527 if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector() && !NInVT.isVector())
528 // The input promotes to the same size. Convert the promoted value.
529 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetPromotedInteger(Op: InOp));
530 break;
531 case TargetLowering::TypeSoftenFloat:
532 // Promote the integer operand by hand.
533 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftenedFloat(Op: InOp));
534 case TargetLowering::TypeSoftPromoteHalf:
535 // Promote the integer operand by hand.
536 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftPromotedHalf(Op: InOp));
537 case TargetLowering::TypeExpandInteger:
538 case TargetLowering::TypeExpandFloat:
539 break;
540 case TargetLowering::TypeScalarizeVector:
541 // Convert the element to an integer and promote it by hand.
542 if (!NOutVT.isVector())
543 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT,
544 Operand: BitConvertToInteger(Op: GetScalarizedVector(Op: InOp)));
545 break;
546 case TargetLowering::TypeScalarizeScalableVector:
547 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
548 case TargetLowering::TypeSplitVector: {
549 if (!NOutVT.isVector()) {
550 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
551 // pieces of the input into integers and reassemble in the final type.
552 SDValue Lo, Hi;
553 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
554 Lo = BitConvertToInteger(Op: Lo);
555 Hi = BitConvertToInteger(Op: Hi);
556
557 if (DAG.getDataLayout().isBigEndian())
558 std::swap(a&: Lo, b&: Hi);
559
560 InOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl,
561 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
562 BitWidth: NOutVT.getSizeInBits()),
563 Operand: JoinIntegers(Lo, Hi));
564 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: InOp);
565 }
566 break;
567 }
568 case TargetLowering::TypeWidenVector:
569 // The input is widened to the same size. Convert to the widened value.
570 // Make sure that the outgoing value is not a vector, because this would
571 // make us bitcast between two vectors which are legalized in different ways.
572 if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector()) {
573 SDValue Res =
574 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetWidenedVector(Op: InOp));
575
576 // For big endian targets we need to shift the casted value or the
577 // interesting bits will end up at the wrong place.
578 if (DAG.getDataLayout().isBigEndian()) {
579 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
580 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
581 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NOutVT, N1: Res,
582 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NOutVT, DL: dl));
583 }
584 return Res;
585 }
586 // If the output type is also a vector and widening it to the same size
587 // as the widened input type would be a legal type, we can widen the bitcast
588 // and handle the promotion after.
589 if (NOutVT.isVector()) {
590 TypeSize WidenInSize = NInVT.getSizeInBits();
591 TypeSize OutSize = OutVT.getSizeInBits();
592 if (WidenInSize.hasKnownScalarFactor(RHS: OutSize)) {
593 unsigned Scale = WidenInSize.getKnownScalarFactor(RHS: OutSize);
594 EVT WideOutVT =
595 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OutVT.getVectorElementType(),
596 EC: OutVT.getVectorElementCount() * Scale);
597 if (isTypeLegal(VT: WideOutVT)) {
598 InOp = DAG.getBitcast(VT: WideOutVT, V: GetWidenedVector(Op: InOp));
599 InOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: InOp,
600 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
601 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: InOp);
602 }
603 }
604 }
605 }
606
607 // TODO: Handle big endian
608 if (!NOutVT.isVector() && InOp.getValueType().isVector() &&
609 DAG.getDataLayout().isLittleEndian()) {
610 // Pad the vector operand with undef and cast to a wider integer.
611 EVT EltVT = InOp.getValueType().getVectorElementType();
612 TypeSize EltSize = EltVT.getSizeInBits();
613 TypeSize OutSize = NOutVT.getSizeInBits();
614
615 if (OutSize.hasKnownScalarFactor(RHS: EltSize)) {
616 unsigned NumEltsWithPadding = OutSize.getKnownScalarFactor(RHS: EltSize);
617 EVT WideVecVT =
618 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumEltsWithPadding);
619
620 if (isTypeLegal(VT: WideVecVT)) {
621 SDValue Inserted = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideVecVT,
622 N1: DAG.getUNDEF(VT: WideVecVT), N2: InOp,
623 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
624
625 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: Inserted);
626 }
627 }
628 }
629
630 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT,
631 Operand: CreateStackStoreLoad(Op: InOp, DestVT: OutVT));
632}
633
634SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
635 SDValue V = GetPromotedInteger(Op: N->getOperand(Num: 0));
636 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N),
637 VT: V.getValueType(), Operand: V);
638}
639
640SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
641 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
642 EVT OVT = N->getValueType(ResNo: 0);
643 EVT NVT = Op.getValueType();
644 SDLoc dl(N);
645
646 // If the larger BSWAP isn't supported by the target, try to expand now.
647 // If we expand later we'll end up with more operations since we lost the
648 // original type. We only do this for scalars since we have a shuffle
649 // based lowering for vectors in LegalizeVectorOps.
650 if (!OVT.isVector() &&
651 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BSWAP, VT: NVT)) {
652 if (SDValue Res = TLI.expandBSWAP(N, DAG))
653 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res);
654 }
655
656 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
657 SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl);
658 if (N->getOpcode() == ISD::BSWAP)
659 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: NVT, Operand: Op),
660 N2: ShAmt);
661 SDValue Mask = N->getOperand(Num: 1);
662 SDValue EVL = N->getOperand(Num: 2);
663 return DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT: NVT,
664 N1: DAG.getNode(Opcode: ISD::VP_BSWAP, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), N2: ShAmt,
665 N3: Mask, N4: EVL);
666}
667
668SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
669 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
670 EVT OVT = N->getValueType(ResNo: 0);
671 EVT NVT = Op.getValueType();
672 SDLoc dl(N);
673
674 // If the larger BITREVERSE isn't supported by the target, try to expand now.
675 // If we expand later we'll end up with more operations since we lost the
676 // original type. We only do this for scalars since we have a shuffle
677 // based lowering for vectors in LegalizeVectorOps.
678 if (!OVT.isVector() && OVT.isSimple() &&
679 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BITREVERSE, VT: NVT)) {
680 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
681 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res);
682 }
683
684 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
685 SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl);
686 if (N->getOpcode() == ISD::BITREVERSE)
687 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT,
688 N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: NVT, Operand: Op), N2: ShAmt);
689 SDValue Mask = N->getOperand(Num: 1);
690 SDValue EVL = N->getOperand(Num: 2);
691 return DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT: NVT,
692 N1: DAG.getNode(Opcode: ISD::VP_BITREVERSE, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL),
693 N2: ShAmt, N3: Mask, N4: EVL);
694}
695
696SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
697 // The pair element type may be legal, or may not promote to the same type as
698 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
699 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N),
700 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
701 VT: N->getValueType(ResNo: 0)), Operand: JoinIntegers(Lo: N->getOperand(Num: 0),
702 Hi: N->getOperand(Num: 1)));
703}
704
705SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
706 EVT VT = N->getValueType(ResNo: 0);
707 // FIXME there is no actual debug info here
708 SDLoc dl(N);
709 // Zero extend things like i1, sign extend everything else. It shouldn't
710 // matter in theory which one we pick, but this tends to give better code?
711 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
712 SDValue Result = DAG.getNode(Opcode: Opc, DL: dl,
713 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT),
714 Operand: SDValue(N, 0));
715 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
716 return Result;
717}
718
719SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
720 EVT OVT = N->getValueType(ResNo: 0);
721 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
722 SDLoc dl(N);
723
724 // If the larger CTLZ isn't supported by the target, try to expand now.
725 // If we expand later we'll end up with more operations since we lost the
726 // original type.
727 if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) &&
728 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ, VT: NVT) &&
729 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ_ZERO_UNDEF, VT: NVT)) {
730 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
731 Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result);
732 return Result;
733 }
734 }
735
736 unsigned CtlzOpcode = N->getOpcode();
737 if (CtlzOpcode == ISD::CTLZ || CtlzOpcode == ISD::VP_CTLZ) {
738 // Subtract off the extra leading bits in the bigger type.
739 SDValue ExtractLeadingBits = DAG.getConstant(
740 Val: NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), DL: dl, VT: NVT);
741 // Zero extend to the promoted type and do the count there.
742 SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
743
744 if (!N->isVPOpcode())
745 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT,
746 N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op),
747 N2: ExtractLeadingBits);
748 SDValue Mask = N->getOperand(Num: 1);
749 SDValue EVL = N->getOperand(Num: 2);
750 return DAG.getNode(Opcode: ISD::VP_SUB, DL: dl, VT: NVT,
751 N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL),
752 N2: ExtractLeadingBits, N3: Mask, N4: EVL);
753 }
754 if (CtlzOpcode == ISD::CTLZ_ZERO_UNDEF ||
755 CtlzOpcode == ISD::VP_CTLZ_ZERO_UNDEF) {
756 // Any Extend the argument
757 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
758 // Op = Op << (sizeinbits(NVT) - sizeinbits(Old VT))
759 unsigned SHLAmount = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
760 auto ShiftConst =
761 DAG.getShiftAmountConstant(Val: SHLAmount, VT: Op.getValueType(), DL: dl);
762 if (!N->isVPOpcode()) {
763 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Op, N2: ShiftConst);
764 return DAG.getNode(Opcode: CtlzOpcode, DL: dl, VT: NVT, Operand: Op);
765 }
766
767 SDValue Mask = N->getOperand(Num: 1);
768 SDValue EVL = N->getOperand(Num: 2);
769 Op = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT: NVT, N1: Op, N2: ShiftConst, N3: Mask, N4: EVL);
770 return DAG.getNode(Opcode: CtlzOpcode, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL);
771 }
772 llvm_unreachable("Invalid CTLZ Opcode");
773}
774
775SDValue DAGTypeLegalizer::PromoteIntRes_CTLS(SDNode *N) {
776 EVT OVT = N->getValueType(ResNo: 0);
777 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
778 SDLoc dl(N);
779
780 SDValue ExtractLeadingBits = DAG.getConstant(
781 Val: NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), DL: dl, VT: NVT);
782
783 SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 0));
784 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::CTLS, DL: dl, VT: NVT, Operand: Op),
785 N2: ExtractLeadingBits);
786}
787
788SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
789 EVT OVT = N->getValueType(ResNo: 0);
790 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
791
792 // If the larger CTPOP isn't supported by the target, try to expand now.
793 // If we expand later we'll end up with more operations since we lost the
794 // original type.
795 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
796 // TargetLowering.
797 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(VT: NVT) &&
798 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT: NVT)) {
799 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
800 Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Result);
801 return Result;
802 }
803 }
804
805 // Zero extend to the promoted type and do the count or parity there.
806 SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
807 if (!N->isVPOpcode())
808 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), Operand: Op);
809
810 SDValue Mask = N->getOperand(Num: 1);
811 SDValue EVL = N->getOperand(Num: 2);
812 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op, N2: Mask,
813 N3: EVL);
814}
815
816SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
817 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
818 EVT OVT = N->getValueType(ResNo: 0);
819 EVT NVT = Op.getValueType();
820 SDLoc dl(N);
821
822 // If the larger CTTZ isn't supported by the target, try to expand now.
823 // If we expand later we'll end up with more operations since we lost the
824 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
825 // larger type.
826 if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) &&
827 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ, VT: NVT) &&
828 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ_ZERO_UNDEF, VT: NVT) &&
829 !TLI.isOperationLegal(Op: ISD::CTPOP, VT: NVT) &&
830 !TLI.isOperationLegal(Op: ISD::CTLZ, VT: NVT)) {
831 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
832 Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result);
833 return Result;
834 }
835 }
836
837 unsigned NewOpc = N->getOpcode();
838 if (NewOpc == ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
839 // The count is the same in the promoted type except if the original
840 // value was zero. This can be handled by setting the bit just off
841 // the top of the original type.
842 auto TopBit = APInt::getOneBitSet(numBits: NVT.getScalarSizeInBits(),
843 BitNo: OVT.getScalarSizeInBits());
844 if (NewOpc == ISD::CTTZ) {
845 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
846 NewOpc = ISD::CTTZ_ZERO_UNDEF;
847 } else {
848 Op =
849 DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT),
850 N3: N->getOperand(Num: 1), N4: N->getOperand(Num: 2));
851 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
852 }
853 }
854 if (!N->isVPOpcode())
855 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Op);
856 return DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
857}
858
859SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(SDNode *N) {
860 SDLoc DL(N);
861 EVT NewVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
862 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: NewVT, Ops: N->ops());
863}
864
865SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
866 SDLoc dl(N);
867 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
868
869 SDValue Op0 = N->getOperand(Num: 0);
870 SDValue Op1 = N->getOperand(Num: 1);
871
872 // If the input also needs to be promoted, do that first so we can get a
873 // get a good idea for the output type.
874 if (TLI.getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType())
875 == TargetLowering::TypePromoteInteger) {
876 SDValue In = GetPromotedInteger(Op: Op0);
877
878 // If the new type is larger than NVT, use it. We probably won't need to
879 // promote it again.
880 EVT SVT = In.getValueType().getScalarType();
881 if (SVT.bitsGE(VT: NVT)) {
882 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT, N1: In, N2: Op1);
883 return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NVT);
884 }
885 }
886
887 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NVT, N1: Op0, N2: Op1);
888}
889
890SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
891 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
892 unsigned NewOpc =
893 TLI.getPreferredFPToIntOpcode(Op: N->getOpcode(), FromVT: N->getValueType(ResNo: 0), ToVT: NVT);
894 SDLoc dl(N);
895
896 SDValue Res;
897 if (N->isStrictFPOpcode()) {
898 Res = DAG.getNode(Opcode: NewOpc, DL: dl, ResultTys: {NVT, MVT::Other},
899 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
900 // Legalize the chain result - switch anything that used the old chain to
901 // use the new one.
902 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
903 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
904 Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
905 N->getOperand(Num: 2)});
906 } else {
907 Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
908 }
909
910 // Assert that the converted value fits in the original type. If it doesn't
911 // (eg: because the value being converted is too big), then the result of the
912 // original operation was undefined anyway, so the assert is still correct.
913 //
914 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
915 // before legalization: fp-to-uint16, 65534. -> 0xfffe
916 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
917 return DAG.getNode(Opcode: (N->getOpcode() == ISD::FP_TO_UINT ||
918 N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
919 N->getOpcode() == ISD::VP_FP_TO_UINT)
920 ? ISD::AssertZext
921 : ISD::AssertSext,
922 DL: dl, VT: NVT, N1: Res,
923 N2: DAG.getValueType(N->getValueType(ResNo: 0).getScalarType()));
924}
925
926SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
927 // Promote the result type, while keeping the original width in Op1.
928 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
929 SDLoc dl(N);
930 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0),
931 N2: N->getOperand(Num: 1));
932}
933
934SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
935 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
936 SDLoc dl(N);
937
938 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
939}
940
941SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
942 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
943 SDLoc dl(N);
944
945 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
946 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
947 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
948 return Res;
949}
950
951SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
952 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
953 SDLoc dl(N);
954 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
955}
956
957SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
958 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
959 SDLoc dl(N);
960
961 SDValue Res =
962 DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other}, Ops: N->getOperand(Num: 0));
963
964 // Legalize the chain result - switch anything that used the old chain to
965 // use the new one.
966 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
967 return Res;
968}
969
970SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
971 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
972 SDLoc dl(N);
973
974 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType())
975 == TargetLowering::TypePromoteInteger) {
976 SDValue Res = GetPromotedInteger(Op: N->getOperand(Num: 0));
977 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
978
979 // If the result and operand types are the same after promotion, simplify
980 // to an in-register extension. Unless this is a VP_*_EXTEND.
981 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
982 // The high bits are not guaranteed to be anything. Insert an extend.
983 if (N->getOpcode() == ISD::SIGN_EXTEND)
984 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res,
985 N2: DAG.getValueType(N->getOperand(Num: 0).getValueType()));
986 if (N->getOpcode() == ISD::ZERO_EXTEND)
987 return DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: N->getOperand(Num: 0).getValueType());
988 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
989 return Res;
990 }
991 }
992
993 // Otherwise, just extend the original operand all the way to the larger type.
994 if (N->getNumOperands() != 1) {
995 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
996 assert(N->isVPOpcode() && "Expected VP opcode");
997 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0),
998 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
999 }
1000 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1001}
1002
1003SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
1004 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1005 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1006 ISD::LoadExtType ExtType =
1007 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
1008 SDLoc dl(N);
1009 SDValue Res = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: N->getChain(), Ptr: N->getBasePtr(),
1010 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
1011
1012 // Legalize the chain result - switch anything that used the old chain to
1013 // use the new one.
1014 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1015 return Res;
1016}
1017
1018SDValue DAGTypeLegalizer::PromoteIntRes_VP_LOAD(VPLoadSDNode *N) {
1019 assert(!N->isIndexed() && "Indexed vp_load during type legalization!");
1020 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1021 ISD::LoadExtType ExtType = (N->getExtensionType() == ISD::NON_EXTLOAD)
1022 ? ISD::EXTLOAD
1023 : N->getExtensionType();
1024 SDLoc dl(N);
1025 SDValue Res =
1026 DAG.getExtLoadVP(ExtType, dl, VT: NVT, Chain: N->getChain(), Ptr: N->getBasePtr(),
1027 Mask: N->getMask(), EVL: N->getVectorLength(), MemVT: N->getMemoryVT(),
1028 MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
1029 // Legalize the chain result - switch anything that used the old chain to
1030 // use the new one.
1031 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1032 return Res;
1033}
1034
1035SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
1036 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1037 SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru());
1038
1039 ISD::LoadExtType ExtType = N->getExtensionType();
1040 if (ExtType == ISD::NON_EXTLOAD)
1041 ExtType = ISD::EXTLOAD;
1042
1043 SDLoc dl(N);
1044 SDValue Res = DAG.getMaskedLoad(VT: NVT, dl, Chain: N->getChain(), Base: N->getBasePtr(),
1045 Offset: N->getOffset(), Mask: N->getMask(), Src0: ExtPassThru,
1046 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
1047 AM: N->getAddressingMode(), ExtType,
1048 IsExpanding: N->isExpandingLoad());
1049 // Legalize the chain result - switch anything that used the old chain to
1050 // use the new one.
1051 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1052 return Res;
1053}
1054
1055SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
1056 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1057 SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru());
1058 assert(NVT == ExtPassThru.getValueType() &&
1059 "Gather result type and the passThru argument type should be the same");
1060
1061 ISD::LoadExtType ExtType = N->getExtensionType();
1062 if (ExtType == ISD::NON_EXTLOAD)
1063 ExtType = ISD::EXTLOAD;
1064
1065 SDLoc dl(N);
1066 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
1067 N->getIndex(), N->getScale() };
1068 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
1069 MemVT: N->getMemoryVT(), dl, Ops,
1070 MMO: N->getMemOperand(), IndexType: N->getIndexType(),
1071 ExtTy: ExtType);
1072 // Legalize the chain result - switch anything that used the old chain to
1073 // use the new one.
1074 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1075 return Res;
1076}
1077
1078SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(SDNode *N) {
1079 SDValue Vec = GetPromotedInteger(Op: N->getOperand(Num: 0));
1080 SDValue Passthru = GetPromotedInteger(Op: N->getOperand(Num: 2));
1081 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: Vec.getValueType(), N1: Vec,
1082 N2: N->getOperand(Num: 1), N3: Passthru);
1083}
1084
1085/// Promote the overflow flag of an overflowing arithmetic node.
1086SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
1087 // Change the return type of the boolean result while obeying
1088 // getSetCCResultType.
1089 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1));
1090 EVT VT = N->getValueType(ResNo: 0);
1091 EVT SVT = getSetCCResultType(VT);
1092 SDValue Ops[3] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
1093 unsigned NumOps = N->getNumOperands();
1094 assert(NumOps <= 3 && "Too many operands");
1095 if (NumOps == 3)
1096 Ops[2] = PromoteTargetBoolean(Bool: N->getOperand(Num: 2), ValVT: VT);
1097
1098 SDLoc dl(N);
1099 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: SVT),
1100 Ops: ArrayRef(Ops, NumOps));
1101
1102 // Modified the sum result - switch anything that used the old sum to use
1103 // the new one.
1104 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1105
1106 // Convert to the expected type.
1107 return DAG.getBoolExtOrTrunc(Op: Res.getValue(R: 1), SL: dl, VT: NVT, OpVT: VT);
1108}
1109
1110template <class MatchContextClass>
1111SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
1112 // If the promoted type is legal, we can convert this to:
1113 // 1. ANY_EXTEND iN to iM
1114 // 2. SHL by M-N
1115 // 3. [US][ADD|SUB|SHL]SAT
1116 // 4. L/ASHR by M-N
1117 // Else it is more efficient to convert this to a min and a max
1118 // operation in the higher precision arithmetic.
1119 SDLoc dl(N);
1120 SDValue Op1 = N->getOperand(Num: 0);
1121 SDValue Op2 = N->getOperand(Num: 1);
1122 MatchContextClass matcher(DAG, TLI, N);
1123
1124 unsigned Opcode = matcher.getRootBaseOpcode();
1125 unsigned OldBits = Op1.getScalarValueSizeInBits();
1126
1127 // USUBSAT can always be promoted as long as we have zero/sign-extended the
1128 // args.
1129 if (Opcode == ISD::USUBSAT) {
1130 SExtOrZExtPromotedOperands(LHS&: Op1, RHS&: Op2);
1131 return matcher.getNode(ISD::USUBSAT, dl, Op1.getValueType(), Op1, Op2);
1132 }
1133
1134 if (Opcode == ISD::UADDSAT) {
1135 EVT OVT = Op1.getValueType();
1136 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
1137 // We can promote if we use sign-extend. Do this if the target prefers.
1138 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT)) {
1139 Op1 = SExtPromotedInteger(Op: Op1);
1140 Op2 = SExtPromotedInteger(Op: Op2);
1141 return matcher.getNode(ISD::UADDSAT, dl, NVT, Op1, Op2);
1142 }
1143
1144 Op1 = ZExtPromotedInteger(Op: Op1);
1145 Op2 = ZExtPromotedInteger(Op: Op2);
1146 unsigned NewBits = NVT.getScalarSizeInBits();
1147 APInt MaxVal = APInt::getLowBitsSet(numBits: NewBits, loBitsSet: OldBits);
1148 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: NVT);
1149 SDValue Add = matcher.getNode(ISD::ADD, dl, NVT, Op1, Op2);
1150 return matcher.getNode(ISD::UMIN, dl, NVT, Add, SatMax);
1151 }
1152
1153 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
1154
1155 // FIXME: We need vp-aware PromotedInteger functions.
1156 if (IsShift) {
1157 Op1 = GetPromotedInteger(Op: Op1);
1158 if (getTypeAction(VT: Op2.getValueType()) == TargetLowering::TypePromoteInteger)
1159 Op2 = ZExtPromotedInteger(Op: Op2);
1160 } else {
1161 Op1 = SExtPromotedInteger(Op: Op1);
1162 Op2 = SExtPromotedInteger(Op: Op2);
1163 }
1164 EVT PromotedType = Op1.getValueType();
1165 unsigned NewBits = PromotedType.getScalarSizeInBits();
1166
1167 // Shift cannot use a min/max expansion, we can't detect overflow if all of
1168 // the bits have been shifted out.
1169 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1170 unsigned ShiftOp;
1171 switch (Opcode) {
1172 case ISD::SADDSAT:
1173 case ISD::SSUBSAT:
1174 case ISD::SSHLSAT:
1175 ShiftOp = ISD::SRA;
1176 break;
1177 case ISD::USHLSAT:
1178 ShiftOp = ISD::SRL;
1179 break;
1180 default:
1181 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
1182 "addition, subtraction or left shift");
1183 }
1184
1185 unsigned SHLAmount = NewBits - OldBits;
1186 SDValue ShiftAmount =
1187 DAG.getShiftAmountConstant(Val: SHLAmount, VT: PromotedType, DL: dl);
1188 Op1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1, N2: ShiftAmount);
1189 if (!IsShift)
1190 Op2 = matcher.getNode(ISD::SHL, dl, PromotedType, Op2, ShiftAmount);
1191
1192 SDValue Result = matcher.getNode(Opcode, dl, PromotedType, Op1, Op2);
1193 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1194 }
1195
1196 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1197 APInt MinVal = APInt::getSignedMinValue(numBits: OldBits).sext(width: NewBits);
1198 APInt MaxVal = APInt::getSignedMaxValue(numBits: OldBits).sext(width: NewBits);
1199 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT: PromotedType);
1200 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: PromotedType);
1201 SDValue Result = matcher.getNode(AddOp, dl, PromotedType, Op1, Op2);
1202 Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1203 Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1204 return Result;
1205}
1206
1207SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1208 // Can just promote the operands then continue with operation.
1209 SDLoc dl(N);
1210 SDValue Op1Promoted, Op2Promoted;
1211 bool Signed =
1212 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1213 bool Saturating =
1214 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1215 if (Signed) {
1216 Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1217 Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1));
1218 } else {
1219 Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1220 Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
1221 }
1222 EVT OldType = N->getOperand(Num: 0).getValueType();
1223 EVT PromotedType = Op1Promoted.getValueType();
1224 unsigned DiffSize =
1225 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1226
1227 if (Saturating) {
1228 // Promoting the operand and result values changes the saturation width,
1229 // which is extends the values that we clamp to on saturation. This could be
1230 // resolved by shifting one of the operands the same amount, which would
1231 // also shift the result we compare against, then shifting back.
1232 Op1Promoted =
1233 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted,
1234 N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl));
1235 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted,
1236 N2: Op2Promoted, N3: N->getOperand(Num: 2));
1237 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1238 return DAG.getNode(Opcode: ShiftOp, DL: dl, VT: PromotedType, N1: Result,
1239 N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl));
1240 }
1241 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted,
1242 N3: N->getOperand(Num: 2));
1243}
1244
1245static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
1246 unsigned SatW, bool Signed,
1247 const TargetLowering &TLI,
1248 SelectionDAG &DAG) {
1249 EVT VT = V.getValueType();
1250 unsigned VTW = VT.getScalarSizeInBits();
1251
1252 if (!Signed) {
1253 // Saturate to the unsigned maximum by getting the minimum of V and the
1254 // maximum.
1255 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: V,
1256 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW),
1257 DL: dl, VT));
1258 }
1259
1260 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1261 // signed minimum of it and V.
1262 V = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: V,
1263 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW - 1),
1264 DL: dl, VT));
1265 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1266 // signed maximum of it and V.
1267 V = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: V,
1268 N2: DAG.getConstant(Val: APInt::getHighBitsSet(numBits: VTW, hiBitsSet: VTW - SatW + 1),
1269 DL: dl, VT));
1270 return V;
1271}
1272
1273static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
1274 unsigned Scale, const TargetLowering &TLI,
1275 SelectionDAG &DAG, unsigned SatW = 0) {
1276 EVT VT = LHS.getValueType();
1277 unsigned VTSize = VT.getScalarSizeInBits();
1278 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1279 N->getOpcode() == ISD::SDIVFIXSAT;
1280 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1281 N->getOpcode() == ISD::UDIVFIXSAT;
1282
1283 SDLoc dl(N);
1284 // Widen the types by a factor of two. This is guaranteed to expand, since it
1285 // will always have enough high bits in the LHS to shift into.
1286 EVT WideVT = VT.changeElementType(
1287 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VTSize * 2));
1288 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL: dl, VT: WideVT);
1289 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL: dl, VT: WideVT);
1290 SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS, RHS, Scale,
1291 DAG);
1292 assert(Res && "Expanding DIVFIX with wide type failed?");
1293 if (Saturating) {
1294 // If the caller has told us to saturate at something less, use that width
1295 // instead of the type before doubling. However, it cannot be more than
1296 // what we just widened!
1297 assert(SatW <= VTSize &&
1298 "Tried to saturate to more than the original type?");
1299 Res = SaturateWidenedDIVFIX(V: Res, dl, SatW: SatW == 0 ? VTSize : SatW, Signed,
1300 TLI, DAG);
1301 }
1302 return DAG.getZExtOrTrunc(Op: Res, DL: dl, VT);
1303}
1304
1305SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1306 SDLoc dl(N);
1307 SDValue Op1Promoted, Op2Promoted;
1308 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1309 N->getOpcode() == ISD::SDIVFIXSAT;
1310 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1311 N->getOpcode() == ISD::UDIVFIXSAT;
1312 if (Signed) {
1313 Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1314 Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1));
1315 } else {
1316 Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1317 Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
1318 }
1319 EVT PromotedType = Op1Promoted.getValueType();
1320 unsigned Scale = N->getConstantOperandVal(Num: 2);
1321
1322 // If the type is already legal and the operation is legal in that type, we
1323 // should not early expand.
1324 if (TLI.isTypeLegal(VT: PromotedType)) {
1325 TargetLowering::LegalizeAction Action =
1326 TLI.getFixedPointOperationAction(Op: N->getOpcode(), VT: PromotedType, Scale);
1327 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1328 unsigned Diff = PromotedType.getScalarSizeInBits() -
1329 N->getValueType(ResNo: 0).getScalarSizeInBits();
1330 if (Saturating)
1331 Op1Promoted =
1332 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted,
1333 N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl));
1334 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted,
1335 N2: Op2Promoted, N3: N->getOperand(Num: 2));
1336 if (Saturating)
1337 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT: PromotedType, N1: Res,
1338 N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl));
1339 return Res;
1340 }
1341 }
1342
1343 // See if we can perform the division in this type without expanding.
1344 if (SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: Op1Promoted,
1345 RHS: Op2Promoted, Scale, DAG)) {
1346 if (Saturating)
1347 Res = SaturateWidenedDIVFIX(V: Res, dl,
1348 SatW: N->getValueType(ResNo: 0).getScalarSizeInBits(),
1349 Signed, TLI, DAG);
1350 return Res;
1351 }
1352 // If we cannot, expand it to twice the type width. If we are saturating, give
1353 // it the original width as a saturating width so we don't need to emit
1354 // two saturations.
1355 return earlyExpandDIVFIX(N, LHS: Op1Promoted, RHS: Op2Promoted, Scale, TLI, DAG,
1356 SatW: N->getValueType(ResNo: 0).getScalarSizeInBits());
1357}
1358
1359SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1360 if (ResNo == 1)
1361 return PromoteIntRes_Overflow(N);
1362
1363 // The operation overflowed iff the result in the larger type is not the
1364 // sign extension of its truncation to the original type.
1365 SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1366 SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1));
1367 EVT OVT = N->getOperand(Num: 0).getValueType();
1368 EVT NVT = LHS.getValueType();
1369 SDLoc dl(N);
1370
1371 // Do the arithmetic in the larger type.
1372 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1373 SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS);
1374
1375 // Calculate the overflow flag: sign extend the arithmetic result from
1376 // the original type.
1377 SDValue Ofl = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res,
1378 N2: DAG.getValueType(OVT));
1379 // Overflowed if and only if this is not equal to Res.
1380 Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE);
1381
1382 // Use the calculated overflow everywhere.
1383 ReplaceValueWith(From: SDValue(N, 1), To: Ofl);
1384
1385 return Res;
1386}
1387
1388SDValue DAGTypeLegalizer::PromoteIntRes_CMP(SDNode *N) {
1389 EVT PromotedResultTy =
1390 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1391 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: PromotedResultTy,
1392 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
1393}
1394
1395SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1396 SDValue Mask = N->getOperand(Num: 0);
1397
1398 SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 1));
1399 SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 2));
1400
1401 unsigned Opcode = N->getOpcode();
1402 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1403 return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS,
1404 N4: N->getOperand(Num: 3));
1405 return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS);
1406}
1407
1408SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1409 SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 2));
1410 SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 3));
1411 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
1412 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
1413 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
1414}
1415
1416SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1417 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1418 EVT InVT = N->getOperand(Num: OpNo).getValueType();
1419 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1420
1421 EVT SVT = getSetCCResultType(VT: InVT);
1422
1423 // If we got back a type that needs to be promoted, this likely means the
1424 // the input type also needs to be promoted. So get the promoted type for
1425 // the input and try the query again.
1426 if (getTypeAction(VT: SVT) == TargetLowering::TypePromoteInteger) {
1427 if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) {
1428 InVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT);
1429 SVT = getSetCCResultType(VT: InVT);
1430 } else {
1431 // Input type isn't promoted, just use the default promoted type.
1432 SVT = NVT;
1433 }
1434 }
1435
1436 SDLoc dl(N);
1437 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1438 "Vector compare must return a vector result!");
1439
1440 // Get the SETCC result using the canonical SETCC type.
1441 SDValue SetCC;
1442 if (N->isStrictFPOpcode()) {
1443 SDVTList VTs = DAG.getVTList(VTs: {SVT, MVT::Other});
1444 SDValue Opers[] = {N->getOperand(Num: 0), N->getOperand(Num: 1),
1445 N->getOperand(Num: 2), N->getOperand(Num: 3)};
1446 SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, Ops: Opers, Flags: N->getFlags());
1447 // Legalize the chain result - switch anything that used the old chain to
1448 // use the new one.
1449 ReplaceValueWith(From: SDValue(N, 1), To: SetCC.getValue(R: 1));
1450 } else
1451 SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SVT, N1: N->getOperand(Num: 0),
1452 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), Flags: N->getFlags());
1453
1454 // Convert to the expected type.
1455 return DAG.getSExtOrTrunc(Op: SetCC, DL: dl, VT: NVT);
1456}
1457
1458SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1459 SDLoc DL(N);
1460 SDValue Arg = N->getOperand(Num: 0);
1461 SDValue Test = N->getOperand(Num: 1);
1462 EVT NResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1463 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: NResVT, N1: Arg, N2: Test);
1464}
1465
1466SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1467 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1));
1468 EVT VT = N->getValueType(ResNo: 0);
1469
1470 SDLoc dl(N);
1471 SDValue Res =
1472 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: NVT), N: N->getOperand(Num: 0));
1473
1474 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1475 return Res.getValue(R: 1);
1476}
1477
1478SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1479 SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0));
1480 SDValue RHS = N->getOperand(Num: 1);
1481 if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1482 RHS = ZExtPromotedInteger(Op: RHS);
1483 if (N->getOpcode() != ISD::VP_SHL)
1484 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1485
1486 SDValue Mask = N->getOperand(Num: 2);
1487 SDValue EVL = N->getOperand(Num: 3);
1488 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1489 N3: Mask, N4: EVL);
1490}
1491
1492SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1493 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
1494 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N),
1495 VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1));
1496}
1497
1498SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1499 // The input may have strange things in the top bits of the registers, but
1500 // these operations don't care. They may have weird bits going out, but
1501 // that too is okay if they are integer operations.
1502 SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0));
1503 SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 1));
1504 if (N->getNumOperands() == 2)
1505 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1506 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1507 assert(N->isVPOpcode() && "Expected VP opcode");
1508 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1509 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
1510}
1511
1512SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1513 // Sign extend the input.
1514 SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1515 SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1));
1516 if (N->getNumOperands() == 2)
1517 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1518 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1519 assert(N->isVPOpcode() && "Expected VP opcode");
1520 SDValue Mask = N->getOperand(Num: 2);
1521 SDValue EVL = N->getOperand(Num: 3);
1522 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1523 N3: Mask, N4: EVL);
1524}
1525
1526SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1527 // Zero extend the input.
1528 SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1529 SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
1530 if (N->getNumOperands() == 2)
1531 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1532 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1533 assert(N->isVPOpcode() && "Expected VP opcode");
1534 // Zero extend the input.
1535 SDValue Mask = N->getOperand(Num: 2);
1536 SDValue EVL = N->getOperand(Num: 3);
1537 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1538 N3: Mask, N4: EVL);
1539}
1540
1541SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1542 SDValue LHS = N->getOperand(Num: 0);
1543 SDValue RHS = N->getOperand(Num: 1);
1544
1545 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1546 // whatever is best for the target and the promoted operands.
1547 SExtOrZExtPromotedOperands(LHS, RHS);
1548
1549 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1550 VT: LHS.getValueType(), N1: LHS, N2: RHS);
1551}
1552
1553SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1554 // The input value must be properly sign extended.
1555 SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1556 SDValue RHS = N->getOperand(Num: 1);
1557 if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1558 RHS = ZExtPromotedInteger(Op: RHS);
1559 if (N->getOpcode() != ISD::VP_SRA)
1560 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1561
1562 SDValue Mask = N->getOperand(Num: 2);
1563 SDValue EVL = N->getOperand(Num: 3);
1564 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1565 N3: Mask, N4: EVL);
1566}
1567
1568SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1569 SDValue RHS = N->getOperand(Num: 1);
1570 // The input value must be properly zero extended.
1571 SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1572 if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1573 RHS = ZExtPromotedInteger(Op: RHS);
1574 if (N->getOpcode() != ISD::VP_SRL)
1575 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS);
1576
1577 SDValue Mask = N->getOperand(Num: 2);
1578 SDValue EVL = N->getOperand(Num: 3);
1579 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS,
1580 N3: Mask, N4: EVL);
1581}
1582
1583SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1584 // Lower the rotate to shifts and ORs which can be promoted.
1585 SDValue Res = TLI.expandROT(N, AllowVectorOps: true /*AllowVectorOps*/, DAG);
1586 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1587 return SDValue();
1588}
1589
1590SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1591 SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0));
1592 SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1));
1593 SDValue Amt = N->getOperand(Num: 2);
1594 if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1595 Amt = ZExtPromotedInteger(Op: Amt);
1596 EVT AmtVT = Amt.getValueType();
1597
1598 SDLoc DL(N);
1599 EVT OldVT = N->getOperand(Num: 0).getValueType();
1600 EVT VT = Lo.getValueType();
1601 unsigned Opcode = N->getOpcode();
1602 bool IsFSHR = Opcode == ISD::FSHR;
1603 unsigned OldBits = OldVT.getScalarSizeInBits();
1604 unsigned NewBits = VT.getScalarSizeInBits();
1605
1606 // Amount has to be interpreted modulo the old bit width.
1607 Amt = DAG.getNode(Opcode: ISD::UREM, DL, VT: AmtVT, N1: Amt,
1608 N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT));
1609
1610 // If the promoted type is twice the size (or more), then we use the
1611 // traditional funnel 'double' shift codegen. This isn't necessary if the
1612 // shift amount is constant.
1613 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1614 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1615 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) &&
1616 !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) {
1617 SDValue HiShift = DAG.getShiftAmountConstant(Val: OldBits, VT, DL);
1618 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: HiShift);
1619 Lo = DAG.getZeroExtendInReg(Op: Lo, DL, VT: OldVT);
1620 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Hi, N2: Lo);
1621 Res = DAG.getNode(Opcode: IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, N1: Res, N2: Amt);
1622 if (!IsFSHR)
1623 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res, N2: HiShift);
1624 return Res;
1625 }
1626
1627 // Shift Lo up to occupy the upper bits of the promoted type.
1628 Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo,
1629 N2: DAG.getShiftAmountConstant(Val: NewBits - OldBits, VT, DL));
1630
1631 // Increase Amount to shift the result into the lower bits of the promoted
1632 // type.
1633 if (IsFSHR)
1634 Amt = DAG.getNode(Opcode: ISD::ADD, DL, VT: AmtVT, N1: Amt,
1635 N2: DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT));
1636
1637 return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt);
1638}
1639
1640// A vp version of PromoteIntRes_FunnelShift.
1641SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
1642 SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0));
1643 SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1));
1644 SDValue Amt = N->getOperand(Num: 2);
1645 SDValue Mask = N->getOperand(Num: 3);
1646 SDValue EVL = N->getOperand(Num: 4);
1647 if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1648 Amt = ZExtPromotedInteger(Op: Amt);
1649 EVT AmtVT = Amt.getValueType();
1650
1651 SDLoc DL(N);
1652 EVT OldVT = N->getOperand(Num: 0).getValueType();
1653 EVT VT = Lo.getValueType();
1654 unsigned Opcode = N->getOpcode();
1655 bool IsFSHR = Opcode == ISD::VP_FSHR;
1656 unsigned OldBits = OldVT.getScalarSizeInBits();
1657 unsigned NewBits = VT.getScalarSizeInBits();
1658
1659 // Amount has to be interpreted modulo the old bit width.
1660 Amt = DAG.getNode(Opcode: ISD::VP_UREM, DL, VT: AmtVT, N1: Amt,
1661 N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT), N3: Mask, N4: EVL);
1662
1663 // If the promoted type is twice the size (or more), then we use the
1664 // traditional funnel 'double' shift codegen. This isn't necessary if the
1665 // shift amount is constant.
1666 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1667 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1668 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) &&
1669 !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) {
1670 SDValue HiShift = DAG.getConstant(Val: OldBits, DL, VT);
1671 Hi = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Hi, N2: HiShift, N3: Mask, N4: EVL);
1672 Lo = DAG.getVPZeroExtendInReg(Op: Lo, Mask, EVL, DL, VT: OldVT);
1673 SDValue Res = DAG.getNode(Opcode: ISD::VP_OR, DL, VT, N1: Hi, N2: Lo, N3: Mask, N4: EVL);
1674 Res = DAG.getNode(Opcode: IsFSHR ? ISD::VP_SRL : ISD::VP_SHL, DL, VT, N1: Res, N2: Amt,
1675 N3: Mask, N4: EVL);
1676 if (!IsFSHR)
1677 Res = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT, N1: Res, N2: HiShift, N3: Mask, N4: EVL);
1678 return Res;
1679 }
1680
1681 // Shift Lo up to occupy the upper bits of the promoted type.
1682 SDValue ShiftOffset = DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT);
1683 Lo = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Lo, N2: ShiftOffset, N3: Mask, N4: EVL);
1684
1685 // Increase Amount to shift the result into the lower bits of the promoted
1686 // type.
1687 if (IsFSHR)
1688 Amt = DAG.getNode(Opcode: ISD::VP_ADD, DL, VT: AmtVT, N1: Amt, N2: ShiftOffset, N3: Mask, N4: EVL);
1689
1690 return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt, N4: Mask, N5: EVL);
1691}
1692
1693SDValue DAGTypeLegalizer::PromoteIntRes_CLMUL(SDNode *N) {
1694 unsigned Opcode = N->getOpcode();
1695
1696 SDLoc DL(N);
1697 EVT OldVT = N->getOperand(Num: 0).getValueType();
1698 EVT VT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OldVT);
1699
1700 if (Opcode == ISD::CLMUL) {
1701 if (!TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CLMUL, VT)) {
1702 if (SDValue Res = TLI.expandCLMUL(N, DAG))
1703 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Res);
1704 }
1705 SDValue X = GetPromotedInteger(Op: N->getOperand(Num: 0));
1706 SDValue Y = GetPromotedInteger(Op: N->getOperand(Num: 1));
1707 return DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
1708 }
1709
1710 SDValue X = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1711 SDValue Y = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
1712
1713 unsigned OldBits = OldVT.getScalarSizeInBits();
1714 unsigned NewBits = VT.getScalarSizeInBits();
1715 if (NewBits < 2 * OldBits) {
1716 SDValue Clmul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
1717 unsigned ShAmt = Opcode == ISD::CLMULH ? OldBits : OldBits - 1;
1718 SDValue Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Clmul,
1719 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT, DL));
1720 SDValue Clmulh = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
1721 ShAmt = Opcode == ISD::CLMULH ? NewBits - OldBits : NewBits - OldBits + 1;
1722 SDValue Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Clmulh,
1723 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT, DL));
1724 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
1725 }
1726
1727 SDValue Clmul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
1728 unsigned ShAmt = Opcode == ISD::CLMULH ? OldBits : OldBits - 1;
1729 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Clmul,
1730 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT, DL));
1731}
1732
1733SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1734 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1735 SDValue Res;
1736 SDValue InOp = N->getOperand(Num: 0);
1737 SDLoc dl(N);
1738
1739 switch (getTypeAction(VT: InOp.getValueType())) {
1740 default: llvm_unreachable("Unknown type action!");
1741 case TargetLowering::TypeLegal:
1742 case TargetLowering::TypeExpandInteger:
1743 Res = InOp;
1744 break;
1745 case TargetLowering::TypePromoteInteger:
1746 Res = GetPromotedInteger(Op: InOp);
1747 break;
1748 case TargetLowering::TypeSplitVector: {
1749 EVT InVT = InOp.getValueType();
1750 assert(InVT.isVector() && "Cannot split scalar types");
1751 ElementCount NumElts = InVT.getVectorElementCount();
1752 assert(NumElts == NVT.getVectorElementCount() &&
1753 "Dst and Src must have the same number of elements");
1754 assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
1755 "Promoted vector type must be a power of two");
1756
1757 SDValue EOp1, EOp2;
1758 GetSplitVector(Op: InOp, Lo&: EOp1, Hi&: EOp2);
1759
1760 EVT HalfNVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getScalarType(),
1761 EC: NumElts.divideCoefficientBy(RHS: 2));
1762 if (N->getOpcode() == ISD::TRUNCATE) {
1763 EOp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp1);
1764 EOp2 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp2);
1765 } else {
1766 assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1767 "Expected VP_TRUNCATE opcode");
1768 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1769 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1));
1770 std::tie(args&: EVLLo, args&: EVLHi) =
1771 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl);
1772 EOp1 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp1, N2: MaskLo, N3: EVLLo);
1773 EOp2 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp2, N2: MaskHi, N3: EVLHi);
1774 }
1775 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, N1: EOp1, N2: EOp2);
1776 }
1777 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1778 // targets.
1779 case TargetLowering::TypeWidenVector: {
1780 SDValue WideInOp = GetWidenedVector(Op: InOp);
1781
1782 // Truncate widened InOp.
1783 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1784 EVT TruncVT = EVT::getVectorVT(Context&: *DAG.getContext(),
1785 VT: N->getValueType(ResNo: 0).getScalarType(), NumElements: NumElem);
1786 SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncVT, Operand: WideInOp);
1787
1788 // Zero extend so that the elements are of same type as those of NVT
1789 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
1790 NumElements: NumElem);
1791 SDValue WideExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: ExtVT, Operand: WideTrunc);
1792
1793 // Extract the low NVT subvector.
1794 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: dl);
1795 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: WideExt, N2: ZeroIdx);
1796 }
1797 }
1798
1799 // Truncate to NVT instead of VT
1800 if (N->getOpcode() == ISD::VP_TRUNCATE)
1801 return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: NVT, N1: Res, N2: N->getOperand(Num: 1),
1802 N3: N->getOperand(Num: 2));
1803 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Res);
1804}
1805
1806SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1807 if (ResNo == 1)
1808 return PromoteIntRes_Overflow(N);
1809
1810 // The operation overflowed iff the result in the larger type is not the
1811 // zero extension of its truncation to the original type.
1812 SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
1813 SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
1814 EVT OVT = N->getOperand(Num: 0).getValueType();
1815 EVT NVT = LHS.getValueType();
1816 SDLoc dl(N);
1817
1818 // Do the arithmetic in the larger type.
1819 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1820 SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS);
1821
1822 // Calculate the overflow flag: zero extend the arithmetic result from
1823 // the original type.
1824 SDValue Ofl = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: OVT);
1825 // Overflowed if and only if this is not equal to Res.
1826 Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE);
1827
1828 // Use the calculated overflow everywhere.
1829 ReplaceValueWith(From: SDValue(N, 1), To: Ofl);
1830
1831 return Res;
1832}
1833
1834// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1835// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1836// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
1837SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1838 unsigned ResNo) {
1839 if (ResNo == 1)
1840 return PromoteIntRes_Overflow(N);
1841
1842 // We need to sign-extend the operands so the carry value computed by the
1843 // wide operation will be equivalent to the carry value computed by the
1844 // narrow operation.
1845 // An UADDO_CARRY can generate carry only if any of the operands has its
1846 // most significant bit set. Sign extension propagates the most significant
1847 // bit into the higher bits which means the extra bit that the narrow
1848 // addition would need (i.e. the carry) will be propagated through the higher
1849 // bits of the wide addition.
1850 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1851 // be preserved by sign extension.
1852 SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1853 SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1));
1854
1855 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(ResNo: 1)};
1856
1857 // Do the arithmetic in the wide type.
1858 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: DAG.getVTList(VTs: ValueVTs),
1859 N1: LHS, N2: RHS, N3: N->getOperand(Num: 2));
1860
1861 // Update the users of the original carry/borrow value.
1862 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1863
1864 return SDValue(Res.getNode(), 0);
1865}
1866
1867SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1868 unsigned ResNo) {
1869 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1870 return PromoteIntRes_Overflow(N);
1871}
1872
1873SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1874 EVT OVT = N->getValueType(ResNo: 0);
1875 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
1876
1877 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1878 // If we expand later we'll end up sign extending more than just the sra input
1879 // in sra+xor+sub expansion.
1880 if (!OVT.isVector() &&
1881 !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::ABS, VT: NVT) &&
1882 !TLI.isOperationLegal(Op: ISD::SMAX, VT: NVT)) {
1883 if (SDValue Res = TLI.expandABS(N, DAG))
1884 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Res);
1885 }
1886
1887 SDValue Op0 = SExtPromotedInteger(Op: N->getOperand(Num: 0));
1888 return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(N), VT: Op0.getValueType(), Operand: Op0);
1889}
1890
1891SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1892 // Promote the overflow bit trivially.
1893 if (ResNo == 1)
1894 return PromoteIntRes_Overflow(N);
1895
1896 SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1);
1897 SDLoc DL(N);
1898 EVT SmallVT = LHS.getValueType();
1899
1900 // To determine if the result overflowed in a larger type, we extend the
1901 // input to the larger type, do the multiply (checking if it overflows),
1902 // then also check the high bits of the result to see if overflow happened
1903 // there.
1904 if (N->getOpcode() == ISD::SMULO) {
1905 LHS = SExtPromotedInteger(Op: LHS);
1906 RHS = SExtPromotedInteger(Op: RHS);
1907 } else {
1908 LHS = ZExtPromotedInteger(Op: LHS);
1909 RHS = ZExtPromotedInteger(Op: RHS);
1910 }
1911 SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: N->getValueType(ResNo: 1));
1912 SDValue Mul = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N1: LHS, N2: RHS);
1913
1914 // Overflow occurred if it occurred in the larger type, or if the high part
1915 // of the result does not zero/sign-extend the low part. Check this second
1916 // possibility first.
1917 SDValue Overflow;
1918 if (N->getOpcode() == ISD::UMULO) {
1919 // Unsigned overflow occurred if the high part is non-zero.
1920 unsigned Shift = SmallVT.getScalarSizeInBits();
1921 SDValue Hi =
1922 DAG.getNode(Opcode: ISD::SRL, DL, VT: Mul.getValueType(), N1: Mul,
1923 N2: DAG.getShiftAmountConstant(Val: Shift, VT: Mul.getValueType(), DL));
1924 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Hi,
1925 RHS: DAG.getConstant(Val: 0, DL, VT: Hi.getValueType()),
1926 Cond: ISD::SETNE);
1927 } else {
1928 // Signed overflow occurred if the high part does not sign extend the low.
1929 SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: Mul.getValueType(),
1930 N1: Mul, N2: DAG.getValueType(SmallVT));
1931 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: SExt, RHS: Mul, Cond: ISD::SETNE);
1932 }
1933
1934 // The only other way for overflow to occur is if the multiplication in the
1935 // larger type itself overflowed.
1936 Overflow = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 1), N1: Overflow,
1937 N2: SDValue(Mul.getNode(), 1));
1938
1939 // Use the calculated overflow everywhere.
1940 ReplaceValueWith(From: SDValue(N, 1), To: Overflow);
1941 return Mul;
1942}
1943
1944SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1945 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
1946 VT: N->getValueType(ResNo: 0)));
1947}
1948
1949SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1950 EVT VT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1951
1952 const APInt &MulImm = N->getConstantOperandAPInt(Num: 0);
1953 return DAG.getVScale(DL: SDLoc(N), VT, MulImm: MulImm.sext(width: VT.getSizeInBits()));
1954}
1955
1956SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1957 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
1958 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
1959 EVT VT = N->getValueType(ResNo: 0);
1960 SDLoc dl(N);
1961
1962 MVT RegVT = TLI.getRegisterType(Context&: *DAG.getContext(), VT);
1963 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
1964 // The argument is passed as NumRegs registers of type RegVT.
1965
1966 SmallVector<SDValue, 8> Parts(NumRegs);
1967 for (unsigned i = 0; i < NumRegs; ++i) {
1968 Parts[i] = DAG.getVAArg(VT: RegVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
1969 Align: N->getConstantOperandVal(Num: 3));
1970 Chain = Parts[i].getValue(R: 1);
1971 }
1972
1973 // Handle endianness of the load.
1974 if (DAG.getDataLayout().isBigEndian())
1975 std::reverse(first: Parts.begin(), last: Parts.end());
1976
1977 // Assemble the parts in the promoted type.
1978 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1979 SDValue Res = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[0]);
1980 for (unsigned i = 1; i < NumRegs; ++i) {
1981 SDValue Part = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[i]);
1982 // Shift it to the right position and "or" it in.
1983 Part = DAG.getNode(
1984 Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Part,
1985 N2: DAG.getShiftAmountConstant(Val: i * RegVT.getSizeInBits(), VT: NVT, DL: dl));
1986 Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Res, N2: Part);
1987 }
1988
1989 // Modified the chain result - switch anything that used the old chain to
1990 // use the new one.
1991 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1992
1993 return Res;
1994}
1995
1996//===----------------------------------------------------------------------===//
1997// Integer Operand Promotion
1998//===----------------------------------------------------------------------===//
1999
2000/// PromoteIntegerOperand - This method is called when the specified operand of
2001/// the specified node is found to need promotion. At this point, all of the
2002/// result types of the node are known to be legal, but other operands of the
2003/// node may need promotion or expansion as well as the specified one.
2004bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
2005 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
2006 SDValue Res = SDValue();
2007 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2008 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2009 return false;
2010 }
2011
2012 switch (N->getOpcode()) {
2013 default:
2014 #ifndef NDEBUG
2015 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
2016 N->dump(&DAG); dbgs() << "\n";
2017 #endif
2018 report_fatal_error(reason: "Do not know how to promote this operator's operand!");
2019
2020 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
2021 case ISD::ANY_EXTEND_VECTOR_INREG:
2022 Res = PromoteIntOp_ANY_EXTEND_VECTOR_INREG(N);
2023 break;
2024 case ISD::ATOMIC_STORE:
2025 Res = PromoteIntOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
2026 break;
2027 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
2028 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
2029 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
2030 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
2031 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
2032 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
2033 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
2034 case ISD::FAKE_USE:
2035 Res = PromoteIntOp_FAKE_USE(N);
2036 break;
2037 case ISD::INSERT_VECTOR_ELT:
2038 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
2039 break;
2040 case ISD::SPLAT_VECTOR:
2041 case ISD::SCALAR_TO_VECTOR:
2042 Res = PromoteIntOp_ScalarOp(N);
2043 break;
2044 case ISD::VSELECT:
2045 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
2046 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
2047 case ISD::VP_SETCC:
2048 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
2049 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
2050 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
2051 case ISD::VP_SINT_TO_FP:
2052 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
2053 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
2054 case ISD::STORE: Res = PromoteIntOp_STORE(N: cast<StoreSDNode>(Val: N),
2055 OpNo); break;
2056 case ISD::VP_STORE:
2057 Res = PromoteIntOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
2058 break;
2059 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N),
2060 OpNo); break;
2061 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(N: cast<MaskedLoadSDNode>(Val: N),
2062 OpNo); break;
2063 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(N: cast<MaskedGatherSDNode>(Val: N),
2064 OpNo); break;
2065 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(N: cast<MaskedScatterSDNode>(Val: N),
2066 OpNo); break;
2067 case ISD::VECTOR_COMPRESS:
2068 Res = PromoteIntOp_VECTOR_COMPRESS(N, OpNo);
2069 break;
2070 case ISD::VP_TRUNCATE:
2071 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
2072 case ISD::BF16_TO_FP:
2073 case ISD::FP16_TO_FP:
2074 case ISD::VP_UINT_TO_FP:
2075 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
2076 case ISD::STRICT_FP16_TO_FP:
2077 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
2078 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
2079 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
2080 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
2081 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
2082
2083 case ISD::SHL:
2084 case ISD::SRA:
2085 case ISD::SRL:
2086 case ISD::ROTL:
2087 case ISD::ROTR:
2088 case ISD::SSHLSAT:
2089 case ISD::USHLSAT:
2090 Res = PromoteIntOp_Shift(N);
2091 break;
2092
2093 case ISD::SCMP:
2094 case ISD::UCMP: Res = PromoteIntOp_CMP(N); break;
2095
2096 case ISD::FSHL:
2097 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
2098
2099 case ISD::FRAMEADDR:
2100 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
2101
2102 case ISD::SMULFIX:
2103 case ISD::SMULFIXSAT:
2104 case ISD::UMULFIX:
2105 case ISD::UMULFIXSAT:
2106 case ISD::SDIVFIX:
2107 case ISD::SDIVFIXSAT:
2108 case ISD::UDIVFIX:
2109 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
2110 case ISD::FPOWI:
2111 case ISD::STRICT_FPOWI:
2112 case ISD::FLDEXP:
2113 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
2114 case ISD::VECREDUCE_ADD:
2115 case ISD::VECREDUCE_MUL:
2116 case ISD::VECREDUCE_AND:
2117 case ISD::VECREDUCE_OR:
2118 case ISD::VECREDUCE_XOR:
2119 case ISD::VECREDUCE_SMAX:
2120 case ISD::VECREDUCE_SMIN:
2121 case ISD::VECREDUCE_UMAX:
2122 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
2123 case ISD::VP_REDUCE_ADD:
2124 case ISD::VP_REDUCE_MUL:
2125 case ISD::VP_REDUCE_AND:
2126 case ISD::VP_REDUCE_OR:
2127 case ISD::VP_REDUCE_XOR:
2128 case ISD::VP_REDUCE_SMAX:
2129 case ISD::VP_REDUCE_SMIN:
2130 case ISD::VP_REDUCE_UMAX:
2131 case ISD::VP_REDUCE_UMIN:
2132 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
2133 break;
2134
2135 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
2136 case ISD::STACKMAP:
2137 Res = PromoteIntOp_STACKMAP(N, OpNo);
2138 break;
2139 case ISD::PATCHPOINT:
2140 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
2141 break;
2142 case ISD::WRITE_REGISTER:
2143 Res = PromoteIntOp_WRITE_REGISTER(N, OpNo);
2144 break;
2145 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2146 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2147 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
2148 break;
2149 case ISD::EXPERIMENTAL_VP_SPLICE:
2150 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
2151 break;
2152 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
2153 Res = PromoteIntOp_VECTOR_HISTOGRAM(N, OpNo);
2154 break;
2155 case ISD::VECTOR_FIND_LAST_ACTIVE:
2156 Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(N, OpNo);
2157 break;
2158 case ISD::GET_ACTIVE_LANE_MASK:
2159 Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
2160 break;
2161 case ISD::PARTIAL_REDUCE_UMLA:
2162 case ISD::PARTIAL_REDUCE_SMLA:
2163 case ISD::PARTIAL_REDUCE_SUMLA:
2164 Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
2165 break;
2166 }
2167
2168 // If the result is null, the sub-method took care of registering results etc.
2169 if (!Res.getNode()) return false;
2170
2171 // If the result is N, the sub-method updated N in place. Tell the legalizer
2172 // core about this.
2173 if (Res.getNode() == N)
2174 return true;
2175
2176 const bool IsStrictFp = N->isStrictFPOpcode();
2177 assert(Res.getValueType() == N->getValueType(0) &&
2178 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2179 "Invalid operand expansion");
2180 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
2181 Res.dump());
2182
2183 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2184 if (IsStrictFp)
2185 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
2186
2187 return false;
2188}
2189
2190// These operands can be either sign extended or zero extended as long as we
2191// treat them the same. If an extension is free, choose that. Otherwise, follow
2192// target preference.
2193void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
2194 SDValue OpL = GetPromotedInteger(Op: LHS);
2195 SDValue OpR = GetPromotedInteger(Op: RHS);
2196
2197 if (TLI.isSExtCheaperThanZExt(FromTy: LHS.getValueType(), ToTy: OpL.getValueType())) {
2198 // The target would prefer to promote the comparison operand with sign
2199 // extension. Honor that unless the promoted values are already zero
2200 // extended.
2201 unsigned OpLEffectiveBits =
2202 DAG.computeKnownBits(Op: OpL).countMaxActiveBits();
2203 unsigned OpREffectiveBits =
2204 DAG.computeKnownBits(Op: OpR).countMaxActiveBits();
2205 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2206 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2207 LHS = OpL;
2208 RHS = OpR;
2209 return;
2210 }
2211
2212 // The promoted values aren't zero extended, use a sext_inreg.
2213 LHS = SExtPromotedInteger(Op: LHS);
2214 RHS = SExtPromotedInteger(Op: RHS);
2215 return;
2216 }
2217
2218 // Prefer to promote the comparison operand with zero extension.
2219
2220 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
2221 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
2222 // that we might not be able to remove.
2223 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpL);
2224 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpR);
2225 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
2226 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
2227 LHS = OpL;
2228 RHS = OpR;
2229 return;
2230 }
2231
2232 // Otherwise, use zext_inreg.
2233 LHS = ZExtPromotedInteger(Op: LHS);
2234 RHS = ZExtPromotedInteger(Op: RHS);
2235}
2236
2237/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
2238/// shared among BR_CC, SELECT_CC, and SETCC handlers.
2239void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
2240 ISD::CondCode CCCode) {
2241 // We have to insert explicit sign or zero extends. Note that we could
2242 // insert sign extends for ALL conditions. For those operations where either
2243 // zero or sign extension would be valid, we ask the target which extension
2244 // it would prefer.
2245
2246 // Signed comparisons always require sign extension.
2247 if (ISD::isSignedIntSetCC(Code: CCCode)) {
2248 LHS = SExtPromotedInteger(Op: LHS);
2249 RHS = SExtPromotedInteger(Op: RHS);
2250 return;
2251 }
2252
2253 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) &&
2254 "Unknown integer comparison!");
2255
2256 SExtOrZExtPromotedOperands(LHS, RHS);
2257}
2258
2259SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
2260 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2261 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2262}
2263
2264SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND_VECTOR_INREG(SDNode *N) {
2265 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2266 EVT ResVT = N->getValueType(ResNo: 0);
2267 EVT OpVT = Op.getValueType();
2268 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getScalarType(),
2269 NumElements: ResVT.getVectorNumElements());
2270 Op = DAG.getExtractSubvector(DL: SDLoc(Op), VT: NewVT, Vec: Op, Idx: 0);
2271 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: ResVT, Operand: Op);
2272}
2273
2274SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
2275 SDValue Op1 = GetPromotedInteger(Op: N->getOperand(Num: 1));
2276 return DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(),
2277 Chain: N->getChain(), Ptr: Op1, Val: N->getBasePtr(), MMO: N->getMemOperand());
2278}
2279
2280SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
2281 EVT OutVT = N->getValueType(ResNo: 0);
2282 SDValue InOp = N->getOperand(Num: 0);
2283 EVT InVT = InOp.getValueType();
2284 EVT NInVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT);
2285 SDLoc dl(N);
2286
2287 switch (getTypeAction(VT: InVT)) {
2288 case TargetLowering::TypePromoteInteger: {
2289 // TODO: Handle big endian & vector input type.
2290 if (OutVT.isVector() && !InVT.isVector() &&
2291 DAG.getDataLayout().isLittleEndian()) {
2292 EVT EltVT = OutVT.getVectorElementType();
2293 TypeSize EltSize = EltVT.getSizeInBits();
2294 TypeSize NInSize = NInVT.getSizeInBits();
2295
2296 if (NInSize.hasKnownScalarFactor(RHS: EltSize)) {
2297 unsigned NumEltsWithPadding = NInSize.getKnownScalarFactor(RHS: EltSize);
2298 EVT WideVecVT =
2299 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumEltsWithPadding);
2300
2301 if (isTypeLegal(VT: WideVecVT)) {
2302 SDValue Promoted = GetPromotedInteger(Op: InOp);
2303 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WideVecVT, Operand: Promoted);
2304 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: Cast,
2305 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2306 }
2307 }
2308 }
2309
2310 break;
2311 }
2312 default:
2313 break;
2314 }
2315
2316 // This should only occur in unusual situations like bitcasting to an
2317 // x86_fp80, so just turn it into a store+load
2318 return CreateStackStoreLoad(Op: InOp, DestVT: OutVT);
2319}
2320
2321SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
2322 assert(OpNo == 2 && "Don't know how to promote this operand!");
2323
2324 SDValue LHS = N->getOperand(Num: 2);
2325 SDValue RHS = N->getOperand(Num: 3);
2326 PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get());
2327
2328 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2329 // legal types.
2330 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2331 Op2: N->getOperand(Num: 1), Op3: LHS, Op4: RHS, Op5: N->getOperand(Num: 4)),
2332 0);
2333}
2334
2335SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2336 assert(OpNo == 1 && "only know how to promote condition");
2337
2338 // Promote all the way up to the canonical SetCC type.
2339 SDValue Cond = PromoteTargetBoolean(Bool: N->getOperand(Num: 1), ValVT: MVT::Other);
2340
2341 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2342 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Cond,
2343 Op3: N->getOperand(Num: 2)), 0);
2344}
2345
2346SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2347 // Since the result type is legal, the operands must promote to it.
2348 EVT OVT = N->getOperand(Num: 0).getValueType();
2349 SDValue Lo = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
2350 SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 1));
2351 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2352 SDLoc dl(N);
2353
2354 Hi = DAG.getNode(
2355 Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: Hi,
2356 N2: DAG.getShiftAmountConstant(Val: OVT.getSizeInBits(), VT: N->getValueType(ResNo: 0), DL: dl));
2357 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
2358}
2359
2360SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2361 // The vector type is legal but the element type is not. This implies
2362 // that the vector is a power-of-two in length and that the element
2363 // type does not have a strange size (eg: it is not i1).
2364 EVT VecVT = N->getValueType(ResNo: 0);
2365 unsigned NumElts = VecVT.getVectorNumElements();
2366 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2367 "Legal vector of one illegal element?");
2368
2369 // Promote the inserted value. The type does not need to match the
2370 // vector element type. Check that any extra bits introduced will be
2371 // truncated away.
2372 assert(N->getOperand(0).getValueSizeInBits() >=
2373 N->getValueType(0).getScalarSizeInBits() &&
2374 "Type of inserted value narrower than vector element type!");
2375
2376 SmallVector<SDValue, 16> NewOps;
2377 for (unsigned i = 0; i < NumElts; ++i)
2378 NewOps.push_back(Elt: GetPromotedInteger(Op: N->getOperand(Num: i)));
2379
2380 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2381}
2382
2383SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2384 unsigned OpNo) {
2385 if (OpNo == 1) {
2386 // Promote the inserted value. This is valid because the type does not
2387 // have to match the vector element type.
2388
2389 // Check that any extra bits introduced will be truncated away.
2390 assert(N->getOperand(1).getValueSizeInBits() >=
2391 N->getValueType(0).getScalarSizeInBits() &&
2392 "Type of inserted value narrower than vector element type!");
2393 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2394 Op2: GetPromotedInteger(Op: N->getOperand(Num: 1)),
2395 Op3: N->getOperand(Num: 2)),
2396 0);
2397 }
2398
2399 assert(OpNo == 2 && "Different operand and result vector types?");
2400
2401 // Promote the index.
2402 SDValue Idx = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 2), DL: SDLoc(N),
2403 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
2404 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2405 Op2: N->getOperand(Num: 1), Op3: Idx), 0);
2406}
2407
2408SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2409 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2410
2411 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2412 // so just promote the operand in place.
2413 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2414}
2415
2416SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2417 assert(OpNo == 0 && "Only know how to promote the condition!");
2418 SDValue Cond = N->getOperand(Num: 0);
2419 EVT OpTy = N->getOperand(Num: 1).getValueType();
2420
2421 if (N->getOpcode() == ISD::VSELECT)
2422 if (SDValue Res = WidenVSELECTMask(N))
2423 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2424 N1: Res, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2425
2426 // Promote all the way up to the canonical SetCC type.
2427 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2428 Cond = PromoteTargetBoolean(Bool: Cond, ValVT: OpVT);
2429
2430 return SDValue(DAG.UpdateNodeOperands(N, Op1: Cond, Op2: N->getOperand(Num: 1),
2431 Op3: N->getOperand(Num: 2)), 0);
2432}
2433
2434SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2435 assert(OpNo == 0 && "Don't know how to promote this operand!");
2436
2437 SDValue LHS = N->getOperand(Num: 0);
2438 SDValue RHS = N->getOperand(Num: 1);
2439 PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get());
2440
2441 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2442 return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2),
2443 Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)), 0);
2444}
2445
2446SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2447 assert(OpNo == 0 && "Don't know how to promote this operand!");
2448
2449 SDValue LHS = N->getOperand(Num: 0);
2450 SDValue RHS = N->getOperand(Num: 1);
2451 PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get());
2452
2453 // The CC (#2) is always legal.
2454 if (N->getOpcode() == ISD::SETCC)
2455 return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2)), 0);
2456
2457 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2458
2459 return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2),
2460 Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)),
2461 0);
2462}
2463
2464SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2465 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2466 Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0);
2467}
2468
2469SDValue DAGTypeLegalizer::PromoteIntOp_CMP(SDNode *N) {
2470 SDValue LHS = N->getOperand(Num: 0);
2471 SDValue RHS = N->getOperand(Num: 1);
2472
2473 if (N->getOpcode() == ISD::SCMP) {
2474 LHS = SExtPromotedInteger(Op: LHS);
2475 RHS = SExtPromotedInteger(Op: RHS);
2476 } else {
2477 SExtOrZExtPromotedOperands(LHS, RHS);
2478 }
2479
2480 return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS), 0);
2481}
2482
2483SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2484 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1),
2485 Op3: ZExtPromotedInteger(Op: N->getOperand(Num: 2))), 0);
2486}
2487
2488SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2489 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2490 SDLoc dl(N);
2491 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op);
2492 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Op.getValueType(),
2493 N1: Op, N2: DAG.getValueType(N->getOperand(Num: 0).getValueType()));
2494}
2495
2496SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
2497 SDLoc dl(N);
2498 EVT VT = N->getValueType(ResNo: 0);
2499 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2500 // FIXME: There is no VP_ANY_EXTEND yet.
2501 Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1),
2502 N3: N->getOperand(Num: 2));
2503 unsigned Diff =
2504 VT.getScalarSizeInBits() - N->getOperand(Num: 0).getScalarValueSizeInBits();
2505 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Diff, VT, DL: dl);
2506 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
2507 SDValue Shl = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: ShAmt, N3: N->getOperand(Num: 1),
2508 N4: N->getOperand(Num: 2));
2509 return DAG.getNode(Opcode: ISD::VP_SRA, DL: dl, VT, N1: Shl, N2: ShAmt, N3: N->getOperand(Num: 1),
2510 N4: N->getOperand(Num: 2));
2511}
2512
2513SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2514 if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2515 return SDValue(DAG.UpdateNodeOperands(N,
2516 Op1: SExtPromotedInteger(Op: N->getOperand(Num: 0)),
2517 Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)),
2518 0);
2519 return SDValue(DAG.UpdateNodeOperands(N,
2520 Op: SExtPromotedInteger(Op: N->getOperand(Num: 0))), 0);
2521}
2522
2523SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2524 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2525 Op2: SExtPromotedInteger(Op: N->getOperand(Num: 1))), 0);
2526}
2527
2528SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2529 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2530 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2531 SDLoc dl(N);
2532
2533 SDValue Val = GetPromotedInteger(Op: N->getValue()); // Get promoted value.
2534
2535 // Truncate the value and store the result.
2536 return DAG.getTruncStore(Chain: Ch, dl, Val, Ptr,
2537 SVT: N->getMemoryVT(), MMO: N->getMemOperand());
2538}
2539
2540SDValue DAGTypeLegalizer::PromoteIntOp_VP_STORE(VPStoreSDNode *N,
2541 unsigned OpNo) {
2542
2543 assert(OpNo == 1 && "Unexpected operand for promotion");
2544 assert(!N->isIndexed() && "expecting unindexed vp_store!");
2545
2546 SDValue DataOp = GetPromotedInteger(Op: N->getValue());
2547 return DAG.getTruncStoreVP(Chain: N->getChain(), dl: SDLoc(N), Val: DataOp, Ptr: N->getBasePtr(),
2548 Mask: N->getMask(), EVL: N->getVectorLength(),
2549 SVT: N->getMemoryVT(), MMO: N->getMemOperand(),
2550 IsCompressing: N->isCompressingStore());
2551}
2552
2553SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2554 unsigned OpNo) {
2555 SDValue DataOp = N->getValue();
2556 SDValue Mask = N->getMask();
2557
2558 if (OpNo == 4) {
2559 // The Mask. Update in place.
2560 EVT DataVT = DataOp.getValueType();
2561 Mask = PromoteTargetBoolean(Bool: Mask, ValVT: DataVT);
2562 SmallVector<SDValue, 4> NewOps(N->ops());
2563 NewOps[4] = Mask;
2564 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2565 }
2566
2567 assert(OpNo == 1 && "Unexpected operand for promotion");
2568 DataOp = GetPromotedInteger(Op: DataOp);
2569
2570 return DAG.getMaskedStore(Chain: N->getChain(), dl: SDLoc(N), Val: DataOp, Base: N->getBasePtr(),
2571 Offset: N->getOffset(), Mask, MemVT: N->getMemoryVT(),
2572 MMO: N->getMemOperand(), AM: N->getAddressingMode(),
2573 /*IsTruncating*/ true, IsCompressing: N->isCompressingStore());
2574}
2575
2576SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2577 unsigned OpNo) {
2578 assert(OpNo == 3 && "Only know how to promote the mask!");
2579 EVT DataVT = N->getValueType(ResNo: 0);
2580 SDValue Mask = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT);
2581 SmallVector<SDValue, 4> NewOps(N->ops());
2582 NewOps[OpNo] = Mask;
2583 SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps);
2584 if (Res == N)
2585 return SDValue(Res, 0);
2586
2587 // Update triggered CSE, do our own replacement since caller can't.
2588 ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0));
2589 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1));
2590 return SDValue();
2591}
2592
2593SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2594 unsigned OpNo) {
2595 SmallVector<SDValue, 5> NewOps(N->ops());
2596
2597 if (OpNo == 2) {
2598 // The Mask
2599 EVT DataVT = N->getValueType(ResNo: 0);
2600 NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT);
2601 } else if (OpNo == 4) {
2602 // The Index
2603 if (N->isIndexSigned())
2604 // Need to sign extend the index since the bits will likely be used.
2605 NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2606 else
2607 NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2608 } else
2609 NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo));
2610
2611 SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps);
2612 if (Res == N)
2613 return SDValue(Res, 0);
2614
2615 // Update triggered CSE, do our own replacement since caller can't.
2616 ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0));
2617 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1));
2618 return SDValue();
2619}
2620
2621SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2622 unsigned OpNo) {
2623 bool TruncateStore = N->isTruncatingStore();
2624 SmallVector<SDValue, 5> NewOps(N->ops());
2625
2626 if (OpNo == 2) {
2627 // The Mask
2628 EVT DataVT = N->getValue().getValueType();
2629 NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT);
2630 } else if (OpNo == 4) {
2631 // The Index
2632 if (N->isIndexSigned())
2633 // Need to sign extend the index since the bits will likely be used.
2634 NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2635 else
2636 NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2637 } else {
2638 NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo));
2639 TruncateStore = true;
2640 }
2641
2642 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: N->getMemoryVT(),
2643 dl: SDLoc(N), Ops: NewOps, MMO: N->getMemOperand(),
2644 IndexType: N->getIndexType(), IsTruncating: TruncateStore);
2645}
2646
2647SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_COMPRESS(SDNode *N,
2648 unsigned OpNo) {
2649 assert(OpNo == 1 && "Can only promote VECTOR_COMPRESS mask.");
2650 SDValue Vec = N->getOperand(Num: 0);
2651 EVT VT = Vec.getValueType();
2652 SDValue Passthru = N->getOperand(Num: 2);
2653 SDValue Mask = PromoteTargetBoolean(Bool: N->getOperand(Num: 1), ValVT: VT);
2654 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT, N1: Vec, N2: Mask, N3: Passthru);
2655}
2656
2657SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2658 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2659 if (N->getOpcode() == ISD::VP_TRUNCATE)
2660 return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op,
2661 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
2662 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2663}
2664
2665SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2666 if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2667 return SDValue(DAG.UpdateNodeOperands(N,
2668 Op1: ZExtPromotedInteger(Op: N->getOperand(Num: 0)),
2669 Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)),
2670 0);
2671 return SDValue(DAG.UpdateNodeOperands(N,
2672 Op: ZExtPromotedInteger(Op: N->getOperand(Num: 0))), 0);
2673}
2674
2675SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2676 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2677 Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0);
2678}
2679
2680SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2681 SDLoc dl(N);
2682 SDValue Src = N->getOperand(Num: 0);
2683 SDValue Op = GetPromotedInteger(Op: Src);
2684 EVT VT = N->getValueType(ResNo: 0);
2685
2686 // If this zext has the nneg flag and the target prefers sext, see if the
2687 // promoted input is already sign extended.
2688 // TODO: Should we have some way to set nneg on ISD::AND instead?
2689 if (N->getFlags().hasNonNeg() && Op.getValueType() == VT &&
2690 TLI.isSExtCheaperThanZExt(FromTy: Src.getValueType(), ToTy: VT)) {
2691 unsigned OpEffectiveBits = DAG.ComputeMaxSignificantBits(Op);
2692 if (OpEffectiveBits <= Src.getScalarValueSizeInBits())
2693 return Op;
2694 }
2695
2696 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op);
2697 return DAG.getZeroExtendInReg(Op, DL: dl, VT: Src.getValueType());
2698}
2699
2700SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) {
2701 SDLoc dl(N);
2702 EVT VT = N->getValueType(ResNo: 0);
2703 SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0));
2704 // FIXME: There is no VP_ANY_EXTEND yet.
2705 Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1),
2706 N3: N->getOperand(Num: 2));
2707 return DAG.getVPZeroExtendInReg(Op, Mask: N->getOperand(Num: 1), EVL: N->getOperand(Num: 2), DL: dl,
2708 VT: N->getOperand(Num: 0).getValueType());
2709}
2710
2711SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2712 SDValue Op2 = ZExtPromotedInteger(Op: N->getOperand(Num: 2));
2713 return SDValue(
2714 DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1), Op3: Op2), 0);
2715}
2716
2717SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2718 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2719 SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
2720 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2721}
2722
2723SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2724 bool IsStrict = N->isStrictFPOpcode();
2725 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2726
2727 bool IsPowI =
2728 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2729 unsigned OpOffset = IsStrict ? 1 : 0;
2730
2731 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2732 // and floating point operand is already type legalized).
2733 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
2734 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
2735
2736 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2737 if (LCImpl == RTLIB::Unsupported) {
2738 // Scalarize vector FPOWI instead of promoting the type. This allows the
2739 // scalar FPOWIs to be visited and converted to libcalls before promoting
2740 // the type.
2741 // FIXME: This should be done in LegalizeVectorOps/LegalizeDAG, but call
2742 // lowering needs the unpromoted EVT.
2743 if (IsPowI && N->getValueType(ResNo: 0).isVector())
2744 return DAG.UnrollVectorOp(N);
2745 SmallVector<SDValue, 3> NewOps(N->ops());
2746 NewOps[1 + OpOffset] = SExtPromotedInteger(Op: N->getOperand(Num: 1 + OpOffset));
2747 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2748 }
2749
2750 // We can't just promote the exponent type in FPOWI, since we want to lower
2751 // the node to a libcall and we if we promote to a type larger than
2752 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2753 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2754 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2755
2756 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2757 assert(DAG.getLibInfo().getIntSize() ==
2758 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2759 "POWI exponent should match with sizeof(int) when doing the libcall.");
2760 TargetLowering::MakeLibCallOptions CallOptions;
2761 CallOptions.setIsSigned(true);
2762 SDValue Ops[2] = {N->getOperand(Num: 0 + OpOffset), N->getOperand(Num: 1 + OpOffset)};
2763 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2764 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
2765 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2766 if (IsStrict)
2767 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2768 return SDValue();
2769}
2770
2771static unsigned getExtendForIntVecReduction(SDNode *N) {
2772 switch (N->getOpcode()) {
2773 default:
2774 llvm_unreachable("Expected integer vector reduction");
2775 case ISD::VECREDUCE_ADD:
2776 case ISD::VECREDUCE_MUL:
2777 case ISD::VECREDUCE_AND:
2778 case ISD::VECREDUCE_OR:
2779 case ISD::VECREDUCE_XOR:
2780 case ISD::VP_REDUCE_ADD:
2781 case ISD::VP_REDUCE_MUL:
2782 case ISD::VP_REDUCE_AND:
2783 case ISD::VP_REDUCE_OR:
2784 case ISD::VP_REDUCE_XOR:
2785 return ISD::ANY_EXTEND;
2786 case ISD::VECREDUCE_SMAX:
2787 case ISD::VECREDUCE_SMIN:
2788 case ISD::VP_REDUCE_SMAX:
2789 case ISD::VP_REDUCE_SMIN:
2790 return ISD::SIGN_EXTEND;
2791 case ISD::VECREDUCE_UMAX:
2792 case ISD::VECREDUCE_UMIN:
2793 case ISD::VP_REDUCE_UMAX:
2794 case ISD::VP_REDUCE_UMIN:
2795 return ISD::ZERO_EXTEND;
2796 }
2797}
2798
2799SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2800 switch (getExtendForIntVecReduction(N)) {
2801 default:
2802 llvm_unreachable("Impossible extension kind for integer reduction");
2803 case ISD::ANY_EXTEND:
2804 return GetPromotedInteger(Op: V);
2805 case ISD::SIGN_EXTEND:
2806 return SExtPromotedInteger(Op: V);
2807 case ISD::ZERO_EXTEND:
2808 return ZExtPromotedInteger(Op: V);
2809 }
2810}
2811
2812SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2813 SDLoc dl(N);
2814 SDValue Op = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0));
2815
2816 EVT OrigEltVT = N->getOperand(Num: 0).getValueType().getVectorElementType();
2817 EVT InVT = Op.getValueType();
2818 EVT EltVT = InVT.getVectorElementType();
2819 EVT ResVT = N->getValueType(ResNo: 0);
2820 unsigned Opcode = N->getOpcode();
2821
2822 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2823 // vecreduce_xor is not legal
2824 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2825 !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_XOR, VT: InVT) &&
2826 TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_ADD, VT: InVT))
2827 Opcode = ISD::VECREDUCE_ADD;
2828
2829 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2830 // vecreduce_or is not legal
2831 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2832 !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_OR, VT: InVT) &&
2833 TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMAX, VT: InVT)) {
2834 Opcode = ISD::VECREDUCE_UMAX;
2835 // Can't use promoteTargetBoolean here because we still need
2836 // to either sign_ext or zero_ext in the undefined case.
2837 switch (TLI.getBooleanContents(Type: InVT)) {
2838 case TargetLoweringBase::UndefinedBooleanContent:
2839 case TargetLoweringBase::ZeroOrOneBooleanContent:
2840 Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
2841 break;
2842 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2843 Op = SExtPromotedInteger(Op: N->getOperand(Num: 0));
2844 break;
2845 }
2846 }
2847
2848 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2849 // vecreduce_and is not legal
2850 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2851 !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_AND, VT: InVT) &&
2852 TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMIN, VT: InVT)) {
2853 Opcode = ISD::VECREDUCE_UMIN;
2854 // Can't use promoteTargetBoolean here because we still need
2855 // to either sign_ext or zero_ext in the undefined case.
2856 switch (TLI.getBooleanContents(Type: InVT)) {
2857 case TargetLoweringBase::UndefinedBooleanContent:
2858 case TargetLoweringBase::ZeroOrOneBooleanContent:
2859 Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
2860 break;
2861 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2862 Op = SExtPromotedInteger(Op: N->getOperand(Num: 0));
2863 break;
2864 }
2865 }
2866
2867 if (ResVT.bitsGE(VT: EltVT))
2868 return DAG.getNode(Opcode, DL: SDLoc(N), VT: ResVT, Operand: Op);
2869
2870 // Result size must be >= element size. If this is not the case after
2871 // promotion, also promote the result type and then truncate.
2872 SDValue Reduce = DAG.getNode(Opcode, DL: dl, VT: EltVT, Operand: Op);
2873 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Reduce);
2874}
2875
2876SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2877 SDLoc DL(N);
2878 SDValue Op = N->getOperand(Num: OpNo);
2879 SmallVector<SDValue, 4> NewOps(N->ops());
2880
2881 if (OpNo == 2) { // Mask
2882 // Update in place.
2883 NewOps[2] = PromoteTargetBoolean(Bool: Op, ValVT: N->getOperand(Num: 1).getValueType());
2884 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2885 }
2886
2887 assert(OpNo == 1 && "Unexpected operand for promotion");
2888
2889 Op = PromoteIntOpVectorReduction(N, V: Op);
2890
2891 NewOps[OpNo] = Op;
2892
2893 EVT VT = N->getValueType(ResNo: 0);
2894 EVT EltVT = Op.getValueType().getScalarType();
2895
2896 if (VT.bitsGE(VT: EltVT))
2897 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, Ops: NewOps);
2898
2899 // Result size must be >= element/start-value size. If this is not the case
2900 // after promotion, also promote both the start value and result type and
2901 // then truncate.
2902 NewOps[0] =
2903 DAG.getNode(Opcode: getExtendForIntVecReduction(N), DL, VT: EltVT, Operand: N->getOperand(Num: 0));
2904 SDValue Reduce = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, Ops: NewOps);
2905 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Reduce);
2906}
2907
2908SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2909 SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
2910 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Op), 0);
2911}
2912
2913SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2914 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2915 SmallVector<SDValue> NewOps(N->ops());
2916 NewOps[OpNo] = GetPromotedInteger(Op: NewOps[OpNo]);
2917 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2918}
2919
2920SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2921 assert(OpNo >= 7);
2922 SmallVector<SDValue> NewOps(N->ops());
2923 NewOps[OpNo] = GetPromotedInteger(Op: NewOps[OpNo]);
2924 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2925}
2926
2927SDValue DAGTypeLegalizer::PromoteIntOp_WRITE_REGISTER(SDNode *N,
2928 unsigned OpNo) {
2929 const Function &Fn = DAG.getMachineFunction().getFunction();
2930 Fn.getContext().diagnose(DI: DiagnosticInfoLegalizationFailure(
2931 "cannot use llvm.write_register with illegal type", Fn,
2932 N->getDebugLoc()));
2933 return N->getOperand(Num: 0);
2934}
2935
2936SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2937 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2938 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2939
2940 SmallVector<SDValue, 8> NewOps(N->ops());
2941 NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2942
2943 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2944}
2945
2946SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
2947 SmallVector<SDValue, 6> NewOps(N->ops());
2948
2949 if (OpNo == 2) { // Offset operand
2950 NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2951 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2952 }
2953
2954 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
2955
2956 NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo));
2957 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2958}
2959
2960SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N,
2961 unsigned OpNo) {
2962 assert(OpNo == 1 && "Unexpected operand for promotion");
2963 SmallVector<SDValue, 7> NewOps(N->ops());
2964 NewOps[1] = GetPromotedInteger(Op: N->getOperand(Num: 1));
2965 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2966}
2967
2968SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N,
2969 unsigned OpNo) {
2970 SmallVector<SDValue, 1> NewOps(N->ops());
2971 NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo));
2972 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2973}
2974
2975SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
2976 SmallVector<SDValue, 1> NewOps(N->ops());
2977 NewOps[0] = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
2978 NewOps[1] = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
2979 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
2980}
2981
2982SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N) {
2983 SmallVector<SDValue, 1> NewOps(N->ops());
2984 switch (N->getOpcode()) {
2985 case ISD::PARTIAL_REDUCE_SMLA:
2986 NewOps[1] = SExtPromotedInteger(Op: N->getOperand(Num: 1));
2987 NewOps[2] = SExtPromotedInteger(Op: N->getOperand(Num: 2));
2988 break;
2989 case ISD::PARTIAL_REDUCE_UMLA:
2990 NewOps[1] = ZExtPromotedInteger(Op: N->getOperand(Num: 1));
2991 NewOps[2] = ZExtPromotedInteger(Op: N->getOperand(Num: 2));
2992 break;
2993 case ISD::PARTIAL_REDUCE_SUMLA:
2994 NewOps[1] = SExtPromotedInteger(Op: N->getOperand(Num: 1));
2995 NewOps[2] = ZExtPromotedInteger(Op: N->getOperand(Num: 2));
2996 break;
2997 default:
2998 llvm_unreachable("unexpected opcode");
2999 }
3000 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
3001}
3002
3003//===----------------------------------------------------------------------===//
3004// Integer Result Expansion
3005//===----------------------------------------------------------------------===//
3006
3007/// ExpandIntegerResult - This method is called when the specified result of the
3008/// specified node is found to need expansion. At this point, the node may also
3009/// have invalid operands or may have other results that need promotion, we just
3010/// know that (at least) one result needs expansion.
3011void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
3012 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
3013 SDValue Lo, Hi;
3014 Lo = Hi = SDValue();
3015
3016 // See if the target wants to custom expand this node.
3017 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
3018 return;
3019
3020 switch (N->getOpcode()) {
3021 default:
3022#ifndef NDEBUG
3023 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
3024 N->dump(&DAG); dbgs() << "\n";
3025#endif
3026 report_fatal_error(reason: "Do not know how to expand the result of this "
3027 "operator!");
3028
3029 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
3030 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
3031 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
3032 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
3033 case ISD::POISON:
3034 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
3035 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
3036 case ISD::SETCC: ExpandIntRes_SETCC(N, Lo, Hi); break;
3037
3038 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
3039 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
3040 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
3041 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
3042 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
3043
3044 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
3045 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
3046 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
3047 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
3048 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
3049 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
3050 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
3051 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
3052 case ISD::ABDS:
3053 case ISD::ABDU: ExpandIntRes_ABD(N, Lo, Hi); break;
3054 case ISD::CTLZ_ZERO_UNDEF:
3055 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
3056 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
3057 case ISD::CTTZ_ZERO_UNDEF:
3058 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
3059 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
3060 case ISD::STRICT_FP_TO_SINT:
3061 case ISD::FP_TO_SINT:
3062 case ISD::STRICT_FP_TO_UINT:
3063 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
3064 case ISD::FP_TO_SINT_SAT:
3065 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
3066 case ISD::STRICT_LROUND:
3067 case ISD::STRICT_LRINT:
3068 case ISD::LROUND:
3069 case ISD::LRINT:
3070 case ISD::STRICT_LLROUND:
3071 case ISD::STRICT_LLRINT:
3072 case ISD::LLROUND:
3073 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
3074 case ISD::LOAD: ExpandIntRes_LOAD(N: cast<LoadSDNode>(Val: N), Lo, Hi); break;
3075 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
3076 case ISD::READCYCLECOUNTER:
3077 case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break;
3078 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
3079 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
3080 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
3081 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
3082 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
3083 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
3084 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
3085 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
3086 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
3087
3088 case ISD::ATOMIC_LOAD_ADD:
3089 case ISD::ATOMIC_LOAD_SUB:
3090 case ISD::ATOMIC_LOAD_AND:
3091 case ISD::ATOMIC_LOAD_CLR:
3092 case ISD::ATOMIC_LOAD_OR:
3093 case ISD::ATOMIC_LOAD_XOR:
3094 case ISD::ATOMIC_LOAD_NAND:
3095 case ISD::ATOMIC_LOAD_MIN:
3096 case ISD::ATOMIC_LOAD_MAX:
3097 case ISD::ATOMIC_LOAD_UMIN:
3098 case ISD::ATOMIC_LOAD_UMAX:
3099 case ISD::ATOMIC_SWAP:
3100 case ISD::ATOMIC_CMP_SWAP: {
3101 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node: N);
3102 SplitInteger(Op: Tmp.first, Lo, Hi);
3103 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
3104 break;
3105 }
3106 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3107 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
3108 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other);
3109 SDValue Tmp = DAG.getAtomicCmpSwap(
3110 Opcode: ISD::ATOMIC_CMP_SWAP, dl: SDLoc(N), MemVT: AN->getMemoryVT(), VTs,
3111 Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 1), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3),
3112 MMO: AN->getMemOperand());
3113
3114 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
3115 // success simply by comparing the loaded value against the ingoing
3116 // comparison.
3117 SDValue Success = DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 1), LHS: Tmp,
3118 RHS: N->getOperand(Num: 2), Cond: ISD::SETEQ);
3119
3120 SplitInteger(Op: Tmp, Lo, Hi);
3121 ReplaceValueWith(From: SDValue(N, 1), To: Success);
3122 ReplaceValueWith(From: SDValue(N, 2), To: Tmp.getValue(R: 1));
3123 break;
3124 }
3125
3126 case ISD::AND:
3127 case ISD::OR:
3128 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
3129
3130 case ISD::UMAX:
3131 case ISD::SMAX:
3132 case ISD::UMIN:
3133 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
3134
3135 case ISD::SCMP:
3136 case ISD::UCMP: ExpandIntRes_CMP(N, Lo, Hi); break;
3137
3138 case ISD::ADD:
3139 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
3140
3141 case ISD::ADDC:
3142 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
3143
3144 case ISD::ADDE:
3145 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
3146
3147 case ISD::UADDO_CARRY:
3148 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
3149
3150 case ISD::SADDO_CARRY:
3151 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
3152
3153 case ISD::SHL:
3154 case ISD::SRA:
3155 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
3156
3157 case ISD::SADDO:
3158 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
3159 case ISD::UADDO:
3160 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
3161 case ISD::UMULO:
3162 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
3163
3164 case ISD::SADDSAT:
3165 case ISD::UADDSAT:
3166 case ISD::SSUBSAT:
3167 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
3168
3169 case ISD::SSHLSAT:
3170 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
3171
3172 case ISD::AVGCEILS:
3173 case ISD::AVGCEILU:
3174 case ISD::AVGFLOORS:
3175 case ISD::AVGFLOORU: ExpandIntRes_AVG(N, Lo, Hi); break;
3176
3177 case ISD::SMULFIX:
3178 case ISD::SMULFIXSAT:
3179 case ISD::UMULFIX:
3180 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
3181
3182 case ISD::SDIVFIX:
3183 case ISD::SDIVFIXSAT:
3184 case ISD::UDIVFIX:
3185 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
3186
3187 case ISD::VECREDUCE_ADD:
3188 case ISD::VECREDUCE_MUL:
3189 case ISD::VECREDUCE_AND:
3190 case ISD::VECREDUCE_OR:
3191 case ISD::VECREDUCE_XOR:
3192 case ISD::VECREDUCE_SMAX:
3193 case ISD::VECREDUCE_SMIN:
3194 case ISD::VECREDUCE_UMAX:
3195 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
3196
3197 case ISD::ROTL:
3198 case ISD::ROTR:
3199 ExpandIntRes_Rotate(N, Lo, Hi);
3200 break;
3201
3202 case ISD::FSHL:
3203 case ISD::FSHR:
3204 ExpandIntRes_FunnelShift(N, Lo, Hi);
3205 break;
3206
3207 case ISD::CLMUL:
3208 case ISD::CLMULR:
3209 case ISD::CLMULH:
3210 ExpandIntRes_CLMUL(N, Lo, Hi);
3211 break;
3212
3213 case ISD::VSCALE:
3214 ExpandIntRes_VSCALE(N, Lo, Hi);
3215 break;
3216
3217 case ISD::READ_REGISTER:
3218 ExpandIntRes_READ_REGISTER(N, Lo, Hi);
3219 break;
3220 }
3221
3222 // If Lo/Hi is null, the sub-method took care of registering results etc.
3223 if (Lo.getNode())
3224 SetExpandedInteger(Op: SDValue(N, ResNo), Lo, Hi);
3225}
3226
3227/// Lower an atomic node to the appropriate builtin call.
3228std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
3229 unsigned Opc = Node->getOpcode();
3230 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
3231 AtomicOrdering order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
3232 // Lower to outline atomic libcall if outline atomics enabled,
3233 // or to sync libcall otherwise
3234 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order: order, VT);
3235 EVT RetVT = Node->getValueType(ResNo: 0);
3236 TargetLowering::MakeLibCallOptions CallOptions;
3237 SmallVector<SDValue, 4> Ops;
3238
3239 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
3240 if (LCImpl != RTLIB::Unsupported) {
3241 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
3242 Ops.push_back(Elt: Node->getOperand(Num: 1));
3243 } else {
3244 LC = RTLIB::getSYNC(Opc, VT);
3245 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3246 "Unexpected atomic op or value type!");
3247 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
3248 LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
3249 }
3250 return TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT, Ops, CallOptions, dl: SDLoc(Node),
3251 Chain: Node->getOperand(Num: 0));
3252}
3253
3254/// N is a shift by a value that needs to be expanded,
3255/// and the shift amount is a constant 'Amt'. Expand the operation.
3256void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
3257 SDValue &Lo, SDValue &Hi) {
3258 SDLoc DL(N);
3259 // Expand the incoming operand to be shifted, so that we have its parts
3260 SDValue InL, InH;
3261 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH);
3262
3263 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
3264 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
3265 if (!Amt) {
3266 Lo = InL;
3267 Hi = InH;
3268 return;
3269 }
3270
3271 EVT NVT = InL.getValueType();
3272 unsigned VTBits = N->getValueType(ResNo: 0).getSizeInBits();
3273 unsigned NVTBits = NVT.getSizeInBits();
3274
3275 if (N->getOpcode() == ISD::SHL) {
3276 if (Amt.uge(RHS: VTBits)) {
3277 Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT);
3278 } else if (Amt.ugt(RHS: NVTBits)) {
3279 Lo = DAG.getConstant(Val: 0, DL, VT: NVT);
3280 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL,
3281 N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL));
3282 } else if (Amt == NVTBits) {
3283 Lo = DAG.getConstant(Val: 0, DL, VT: NVT);
3284 Hi = InL;
3285 } else {
3286 Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL,
3287 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL));
3288 Hi = DAG.getNode(
3289 Opcode: ISD::OR, DL, VT: NVT,
3290 N1: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH,
3291 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)),
3292 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL,
3293 N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL)));
3294 }
3295 return;
3296 }
3297
3298 if (N->getOpcode() == ISD::SRL) {
3299 if (Amt.uge(RHS: VTBits)) {
3300 Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT);
3301 } else if (Amt.ugt(RHS: NVTBits)) {
3302 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH,
3303 N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL));
3304 Hi = DAG.getConstant(Val: 0, DL, VT: NVT);
3305 } else if (Amt == NVTBits) {
3306 Lo = InH;
3307 Hi = DAG.getConstant(Val: 0, DL, VT: NVT);
3308 } else {
3309 Lo = DAG.getNode(
3310 Opcode: ISD::OR, DL, VT: NVT,
3311 N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL,
3312 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)),
3313 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH,
3314 N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL)));
3315 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH,
3316 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL));
3317 }
3318 return;
3319 }
3320
3321 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3322 if (Amt.uge(RHS: VTBits)) {
3323 Hi = Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH,
3324 N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL));
3325 } else if (Amt.ugt(RHS: NVTBits)) {
3326 Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH,
3327 N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL));
3328 Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH,
3329 N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL));
3330 } else if (Amt == NVTBits) {
3331 Lo = InH;
3332 Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH,
3333 N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL));
3334 } else {
3335 Lo = DAG.getNode(
3336 Opcode: ISD::OR, DL, VT: NVT,
3337 N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL,
3338 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)),
3339 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH,
3340 N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL)));
3341 Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH,
3342 N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL));
3343 }
3344}
3345
3346/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
3347/// this shift based on knowledge of the high bit of the shift amount. If we
3348/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
3349/// shift amount.
3350bool DAGTypeLegalizer::
3351ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3352 unsigned Opc = N->getOpcode();
3353 SDValue In = N->getOperand(Num: 0);
3354 SDValue Amt = N->getOperand(Num: 1);
3355 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
3356 EVT ShTy = Amt.getValueType();
3357 unsigned ShBits = ShTy.getScalarSizeInBits();
3358 unsigned NVTBits = NVT.getScalarSizeInBits();
3359 assert(isPowerOf2_32(NVTBits) &&
3360 "Expanded integer type size not a power of two!");
3361 SDLoc dl(N);
3362
3363 APInt HighBitMask = APInt::getHighBitsSet(numBits: ShBits, hiBitsSet: ShBits - Log2_32(Value: NVTBits));
3364 KnownBits Known = DAG.computeKnownBits(Op: Amt);
3365
3366 // If we don't know anything about the high bits, exit.
3367 if (((Known.Zero | Known.One) & HighBitMask) == 0)
3368 return false;
3369
3370 // Get the incoming operand to be shifted.
3371 SDValue InL, InH;
3372 GetExpandedInteger(Op: In, Lo&: InL, Hi&: InH);
3373
3374 // If we know that any of the high bits of the shift amount are one, then we
3375 // can do this as a couple of simple shifts.
3376 if (Known.One.intersects(RHS: HighBitMask)) {
3377 // Mask out the high bit, which we know is set.
3378 Amt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShTy, N1: Amt,
3379 N2: DAG.getConstant(Val: ~HighBitMask, DL: dl, VT: ShTy));
3380
3381 switch (Opc) {
3382 default: llvm_unreachable("Unknown shift");
3383 case ISD::SHL:
3384 Lo = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Low part is zero.
3385 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt); // High part from Lo part.
3386 return true;
3387 case ISD::SRL:
3388 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero.
3389 Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part.
3390 return true;
3391 case ISD::SRA:
3392 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign extend high part.
3393 N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy));
3394 Lo = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part.
3395 return true;
3396 }
3397 }
3398
3399 // If we know that all of the high bits of the shift amount are zero, then we
3400 // can do this as a couple of simple shifts.
3401 if (HighBitMask.isSubsetOf(RHS: Known.Zero)) {
3402 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
3403 // shift if x is zero. We can use XOR here because x is known to be smaller
3404 // than 32.
3405 SDValue Amt2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ShTy, N1: Amt,
3406 N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy));
3407
3408 unsigned Op1, Op2;
3409 switch (Opc) {
3410 default: llvm_unreachable("Unknown shift");
3411 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
3412 case ISD::SRL:
3413 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
3414 }
3415
3416 // When shifting right the arithmetic for Lo and Hi is swapped.
3417 if (Opc != ISD::SHL)
3418 std::swap(a&: InL, b&: InH);
3419
3420 // Use a little trick to get the bits that move from Lo to Hi. First
3421 // shift by one bit.
3422 SDValue Sh1 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: InL, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShTy));
3423 // Then compute the remaining shift with amount-1.
3424 SDValue Sh2 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: Sh1, N2: Amt2);
3425
3426 Lo = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, N1: InL, N2: Amt);
3427 Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: Op1, DL: dl, VT: NVT, N1: InH, N2: Amt),N2: Sh2);
3428
3429 if (Opc != ISD::SHL)
3430 std::swap(a&: Hi, b&: Lo);
3431 return true;
3432 }
3433
3434 return false;
3435}
3436
3437/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
3438/// of any size.
3439bool DAGTypeLegalizer::
3440ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3441 SDValue Amt = N->getOperand(Num: 1);
3442 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
3443 EVT ShTy = Amt.getValueType();
3444 unsigned NVTBits = NVT.getSizeInBits();
3445 assert(isPowerOf2_32(NVTBits) &&
3446 "Expanded integer type size not a power of two!");
3447 SDLoc dl(N);
3448
3449 // Get the incoming operand to be shifted.
3450 SDValue InL, InH;
3451 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH);
3452
3453 SDValue NVBitsNode = DAG.getConstant(Val: NVTBits, DL: dl, VT: ShTy);
3454 SDValue AmtExcess = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: Amt, N2: NVBitsNode);
3455 SDValue AmtLack = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: NVBitsNode, N2: Amt);
3456 SDValue isShort = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy),
3457 LHS: Amt, RHS: NVBitsNode, Cond: ISD::SETULT);
3458 SDValue isZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy),
3459 LHS: Amt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShTy),
3460 Cond: ISD::SETEQ);
3461
3462 SDValue LoS, HiS, LoL, HiL;
3463 switch (N->getOpcode()) {
3464 default: llvm_unreachable("Unknown shift");
3465 case ISD::SHL:
3466 // Short: ShAmt < NVTBits
3467 LoS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt);
3468 HiS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT,
3469 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: Amt),
3470 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: AmtLack));
3471
3472 // Long: ShAmt >= NVTBits
3473 LoL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Lo part is zero.
3474 HiL = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: AmtExcess); // Hi from Lo part.
3475
3476 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL);
3477 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InH,
3478 RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL));
3479 return true;
3480 case ISD::SRL:
3481 // Short: ShAmt < NVTBits
3482 HiS = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt);
3483 LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT,
3484 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt),
3485 // FIXME: If Amt is zero, the following shift generates an undefined result
3486 // on some architectures.
3487 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack));
3488
3489 // Long: ShAmt >= NVTBits
3490 HiL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero.
3491 LoL = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part.
3492
3493 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL,
3494 RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL));
3495 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL);
3496 return true;
3497 case ISD::SRA:
3498 // Short: ShAmt < NVTBits
3499 HiS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt);
3500 LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT,
3501 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt),
3502 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack));
3503
3504 // Long: ShAmt >= NVTBits
3505 HiL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign of Hi part.
3506 N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy));
3507 LoL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part.
3508
3509 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL,
3510 RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL));
3511 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL);
3512 return true;
3513 }
3514}
3515
3516static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3517
3518 switch (Op) {
3519 default: llvm_unreachable("invalid min/max opcode");
3520 case ISD::SMAX:
3521 return std::make_pair(x: ISD::SETGT, y: ISD::UMAX);
3522 case ISD::UMAX:
3523 return std::make_pair(x: ISD::SETUGT, y: ISD::UMAX);
3524 case ISD::SMIN:
3525 return std::make_pair(x: ISD::SETLT, y: ISD::UMIN);
3526 case ISD::UMIN:
3527 return std::make_pair(x: ISD::SETULT, y: ISD::UMIN);
3528 }
3529}
3530
3531void DAGTypeLegalizer::ExpandIntRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3532 SDLoc DL(N);
3533
3534 SDValue LHS = N->getOperand(Num: 0);
3535 SDValue RHS = N->getOperand(Num: 1);
3536 EVT NewVT = getSetCCResultType(VT: LHS.getValueType());
3537
3538 // Taking the same approach as ScalarizeVecRes_SETCC
3539 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: NewVT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2));
3540
3541 Res = DAG.getBoolExtOrTrunc(Op: Res, SL: DL, VT: N->getValueType(ResNo: 0), OpVT: NewVT);
3542 SplitInteger(Op: Res, Lo, Hi);
3543}
3544
3545void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3546 SDValue &Lo, SDValue &Hi) {
3547 SDLoc DL(N);
3548
3549 SDValue LHS = N->getOperand(Num: 0);
3550 SDValue RHS = N->getOperand(Num: 1);
3551
3552 // If the upper halves are all sign bits, then we can perform the MINMAX on
3553 // the lower half and sign-extend the result to the upper half.
3554 unsigned NumBits = N->getValueType(ResNo: 0).getScalarSizeInBits();
3555 unsigned NumHalfBits = NumBits / 2;
3556 if (DAG.ComputeNumSignBits(Op: LHS) > NumHalfBits &&
3557 DAG.ComputeNumSignBits(Op: RHS) > NumHalfBits) {
3558 SDValue LHSL, LHSH, RHSL, RHSH;
3559 GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH);
3560 GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH);
3561 EVT NVT = LHSL.getValueType();
3562
3563 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: LHSL, N2: RHSL);
3564 Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: Lo,
3565 N2: DAG.getShiftAmountConstant(Val: NumHalfBits - 1, VT: NVT, DL));
3566 return;
3567 }
3568
3569 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3570 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3571 if ((N->getOpcode() == ISD::SMAX && isNullConstant(V: RHS)) ||
3572 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(V: RHS))) {
3573 SDValue LHSL, LHSH, RHSL, RHSH;
3574 GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH);
3575 GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH);
3576 EVT NVT = LHSL.getValueType();
3577 EVT CCT = getSetCCResultType(VT: NVT);
3578
3579 SDValue HiNeg =
3580 DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: DAG.getConstant(Val: 0, DL, VT: NVT), Cond: ISD::SETLT);
3581 if (N->getOpcode() == ISD::SMIN) {
3582 Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: LHSL, RHS: DAG.getAllOnesConstant(DL, VT: NVT));
3583 } else {
3584 Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: DAG.getConstant(Val: 0, DL, VT: NVT), RHS: LHSL);
3585 }
3586 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH});
3587 return;
3588 }
3589
3590 const APInt *RHSVal = nullptr;
3591 if (auto *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS))
3592 RHSVal = &RHSConst->getAPIntValue();
3593
3594 // The high half of MIN/MAX is always just the the MIN/MAX of the
3595 // high halves of the operands. Expand this way if it appears profitable.
3596 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3597 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3598 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3599 SDValue LHSL, LHSH, RHSL, RHSH;
3600 GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH);
3601 GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH);
3602 EVT NVT = LHSL.getValueType();
3603 EVT CCT = getSetCCResultType(VT: NVT);
3604
3605 ISD::NodeType LoOpc;
3606 ISD::CondCode CondC;
3607 std::tie(args&: CondC, args&: LoOpc) = getExpandedMinMaxOps(Op: N->getOpcode());
3608
3609 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH});
3610 // We need to know whether to select Lo part that corresponds to 'winning'
3611 // Hi part or if Hi parts are equal.
3612 SDValue IsHiLeft = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: CondC);
3613 SDValue IsHiEq = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: ISD::SETEQ);
3614
3615 // Lo part corresponding to the 'winning' Hi part
3616 SDValue LoCmp = DAG.getSelect(DL, VT: NVT, Cond: IsHiLeft, LHS: LHSL, RHS: RHSL);
3617
3618 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3619 SDValue LoMinMax = DAG.getNode(Opcode: LoOpc, DL, VT: NVT, Ops: {LHSL, RHSL});
3620
3621 Lo = DAG.getSelect(DL, VT: NVT, Cond: IsHiEq, LHS: LoMinMax, RHS: LoCmp);
3622 return;
3623 }
3624
3625 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3626 // the compare.
3627 ISD::CondCode Pred;
3628 switch (N->getOpcode()) {
3629 default: llvm_unreachable("How did we get here?");
3630 case ISD::SMAX:
3631 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3632 Pred = ISD::SETGE;
3633 else
3634 Pred = ISD::SETGT;
3635 break;
3636 case ISD::SMIN:
3637 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3638 Pred = ISD::SETLE;
3639 else
3640 Pred = ISD::SETLT;
3641 break;
3642 case ISD::UMAX:
3643 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3644 Pred = ISD::SETUGE;
3645 else
3646 Pred = ISD::SETUGT;
3647 break;
3648 case ISD::UMIN:
3649 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3650 Pred = ISD::SETULE;
3651 else
3652 Pred = ISD::SETULT;
3653 break;
3654 }
3655 EVT VT = N->getValueType(ResNo: 0);
3656 EVT CCT = getSetCCResultType(VT);
3657 SDValue Cond = DAG.getSetCC(DL, VT: CCT, LHS, RHS, Cond: Pred);
3658 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3659 SplitInteger(Op: Result, Lo, Hi);
3660}
3661
3662void DAGTypeLegalizer::ExpandIntRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
3663 SDValue ExpandedCMP = TLI.expandCMP(Node: N, DAG);
3664 SplitInteger(Op: ExpandedCMP, Lo, Hi);
3665}
3666
3667void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3668 SDValue &Lo, SDValue &Hi) {
3669 SDLoc dl(N);
3670 // Expand the subcomponents.
3671 SDValue LHSL, LHSH, RHSL, RHSH;
3672 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
3673 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH);
3674
3675 EVT NVT = LHSL.getValueType();
3676 SDValue LoOps[2] = { LHSL, RHSL };
3677 SDValue HiOps[3] = { LHSH, RHSH };
3678
3679 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3680 Op: N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3681 VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT));
3682 if (HasOpCarry) {
3683 SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT));
3684 if (N->getOpcode() == ISD::ADD) {
3685 Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps);
3686 HiOps[2] = Lo.getValue(R: 1);
3687 Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero()
3688 ? DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2))
3689 : DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, Ops: HiOps);
3690 } else {
3691 Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps);
3692 HiOps[2] = Lo.getValue(R: 1);
3693 Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero()
3694 ? DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2))
3695 : DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, Ops: HiOps);
3696 }
3697 return;
3698 }
3699
3700 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3701 // them. TODO: Teach operation legalization how to expand unsupported
3702 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3703 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3704 // generate a value of this type in the expanded code sequence.
3705 bool hasCarry =
3706 TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ?
3707 ISD::ADDC : ISD::SUBC,
3708 VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT));
3709
3710 if (hasCarry) {
3711 SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: MVT::Glue);
3712 if (N->getOpcode() == ISD::ADD) {
3713 Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps);
3714 HiOps[2] = Lo.getValue(R: 1);
3715 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps);
3716 } else {
3717 Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps);
3718 HiOps[2] = Lo.getValue(R: 1);
3719 Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps);
3720 }
3721 return;
3722 }
3723
3724 bool hasOVF =
3725 TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ?
3726 ISD::UADDO : ISD::USUBO,
3727 VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT));
3728 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(Type: NVT);
3729
3730 if (hasOVF) {
3731 EVT OvfVT = getSetCCResultType(VT: NVT);
3732 SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: OvfVT);
3733 int RevOpc;
3734 if (N->getOpcode() == ISD::ADD) {
3735 RevOpc = ISD::SUB;
3736 Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps);
3737 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2));
3738 } else {
3739 RevOpc = ISD::ADD;
3740 Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps);
3741 Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2));
3742 }
3743 SDValue OVF = Lo.getValue(R: 1);
3744
3745 switch (BoolType) {
3746 case TargetLoweringBase::UndefinedBooleanContent:
3747 OVF = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OvfVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: OvfVT), N2: OVF);
3748 [[fallthrough]];
3749 case TargetLoweringBase::ZeroOrOneBooleanContent:
3750 OVF = DAG.getZExtOrTrunc(Op: OVF, DL: dl, VT: NVT);
3751 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Hi, N2: OVF);
3752 break;
3753 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
3754 OVF = DAG.getSExtOrTrunc(Op: OVF, DL: dl, VT: NVT);
3755 Hi = DAG.getNode(Opcode: RevOpc, DL: dl, VT: NVT, N1: Hi, N2: OVF);
3756 }
3757 return;
3758 }
3759
3760 if (N->getOpcode() == ISD::ADD) {
3761 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: LoOps);
3762 SDValue Cmp;
3763 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3764 // range of X. We assume comparing with 0 is cheap.
3765 if (isOneConstant(V: LoOps[1]))
3766 Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo,
3767 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ);
3768 else if (isAllOnesConstant(V: LoOps[1])) {
3769 if (isAllOnesConstant(V: HiOps[1]))
3770 Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0],
3771 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ);
3772 else
3773 Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0],
3774 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE);
3775 } else
3776 Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, RHS: LoOps[0],
3777 Cond: ISD::SETULT);
3778
3779 SDValue Carry;
3780 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
3781 Carry = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT);
3782 else
3783 Carry = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT),
3784 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT));
3785
3786 if (isAllOnesConstant(V: LoOps[1]) && isAllOnesConstant(V: HiOps[1])) {
3787 Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: HiOps[0], N2: Carry);
3788 } else {
3789 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2));
3790 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: Hi, N2: Carry);
3791 }
3792 } else {
3793 Lo = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: LoOps);
3794 Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2));
3795 SDValue Cmp =
3796 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LoOps[0].getValueType()),
3797 LHS: LoOps[0], RHS: LoOps[1], Cond: ISD::SETULT);
3798
3799 SDValue Borrow;
3800 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
3801 Borrow = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT);
3802 else
3803 Borrow = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT),
3804 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT));
3805
3806 Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Hi, N2: Borrow);
3807 }
3808}
3809
3810void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3811 SDValue &Lo, SDValue &Hi) {
3812 // Expand the subcomponents.
3813 SDValue LHSL, LHSH, RHSL, RHSH;
3814 SDLoc dl(N);
3815 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
3816 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH);
3817 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: MVT::Glue);
3818 SDValue LoOps[2] = { LHSL, RHSL };
3819 SDValue HiOps[3] = { LHSH, RHSH };
3820
3821 if (N->getOpcode() == ISD::ADDC) {
3822 Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps);
3823 HiOps[2] = Lo.getValue(R: 1);
3824 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps);
3825 } else {
3826 Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps);
3827 HiOps[2] = Lo.getValue(R: 1);
3828 Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps);
3829 }
3830
3831 // Legalized the flag result - switch anything that used the old flag to
3832 // use the new one.
3833 ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1));
3834}
3835
3836void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3837 SDValue &Lo, SDValue &Hi) {
3838 // Expand the subcomponents.
3839 SDValue LHSL, LHSH, RHSL, RHSH;
3840 SDLoc dl(N);
3841 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
3842 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH);
3843 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: MVT::Glue);
3844 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) };
3845 SDValue HiOps[3] = { LHSH, RHSH };
3846
3847 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps);
3848 HiOps[2] = Lo.getValue(R: 1);
3849 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps);
3850
3851 // Legalized the flag result - switch anything that used the old flag to
3852 // use the new one.
3853 ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1));
3854}
3855
3856void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3857 SDValue &Lo, SDValue &Hi) {
3858 SDValue LHS = N->getOperand(Num: 0);
3859 SDValue RHS = N->getOperand(Num: 1);
3860 SDLoc dl(N);
3861
3862 SDValue Ovf;
3863
3864 unsigned CarryOp, NoCarryOp;
3865 ISD::CondCode Cond;
3866 switch(N->getOpcode()) {
3867 case ISD::UADDO:
3868 CarryOp = ISD::UADDO_CARRY;
3869 NoCarryOp = ISD::ADD;
3870 Cond = ISD::SETULT;
3871 break;
3872 case ISD::USUBO:
3873 CarryOp = ISD::USUBO_CARRY;
3874 NoCarryOp = ISD::SUB;
3875 Cond = ISD::SETUGT;
3876 break;
3877 default:
3878 llvm_unreachable("Node has unexpected Opcode");
3879 }
3880
3881 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3882 Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType()));
3883
3884 if (HasCarryOp) {
3885 // Expand the subcomponents.
3886 SDValue LHSL, LHSH, RHSL, RHSH;
3887 GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH);
3888 GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH);
3889 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1));
3890 SDValue LoOps[2] = { LHSL, RHSL };
3891 SDValue HiOps[3] = { LHSH, RHSH };
3892
3893 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps);
3894 HiOps[2] = Lo.getValue(R: 1);
3895 Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: HiOps);
3896
3897 Ovf = Hi.getValue(R: 1);
3898 } else {
3899 // Expand the result by simply replacing it with the equivalent
3900 // non-overflow-checking operation.
3901 SDValue Sum = DAG.getNode(Opcode: NoCarryOp, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
3902 SplitInteger(Op: Sum, Lo, Hi);
3903
3904 if (N->getOpcode() == ISD::UADDO && isOneConstant(V: RHS)) {
3905 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
3906 // with (Lo | Hi) == 0.
3907 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Lo.getValueType(), N1: Lo, N2: Hi);
3908 Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Or,
3909 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Lo.getValueType()), Cond: ISD::SETEQ);
3910 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(V: RHS)) {
3911 // Special case: uaddo X, -1 overflows if X == 0.
3912 Ovf =
3913 DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS,
3914 RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType()), Cond: ISD::SETNE);
3915 } else {
3916 // Calculate the overflow: addition overflows iff a + b < a, and
3917 // subtraction overflows iff a - b > a.
3918 Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Sum, RHS: LHS, Cond);
3919 }
3920 }
3921
3922 // Legalized the flag result - switch anything that used the old flag to
3923 // use the new one.
3924 ReplaceValueWith(From: SDValue(N, 1), To: Ovf);
3925}
3926
3927void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
3928 SDValue &Hi) {
3929 // Expand the subcomponents.
3930 SDValue LHSL, LHSH, RHSL, RHSH;
3931 SDLoc dl(N);
3932 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
3933 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH);
3934 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1));
3935 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) };
3936 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3937
3938 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps);
3939 HiOps[2] = Lo.getValue(R: 1);
3940 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps);
3941
3942 // Legalized the flag result - switch anything that used the old flag to
3943 // use the new one.
3944 ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1));
3945}
3946
3947void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3948 SDValue &Lo, SDValue &Hi) {
3949 // Expand the subcomponents.
3950 SDValue LHSL, LHSH, RHSL, RHSH;
3951 SDLoc dl(N);
3952 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
3953 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH);
3954 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1));
3955
3956 // We need to use an unsigned carry op for the lo part.
3957 unsigned CarryOp =
3958 N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
3959 Lo = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSL, RHSL, N->getOperand(Num: 2) });
3960 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) });
3961
3962 // Legalized the flag result - switch anything that used the old flag to
3963 // use the new one.
3964 ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1));
3965}
3966
3967void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3968 SDValue &Lo, SDValue &Hi) {
3969 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
3970 SDLoc dl(N);
3971 SDValue Op = N->getOperand(Num: 0);
3972 if (Op.getValueType().bitsLE(VT: NVT)) {
3973 // The low part is any extension of the input (which degenerates to a copy).
3974 Lo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Op);
3975 Hi = DAG.getUNDEF(VT: NVT); // The high part is undefined.
3976 } else {
3977 // For example, extension of an i48 to an i64. The operand type necessarily
3978 // promotes to the result type, so will end up being expanded too.
3979 assert(getTypeAction(Op.getValueType()) ==
3980 TargetLowering::TypePromoteInteger &&
3981 "Only know how to promote this result!");
3982 SDValue Res = GetPromotedInteger(Op);
3983 assert(Res.getValueType() == N->getValueType(0) &&
3984 "Operand over promoted?");
3985 // Split the promoted operand. This will simplify when it is expanded.
3986 SplitInteger(Op: Res, Lo, Hi);
3987 }
3988}
3989
3990void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3991 SDValue &Lo, SDValue &Hi) {
3992 SDLoc dl(N);
3993 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
3994 EVT NVT = Lo.getValueType();
3995 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
3996 unsigned NVTBits = NVT.getSizeInBits();
3997 unsigned EVTBits = EVT.getSizeInBits();
3998
3999 if (NVTBits < EVTBits) {
4000 Hi = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Hi,
4001 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(),
4002 BitWidth: EVTBits - NVTBits)));
4003 } else {
4004 Lo = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT));
4005 // The high part replicates the sign bit of Lo, make it explicit.
4006 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo,
4007 N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL: dl));
4008 }
4009}
4010
4011void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
4012 SDValue &Lo, SDValue &Hi) {
4013 SDLoc dl(N);
4014 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
4015 EVT NVT = Lo.getValueType();
4016 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
4017 unsigned NVTBits = NVT.getSizeInBits();
4018 unsigned EVTBits = EVT.getSizeInBits();
4019
4020 if (NVTBits < EVTBits) {
4021 Hi = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Hi,
4022 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(),
4023 BitWidth: EVTBits - NVTBits)));
4024 } else {
4025 Lo = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT));
4026 // The high part must be zero, make it explicit.
4027 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4028 }
4029}
4030
4031void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
4032 SDValue &Lo, SDValue &Hi) {
4033 SDLoc dl(N);
4034 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands.
4035 Lo = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
4036 Hi = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
4037}
4038
4039void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
4040 SDValue &Lo, SDValue &Hi) {
4041 SDLoc dl(N);
4042 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands.
4043 Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Lo.getValueType(), Operand: Lo);
4044 Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Hi.getValueType(), Operand: Hi);
4045}
4046
4047void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
4048 SDValue &Hi) {
4049 SDLoc dl(N);
4050 // parity(HiLo) -> parity(Lo^Hi)
4051 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
4052 EVT NVT = Lo.getValueType();
4053 Lo =
4054 DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT: NVT, Operand: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Hi));
4055 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4056}
4057
4058void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
4059 SDValue &Lo, SDValue &Hi) {
4060 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4061 unsigned NBitWidth = NVT.getSizeInBits();
4062 auto Constant = cast<ConstantSDNode>(Val: N);
4063 const APInt &Cst = Constant->getAPIntValue();
4064 bool IsTarget = Constant->isTargetOpcode();
4065 bool IsOpaque = Constant->isOpaque();
4066 SDLoc dl(N);
4067 Lo = DAG.getConstant(Val: Cst.trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget, isOpaque: IsOpaque);
4068 Hi = DAG.getConstant(Val: Cst.lshr(shiftAmt: NBitWidth).trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget,
4069 isOpaque: IsOpaque);
4070}
4071
4072void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
4073 SDLoc dl(N);
4074
4075 SDValue N0 = N->getOperand(Num: 0);
4076 GetExpandedInteger(Op: N0, Lo, Hi);
4077 EVT NVT = Lo.getValueType();
4078
4079 // If the upper half is all sign bits, then we can perform the ABS on the
4080 // lower half and zero-extend.
4081 if (DAG.ComputeNumSignBits(Op: N0) > NVT.getScalarSizeInBits()) {
4082 Lo = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: NVT, Operand: Lo);
4083 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4084 return;
4085 }
4086
4087 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
4088 // we use in LegalizeDAG. The SUB part of the expansion is based on
4089 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
4090 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
4091 // expanded if needed. Shift expansion has a special case for filling with
4092 // sign bits so that we will only end up with one SRA.
4093 bool HasSubCarry = TLI.isOperationLegalOrCustom(
4094 Op: ISD::USUBO_CARRY, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT));
4095 if (HasSubCarry) {
4096 SDValue Sign = DAG.getNode(
4097 Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Hi,
4098 N2: DAG.getShiftAmountConstant(Val: NVT.getSizeInBits() - 1, VT: NVT, DL: dl));
4099 SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT));
4100 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Sign);
4101 Hi = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Hi, N2: Sign);
4102 Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: Lo, N2: Sign);
4103 Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: Hi, N2: Sign, N3: Lo.getValue(R: 1));
4104 return;
4105 }
4106
4107 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
4108 EVT VT = N->getValueType(ResNo: 0);
4109 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT,
4110 N1: DAG.getConstant(Val: 0, DL: dl, VT), N2: N0);
4111 SDValue NegLo, NegHi;
4112 SplitInteger(Op: Neg, Lo&: NegLo, Hi&: NegHi);
4113
4114 SDValue HiIsNeg = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi,
4115 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETLT);
4116 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegLo, RHS: Lo);
4117 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegHi, RHS: Hi);
4118}
4119
4120void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
4121 SDValue &Lo, SDValue &Hi) {
4122 SDLoc dl(N);
4123 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
4124 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
4125 EVT NVT = Lo.getValueType();
4126
4127 SDValue HiNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi,
4128 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE);
4129
4130 SDValue LoLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Lo);
4131 SDValue HiLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Hi);
4132
4133 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiNotZero, LHS: HiLZ,
4134 RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: LoLZ,
4135 N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl,
4136 VT: NVT)));
4137 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4138}
4139
4140void DAGTypeLegalizer::ExpandIntRes_ABD(SDNode *N, SDValue &Lo, SDValue &Hi) {
4141 SDValue Result = TLI.expandABD(N, DAG);
4142 SplitInteger(Op: Result, Lo, Hi);
4143}
4144
4145void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
4146 SDValue Op = N->getOperand(Num: 0);
4147 EVT VT = N->getValueType(ResNo: 0);
4148 SDLoc DL(N);
4149
4150 if (TLI.getOperationAction(Op: ISD::CTPOP, VT) == TargetLoweringBase::LibCall) {
4151 RTLIB::Libcall LC = RTLIB::getCTPOP(VT);
4152 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4153 "LibCall explicitly requested, but not available");
4154
4155 if (RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC)) {
4156 TargetLowering::MakeLibCallOptions CallOptions;
4157 EVT IntVT =
4158 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
4159 SDValue Res =
4160 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: IntVT, Ops: Op, CallOptions, dl: DL).first;
4161 SplitInteger(Op: DAG.getSExtOrTrunc(Op: Res, DL, VT), Lo, Hi);
4162 return;
4163 }
4164
4165 // If the function is not available, fall back on the expansion.
4166 }
4167
4168 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
4169 GetExpandedInteger(Op, Lo, Hi);
4170 EVT NVT = Lo.getValueType();
4171 Lo = DAG.getNode(Opcode: ISD::ADD, DL, VT: NVT, N1: DAG.getNode(Opcode: ISD::CTPOP, DL, VT: NVT, Operand: Lo),
4172 N2: DAG.getNode(Opcode: ISD::CTPOP, DL, VT: NVT, Operand: Hi));
4173 Hi = DAG.getConstant(Val: 0, DL, VT: NVT);
4174}
4175
4176void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
4177 SDValue &Lo, SDValue &Hi) {
4178 SDLoc dl(N);
4179 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
4180 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
4181 EVT NVT = Lo.getValueType();
4182
4183 SDValue LoNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo,
4184 RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE);
4185
4186 SDValue LoLZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Lo);
4187 SDValue HiLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Hi);
4188
4189 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: LoNotZero, LHS: LoLZ,
4190 RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: HiLZ,
4191 N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl,
4192 VT: NVT)));
4193 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4194}
4195
4196void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
4197 SDValue &Hi) {
4198 SDLoc dl(N);
4199 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4200 unsigned NBitWidth = NVT.getSizeInBits();
4201
4202 Lo = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL: dl, ResultTys: {NVT, MVT::Other}, Ops: N->getOperand(Num: 0));
4203 SDValue Chain = Lo.getValue(R: 1);
4204 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
4205 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo,
4206 N2: DAG.getShiftAmountConstant(Val: NBitWidth - 1, VT: NVT, DL: dl));
4207
4208 // Legalize the chain result - switch anything that used the old chain to
4209 // use the new one.
4210 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
4211}
4212
4213// Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
4214static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
4215 SDLoc DL, SelectionDAG &DAG) {
4216 if (IsStrict) {
4217 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op});
4218 Chain = Op.getValue(R: 1);
4219 return Op;
4220 }
4221 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op);
4222}
4223
4224void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
4225 SDValue &Hi) {
4226 SDLoc dl(N);
4227 EVT VT = N->getValueType(ResNo: 0);
4228
4229 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
4230 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
4231 bool IsStrict = N->isStrictFPOpcode();
4232 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
4233 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
4234
4235 // If the input is bf16 or needs to be soft promoted, extend to f32.
4236 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf ||
4237 Op.getValueType() == MVT::bf16) {
4238 Op = fpExtendHelper(Op, Chain, IsStrict, VT: MVT::f32, DL: dl, DAG);
4239 }
4240
4241 // NOTE: We need a variable that lives across makeLibCall so
4242 // CallOptions.setTypeListBeforeSoften can save a reference to it.
4243 EVT OpVT = Op.getValueType();
4244
4245 RTLIB::Libcall LC =
4246 IsSigned ? RTLIB::getFPTOSINT(OpVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT, RetVT: VT);
4247 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
4248 TargetLowering::MakeLibCallOptions CallOptions;
4249 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeSoftenFloat)
4250 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: VT);
4251 else
4252 CallOptions.setIsSigned(true); // FIXME: Is this needed?
4253 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op,
4254 CallOptions, dl, Chain);
4255 SplitInteger(Op: Tmp.first, Lo, Hi);
4256
4257 if (IsStrict)
4258 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
4259}
4260
4261void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
4262 SDValue &Hi) {
4263 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
4264 SplitInteger(Op: Res, Lo, Hi);
4265}
4266
4267void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
4268 SDValue &Hi) {
4269 SDLoc dl(N);
4270 bool IsStrict = N->isStrictFPOpcode();
4271 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
4272 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
4273
4274 EVT VT = Op.getValueType();
4275
4276 if (VT == MVT::f16) {
4277 // Extend to f32.
4278 VT = MVT::f32;
4279 Op = fpExtendHelper(Op, Chain, IsStrict, VT, DL: dl, DAG);
4280 }
4281
4282 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4283 if (N->getOpcode() == ISD::LROUND ||
4284 N->getOpcode() == ISD::STRICT_LROUND) {
4285 LC = RTLIB::getLROUND(VT);
4286 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
4287 } else if (N->getOpcode() == ISD::LRINT ||
4288 N->getOpcode() == ISD::STRICT_LRINT) {
4289 LC = RTLIB::getLRINT(RetVT: VT);
4290 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
4291 } else if (N->getOpcode() == ISD::LLROUND ||
4292 N->getOpcode() == ISD::STRICT_LLROUND) {
4293 LC = RTLIB::getLLROUND(VT);
4294 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
4295 } else if (N->getOpcode() == ISD::LLRINT ||
4296 N->getOpcode() == ISD::STRICT_LLRINT) {
4297 LC = RTLIB::getLLRINT(RetVT: VT);
4298 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
4299 } else
4300 llvm_unreachable("Unexpected opcode!");
4301
4302 EVT RetVT = N->getValueType(ResNo: 0);
4303
4304 TargetLowering::MakeLibCallOptions CallOptions;
4305 CallOptions.setIsSigned(true);
4306 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4307 Ops: Op, CallOptions, dl,
4308 Chain);
4309 SplitInteger(Op: Tmp.first, Lo, Hi);
4310
4311 if (N->isStrictFPOpcode())
4312 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
4313}
4314
4315void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
4316 SDValue &Lo, SDValue &Hi) {
4317 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?");
4318
4319 if (ISD::isNormalLoad(N)) {
4320 ExpandRes_NormalLoad(N, Lo, Hi);
4321 return;
4322 }
4323
4324 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
4325
4326 EVT VT = N->getValueType(ResNo: 0);
4327 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4328 SDValue Ch = N->getChain();
4329 SDValue Ptr = N->getBasePtr();
4330 ISD::LoadExtType ExtType = N->getExtensionType();
4331 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4332 AAMDNodes AAInfo = N->getAAInfo();
4333 SDLoc dl(N);
4334
4335 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4336
4337 if (N->getMemoryVT().bitsLE(VT: NVT)) {
4338 EVT MemVT = N->getMemoryVT();
4339
4340 Lo = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), MemVT,
4341 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
4342
4343 // Remember the chain.
4344 Ch = Lo.getValue(R: 1);
4345
4346 if (ExtType == ISD::SEXTLOAD) {
4347 // The high part is obtained by SRA'ing all but one of the bits of the
4348 // lo part.
4349 unsigned LoSize = Lo.getValueSizeInBits();
4350 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo,
4351 N2: DAG.getShiftAmountConstant(Val: LoSize - 1, VT: NVT, DL: dl));
4352 } else if (ExtType == ISD::ZEXTLOAD) {
4353 // The high part is just a zero.
4354 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4355 } else {
4356 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
4357 // The high part is undefined.
4358 Hi = DAG.getUNDEF(VT: NVT);
4359 }
4360 } else if (DAG.getDataLayout().isLittleEndian()) {
4361 // Little-endian - low bits are at low addresses.
4362 Lo = DAG.getLoad(VT: NVT, dl, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
4363 MMOFlags, AAInfo);
4364
4365 unsigned ExcessBits =
4366 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4367 EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits);
4368
4369 // Increment the pointer to the other half.
4370 unsigned IncrementSize = NVT.getSizeInBits()/8;
4371 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
4372 Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr,
4373 PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), MemVT: NEVT,
4374 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
4375
4376 // Build a factor node to remember that this load is independent of the
4377 // other one.
4378 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4379 N2: Hi.getValue(R: 1));
4380 } else {
4381 // Big-endian - high bits are at low addresses. Favor aligned loads at
4382 // the cost of some bit-fiddling.
4383 EVT MemVT = N->getMemoryVT();
4384 unsigned EBytes = MemVT.getStoreSize();
4385 unsigned IncrementSize = NVT.getSizeInBits()/8;
4386 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4387
4388 // Load both the high bits and maybe some of the low bits.
4389 Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(),
4390 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(),
4391 BitWidth: MemVT.getSizeInBits() - ExcessBits),
4392 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
4393
4394 // Increment the pointer to the other half.
4395 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
4396 // Load the rest of the low bits.
4397 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: NVT, Chain: Ch, Ptr,
4398 PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize),
4399 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits),
4400 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
4401
4402 // Build a factor node to remember that this load is independent of the
4403 // other one.
4404 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4405 N2: Hi.getValue(R: 1));
4406
4407 if (ExcessBits < NVT.getSizeInBits()) {
4408 // Transfer low bits from the bottom of Hi to the top of Lo.
4409 Lo = DAG.getNode(
4410 Opcode: ISD::OR, DL: dl, VT: NVT, N1: Lo,
4411 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi,
4412 N2: DAG.getShiftAmountConstant(Val: ExcessBits, VT: NVT, DL: dl)));
4413 // Move high bits to the right position in Hi.
4414 Hi = DAG.getNode(Opcode: ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, DL: dl, VT: NVT,
4415 N1: Hi,
4416 N2: DAG.getShiftAmountConstant(
4417 Val: NVT.getSizeInBits() - ExcessBits, VT: NVT, DL: dl));
4418 }
4419 }
4420
4421 // Legalize the chain result - switch anything that used the old chain to
4422 // use the new one.
4423 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4424}
4425
4426void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
4427 SDValue &Lo, SDValue &Hi) {
4428 SDLoc dl(N);
4429 SDValue LL, LH, RL, RH;
4430 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
4431 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
4432
4433 SDNodeFlags Flags;
4434 if (N->getOpcode() == ISD::OR)
4435 Flags.setDisjoint(N->getFlags().hasDisjoint());
4436
4437 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LL, N2: RL, Flags);
4438 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LH, N2: RH, Flags);
4439}
4440
4441void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
4442 SDValue &Lo, SDValue &Hi) {
4443 EVT VT = N->getValueType(ResNo: 0);
4444 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4445 SDLoc dl(N);
4446
4447 SDValue LL, LH, RL, RH;
4448 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
4449 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
4450
4451 if (TLI.expandMUL(N, Lo, Hi, HiLoVT: NVT, DAG,
4452 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
4453 LL, LH, RL, RH))
4454 return;
4455
4456 // If nothing else, we can make a libcall.
4457 RTLIB::Libcall LC = RTLIB::getMUL(VT);
4458 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
4459 if (LCImpl == RTLIB::Unsupported) {
4460 // Perform a wide multiplication where the wide type is the original VT and
4461 // the 4 parts are the split arguments.
4462 TLI.forceExpandMultiply(DAG, dl, /*Signed=*/false, Lo, Hi, LHS: LL, RHS: RL, HiLHS: LH, HiRHS: RH);
4463 return;
4464 }
4465
4466 // Note that we don't need to do a wide MUL here since we don't care about the
4467 // upper half of the result if it exceeds VT.
4468 SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
4469 TargetLowering::MakeLibCallOptions CallOptions;
4470 CallOptions.setIsSigned(true);
4471 SplitInteger(Op: TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: VT, Ops, CallOptions, dl).first, Lo,
4472 Hi);
4473}
4474
4475void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo,
4476 SDValue &Hi) {
4477 SDLoc DL(N);
4478 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
4479 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: NVT, VT3: MVT::Other);
4480 SDValue R = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N: N->getOperand(Num: 0));
4481 Lo = R.getValue(R: 0);
4482 Hi = R.getValue(R: 1);
4483 ReplaceValueWith(From: SDValue(N, 1), To: R.getValue(R: 2));
4484}
4485
4486void DAGTypeLegalizer::ExpandIntRes_AVG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4487 SDValue Result = TLI.expandAVG(N, DAG);
4488 SplitInteger(Op: Result, Lo, Hi);
4489}
4490
4491void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
4492 SDValue &Hi) {
4493 SDValue Result = TLI.expandAddSubSat(Node: N, DAG);
4494 SplitInteger(Op: Result, Lo, Hi);
4495}
4496
4497void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
4498 SDValue &Hi) {
4499 SDValue Result = TLI.expandShlSat(Node: N, DAG);
4500 SplitInteger(Op: Result, Lo, Hi);
4501}
4502
4503/// This performs an expansion of the integer result for a fixed point
4504/// multiplication. The default expansion performs rounding down towards
4505/// negative infinity, though targets that do care about rounding should specify
4506/// a target hook for rounding and provide their own expansion or lowering of
4507/// fixed point multiplication to be consistent with rounding.
4508void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
4509 SDValue &Hi) {
4510 SDLoc dl(N);
4511 EVT VT = N->getValueType(ResNo: 0);
4512 unsigned VTSize = VT.getScalarSizeInBits();
4513 SDValue LHS = N->getOperand(Num: 0);
4514 SDValue RHS = N->getOperand(Num: 1);
4515 uint64_t Scale = N->getConstantOperandVal(Num: 2);
4516 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
4517 N->getOpcode() == ISD::UMULFIXSAT);
4518 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
4519 N->getOpcode() == ISD::SMULFIXSAT);
4520
4521 // Handle special case when scale is equal to zero.
4522 if (!Scale) {
4523 SDValue Result;
4524 if (!Saturating) {
4525 Result = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
4526 } else {
4527 EVT BoolVT = getSetCCResultType(VT);
4528 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
4529 Result = DAG.getNode(Opcode: MulOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
4530 SDValue Product = Result.getValue(R: 0);
4531 SDValue Overflow = Result.getValue(R: 1);
4532 if (Signed) {
4533 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
4534 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
4535 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
4536 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
4537 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4538 // Xor the inputs, if resulting sign bit is 0 the product will be
4539 // positive, else negative.
4540 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
4541 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
4542 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
4543 Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
4544 } else {
4545 // For unsigned multiplication, we only need to check the max since we
4546 // can't really overflow towards zero.
4547 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
4548 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
4549 Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
4550 }
4551 }
4552 SplitInteger(Op: Result, Lo, Hi);
4553 return;
4554 }
4555
4556 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
4557 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
4558 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
4559
4560 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4561 SDValue LL, LH, RL, RH;
4562 GetExpandedInteger(Op: LHS, Lo&: LL, Hi&: LH);
4563 GetExpandedInteger(Op: RHS, Lo&: RL, Hi&: RH);
4564 SmallVector<SDValue, 4> Result;
4565
4566 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
4567 if (!TLI.expandMUL_LOHI(Opcode: LoHiOp, VT, dl, LHS, RHS, Result, HiLoVT: NVT, DAG,
4568 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
4569 LL, LH, RL, RH)) {
4570 Result.clear();
4571 Result.resize(N: 4);
4572
4573 SDValue LoTmp, HiTmp;
4574 TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo&: LoTmp, Hi&: HiTmp);
4575 SplitInteger(Op: LoTmp, Lo&: Result[0], Hi&: Result[1]);
4576 SplitInteger(Op: HiTmp, Lo&: Result[2], Hi&: Result[3]);
4577 }
4578 assert(Result.size() == 4 && "Unexpected number of partlets in the result");
4579
4580 unsigned NVTSize = NVT.getScalarSizeInBits();
4581 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
4582 "the size of the current value type");
4583
4584 // After getting the multiplication result in 4 parts, we need to perform a
4585 // shift right by the amount of the scale to get the result in that scale.
4586 //
4587 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
4588 // 128 bits that are cut into 4 32-bit parts:
4589 //
4590 // HH HL LH LL
4591 // |---32---|---32---|---32---|---32---|
4592 // 128 96 64 32 0
4593 //
4594 // |------VTSize-----|
4595 //
4596 // |NVTSize-|
4597 //
4598 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
4599 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
4600 // amount and get Lo and Hi using two funnel shifts. Or for the special case
4601 // when Scale is a multiple of NVTSize we can just pick the result without
4602 // shifting.
4603 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
4604 if (Scale % NVTSize) {
4605 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: Scale % NVTSize, VT: NVT, DL: dl);
4606 Lo = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 1], N2: Result[Part0],
4607 N3: ShiftAmount);
4608 Hi = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 2], N2: Result[Part0 + 1],
4609 N3: ShiftAmount);
4610 } else {
4611 Lo = Result[Part0];
4612 Hi = Result[Part0 + 1];
4613 }
4614
4615 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
4616 if (!Saturating)
4617 return;
4618
4619 // Can not overflow when there is no integer part.
4620 if (Scale == VTSize)
4621 return;
4622
4623 // To handle saturation we must check for overflow in the multiplication.
4624 //
4625 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
4626 // aren't all zeroes.
4627 //
4628 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
4629 // aren't all ones or all zeroes.
4630 //
4631 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
4632 // highest bit of HH determines saturation direction in the event of signed
4633 // saturation.
4634
4635 SDValue ResultHL = Result[2];
4636 SDValue ResultHH = Result[3];
4637
4638 SDValue SatMax, SatMin;
4639 SDValue NVTZero = DAG.getConstant(Val: 0, DL: dl, VT: NVT);
4640 SDValue NVTNeg1 = DAG.getAllOnesConstant(DL: dl, VT: NVT);
4641 EVT BoolNVT = getSetCCResultType(VT: NVT);
4642
4643 if (!Signed) {
4644 if (Scale < NVTSize) {
4645 // Overflow happened if ((HH | (HL >> Scale)) != 0).
4646 SDValue HLAdjusted =
4647 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL,
4648 N2: DAG.getShiftAmountConstant(Val: Scale, VT: NVT, DL: dl));
4649 SDValue Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: HLAdjusted, N2: ResultHH);
4650 SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: Tmp, RHS: NVTZero, Cond: ISD::SETNE);
4651 } else if (Scale == NVTSize) {
4652 // Overflow happened if (HH != 0).
4653 SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETNE);
4654 } else if (Scale < VTSize) {
4655 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
4656 SDValue HLAdjusted =
4657 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL,
4658 N2: DAG.getShiftAmountConstant(Val: Scale - NVTSize, VT: NVT, DL: dl));
4659 SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: HLAdjusted, RHS: NVTZero, Cond: ISD::SETNE);
4660 } else
4661 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
4662 "(and saturation can't happen with Scale==VTSize).");
4663
4664 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Hi);
4665 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Lo);
4666 return;
4667 }
4668
4669 if (Scale < NVTSize) {
4670 // The number of overflow bits we can check are VTSize - Scale + 1 (we
4671 // include the sign bit). If these top bits are > 0, then we overflowed past
4672 // the max value. If these top bits are < -1, then we overflowed past the
4673 // min value. Otherwise, we did not overflow.
4674 unsigned OverflowBits = VTSize - Scale + 1;
4675 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4676 "Extent of overflow bits must start within HL");
4677 SDValue HLHiMask = DAG.getConstant(
4678 Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits - NVTSize), DL: dl, VT: NVT);
4679 SDValue HLLoMask = DAG.getConstant(
4680 Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: VTSize - OverflowBits), DL: dl, VT: NVT);
4681 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
4682 SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT);
4683 SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ);
4684 SDValue HLUGT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLLoMask, Cond: ISD::SETUGT);
4685 SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0,
4686 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLUGT));
4687 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
4688 SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT);
4689 SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ);
4690 SDValue HLULT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLHiMask, Cond: ISD::SETULT);
4691 SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT,
4692 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLULT));
4693 } else if (Scale == NVTSize) {
4694 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
4695 SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT);
4696 SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ);
4697 SDValue HLNeg = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETLT);
4698 SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0,
4699 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLNeg));
4700 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
4701 SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT);
4702 SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ);
4703 SDValue HLPos = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETGE);
4704 SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT,
4705 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLPos));
4706 } else if (Scale < VTSize) {
4707 // This is similar to the case when we saturate if Scale < NVTSize, but we
4708 // only need to check HH.
4709 unsigned OverflowBits = VTSize - Scale + 1;
4710 SDValue HHHiMask = DAG.getConstant(
4711 Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits), DL: dl, VT: NVT);
4712 SDValue HHLoMask = DAG.getConstant(
4713 Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: NVTSize - OverflowBits), DL: dl, VT: NVT);
4714 SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHLoMask, Cond: ISD::SETGT);
4715 SatMin = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHHiMask, Cond: ISD::SETLT);
4716 } else
4717 llvm_unreachable("Illegal scale for signed fixed point mul.");
4718
4719 // Saturate to signed maximum.
4720 APInt MaxHi = APInt::getSignedMaxValue(numBits: NVTSize);
4721 APInt MaxLo = APInt::getAllOnes(numBits: NVTSize);
4722 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxHi, DL: dl, VT: NVT), RHS: Hi);
4723 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxLo, DL: dl, VT: NVT), RHS: Lo);
4724 // Saturate to signed minimum.
4725 APInt MinHi = APInt::getSignedMinValue(numBits: NVTSize);
4726 Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: DAG.getConstant(Val: MinHi, DL: dl, VT: NVT), RHS: Hi);
4727 Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: NVTZero, RHS: Lo);
4728}
4729
4730void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
4731 SDValue &Hi) {
4732 SDLoc dl(N);
4733 // Try expanding in the existing type first.
4734 SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: N->getOperand(Num: 0),
4735 RHS: N->getOperand(Num: 1),
4736 Scale: N->getConstantOperandVal(Num: 2), DAG);
4737
4738 if (!Res)
4739 Res = earlyExpandDIVFIX(N, LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1),
4740 Scale: N->getConstantOperandVal(Num: 2), TLI, DAG);
4741 SplitInteger(Op: Res, Lo, Hi);
4742}
4743
4744void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
4745 SDValue &Lo, SDValue &Hi) {
4746 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
4747 "Node has unexpected Opcode");
4748 SDValue LHS = Node->getOperand(Num: 0);
4749 SDValue RHS = Node->getOperand(Num: 1);
4750 SDLoc dl(Node);
4751
4752 SDValue Ovf;
4753
4754 bool IsAdd = Node->getOpcode() == ISD::SADDO;
4755 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
4756
4757 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4758 Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType()));
4759
4760 if (HasCarryOp) {
4761 // Expand the subcomponents.
4762 SDValue LHSL, LHSH, RHSL, RHSH;
4763 GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH);
4764 GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH);
4765 SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: Node->getValueType(ResNo: 1));
4766
4767 Lo = DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: dl, VTList, Ops: {LHSL, RHSL});
4768 Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) });
4769
4770 Ovf = Hi.getValue(R: 1);
4771 } else {
4772 // Expand the result by simply replacing it with the equivalent
4773 // non-overflow-checking operation.
4774 SDValue Sum = DAG.getNode(Opcode: Node->getOpcode() == ISD::SADDO ?
4775 ISD::ADD : ISD::SUB, DL: dl, VT: LHS.getValueType(),
4776 N1: LHS, N2: RHS);
4777 SplitInteger(Op: Sum, Lo, Hi);
4778
4779 // Compute the overflow.
4780 //
4781 // LHSSign -> LHS < 0
4782 // RHSSign -> RHS < 0
4783 // SumSign -> Sum < 0
4784 //
4785 // Add:
4786 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
4787 // Sub:
4788 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
4789 //
4790 // To get better codegen we can rewrite this by doing bitwise math on
4791 // the integers and extract the final sign bit at the end. So the
4792 // above becomes:
4793 //
4794 // Add:
4795 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
4796 // Sub:
4797 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
4798 //
4799 // NOTE: This is different than the expansion we do in expandSADDSUBO
4800 // because it is more costly to determine the RHS is > 0 for SSUBO with the
4801 // integers split.
4802 EVT VT = LHS.getValueType();
4803 SDValue SignsMatch = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
4804 if (IsAdd)
4805 SignsMatch = DAG.getNOT(DL: dl, Val: SignsMatch, VT);
4806
4807 SDValue SumSignNE = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: Sum);
4808 Ovf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SignsMatch, N2: SumSignNE);
4809 EVT OType = Node->getValueType(ResNo: 1);
4810 Ovf = DAG.getSetCC(DL: dl, VT: OType, LHS: Ovf, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
4811 }
4812
4813 // Use the calculated overflow everywhere.
4814 ReplaceValueWith(From: SDValue(Node, 1), To: Ovf);
4815}
4816
4817void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
4818 SDValue &Lo, SDValue &Hi) {
4819 EVT VT = N->getValueType(ResNo: 0);
4820 SDLoc dl(N);
4821 SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
4822
4823 if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) {
4824 SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops);
4825 SplitInteger(Op: Res.getValue(R: 0), Lo, Hi);
4826 return;
4827 }
4828
4829 RTLIB::Libcall LC = RTLIB::getSDIV(VT);
4830 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
4831
4832 TargetLowering::MakeLibCallOptions CallOptions;
4833 CallOptions.setIsSigned(true);
4834 SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi);
4835}
4836
4837void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
4838 SDValue &Hi) {
4839 SDLoc dl(N);
4840 SDValue Shiftee = N->getOperand(Num: 0);
4841 EVT VT = Shiftee.getValueType();
4842 SDValue ShAmt = N->getOperand(Num: 1);
4843 EVT ShAmtVT = ShAmt.getValueType();
4844
4845 EVT LoadVT = VT;
4846 do {
4847 LoadVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LoadVT);
4848 } while (!TLI.isTypeLegal(VT: LoadVT));
4849
4850 const unsigned ShiftUnitInBits = LoadVT.getStoreSizeInBits();
4851 assert(ShiftUnitInBits <= VT.getScalarSizeInBits());
4852 assert(isPowerOf2_32(ShiftUnitInBits) &&
4853 "Shifting unit is not a a power of two!");
4854
4855 const bool IsOneStepShift =
4856 DAG.computeKnownBits(Op: ShAmt).countMinTrailingZeros() >=
4857 Log2_32(Value: ShiftUnitInBits);
4858
4859 // If we can't do it as one step, we'll have two uses of shift amount,
4860 // and thus must freeze it.
4861 if (!IsOneStepShift)
4862 ShAmt = DAG.getFreeze(V: ShAmt);
4863
4864 unsigned VTBitWidth = VT.getScalarSizeInBits();
4865 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?");
4866 unsigned VTByteWidth = VTBitWidth / 8;
4867 assert(isPowerOf2_32(VTByteWidth) &&
4868 "Shiftee type size is not a power of two!");
4869 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4870 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4871 EVT StackSlotVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StackSlotBitWidth);
4872
4873 // Get a temporary stack slot 2x the width of our VT.
4874 // FIXME: reuse stack slots?
4875 Align StackAlign = DAG.getReducedAlign(VT: StackSlotVT, /*UseABI=*/false);
4876 SDValue StackPtr =
4877 DAG.CreateStackTemporary(Bytes: StackSlotVT.getStoreSize(), Alignment: StackAlign);
4878 EVT PtrTy = StackPtr.getValueType();
4879 SDValue Ch = DAG.getEntryNode();
4880
4881 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack(
4882 MF&: DAG.getMachineFunction(),
4883 FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
4884
4885 // Extend the value, that is being shifted, to the entire stack slot's width.
4886 SDValue Init;
4887 if (N->getOpcode() != ISD::SHL) {
4888 unsigned WideningOpc =
4889 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4890 Init = DAG.getNode(Opcode: WideningOpc, DL: dl, VT: StackSlotVT, Operand: Shiftee);
4891 } else {
4892 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width.
4893 SDValue AllZeros = DAG.getConstant(Val: 0, DL: dl, VT);
4894 Init = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: StackSlotVT, N1: AllZeros, N2: Shiftee);
4895 }
4896 // And spill it into the stack slot.
4897 Ch = DAG.getStore(Chain: Ch, dl, Val: Init, Ptr: StackPtr, PtrInfo: StackPtrInfo, Alignment: StackAlign);
4898
4899 // Now, compute the full-byte offset into stack slot from where we can load.
4900 // We have shift amount, which is in bits. Offset should point to an aligned
4901 // address.
4902 SDNodeFlags Flags;
4903 Flags.setExact(IsOneStepShift);
4904 SDValue SrlTmp = DAG.getNode(
4905 Opcode: ISD::SRL, DL: dl, VT: ShAmtVT, N1: ShAmt,
4906 N2: DAG.getConstant(Val: Log2_32(Value: ShiftUnitInBits), DL: dl, VT: ShAmtVT), Flags);
4907 SDValue BitOffset =
4908 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ShAmtVT, N1: SrlTmp,
4909 N2: DAG.getConstant(Val: Log2_32(Value: ShiftUnitInBits), DL: dl, VT: ShAmtVT));
4910
4911 SDValue ByteOffset =
4912 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShAmtVT, N1: BitOffset,
4913 N2: DAG.getConstant(Val: 3, DL: dl, VT: ShAmtVT), Flags: SDNodeFlags::Exact);
4914 // And clamp it, because OOB load is an immediate UB,
4915 // while shift overflow would have *just* been poison.
4916 ByteOffset = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ByteOffset,
4917 N2: DAG.getConstant(Val: VTByteWidth - 1, DL: dl, VT: ShAmtVT));
4918 // We have exactly two strategies on indexing into stack slot here:
4919 // 1. upwards starting from the beginning of the slot
4920 // 2. downwards starting from the middle of the slot
4921 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift
4922 // and vice versa on big-endian machine.
4923 bool WillIndexUpwards = N->getOpcode() != ISD::SHL;
4924 if (DAG.getDataLayout().isBigEndian())
4925 WillIndexUpwards = !WillIndexUpwards;
4926
4927 SDValue AdjStackPtr;
4928 if (WillIndexUpwards) {
4929 AdjStackPtr = StackPtr;
4930 } else {
4931 AdjStackPtr = DAG.getMemBasePlusOffset(
4932 Base: StackPtr, Offset: DAG.getConstant(Val: VTByteWidth, DL: dl, VT: PtrTy), DL: dl);
4933 ByteOffset = DAG.getNegative(Val: ByteOffset, DL: dl, VT: ShAmtVT);
4934 }
4935
4936 // Get the pointer somewhere into the stack slot from which we need to load.
4937 ByteOffset = DAG.getSExtOrTrunc(Op: ByteOffset, DL: dl, VT: PtrTy);
4938 AdjStackPtr = DAG.getMemBasePlusOffset(Base: AdjStackPtr, Offset: ByteOffset, DL: dl);
4939
4940 // And load it! While the load is not legal, legalizing it is obvious.
4941 SDValue Res =
4942 DAG.getLoad(VT, dl, Chain: Ch, Ptr: AdjStackPtr,
4943 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
4944 Alignment: commonAlignment(A: StackAlign, Offset: LoadVT.getStoreSize()));
4945
4946 // If we may still have a remaining bits to shift by, do so now.
4947 if (!IsOneStepShift) {
4948 SDValue ShAmtRem =
4949 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
4950 N2: DAG.getConstant(Val: ShiftUnitInBits - 1, DL: dl, VT: ShAmtVT));
4951 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: Res, N2: ShAmtRem);
4952 }
4953
4954 // Finally, split the computed value.
4955 SplitInteger(Op: Res, Lo, Hi);
4956}
4957
4958void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
4959 SDValue &Lo, SDValue &Hi) {
4960 EVT VT = N->getValueType(ResNo: 0);
4961 unsigned Opc = N->getOpcode();
4962 SDLoc dl(N);
4963
4964 // If we can emit an efficient shift operation, do so now. Check to see if
4965 // the RHS is a constant.
4966 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1)))
4967 return ExpandShiftByConstant(N, Amt: CN->getAPIntValue(), Lo, Hi);
4968
4969 // If we can determine that the high bit of the shift is zero or one, even if
4970 // the low bits are variable, emit this shift in an optimized form.
4971 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
4972 return;
4973
4974 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
4975 unsigned PartsOpc;
4976 if (Opc == ISD::SHL) {
4977 PartsOpc = ISD::SHL_PARTS;
4978 } else if (Opc == ISD::SRL) {
4979 PartsOpc = ISD::SRL_PARTS;
4980 } else {
4981 assert(Opc == ISD::SRA && "Unknown shift!");
4982 PartsOpc = ISD::SRA_PARTS;
4983 }
4984
4985 // Next check to see if the target supports this SHL_PARTS operation or if it
4986 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
4987 // size, but create a libcall instead.
4988 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
4989 TargetLowering::LegalizeAction Action = TLI.getOperationAction(Op: PartsOpc, VT: NVT);
4990 const bool LegalOrCustom =
4991 (Action == TargetLowering::Legal && TLI.isTypeLegal(VT: NVT)) ||
4992 Action == TargetLowering::Custom;
4993
4994 unsigned ExpansionFactor = 1;
4995 // That VT->NVT expansion is one step. But will we re-expand NVT?
4996 for (EVT TmpVT = NVT;;) {
4997 EVT NewTMPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: TmpVT);
4998 if (NewTMPVT == TmpVT)
4999 break;
5000 TmpVT = NewTMPVT;
5001 ++ExpansionFactor;
5002 }
5003
5004 TargetLowering::ShiftLegalizationStrategy S =
5005 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor);
5006
5007 if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack)
5008 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi);
5009
5010 if (LegalOrCustom &&
5011 S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) {
5012 // Expand the subcomponents.
5013 SDValue LHSL, LHSH;
5014 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH);
5015 EVT VT = LHSL.getValueType();
5016
5017 // If the shift amount operand is coming from a vector legalization it may
5018 // have an illegal type. Fix that first by casting the operand, otherwise
5019 // the new SHL_PARTS operation would need further legalization.
5020 SDValue ShiftOp = N->getOperand(Num: 1);
5021 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
5022 if (ShiftOp.getValueType() != ShiftTy)
5023 ShiftOp = DAG.getZExtOrTrunc(Op: ShiftOp, DL: dl, VT: ShiftTy);
5024
5025 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
5026 Lo = DAG.getNode(Opcode: PartsOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops);
5027 Hi = Lo.getValue(R: 1);
5028 return;
5029 }
5030
5031 // Otherwise, emit a libcall.
5032 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5033 bool isSigned;
5034 if (Opc == ISD::SHL) {
5035 isSigned = false; /*sign irrelevant*/
5036 LC = RTLIB::getSHL(VT);
5037 } else if (Opc == ISD::SRL) {
5038 isSigned = false;
5039 LC = RTLIB::getSRL(VT);
5040 } else {
5041 assert(Opc == ISD::SRA && "Unknown shift!");
5042 isSigned = true;
5043 LC = RTLIB::getSRA(VT);
5044 }
5045
5046 if (RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC)) {
5047 EVT ShAmtTy =
5048 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
5049 SDValue ShAmt = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl, VT: ShAmtTy);
5050 SDValue Ops[2] = {N->getOperand(Num: 0), ShAmt};
5051 TargetLowering::MakeLibCallOptions CallOptions;
5052 CallOptions.setIsSigned(isSigned);
5053 SplitInteger(
5054 Op: TLI.makeLibCall(DAG, LibcallImpl, RetVT: VT, Ops, CallOptions, dl).first, Lo,
5055 Hi);
5056 return;
5057 }
5058
5059 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
5060 llvm_unreachable("Unsupported shift!");
5061}
5062
5063void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
5064 SDValue &Lo, SDValue &Hi) {
5065 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5066 SDLoc dl(N);
5067 SDValue Op = N->getOperand(Num: 0);
5068 if (Op.getValueType().bitsLE(VT: NVT)) {
5069 // The low part is sign extension of the input (degenerates to a copy).
5070 Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
5071 // The high part is obtained by SRA'ing all but one of the bits of low part.
5072 unsigned LoSize = NVT.getSizeInBits();
5073 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo,
5074 N2: DAG.getShiftAmountConstant(Val: LoSize - 1, VT: NVT, DL: dl));
5075 } else {
5076 // For example, extension of an i48 to an i64. The operand type necessarily
5077 // promotes to the result type, so will end up being expanded too.
5078 assert(getTypeAction(Op.getValueType()) ==
5079 TargetLowering::TypePromoteInteger &&
5080 "Only know how to promote this result!");
5081 SDValue Res = GetPromotedInteger(Op);
5082 assert(Res.getValueType() == N->getValueType(0) &&
5083 "Operand over promoted?");
5084 // Split the promoted operand. This will simplify when it is expanded.
5085 SplitInteger(Op: Res, Lo, Hi);
5086 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5087 Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi,
5088 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(),
5089 BitWidth: ExcessBits)));
5090 }
5091}
5092
5093void DAGTypeLegalizer::
5094ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
5095 SDLoc dl(N);
5096 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
5097 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
5098
5099 if (EVT.bitsLE(VT: Lo.getValueType())) {
5100 // sext_inreg the low part if needed.
5101 Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Lo.getValueType(), N1: Lo,
5102 N2: N->getOperand(Num: 1));
5103
5104 // The high part gets the sign extension from the lo-part. This handles
5105 // things like sextinreg V:i64 from i8.
5106 Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Hi.getValueType(), N1: Lo,
5107 N2: DAG.getShiftAmountConstant(Val: Hi.getValueSizeInBits() - 1,
5108 VT: Hi.getValueType(), DL: dl));
5109 } else {
5110 // For example, extension of an i48 to an i64. Leave the low part alone,
5111 // sext_inreg the high part.
5112 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
5113 Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi,
5114 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(),
5115 BitWidth: ExcessBits)));
5116 }
5117}
5118
5119void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
5120 SDValue &Lo, SDValue &Hi) {
5121 EVT VT = N->getValueType(ResNo: 0);
5122 SDLoc dl(N);
5123 SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
5124
5125 if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) {
5126 SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops);
5127 SplitInteger(Op: Res.getValue(R: 1), Lo, Hi);
5128 return;
5129 }
5130
5131 RTLIB::Libcall LC = RTLIB::getSREM(VT);
5132 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
5133
5134 TargetLowering::MakeLibCallOptions CallOptions;
5135 CallOptions.setIsSigned(true);
5136 SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi);
5137}
5138
5139void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
5140 SDValue &Lo, SDValue &Hi) {
5141 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5142 SDValue InOp = N->getOperand(Num: 0);
5143 EVT InVT = InOp.getValueType();
5144 SDLoc dl(N);
5145 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: InOp);
5146 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: InVT, N1: InOp,
5147 N2: DAG.getShiftAmountConstant(Val: NVT.getSizeInBits(), VT: InVT, DL: dl));
5148 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Hi);
5149}
5150
5151void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
5152 SDValue &Lo, SDValue &Hi) {
5153 EVT VT = N->getValueType(ResNo: 0);
5154 SDLoc dl(N);
5155
5156 if (N->getOpcode() == ISD::UMULO) {
5157 // This section expands the operation into the following sequence of
5158 // instructions. `iNh` here refers to a type which has half the bit width of
5159 // the type the original operation operated on.
5160 //
5161 // %0 = %LHS.HI != 0 && %RHS.HI != 0
5162 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
5163 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
5164 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
5165 // %4 = add iNh %1.0, %2.0 as iN
5166 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
5167 //
5168 // %lo = %3.LO
5169 // %hi = %5.0
5170 // %ovf = %0 || %1.1 || %2.1 || %5.1
5171 SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1);
5172 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
5173 GetExpandedInteger(Op: LHS, Lo&: LHSLow, Hi&: LHSHigh);
5174 GetExpandedInteger(Op: RHS, Lo&: RHSLow, Hi&: RHSHigh);
5175 EVT HalfVT = LHSLow.getValueType();
5176 EVT BitVT = N->getValueType(ResNo: 1);
5177 SDVTList VTHalfWithO = DAG.getVTList(VT1: HalfVT, VT2: BitVT);
5178
5179 SDValue HalfZero = DAG.getConstant(Val: 0, DL: dl, VT: HalfVT);
5180 SDValue Overflow = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BitVT,
5181 N1: DAG.getSetCC(DL: dl, VT: BitVT, LHS: LHSHigh, RHS: HalfZero, Cond: ISD::SETNE),
5182 N2: DAG.getSetCC(DL: dl, VT: BitVT, LHS: RHSHigh, RHS: HalfZero, Cond: ISD::SETNE));
5183
5184 SDValue One = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: LHSHigh, N2: RHSLow);
5185 Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: One.getValue(R: 1));
5186
5187 SDValue Two = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: RHSHigh, N2: LHSLow);
5188 Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Two.getValue(R: 1));
5189
5190 SDValue HighSum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HalfVT, N1: One, N2: Two);
5191
5192 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
5193 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
5194 // operation recursively legalized?).
5195 //
5196 // Many backends understand this pattern and will convert into LOHI
5197 // themselves, if applicable.
5198 SDValue Three = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT,
5199 N1: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LHSLow),
5200 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RHSLow));
5201 SplitInteger(Op: Three, Lo, Hi);
5202
5203 Hi = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: VTHalfWithO, N1: Hi, N2: HighSum);
5204 Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Hi.getValue(R: 1));
5205 ReplaceValueWith(From: SDValue(N, 1), To: Overflow);
5206 return;
5207 }
5208
5209 Type *RetTy = VT.getTypeForEVT(Context&: *DAG.getContext());
5210 EVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
5211 Type *PtrTy = PtrVT.getTypeForEVT(Context&: *DAG.getContext());
5212
5213 // Replace this with a libcall that will check overflow.
5214 RTLIB::Libcall LC = RTLIB::getMULO(VT);
5215 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
5216
5217 // If we don't have the libcall or if the function we are compiling is the
5218 // implementation of the expected libcall (avoid inf-loop), expand inline.
5219 if (LCImpl == RTLIB::Unsupported ||
5220 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LCImpl) ==
5221 DAG.getMachineFunction().getName()) {
5222 // FIXME: This is not an optimal expansion, but better than crashing.
5223 SDValue MulLo, MulHi;
5224 TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, LHS: N->getOperand(Num: 0),
5225 RHS: N->getOperand(Num: 1), Lo&: MulLo, Hi&: MulHi);
5226 SDValue SRA = DAG.getNode(
5227 Opcode: ISD::SRA, DL: dl, VT, N1: MulLo,
5228 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
5229 SDValue Overflow =
5230 DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: MulHi, RHS: SRA, Cond: ISD::SETNE);
5231 SplitInteger(Op: MulLo, Lo, Hi);
5232 ReplaceValueWith(From: SDValue(N, 1), To: Overflow);
5233 return;
5234 }
5235
5236 SDValue Temp = DAG.CreateStackTemporary(VT: PtrVT);
5237 // Temporary for the overflow value, default it to zero.
5238 SDValue Chain =
5239 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), Ptr: Temp,
5240 PtrInfo: MachinePointerInfo());
5241
5242 TargetLowering::ArgListTy Args;
5243 for (const SDValue &Op : N->op_values()) {
5244 EVT ArgVT = Op.getValueType();
5245 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
5246 TargetLowering::ArgListEntry Entry(Op, ArgTy);
5247 Entry.IsSExt = true;
5248 Entry.IsZExt = false;
5249 Args.push_back(x: Entry);
5250 }
5251
5252 // Also pass the address of the overflow check.
5253 TargetLowering::ArgListEntry Entry(
5254 Temp, PointerType::getUnqual(C&: PtrTy->getContext()));
5255 Entry.IsSExt = true;
5256 Entry.IsZExt = false;
5257 Args.push_back(x: Entry);
5258
5259 SDValue Func = DAG.getExternalSymbol(LCImpl, VT: PtrVT);
5260
5261 TargetLowering::CallLoweringInfo CLI(DAG);
5262 CLI.setDebugLoc(dl)
5263 .setChain(Chain)
5264 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
5265 Target: Func, ArgsList: std::move(Args))
5266 .setSExtResult();
5267
5268 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
5269
5270 SplitInteger(Op: CallInfo.first, Lo, Hi);
5271 SDValue Temp2 =
5272 DAG.getLoad(VT: PtrVT, dl, Chain: CallInfo.second, Ptr: Temp, PtrInfo: MachinePointerInfo());
5273 SDValue Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Temp2,
5274 RHS: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT),
5275 Cond: ISD::SETNE);
5276 // Use the overflow from the libcall everywhere.
5277 ReplaceValueWith(From: SDValue(N, 1), To: Ofl);
5278}
5279
5280void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
5281 SDValue &Lo, SDValue &Hi) {
5282 EVT VT = N->getValueType(ResNo: 0);
5283 SDLoc dl(N);
5284 SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
5285
5286 if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) {
5287 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops);
5288 SplitInteger(Op: Res.getValue(R: 0), Lo, Hi);
5289 return;
5290 }
5291
5292 // Try to expand UDIV by constant.
5293 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
5294 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5295 // Only if the new type is legal.
5296 if (isTypeLegal(VT: NVT)) {
5297 SDValue InL, InH;
5298 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH);
5299 SmallVector<SDValue> Result;
5300 if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) {
5301 Lo = Result[0];
5302 Hi = Result[1];
5303 return;
5304 }
5305 }
5306 }
5307
5308 RTLIB::Libcall LC = RTLIB::getUDIV(VT);
5309 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
5310
5311 TargetLowering::MakeLibCallOptions CallOptions;
5312 SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi);
5313}
5314
5315void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
5316 SDValue &Lo, SDValue &Hi) {
5317 EVT VT = N->getValueType(ResNo: 0);
5318 SDLoc dl(N);
5319 SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
5320
5321 if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) {
5322 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops);
5323 SplitInteger(Op: Res.getValue(R: 1), Lo, Hi);
5324 return;
5325 }
5326
5327 // Try to expand UREM by constant.
5328 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
5329 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5330 // Only if the new type is legal.
5331 if (isTypeLegal(VT: NVT)) {
5332 SDValue InL, InH;
5333 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH);
5334 SmallVector<SDValue> Result;
5335 if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) {
5336 Lo = Result[0];
5337 Hi = Result[1];
5338 return;
5339 }
5340 }
5341 }
5342
5343 RTLIB::Libcall LC = RTLIB::getUREM(VT);
5344 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
5345
5346 TargetLowering::MakeLibCallOptions CallOptions;
5347 SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi);
5348}
5349
5350void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
5351 SDValue &Lo, SDValue &Hi) {
5352 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5353 SDLoc dl(N);
5354 SDValue Op = N->getOperand(Num: 0);
5355 if (Op.getValueType().bitsLE(VT: NVT)) {
5356 // The low part is zero extension of the input (degenerates to a copy).
5357 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
5358 Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // The high part is just a zero.
5359 } else {
5360 // For example, extension of an i48 to an i64. The operand type necessarily
5361 // promotes to the result type, so will end up being expanded too.
5362 assert(getTypeAction(Op.getValueType()) ==
5363 TargetLowering::TypePromoteInteger &&
5364 "Only know how to promote this result!");
5365 SDValue Res = GetPromotedInteger(Op);
5366 assert(Res.getValueType() == N->getValueType(0) &&
5367 "Operand over promoted?");
5368 // Split the promoted operand. This will simplify when it is expanded.
5369 SplitInteger(Op: Res, Lo, Hi);
5370 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
5371 Hi = DAG.getZeroExtendInReg(Op: Hi, DL: dl,
5372 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
5373 BitWidth: ExcessBits));
5374 }
5375}
5376
5377void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
5378 SDValue &Lo, SDValue &Hi) {
5379 SDLoc dl(N);
5380 EVT VT = cast<AtomicSDNode>(Val: N)->getMemoryVT();
5381 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i1, VT3: MVT::Other);
5382 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
5383 SDValue Swap = DAG.getAtomicCmpSwap(
5384 Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
5385 MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(), VTs, Chain: N->getOperand(Num: 0),
5386 Ptr: N->getOperand(Num: 1), Cmp: Zero, Swp: Zero, MMO: cast<AtomicSDNode>(Val: N)->getMemOperand());
5387
5388 ReplaceValueWith(From: SDValue(N, 0), To: Swap.getValue(R: 0));
5389 ReplaceValueWith(From: SDValue(N, 1), To: Swap.getValue(R: 2));
5390}
5391
5392void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
5393 SDValue &Lo, SDValue &Hi) {
5394 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
5395 // both halves independently.
5396 SDValue Res = TLI.expandVecReduce(Node: N, DAG);
5397 SplitInteger(Op: Res, Lo, Hi);
5398}
5399
5400void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
5401 SDValue &Lo, SDValue &Hi) {
5402 // Delegate to funnel-shift expansion.
5403 SDLoc DL(N);
5404 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
5405 SDValue Res = DAG.getNode(Opcode, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5406 N2: N->getOperand(Num: 0), N3: N->getOperand(Num: 1));
5407 SplitInteger(Op: Res, Lo, Hi);
5408}
5409
5410void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
5411 SDValue &Hi) {
5412 // Values numbered from least significant to most significant.
5413 SDValue In1, In2, In3, In4;
5414 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: In3, Hi&: In4);
5415 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: In1, Hi&: In2);
5416 EVT HalfVT = In1.getValueType();
5417
5418 SDLoc DL(N);
5419 unsigned Opc = N->getOpcode();
5420 SDValue ShAmt = N->getOperand(Num: 2);
5421 EVT ShAmtVT = ShAmt.getValueType();
5422 EVT ShAmtCCVT = getSetCCResultType(VT: ShAmtVT);
5423
5424 // If the shift amount is at least half the bitwidth, swap the inputs.
5425 unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
5426 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL, VT: ShAmtVT, N1: ShAmt,
5427 N2: DAG.getConstant(Val: HalfVTBits, DL, VT: ShAmtVT));
5428 SDValue Cond =
5429 DAG.getSetCC(DL, VT: ShAmtCCVT, LHS: AndNode, RHS: DAG.getConstant(Val: 0, DL, VT: ShAmtVT),
5430 Cond: Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ);
5431
5432 // Expand to a pair of funnel shifts.
5433 EVT NewShAmtVT = TLI.getShiftAmountTy(LHSTy: HalfVT, DL: DAG.getDataLayout());
5434 SDValue NewShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: NewShAmtVT);
5435
5436 SDValue Select1 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In1, N3: In2);
5437 SDValue Select2 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In2, N3: In3);
5438 SDValue Select3 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In3, N3: In4);
5439 Lo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select2, N2: Select1, N3: NewShAmt);
5440 Hi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select3, N2: Select2, N3: NewShAmt);
5441}
5442
5443void DAGTypeLegalizer::ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo, SDValue &Hi) {
5444 if (N->getOpcode() != ISD::CLMUL) {
5445 SDValue Res = TLI.expandCLMUL(N, DAG);
5446 return SplitInteger(Op: Res, Lo, Hi);
5447 }
5448
5449 SDValue LL, LH, RL, RH;
5450 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
5451 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
5452 EVT HalfVT = LL.getValueType();
5453 SDLoc DL(N);
5454
5455 // The low bits are a direct CLMUL of the the low bits.
5456 Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: LL, N2: RL);
5457
5458 // We compute two Hi-Lo cross-products, XOR them, and XOR it with the overflow
5459 // of the CLMUL of the low bits (given by CLMULH of the low bits) to yield the
5460 // final high bits.
5461 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: LL, N2: RL);
5462 SDValue HiLoCross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: LL, N2: RH);
5463 SDValue HiLoCross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: LH, N2: RL);
5464 SDValue HiLoCross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: HiLoCross1, N2: HiLoCross2);
5465 Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: HiLoCross);
5466}
5467
5468void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
5469 SDValue &Hi) {
5470 EVT VT = N->getValueType(ResNo: 0);
5471 EVT HalfVT =
5472 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: N->getValueSizeInBits(ResNo: 0) / 2);
5473 SDLoc dl(N);
5474
5475 // We assume VSCALE(1) fits into a legal integer.
5476 APInt One(HalfVT.getSizeInBits(), 1);
5477 SDValue VScaleBase = DAG.getVScale(DL: dl, VT: HalfVT, MulImm: One);
5478 VScaleBase = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: VScaleBase);
5479 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: VScaleBase, N2: N->getOperand(Num: 0));
5480 SplitInteger(Op: Res, Lo, Hi);
5481}
5482
5483void DAGTypeLegalizer::ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo,
5484 SDValue &Hi) {
5485 const Function &Fn = DAG.getMachineFunction().getFunction();
5486 Fn.getContext().diagnose(DI: DiagnosticInfoLegalizationFailure(
5487 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
5488 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
5489 EVT LoVT, HiVT;
5490 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5491 Lo = DAG.getPOISON(VT: LoVT);
5492 Hi = DAG.getPOISON(VT: HiVT);
5493}
5494
5495//===----------------------------------------------------------------------===//
5496// Integer Operand Expansion
5497//===----------------------------------------------------------------------===//
5498
5499/// ExpandIntegerOperand - This method is called when the specified operand of
5500/// the specified node is found to need expansion. At this point, all of the
5501/// result types of the node are known to be legal, but other operands of the
5502/// node may need promotion or expansion as well as the specified one.
5503bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
5504 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG));
5505 SDValue Res = SDValue();
5506
5507 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
5508 return false;
5509
5510 switch (N->getOpcode()) {
5511 default:
5512 #ifndef NDEBUG
5513 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
5514 N->dump(&DAG); dbgs() << "\n";
5515 #endif
5516 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
5517
5518 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
5519 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
5520 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
5521 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
5522 case ISD::FAKE_USE:
5523 Res = ExpandOp_FAKE_USE(N);
5524 break;
5525 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
5526 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5527 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
5528 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
5529 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
5530 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
5531 case ISD::STRICT_SINT_TO_FP:
5532 case ISD::SINT_TO_FP:
5533 case ISD::STRICT_UINT_TO_FP:
5534 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break;
5535 case ISD::STORE: Res = ExpandIntOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo); break;
5536 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
5537
5538 case ISD::SHL:
5539 case ISD::SRA:
5540 case ISD::SRL:
5541 case ISD::ROTL:
5542 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
5543 case ISD::RETURNADDR:
5544 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
5545
5546 case ISD::SCMP:
5547 case ISD::UCMP: Res = ExpandIntOp_CMP(N); break;
5548
5549 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
5550 case ISD::STACKMAP:
5551 Res = ExpandIntOp_STACKMAP(N, OpNo);
5552 break;
5553 case ISD::PATCHPOINT:
5554 Res = ExpandIntOp_PATCHPOINT(N, OpNo);
5555 break;
5556 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5557 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5558 Res = ExpandIntOp_VP_STRIDED(N, OpNo);
5559 break;
5560 case ISD::WRITE_REGISTER:
5561 Res = ExpandIntOp_WRITE_REGISTER(N, OpNo);
5562 break;
5563 }
5564
5565 // If the result is null, the sub-method took care of registering results etc.
5566 if (!Res.getNode()) return false;
5567
5568 // If the result is N, the sub-method updated N in place. Tell the legalizer
5569 // core about this.
5570 if (Res.getNode() == N)
5571 return true;
5572
5573 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5574 "Invalid operand expansion");
5575
5576 ReplaceValueWith(From: SDValue(N, 0), To: Res);
5577 return false;
5578}
5579
5580/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
5581/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
5582void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
5583 SDValue &NewRHS,
5584 ISD::CondCode &CCCode,
5585 const SDLoc &dl) {
5586 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5587 GetExpandedInteger(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
5588 GetExpandedInteger(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
5589
5590 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
5591 if (RHSLo == RHSHi && isAllOnesConstant(V: RHSLo)) {
5592 // Equality comparison to -1.
5593 NewLHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: LHSHi);
5594 NewRHS = RHSLo;
5595 return;
5596 }
5597
5598 NewLHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
5599 NewRHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSHi, N2: RHSHi);
5600 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NewLHS.getValueType(), N1: NewLHS, N2: NewRHS);
5601 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: NewLHS.getValueType());
5602 return;
5603 }
5604
5605 // If this is a comparison of the sign bit, just look at the top part.
5606 // X > -1, x < 0
5607 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: NewRHS))
5608 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
5609 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
5610 NewLHS = LHSHi;
5611 NewRHS = RHSHi;
5612 return;
5613 }
5614
5615 // FIXME: This generated code sucks.
5616 ISD::CondCode LowCC;
5617 switch (CCCode) {
5618 default: llvm_unreachable("Unknown integer setcc!");
5619 case ISD::SETLT:
5620 case ISD::SETULT: LowCC = ISD::SETULT; break;
5621 case ISD::SETGT:
5622 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
5623 case ISD::SETLE:
5624 case ISD::SETULE: LowCC = ISD::SETULE; break;
5625 case ISD::SETGE:
5626 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
5627 }
5628
5629 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
5630 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
5631 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
5632
5633 // NOTE: on targets without efficient SELECT of bools, we can always use
5634 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
5635 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
5636 nullptr);
5637 SDValue LoCmp, HiCmp;
5638 if (TLI.isTypeLegal(VT: LHSLo.getValueType()) &&
5639 TLI.isTypeLegal(VT: RHSLo.getValueType()))
5640 LoCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSLo.getValueType()), N0: LHSLo,
5641 N1: RHSLo, Cond: LowCC, foldBooleans: false, DCI&: DagCombineInfo, dl);
5642 if (!LoCmp.getNode())
5643 LoCmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
5644 RHS: RHSLo, Cond: LowCC);
5645 if (TLI.isTypeLegal(VT: LHSHi.getValueType()) &&
5646 TLI.isTypeLegal(VT: RHSHi.getValueType()))
5647 HiCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSHi.getValueType()), N0: LHSHi,
5648 N1: RHSHi, Cond: CCCode, foldBooleans: false, DCI&: DagCombineInfo, dl);
5649 if (!HiCmp.getNode())
5650 HiCmp =
5651 DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()),
5652 N1: LHSHi, N2: RHSHi, N3: DAG.getCondCode(Cond: CCCode));
5653
5654 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(Val: LoCmp.getNode());
5655 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(Val: HiCmp.getNode());
5656
5657 bool EqAllowed = ISD::isTrueWhenEqual(Cond: CCCode);
5658
5659 // FIXME: Is the HiCmpC->isOne() here correct for
5660 // ZeroOrNegativeOneBooleanContent.
5661 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
5662 (!EqAllowed &&
5663 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
5664 // For LE / GE, if high part is known false, ignore the low part.
5665 // For LT / GT: if low part is known false, return the high part.
5666 // if high part is known true, ignore the low part.
5667 NewLHS = HiCmp;
5668 NewRHS = SDValue();
5669 return;
5670 }
5671
5672 if (LHSHi == RHSHi) {
5673 // Comparing the low bits is enough.
5674 NewLHS = LoCmp;
5675 NewRHS = SDValue();
5676 return;
5677 }
5678
5679 // Lower with SETCCCARRY if the target supports it.
5680 EVT HiVT = LHSHi.getValueType();
5681 EVT ExpandVT = TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: HiVT);
5682 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(Op: ISD::SETCCCARRY, VT: ExpandVT);
5683
5684 // FIXME: Make all targets support this, then remove the other lowering.
5685 if (HasSETCCCARRY) {
5686 // SETCCCARRY can detect < and >= directly. For > and <=, flip
5687 // operands and condition code.
5688 bool FlipOperands = false;
5689 switch (CCCode) {
5690 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
5691 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
5692 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
5693 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
5694 default: break;
5695 }
5696 if (FlipOperands) {
5697 std::swap(a&: LHSLo, b&: RHSLo);
5698 std::swap(a&: LHSHi, b&: RHSHi);
5699 }
5700 // Perform a wide subtraction, feeding the carry from the low part into
5701 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
5702 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
5703 // zero or positive iff LHS >= RHS.
5704 EVT LoVT = LHSLo.getValueType();
5705 SDVTList VTList = DAG.getVTList(VT1: LoVT, VT2: getSetCCResultType(VT: LoVT));
5706 SDValue LowCmp = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: LHSLo, N2: RHSLo);
5707 SDValue Res = DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: getSetCCResultType(VT: HiVT),
5708 N1: LHSHi, N2: RHSHi, N3: LowCmp.getValue(R: 1),
5709 N4: DAG.getCondCode(Cond: CCCode));
5710 NewLHS = Res;
5711 NewRHS = SDValue();
5712 return;
5713 }
5714
5715 NewLHS = TLI.SimplifySetCC(VT: getSetCCResultType(VT: HiVT), N0: LHSHi, N1: RHSHi, Cond: ISD::SETEQ,
5716 foldBooleans: false, DCI&: DagCombineInfo, dl);
5717 if (!NewLHS.getNode())
5718 NewLHS =
5719 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: HiVT), LHS: LHSHi, RHS: RHSHi, Cond: ISD::SETEQ);
5720 NewLHS = DAG.getSelect(DL: dl, VT: LoCmp.getValueType(), Cond: NewLHS, LHS: LoCmp, RHS: HiCmp);
5721 NewRHS = SDValue();
5722}
5723
5724SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
5725 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
5726 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
5727 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N));
5728
5729 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5730 // against zero to select between true and false values.
5731 if (!NewRHS.getNode()) {
5732 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
5733 CCCode = ISD::SETNE;
5734 }
5735
5736 // Update N to have the operands specified.
5737 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
5738 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
5739 Op5: N->getOperand(Num: 4)), 0);
5740}
5741
5742SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
5743 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
5744 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
5745 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N));
5746
5747 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5748 // against zero to select between true and false values.
5749 if (!NewRHS.getNode()) {
5750 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
5751 CCCode = ISD::SETNE;
5752 }
5753
5754 // Update N to have the operands specified.
5755 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
5756 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
5757 Op5: DAG.getCondCode(Cond: CCCode)), 0);
5758}
5759
5760SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
5761 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
5762 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
5763 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N));
5764
5765 // If ExpandSetCCOperands returned a scalar, use it.
5766 if (!NewRHS.getNode()) {
5767 assert(NewLHS.getValueType() == N->getValueType(0) &&
5768 "Unexpected setcc expansion!");
5769 return NewLHS;
5770 }
5771
5772 // Otherwise, update N to have the operands specified.
5773 return SDValue(
5774 DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, Op3: DAG.getCondCode(Cond: CCCode)), 0);
5775}
5776
5777SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
5778 SDValue LHS = N->getOperand(Num: 0);
5779 SDValue RHS = N->getOperand(Num: 1);
5780 SDValue Carry = N->getOperand(Num: 2);
5781 SDValue Cond = N->getOperand(Num: 3);
5782 SDLoc dl = SDLoc(N);
5783
5784 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5785 GetExpandedInteger(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
5786 GetExpandedInteger(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
5787
5788 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
5789 SDVTList VTList = DAG.getVTList(VT1: LHSLo.getValueType(), VT2: Carry.getValueType());
5790 SDValue LowCmp =
5791 DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: LHSLo, N2: RHSLo, N3: Carry);
5792 return DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: N->getValueType(ResNo: 0), N1: LHSHi, N2: RHSHi,
5793 N3: LowCmp.getValue(R: 1), N4: Cond);
5794}
5795
5796SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
5797 // Split the operand and replace with SPLAT_VECTOR_PARTS.
5798 SDValue Lo, Hi;
5799 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
5800 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo,
5801 N2: Hi);
5802}
5803
5804SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
5805 // The value being shifted is legal, but the shift amount is too big.
5806 // It follows that either the result of the shift is undefined, or the
5807 // upper half of the shift amount is zero. Just use the lower half.
5808 SDValue Lo, Hi;
5809 GetExpandedInteger(Op: N->getOperand(Num: 1), Lo, Hi);
5810 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Lo), 0);
5811}
5812
5813SDValue DAGTypeLegalizer::ExpandIntOp_CMP(SDNode *N) {
5814 return TLI.expandCMP(Node: N, DAG);
5815}
5816
5817SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
5818 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
5819 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
5820 // constant to valid type.
5821 SDValue Lo, Hi;
5822 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi);
5823 return SDValue(DAG.UpdateNodeOperands(N, Op: Lo), 0);
5824}
5825
5826SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
5827 bool IsStrict = N->isStrictFPOpcode();
5828 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
5829 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
5830 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
5831 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
5832 EVT DstVT = N->getValueType(ResNo: 0);
5833 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT)
5834 : RTLIB::getUINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT);
5835 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5836 "Don't know how to expand this XINT_TO_FP!");
5837 TargetLowering::MakeLibCallOptions CallOptions;
5838 CallOptions.setIsSigned(true);
5839 std::pair<SDValue, SDValue> Tmp =
5840 TLI.makeLibCall(DAG, LC, RetVT: DstVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
5841
5842 if (!IsStrict)
5843 return Tmp.first;
5844
5845 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
5846 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
5847 return SDValue();
5848}
5849
5850SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
5851 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?");
5852
5853 if (ISD::isNormalStore(N))
5854 return ExpandOp_NormalStore(N, OpNo);
5855
5856 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
5857 assert(OpNo == 1 && "Can only expand the stored value so far");
5858
5859 EVT VT = N->getOperand(Num: 1).getValueType();
5860 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5861 SDValue Ch = N->getChain();
5862 SDValue Ptr = N->getBasePtr();
5863 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
5864 AAMDNodes AAInfo = N->getAAInfo();
5865 SDLoc dl(N);
5866 SDValue Lo, Hi;
5867
5868 assert(NVT.isByteSized() && "Expanded type not byte sized!");
5869
5870 if (N->getMemoryVT().bitsLE(VT: NVT)) {
5871 GetExpandedInteger(Op: N->getValue(), Lo, Hi);
5872 return DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(),
5873 SVT: N->getMemoryVT(), Alignment: N->getBaseAlign(), MMOFlags,
5874 AAInfo);
5875 }
5876
5877 if (DAG.getDataLayout().isLittleEndian()) {
5878 // Little-endian - low bits are at low addresses.
5879 GetExpandedInteger(Op: N->getValue(), Lo, Hi);
5880
5881 Lo = DAG.getStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
5882 MMOFlags, AAInfo);
5883
5884 unsigned ExcessBits =
5885 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
5886 EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits);
5887
5888 // Increment the pointer to the other half.
5889 unsigned IncrementSize = NVT.getSizeInBits()/8;
5890 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
5891 Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr,
5892 PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize),
5893 SVT: NEVT, Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
5894 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
5895 }
5896
5897 // Big-endian - high bits are at low addresses. Favor aligned stores at
5898 // the cost of some bit-fiddling.
5899 GetExpandedInteger(Op: N->getValue(), Lo, Hi);
5900
5901 EVT ExtVT = N->getMemoryVT();
5902 unsigned EBytes = ExtVT.getStoreSize();
5903 unsigned IncrementSize = NVT.getSizeInBits()/8;
5904 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5905 EVT HiVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
5906 BitWidth: ExtVT.getSizeInBits() - ExcessBits);
5907
5908 if (ExcessBits < NVT.getSizeInBits()) {
5909 // Transfer high bits from the top of Lo to the bottom of Hi.
5910 Hi = DAG.getNode(
5911 Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi,
5912 N2: DAG.getShiftAmountConstant(Val: NVT.getSizeInBits() - ExcessBits, VT: NVT, DL: dl));
5913 Hi = DAG.getNode(
5914 Opcode: ISD::OR, DL: dl, VT: NVT, N1: Hi,
5915 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Lo,
5916 N2: DAG.getShiftAmountConstant(Val: ExcessBits, VT: NVT, DL: dl)));
5917 }
5918
5919 // Store both the high bits and maybe some of the low bits.
5920 Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr, PtrInfo: N->getPointerInfo(), SVT: HiVT,
5921 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
5922
5923 // Increment the pointer to the other half.
5924 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
5925 // Store the lowest ExcessBits bits in the second half.
5926 Lo = DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr,
5927 PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize),
5928 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits),
5929 Alignment: N->getBaseAlign(), MMOFlags, AAInfo);
5930 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
5931}
5932
5933SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
5934 SDValue InL, InH;
5935 GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH);
5936 // Just truncate the low part of the source.
5937 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InL);
5938}
5939
5940SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
5941 SDLoc dl(N);
5942 SDValue Swap =
5943 DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(),
5944 Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 2), Val: N->getOperand(Num: 1),
5945 MMO: cast<AtomicSDNode>(Val: N)->getMemOperand());
5946 return Swap.getValue(R: 1);
5947}
5948
5949SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
5950 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5951 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5952
5953 SDValue Hi; // The upper half is dropped out.
5954 SmallVector<SDValue, 8> NewOps(N->ops());
5955 GetExpandedInteger(Op: NewOps[OpNo], Lo&: NewOps[OpNo], Hi);
5956
5957 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
5958}
5959
5960SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo) {
5961 const Function &Fn = DAG.getMachineFunction().getFunction();
5962 Fn.getContext().diagnose(DI: DiagnosticInfoLegalizationFailure(
5963 "cannot use llvm.write_register with illegal type", Fn,
5964 N->getDebugLoc()));
5965
5966 return N->getOperand(Num: 0);
5967}
5968
5969SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
5970 SDLoc dl(N);
5971
5972 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
5973 SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1));
5974 EVT OutVT = V0.getValueType();
5975
5976 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, N1: V0, N2: V1, N3: N->getOperand(Num: 2));
5977}
5978
5979SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
5980 SDLoc DL(N);
5981 unsigned Factor = N->getNumOperands();
5982
5983 SmallVector<SDValue, 8> Ops(Factor);
5984 for (unsigned i = 0; i != Factor; i++)
5985 Ops[i] = GetPromotedInteger(Op: N->getOperand(Num: i));
5986
5987 SmallVector<EVT, 8> ResVTs(Factor, Ops[0].getValueType());
5988 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VTs: ResVTs), Ops);
5989
5990 for (unsigned i = 0; i != Factor; i++)
5991 SetPromotedInteger(Op: SDValue(N, i), Result: Res.getValue(R: i));
5992
5993 return SDValue();
5994}
5995
5996SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
5997
5998 EVT OutVT = N->getValueType(ResNo: 0);
5999 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6000 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6001 EVT NOutVTElem = NOutVT.getVectorElementType();
6002
6003 SDLoc dl(N);
6004 SDValue BaseIdx = N->getOperand(Num: 1);
6005
6006 // TODO: We may be able to use this for types other than scalable
6007 // vectors and fix those tests that expect BUILD_VECTOR to be used
6008 if (OutVT.isScalableVector()) {
6009 SDValue InOp0 = N->getOperand(Num: 0);
6010 EVT InVT = InOp0.getValueType();
6011
6012 // Try and extract from a smaller type so that it eventually falls
6013 // into the promotion code below.
6014 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector ||
6015 getTypeAction(VT: InVT) == TargetLowering::TypeLegal) {
6016 EVT NInVT = InVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
6017 unsigned NElts = NInVT.getVectorMinNumElements();
6018 uint64_t IdxVal = BaseIdx->getAsZExtVal();
6019
6020 SDValue Step1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NInVT, N1: InOp0,
6021 N2: DAG.getConstant(Val: alignDown(Value: IdxVal, Align: NElts), DL: dl,
6022 VT: BaseIdx.getValueType()));
6023 SDValue Step2 = DAG.getNode(
6024 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: Step1,
6025 N2: DAG.getConstant(Val: IdxVal % NElts, DL: dl, VT: BaseIdx.getValueType()));
6026 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Step2);
6027 }
6028
6029 // Try and extract from a widened type.
6030 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6031 SDValue Ops[] = {GetWidenedVector(Op: InOp0), BaseIdx};
6032 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: OutVT, Ops);
6033 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext);
6034 }
6035
6036 // Promote operands and see if this is handled by target lowering,
6037 // Otherwise, use the BUILD_VECTOR approach below
6038 if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) {
6039 // Collect the (promoted) operands
6040 SDValue Ops[] = { GetPromotedInteger(Op: InOp0), BaseIdx };
6041
6042 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
6043 assert(PromEltVT.bitsLE(NOutVTElem) &&
6044 "Promoted operand has an element type greater than result");
6045
6046 EVT ExtVT = NOutVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PromEltVT);
6047 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: ExtVT, Ops);
6048 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext);
6049 }
6050 }
6051
6052 if (OutVT.isScalableVector())
6053 report_fatal_error(reason: "Unable to promote scalable types using BUILD_VECTOR");
6054
6055 SDValue InOp0 = N->getOperand(Num: 0);
6056 if (getTypeAction(VT: InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
6057 InOp0 = GetPromotedInteger(Op: InOp0);
6058
6059 EVT InVT = InOp0.getValueType();
6060 EVT InSVT = InVT.getVectorElementType();
6061
6062 unsigned OutNumElems = OutVT.getVectorNumElements();
6063 SmallVector<SDValue, 8> Ops;
6064 Ops.reserve(N: OutNumElems);
6065 for (unsigned i = 0; i != OutNumElems; ++i) {
6066 // Extract the element from the original vector.
6067 SDValue Index = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BaseIdx.getValueType(), N1: BaseIdx,
6068 N2: DAG.getConstant(Val: i, DL: dl, VT: BaseIdx.getValueType()));
6069 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: InSVT,
6070 N1: N->getOperand(Num: 0), N2: Index);
6071 SDValue Op = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NOutVTElem);
6072 // Insert the converted element to the new vector.
6073 Ops.push_back(Elt: Op);
6074 }
6075
6076 return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops);
6077}
6078
6079SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
6080 EVT OutVT = N->getValueType(ResNo: 0);
6081 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6082 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6083
6084 SDLoc dl(N);
6085 SDValue Vec = N->getOperand(Num: 0);
6086 SDValue SubVec = N->getOperand(Num: 1);
6087 SDValue Idx = N->getOperand(Num: 2);
6088
6089 EVT SubVecVT = SubVec.getValueType();
6090 EVT NSubVT =
6091 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NOutVT.getVectorElementType(),
6092 EC: SubVecVT.getVectorElementCount());
6093
6094 Vec = GetPromotedInteger(Op: Vec);
6095 SubVec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NSubVT, Operand: SubVec);
6096
6097 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: NOutVT, N1: Vec, N2: SubVec, N3: Idx);
6098}
6099
6100SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
6101 SDLoc dl(N);
6102
6103 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
6104 EVT OutVT = V0.getValueType();
6105
6106 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: OutVT, Operand: V0);
6107}
6108
6109SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
6110 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(Val: N);
6111 EVT VT = N->getValueType(ResNo: 0);
6112 SDLoc dl(N);
6113
6114 ArrayRef<int> NewMask = SV->getMask().slice(N: 0, M: VT.getVectorNumElements());
6115
6116 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
6117 SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1));
6118 EVT OutVT = V0.getValueType();
6119
6120 return DAG.getVectorShuffle(VT: OutVT, dl, N1: V0, N2: V1, Mask: NewMask);
6121}
6122
6123SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
6124 EVT OutVT = N->getValueType(ResNo: 0);
6125 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6126 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6127 unsigned NumElems = N->getNumOperands();
6128 EVT NOutVTElem = NOutVT.getVectorElementType();
6129 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(Type: NOutVT);
6130 unsigned NOutExtOpc = TargetLowering::getExtendForContent(Content: NOutBoolType);
6131 SDLoc dl(N);
6132
6133 SmallVector<SDValue, 8> Ops;
6134 Ops.reserve(N: NumElems);
6135 for (unsigned i = 0; i != NumElems; ++i) {
6136 SDValue Op = N->getOperand(Num: i);
6137 EVT OpVT = Op.getValueType();
6138 // BUILD_VECTOR integer operand types are allowed to be larger than the
6139 // result's element type. This may still be true after the promotion. For
6140 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
6141 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
6142 if (OpVT.bitsLT(VT: NOutVTElem)) {
6143 unsigned ExtOpc = ISD::ANY_EXTEND;
6144 // Attempt to extend constant bool vectors to match target's BooleanContent.
6145 // While not necessary, this improves chances of the constant correctly
6146 // folding with compare results (e.g. for NOT patterns).
6147 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant)
6148 ExtOpc = NOutExtOpc;
6149 Op = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: NOutVTElem, Operand: Op);
6150 }
6151 Ops.push_back(Elt: Op);
6152 }
6153
6154 return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops);
6155}
6156
6157SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
6158
6159 SDLoc dl(N);
6160
6161 assert(!N->getOperand(0).getValueType().isVector() &&
6162 "Input must be a scalar");
6163
6164 EVT OutVT = N->getValueType(ResNo: 0);
6165 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6166 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6167 EVT NOutElemVT = NOutVT.getVectorElementType();
6168
6169 SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutElemVT, Operand: N->getOperand(Num: 0));
6170 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NOutVT, Operand: Op);
6171}
6172
6173SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
6174 SDLoc dl(N);
6175 EVT OutVT = N->getValueType(ResNo: 0);
6176 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6177 assert(NOutVT.isScalableVector() &&
6178 "Type must be promoted to a scalable vector type");
6179 const APInt &StepVal = N->getConstantOperandAPInt(Num: 0);
6180 return DAG.getStepVector(DL: dl, ResVT: NOutVT,
6181 StepVal: StepVal.sext(width: NOutVT.getScalarSizeInBits()));
6182}
6183
6184SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
6185 SDLoc dl(N);
6186
6187 EVT OutVT = N->getValueType(ResNo: 0);
6188 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6189 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6190
6191 unsigned NumOperands = N->getNumOperands();
6192 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
6193 EVT OutElemTy = NOutVT.getVectorElementType();
6194 if (OutVT.isScalableVector()) {
6195 // Find the largest promoted element type for each of the operands.
6196 SDUse *MaxSizedValue = std::max_element(
6197 first: N->op_begin(), last: N->op_end(), comp: [](const SDValue &A, const SDValue &B) {
6198 EVT AVT = A.getValueType().getVectorElementType();
6199 EVT BVT = B.getValueType().getVectorElementType();
6200 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
6201 });
6202 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();
6203
6204 // Then promote all vectors to the largest element type.
6205 SmallVector<SDValue, 8> Ops;
6206 for (unsigned I = 0; I < NumOperands; ++I) {
6207 SDValue Op = N->getOperand(Num: I);
6208 EVT OpVT = Op.getValueType();
6209 if (getTypeAction(VT: OpVT) == TargetLowering::TypePromoteInteger)
6210 Op = GetPromotedInteger(Op);
6211 else
6212 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&
6213 "Unhandled legalization type");
6214
6215 if (OpVT.getVectorElementType().getScalarSizeInBits() <
6216 MaxElementVT.getScalarSizeInBits())
6217 Op = DAG.getAnyExtOrTrunc(
6218 Op, DL: dl,
6219 VT: OpVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MaxElementVT));
6220 Ops.push_back(Elt: Op);
6221 }
6222
6223 // Do the CONCAT on the promoted type and finally truncate to (the promoted)
6224 // NOutVT.
6225 return DAG.getAnyExtOrTrunc(
6226 Op: DAG.getNode(
6227 Opcode: ISD::CONCAT_VECTORS, DL: dl,
6228 VT: OutVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MaxElementVT),
6229 Ops),
6230 DL: dl, VT: NOutVT);
6231 }
6232
6233 unsigned NumElem = N->getOperand(Num: 0).getValueType().getVectorNumElements();
6234 assert(NumElem * NumOperands == NumOutElem &&
6235 "Unexpected number of elements");
6236
6237 // Take the elements from the first vector.
6238 SmallVector<SDValue, 8> Ops(NumOutElem);
6239 for (unsigned i = 0; i < NumOperands; ++i) {
6240 SDValue Op = N->getOperand(Num: i);
6241 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteInteger)
6242 Op = GetPromotedInteger(Op);
6243 EVT SclrTy = Op.getValueType().getVectorElementType();
6244 assert(NumElem == Op.getValueType().getVectorNumElements() &&
6245 "Unexpected number of elements");
6246
6247 for (unsigned j = 0; j < NumElem; ++j) {
6248 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Op,
6249 N2: DAG.getVectorIdxConstant(Val: j, DL: dl));
6250 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: OutElemTy);
6251 }
6252 }
6253
6254 return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops);
6255}
6256
6257SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
6258 EVT VT = N->getValueType(ResNo: 0);
6259 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6260 assert(NVT.isVector() && "This type must be promoted to a vector type");
6261
6262 SDLoc dl(N);
6263
6264 // For operands whose TypeAction is to promote, extend the promoted node
6265 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
6266 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
6267 // type..
6268 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType())
6269 == TargetLowering::TypePromoteInteger) {
6270 SDValue Promoted;
6271
6272 switch(N->getOpcode()) {
6273 case ISD::SIGN_EXTEND_VECTOR_INREG:
6274 Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0));
6275 break;
6276 case ISD::ZERO_EXTEND_VECTOR_INREG:
6277 Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0));
6278 break;
6279 case ISD::ANY_EXTEND_VECTOR_INREG:
6280 Promoted = GetPromotedInteger(Op: N->getOperand(Num: 0));
6281 break;
6282 default:
6283 llvm_unreachable("Node has unexpected Opcode");
6284 }
6285 unsigned NewSize = NVT.getSizeInBits();
6286 if (Promoted.getValueType().getSizeInBits() > NewSize) {
6287 EVT ExtractVT = EVT::getVectorVT(
6288 Context&: *DAG.getContext(), VT: Promoted.getValueType().getVectorElementType(),
6289 NumElements: NewSize / Promoted.getScalarValueSizeInBits());
6290
6291 Promoted = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: ExtractVT, N1: Promoted,
6292 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6293 }
6294 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Promoted);
6295 }
6296
6297 // Directly extend to the appropriate transform-to type.
6298 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
6299}
6300
6301SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
6302 EVT VT = N->getValueType(ResNo: 0);
6303 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6304 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: SDLoc(N), VT: NVT, Ops: N->ops());
6305}
6306
6307SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
6308 EVT VT = N->getValueType(ResNo: 0);
6309 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6310 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
6311}
6312
6313SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
6314 SDLoc DL(N);
6315 EVT VT = N->getValueType(ResNo: 0);
6316 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6317 SDValue ExtAcc = GetPromotedInteger(Op: N->getOperand(Num: 0));
6318 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: ExtAcc, N2: N->getOperand(Num: 1),
6319 N3: N->getOperand(Num: 2));
6320}
6321
6322SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
6323 EVT OutVT = N->getValueType(ResNo: 0);
6324 EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT);
6325 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
6326
6327 EVT NOutVTElem = NOutVT.getVectorElementType();
6328
6329 SDLoc dl(N);
6330 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
6331
6332 SDValue ConvElem = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl,
6333 VT: NOutVTElem, Operand: N->getOperand(Num: 1));
6334 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NOutVT,
6335 N1: V0, N2: ConvElem, N3: N->getOperand(Num: 2));
6336}
6337
6338SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
6339 // The VECREDUCE result size may be larger than the element size, so
6340 // we can simply change the result type.
6341 SDLoc dl(N);
6342 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6343 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Ops: N->ops());
6344}
6345
6346SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
6347 // The VP_REDUCE result size may be larger than the element size, so we can
6348 // simply change the result type. However the start value and result must be
6349 // the same.
6350 SDLoc DL(N);
6351 SDValue Start = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0));
6352 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: Start.getValueType(), N1: Start,
6353 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6354}
6355
6356SDValue DAGTypeLegalizer::PromoteIntRes_PATCHPOINT(SDNode *N) {
6357 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6358 SDLoc dl(N);
6359
6360 assert(N->getNumValues() == 3 && "Expected 3 values for PATCHPOINT");
6361 SDVTList VTList = DAG.getVTList(VTs: {NVT, MVT::Other, MVT::Glue});
6362
6363 SmallVector<SDValue> Ops(N->ops());
6364 SDValue Res = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList, Ops);
6365
6366 // Replace chain and glue uses with the new patchpoint.
6367 SDValue From[] = {SDValue(N, 1), SDValue(N, 2)};
6368 SDValue To[] = {Res.getValue(R: 1), Res.getValue(R: 2)};
6369 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
6370
6371 return Res.getValue(R: 0);
6372}
6373
6374SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(SDNode *N) {
6375 const Function &Fn = DAG.getMachineFunction().getFunction();
6376 Fn.getContext().diagnose(DI: DiagnosticInfoLegalizationFailure(
6377 "cannot use llvm.read_register with illegal type", Fn, N->getDebugLoc()));
6378
6379 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6380 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
6381 return DAG.getPOISON(VT: NVT);
6382}
6383
6384SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6385 SDLoc dl(N);
6386 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
6387 SDValue V1 = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl,
6388 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
6389 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl,
6390 VT: V0->getValueType(ResNo: 0).getScalarType(), N1: V0, N2: V1);
6391
6392 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
6393 // element types. If this is the case then we need to expand the outgoing
6394 // value and not truncate it.
6395 return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0));
6396}
6397
6398SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
6399 SDLoc dl(N);
6400 // The result type is equal to the first input operand's type, so the
6401 // type that needs promoting must be the second source vector.
6402 SDValue V0 = N->getOperand(Num: 0);
6403 SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1));
6404 SDValue Idx = N->getOperand(Num: 2);
6405 EVT PromVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6406 VT: V1.getValueType().getVectorElementType(),
6407 EC: V0.getValueType().getVectorElementCount());
6408 V0 = DAG.getAnyExtOrTrunc(Op: V0, DL: dl, VT: PromVT);
6409 SDValue Ext = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: PromVT, N1: V0, N2: V1, N3: Idx);
6410 return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0));
6411}
6412
6413// FIXME: We wouldn't need this if clang could promote short integers
6414// that are arguments to FAKE_USE.
6415SDValue DAGTypeLegalizer::PromoteIntOp_FAKE_USE(SDNode *N) {
6416 SDLoc dl(N);
6417 SDValue V0 = N->getOperand(Num: 0);
6418 SDValue V1 = N->getOperand(Num: 1);
6419 EVT InVT1 = V1.getValueType();
6420 SDValue VPromoted =
6421 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl,
6422 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT1), Operand: V1);
6423 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: V0, N2: VPromoted);
6424}
6425
6426SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
6427 SDLoc dl(N);
6428 SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0));
6429 MVT InVT = V0.getValueType().getSimpleVT();
6430 MVT OutVT = MVT::getVectorVT(VT: InVT.getVectorElementType(),
6431 NumElements: N->getValueType(ResNo: 0).getVectorNumElements());
6432 SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: V0, N2: N->getOperand(Num: 1));
6433 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Ext);
6434}
6435
6436SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
6437 SDLoc dl(N);
6438
6439 EVT ResVT = N->getValueType(ResNo: 0);
6440 unsigned NumElems = N->getNumOperands();
6441
6442 if (ResVT.isScalableVector()) {
6443 SDValue ResVec = DAG.getUNDEF(VT: ResVT);
6444
6445 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6446 SDValue Op = N->getOperand(Num: OpIdx);
6447 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
6448 ResVec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: ResVec, N2: Op,
6449 N3: DAG.getIntPtrConstant(Val: OpIdx * OpNumElts, DL: dl));
6450 }
6451
6452 return ResVec;
6453 }
6454
6455 EVT RetSclrTy = N->getValueType(ResNo: 0).getVectorElementType();
6456
6457 SmallVector<SDValue, 8> NewOps;
6458 NewOps.reserve(N: NumElems);
6459
6460 // For each incoming vector
6461 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6462 SDValue Incoming = GetPromotedInteger(Op: N->getOperand(Num: VecIdx));
6463 EVT SclrTy = Incoming->getValueType(ResNo: 0).getVectorElementType();
6464 unsigned NumElem = Incoming->getValueType(ResNo: 0).getVectorNumElements();
6465
6466 for (unsigned i=0; i<NumElem; ++i) {
6467 // Extract element from incoming vector
6468 SDValue Ex = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Incoming,
6469 N2: DAG.getVectorIdxConstant(Val: i, DL: dl));
6470 SDValue Tr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RetSclrTy, Operand: Ex);
6471 NewOps.push_back(Elt: Tr);
6472 }
6473 }
6474
6475 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: dl, Ops: NewOps);
6476}
6477
6478SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
6479 assert(OpNo > 1);
6480 SDValue Op = N->getOperand(Num: OpNo);
6481
6482 // FIXME: Non-constant operands are not yet handled:
6483 // - https://github.com/llvm/llvm-project/issues/26431
6484 // - https://github.com/llvm/llvm-project/issues/55957
6485 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op);
6486 if (!CN)
6487 return SDValue();
6488
6489 // Copy operands before the one being expanded.
6490 SmallVector<SDValue> NewOps;
6491 for (unsigned I = 0; I < OpNo; I++)
6492 NewOps.push_back(Elt: N->getOperand(Num: I));
6493
6494 EVT Ty = Op.getValueType();
6495 SDLoc DL = SDLoc(N);
6496 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6497 NewOps.push_back(
6498 Elt: DAG.getTargetConstant(Val: StackMaps::ConstantOp, DL, VT: MVT::i64));
6499 NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty));
6500 } else {
6501 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6502 return SDValue();
6503 }
6504
6505 // Copy remaining operands.
6506 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6507 NewOps.push_back(Elt: N->getOperand(Num: I));
6508
6509 SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps);
6510
6511 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6512 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
6513
6514 return SDValue(); // Signal that we have replaced the node already.
6515}
6516
6517SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
6518 assert(OpNo >= 7);
6519 SDValue Op = N->getOperand(Num: OpNo);
6520
6521 // FIXME: Non-constant operands are not yet handled:
6522 // - https://github.com/llvm/llvm-project/issues/26431
6523 // - https://github.com/llvm/llvm-project/issues/55957
6524 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op);
6525 if (!CN)
6526 return SDValue();
6527
6528 // Copy operands before the one being expanded.
6529 SmallVector<SDValue> NewOps;
6530 for (unsigned I = 0; I < OpNo; I++)
6531 NewOps.push_back(Elt: N->getOperand(Num: I));
6532
6533 EVT Ty = Op.getValueType();
6534 SDLoc DL = SDLoc(N);
6535 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6536 NewOps.push_back(
6537 Elt: DAG.getTargetConstant(Val: StackMaps::ConstantOp, DL, VT: MVT::i64));
6538 NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty));
6539 } else {
6540 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6541 return SDValue();
6542 }
6543
6544 // Copy remaining operands.
6545 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6546 NewOps.push_back(Elt: N->getOperand(Num: I));
6547
6548 SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps);
6549
6550 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6551 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
6552
6553 return SDValue(); // Signal that we have replaced the node already.
6554}
6555