1//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that LoongArch uses to lower LLVM code into
10// a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LoongArchISelLowering.h"
15#include "LoongArch.h"
16#include "LoongArchMachineFunctionInfo.h"
17#include "LoongArchRegisterInfo.h"
18#include "LoongArchSelectionDAGInfo.h"
19#include "LoongArchSubtarget.h"
20#include "MCTargetDesc/LoongArchBaseInfo.h"
21#include "MCTargetDesc/LoongArchMCTargetDesc.h"
22#include "MCTargetDesc/LoongArchMatInt.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/IntrinsicsLoongArch.h"
33#include "llvm/Support/CodeGen.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include <llvm/Analysis/VectorUtils.h>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "loongarch-isel-lowering"
43
44STATISTIC(NumTailCalls, "Number of tail calls");
45
46enum MaterializeFPImm {
47 NoMaterializeFPImm = 0,
48 MaterializeFPImm2Ins = 2,
49 MaterializeFPImm3Ins = 3,
50 MaterializeFPImm4Ins = 4,
51 MaterializeFPImm5Ins = 5,
52 MaterializeFPImm6Ins = 6
53};
54
55static cl::opt<MaterializeFPImm> MaterializeFPImmInsNum(
56 "loongarch-materialize-float-imm", cl::Hidden,
57 cl::desc("Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
60 cl::init(Val: MaterializeFPImm3Ins),
61 cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"),
62 clEnumValN(MaterializeFPImm2Ins, "2",
63 "Materialize FP immediate within 2 instructions"),
64 clEnumValN(MaterializeFPImm3Ins, "3",
65 "Materialize FP immediate within 3 instructions"),
66 clEnumValN(MaterializeFPImm4Ins, "4",
67 "Materialize FP immediate within 4 instructions"),
68 clEnumValN(MaterializeFPImm5Ins, "5",
69 "Materialize FP immediate within 5 instructions"),
70 clEnumValN(MaterializeFPImm6Ins, "6",
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
73
74static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
75 cl::desc("Trap on integer division by zero."),
76 cl::init(Val: false));
77
78LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
79 const LoongArchSubtarget &STI)
80 : TargetLowering(TM, STI), Subtarget(STI) {
81
82 MVT GRLenVT = Subtarget.getGRLenVT();
83
84 // Set up the register classes.
85
86 addRegisterClass(VT: GRLenVT, RC: &LoongArch::GPRRegClass);
87 if (Subtarget.hasBasicF())
88 addRegisterClass(VT: MVT::f32, RC: &LoongArch::FPR32RegClass);
89 if (Subtarget.hasBasicD())
90 addRegisterClass(VT: MVT::f64, RC: &LoongArch::FPR64RegClass);
91
92 static const MVT::SimpleValueType LSXVTs[] = {
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
94 static const MVT::SimpleValueType LASXVTs[] = {
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
96
97 if (Subtarget.hasExtLSX())
98 for (MVT VT : LSXVTs)
99 addRegisterClass(VT, RC: &LoongArch::LSX128RegClass);
100
101 if (Subtarget.hasExtLASX())
102 for (MVT VT : LASXVTs)
103 addRegisterClass(VT, RC: &LoongArch::LASX256RegClass);
104
105 // Set operations for LA32 and LA64.
106
107 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: GRLenVT,
108 MemVT: MVT::i1, Action: Promote);
109
110 setOperationAction(Op: ISD::SHL_PARTS, VT: GRLenVT, Action: Custom);
111 setOperationAction(Op: ISD::SRA_PARTS, VT: GRLenVT, Action: Custom);
112 setOperationAction(Op: ISD::SRL_PARTS, VT: GRLenVT, Action: Custom);
113 setOperationAction(Op: ISD::FP_TO_SINT, VT: GRLenVT, Action: Custom);
114 setOperationAction(Op: ISD::ROTL, VT: GRLenVT, Action: Expand);
115 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Expand);
116
117 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
118 ISD::JumpTable, ISD::GlobalTLSAddress},
119 VT: GRLenVT, Action: Custom);
120
121 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: GRLenVT, Action: Custom);
122
123 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: GRLenVT, Action: Custom);
124 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
125 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
126 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
127
128 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
129 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
130
131 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
132 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
133 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
134
135 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
136
137 // BITREV/REVB requires the 32S feature.
138 if (STI.has32S()) {
139 // Expand bitreverse.i16 with native-width bitrev and shift for now, before
140 // we get to know which of sll and revb.2h is faster.
141 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
142 setOperationAction(Op: ISD::BITREVERSE, VT: GRLenVT, Action: Legal);
143
144 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
145 // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
146 // and i32 could still be byte-swapped relatively cheaply.
147 setOperationAction(Op: ISD::BSWAP, VT: MVT::i16, Action: Custom);
148 } else {
149 setOperationAction(Op: ISD::BSWAP, VT: GRLenVT, Action: Expand);
150 setOperationAction(Op: ISD::CTTZ, VT: GRLenVT, Action: Expand);
151 setOperationAction(Op: ISD::CTLZ, VT: GRLenVT, Action: Expand);
152 setOperationAction(Op: ISD::ROTR, VT: GRLenVT, Action: Expand);
153 setOperationAction(Op: ISD::SELECT, VT: GRLenVT, Action: Custom);
154 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
155 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
156 }
157
158 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
159 setOperationAction(Op: ISD::BR_CC, VT: GRLenVT, Action: Expand);
160 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
161 setOperationAction(Op: ISD::SELECT_CC, VT: GRLenVT, Action: Expand);
162 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
163 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: GRLenVT, Action: Expand);
164
165 setOperationAction(Op: ISD::FP_TO_UINT, VT: GRLenVT, Action: Custom);
166 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Expand);
167
168 // Set operations for LA64 only.
169
170 if (Subtarget.is64Bit()) {
171 setOperationAction(Op: ISD::ADD, VT: MVT::i32, Action: Custom);
172 setOperationAction(Op: ISD::SUB, VT: MVT::i32, Action: Custom);
173 setOperationAction(Op: ISD::SHL, VT: MVT::i32, Action: Custom);
174 setOperationAction(Op: ISD::SRA, VT: MVT::i32, Action: Custom);
175 setOperationAction(Op: ISD::SRL, VT: MVT::i32, Action: Custom);
176 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
177 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
178 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Custom);
179 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Custom);
180 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
181 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Custom);
182 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
183 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i32, Action: Custom);
184 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i32, Action: Custom);
185 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
186 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
187 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i32, Action: Custom);
188
189 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Custom);
190 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Custom);
191 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: MVT::i32,
192 Action: Custom);
193 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
194 }
195
196 // Set operations for LA32 only.
197
198 if (!Subtarget.is64Bit()) {
199 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
200 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
201 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i64, Action: Custom);
202 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
203 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i64, Action: Custom);
204 if (Subtarget.hasBasicD())
205 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
206 }
207
208 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
209
210 static const ISD::CondCode FPCCToExpand[] = {
211 ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
212 ISD::SETGE, ISD::SETNE, ISD::SETGT};
213
214 // Set operations for 'F' feature.
215
216 if (Subtarget.hasBasicF()) {
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
218 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
220 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
221 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
222
223 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
224 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
225 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
226 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Legal);
227 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
228 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
229 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
230 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
231 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
232 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
233 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
234 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Legal);
235 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
236 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
237 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
238 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
239 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
240 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32,
241 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
242 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32,
243 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
244 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
245 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
246 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
247 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
248 setOperationAction(Op: ISD::GET_ROUNDING, VT: GRLenVT, Action: Custom);
249
250 if (Subtarget.is64Bit())
251 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
252
253 if (!Subtarget.hasBasicD()) {
254 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
255 if (Subtarget.is64Bit()) {
256 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
257 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
258 }
259 }
260 }
261
262 // Set operations for 'D' feature.
263
264 if (Subtarget.hasBasicD()) {
265 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
266 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
267 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
268 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
269 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
271 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
272
273 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
274 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
275 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
276 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
277 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
278 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Legal);
279 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
280 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
281 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
282 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
283 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
284 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Legal);
285 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
286 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
287 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
288 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
289 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
290 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
291 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64,
292 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
293 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
294 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
295 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
296
297 if (Subtarget.is64Bit())
298 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
299 }
300
301 // Set operations for 'LSX' feature.
302
303 if (Subtarget.hasExtLSX()) {
304 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
305 // Expand all truncating stores and extending loads.
306 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
307 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
308 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
309 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
310 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
311 }
312 // By default everything must be expanded. Then we will selectively turn
313 // on ones that can be effectively codegen'd.
314 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
315 setOperationAction(Op, VT, Action: Expand);
316 }
317
318 for (MVT VT : LSXVTs) {
319 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
320 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
321 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
322
323 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
324 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
325 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
326
327 setOperationAction(Op: ISD::SETCC, VT, Action: Legal);
328 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
329 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
330 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
331 }
332 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
333 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
334 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
335 Action: Legal);
336 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
337 VT, Action: Legal);
338 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
339 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
340 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
341 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
342 setCondCodeAction(
343 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
344 Action: Expand);
345 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
346 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
347 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
348 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
349 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
350 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
351 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
352 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
353 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
354 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
355 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
356 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
357 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
358 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
359 }
360 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
361 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
362 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
363 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
364 for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
365 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Action: Legal);
366 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
367 }
368 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Custom);
369 for (MVT VT : {MVT::v4f32, MVT::v2f64}) {
370 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
371 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
372 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
373 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
374 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
375 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
376 ISD::SETUGE, ISD::SETUGT},
377 VT, Action: Expand);
378 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
379 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
380 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
381 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
382 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
383 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
384 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
385 }
386 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Legal);
387 setOperationAction(Ops: ISD::FCEIL, VTs: {MVT::f32, MVT::f64}, Action: Legal);
388 setOperationAction(Ops: ISD::FFLOOR, VTs: {MVT::f32, MVT::f64}, Action: Legal);
389 setOperationAction(Ops: ISD::FTRUNC, VTs: {MVT::f32, MVT::f64}, Action: Legal);
390 setOperationAction(Ops: ISD::FROUNDEVEN, VTs: {MVT::f32, MVT::f64}, Action: Legal);
391
392 for (MVT VT :
393 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
394 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
395 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
396 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
397 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
398 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
399 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
400 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
401 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
402 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
403 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
404 }
405 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Custom);
406 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
407 // We want to legalize this to an f64 load rather than an i64 load.
408 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Custom);
409 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
410 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
411 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
412 MVT::v16i64})
413 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Custom);
414 }
415
416 // Set operations for 'LASX' feature.
417
418 if (Subtarget.hasExtLASX()) {
419 for (MVT VT : LASXVTs) {
420 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
421 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
422 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
423
424 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
425 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
426 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
427 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
428 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT, Action: Legal);
429
430 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
431 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
432 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
433 }
434 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
435 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
436 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
437 Action: Legal);
438 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
439 VT, Action: Legal);
440 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
441 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
442 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
443 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
444 setCondCodeAction(
445 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
446 Action: Expand);
447 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
448 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
449 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
450 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
451 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
452 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
453 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
454 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
455 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
456 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
457 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
458 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
459 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
460 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
461 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
462 }
463 for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
464 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
465 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
466 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
467 for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
468 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
469 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Legal);
470 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
471 }
472 for (MVT VT : {MVT::v8f32, MVT::v4f64}) {
473 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
474 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
475 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
476 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
477 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
478 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
479 ISD::SETUGE, ISD::SETUGT},
480 VT, Action: Expand);
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
482 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
483 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
484 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
485 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
486 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
487 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
488 }
489 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v4f32, Action: Custom);
490 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f64, Action: Custom);
491 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
492 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Legal);
493 setOperationAction(Op: ISD::ZERO_EXTEND, VT, Action: Legal);
494 setOperationAction(Op: ISD::ANY_EXTEND, VT, Action: Custom);
495 }
496 for (MVT VT :
497 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
498 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Legal);
499 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Legal);
500 }
501 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
502 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Legal);
503 }
504
505 // Set DAG combine for LA32 and LA64.
506 if (Subtarget.hasBasicF()) {
507 setTargetDAGCombine(ISD::SINT_TO_FP);
508 }
509
510 setTargetDAGCombine(ISD::AND);
511 setTargetDAGCombine(ISD::OR);
512 setTargetDAGCombine(ISD::SRL);
513 setTargetDAGCombine(ISD::SETCC);
514
515 // Set DAG combine for 'LSX' feature.
516
517 if (Subtarget.hasExtLSX()) {
518 setTargetDAGCombine(ISD::ADD);
519 setTargetDAGCombine(ISD::SUB);
520 setTargetDAGCombine(ISD::SHL);
521 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
522 setTargetDAGCombine(ISD::BITCAST);
523 setTargetDAGCombine(ISD::VSELECT);
524 setTargetDAGCombine(ISD::FP_TO_SINT);
525 setTargetDAGCombine(ISD::FP_TO_UINT);
526 setTargetDAGCombine(ISD::UINT_TO_FP);
527 setTargetDAGCombine(ISD::ZERO_EXTEND);
528 setTargetDAGCombine(ISD::SIGN_EXTEND);
529 }
530
531 // Set DAG combine for 'LASX' feature.
532 if (Subtarget.hasExtLASX()) {
533 setTargetDAGCombine(ISD::ANY_EXTEND);
534 setTargetDAGCombine(ISD::CONCAT_VECTORS);
535 }
536
537 // Compute derived properties from the register classes.
538 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
539
540 setStackPointerRegisterToSaveRestore(LoongArch::R3);
541
542 setBooleanContents(ZeroOrOneBooleanContent);
543 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
544
545 setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());
546
547 setMinCmpXchgSizeInBits(32);
548
549 // Function alignments.
550 setMinFunctionAlignment(Align(4));
551 // Set preferred alignments.
552 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
553 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
554 setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
555
556 // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
557 if (Subtarget.hasLAMCAS())
558 setMinCmpXchgSizeInBits(8);
559
560 if (Subtarget.hasSCQ()) {
561 setMaxAtomicSizeInBitsSupported(128);
562 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i128, Action: Custom);
563 }
564
565 // Disable strict node mutation.
566 IsStrictFPEnabled = true;
567}
568
569bool LoongArchTargetLowering::isOffsetFoldingLegal(
570 const GlobalAddressSDNode *GA) const {
571 // In order to maximise the opportunity for common subexpression elimination,
572 // keep a separate ADD node for the global address offset instead of folding
573 // it in the global address node. Later peephole optimisations may choose to
574 // fold it back in when profitable.
575 return false;
576}
577
578SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
579 SelectionDAG &DAG) const {
580 switch (Op.getOpcode()) {
581 case ISD::ATOMIC_FENCE:
582 return lowerATOMIC_FENCE(Op, DAG);
583 case ISD::EH_DWARF_CFA:
584 return lowerEH_DWARF_CFA(Op, DAG);
585 case ISD::GlobalAddress:
586 return lowerGlobalAddress(Op, DAG);
587 case ISD::GlobalTLSAddress:
588 return lowerGlobalTLSAddress(Op, DAG);
589 case ISD::INTRINSIC_WO_CHAIN:
590 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
591 case ISD::INTRINSIC_W_CHAIN:
592 return lowerINTRINSIC_W_CHAIN(Op, DAG);
593 case ISD::INTRINSIC_VOID:
594 return lowerINTRINSIC_VOID(Op, DAG);
595 case ISD::BlockAddress:
596 return lowerBlockAddress(Op, DAG);
597 case ISD::JumpTable:
598 return lowerJumpTable(Op, DAG);
599 case ISD::SHL_PARTS:
600 return lowerShiftLeftParts(Op, DAG);
601 case ISD::SRA_PARTS:
602 return lowerShiftRightParts(Op, DAG, IsSRA: true);
603 case ISD::SRL_PARTS:
604 return lowerShiftRightParts(Op, DAG, IsSRA: false);
605 case ISD::ConstantPool:
606 return lowerConstantPool(Op, DAG);
607 case ISD::FP_TO_SINT:
608 return lowerFP_TO_SINT(Op, DAG);
609 case ISD::FP_TO_UINT:
610 return lowerFP_TO_UINT(Op, DAG);
611 case ISD::BITCAST:
612 return lowerBITCAST(Op, DAG);
613 case ISD::UINT_TO_FP:
614 return lowerUINT_TO_FP(Op, DAG);
615 case ISD::SINT_TO_FP:
616 return lowerSINT_TO_FP(Op, DAG);
617 case ISD::VASTART:
618 return lowerVASTART(Op, DAG);
619 case ISD::FRAMEADDR:
620 return lowerFRAMEADDR(Op, DAG);
621 case ISD::RETURNADDR:
622 return lowerRETURNADDR(Op, DAG);
623 case ISD::SET_ROUNDING:
624 return lowerSET_ROUNDING(Op, DAG);
625 case ISD::GET_ROUNDING:
626 return lowerGET_ROUNDING(Op, DAG);
627 case ISD::WRITE_REGISTER:
628 return lowerWRITE_REGISTER(Op, DAG);
629 case ISD::INSERT_VECTOR_ELT:
630 return lowerINSERT_VECTOR_ELT(Op, DAG);
631 case ISD::EXTRACT_VECTOR_ELT:
632 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
633 case ISD::BUILD_VECTOR:
634 return lowerBUILD_VECTOR(Op, DAG);
635 case ISD::CONCAT_VECTORS:
636 return lowerCONCAT_VECTORS(Op, DAG);
637 case ISD::VECTOR_SHUFFLE:
638 return lowerVECTOR_SHUFFLE(Op, DAG);
639 case ISD::BITREVERSE:
640 return lowerBITREVERSE(Op, DAG);
641 case ISD::SCALAR_TO_VECTOR:
642 return lowerSCALAR_TO_VECTOR(Op, DAG);
643 case ISD::PREFETCH:
644 return lowerPREFETCH(Op, DAG);
645 case ISD::SELECT:
646 return lowerSELECT(Op, DAG);
647 case ISD::BRCOND:
648 return lowerBRCOND(Op, DAG);
649 case ISD::FP_TO_FP16:
650 return lowerFP_TO_FP16(Op, DAG);
651 case ISD::FP16_TO_FP:
652 return lowerFP16_TO_FP(Op, DAG);
653 case ISD::FP_TO_BF16:
654 return lowerFP_TO_BF16(Op, DAG);
655 case ISD::BF16_TO_FP:
656 return lowerBF16_TO_FP(Op, DAG);
657 case ISD::VECREDUCE_ADD:
658 return lowerVECREDUCE_ADD(Op, DAG);
659 case ISD::ROTL:
660 case ISD::ROTR:
661 return lowerRotate(Op, DAG);
662 case ISD::VECREDUCE_AND:
663 case ISD::VECREDUCE_OR:
664 case ISD::VECREDUCE_XOR:
665 case ISD::VECREDUCE_SMAX:
666 case ISD::VECREDUCE_SMIN:
667 case ISD::VECREDUCE_UMAX:
668 case ISD::VECREDUCE_UMIN:
669 return lowerVECREDUCE(Op, DAG);
670 case ISD::ConstantFP:
671 return lowerConstantFP(Op, DAG);
672 case ISD::SETCC:
673 return lowerSETCC(Op, DAG);
674 case ISD::FP_ROUND:
675 return lowerFP_ROUND(Op, DAG);
676 case ISD::FP_EXTEND:
677 return lowerFP_EXTEND(Op, DAG);
678 case ISD::SIGN_EXTEND_VECTOR_INREG:
679 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
680 case ISD::DYNAMIC_STACKALLOC:
681 return lowerDYNAMIC_STACKALLOC(Op, DAG);
682 case ISD::ANY_EXTEND:
683 return lowerANY_EXTEND(Op, DAG);
684 }
685 return SDValue();
686}
687
688// Helper to attempt to return a cheaper, bit-inverted version of \p V.
689static SDValue isNOT(SDValue V, SelectionDAG &DAG) {
690 // TODO: don't always ignore oneuse constraints.
691 V = peekThroughBitcasts(V);
692 EVT VT = V.getValueType();
693
694 // Match not(xor X, -1) -> X.
695 if (V.getOpcode() == ISD::XOR &&
696 (ISD::isBuildVectorAllOnes(N: V.getOperand(i: 1).getNode()) ||
697 isAllOnesConstant(V: V.getOperand(i: 1))))
698 return V.getOperand(i: 0);
699
700 // Match not(extract_subvector(not(X)) -> extract_subvector(X).
701 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
702 (isNullConstant(V: V.getOperand(i: 1)) || V.getOperand(i: 0).hasOneUse())) {
703 if (SDValue Not = isNOT(V: V.getOperand(i: 0), DAG)) {
704 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
705 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Not), VT, N1: Not,
706 N2: V.getOperand(i: 1));
707 }
708 }
709
710 // Match not(SplatVector(not(X)) -> SplatVector(X).
711 if (V.getOpcode() == ISD::BUILD_VECTOR) {
712 if (SDValue SplatValue =
713 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
714 if (!V->isOnlyUserOf(N: SplatValue.getNode()))
715 return SDValue();
716
717 if (SDValue Not = isNOT(V: SplatValue, DAG)) {
718 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
719 return DAG.getSplat(VT, DL: SDLoc(Not), Op: Not);
720 }
721 }
722 }
723
724 // Match not(or(not(X),not(Y))) -> and(X, Y).
725 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
726 V.getOperand(i: 0).hasOneUse() && V.getOperand(i: 1).hasOneUse()) {
727 // TODO: Handle cases with single NOT operand -> VANDN
728 if (SDValue Op1 = isNOT(V: V.getOperand(i: 1), DAG))
729 if (SDValue Op0 = isNOT(V: V.getOperand(i: 0), DAG))
730 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(V), VT, N1: DAG.getBitcast(VT, V: Op0),
731 N2: DAG.getBitcast(VT, V: Op1));
732 }
733
734 // TODO: Add more matching patterns. Such as,
735 // not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
736 // not(slt(C, X)) -> slt(X - 1, C)
737 return SDValue();
738}
739
740// Combine two ISD::FP_ROUND / LoongArchISD::VFCVT nodes with same type to
741// LoongArchISD::VFCVT. For example:
742// x1 = fp_round x, 0
743// y1 = fp_round y, 0
744// z = concat_vectors x1, y1
745// Or
746// x1 = LoongArch::VFCVT undef, x
747// y1 = LoongArch::VFCVT undef, y
748// z = LoongArchISD::VPACKEV y1, x1; or LoongArchISD::VPERMI y1, x1, 68
749// can be combined to:
750// z = LoongArch::VFCVT y, x
751static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG,
752 const LoongArchSubtarget &Subtarget) {
753 assert(((N->getOpcode() == ISD::CONCAT_VECTORS && N->getNumOperands() == 2) ||
754 (N->getOpcode() == LoongArchISD::VPACKEV) ||
755 (N->getOpcode() == LoongArchISD::VPERMI)) &&
756 "Invalid Node");
757
758 SDValue Op0 = peekThroughBitcasts(V: N->getOperand(Num: 0));
759 SDValue Op1 = peekThroughBitcasts(V: N->getOperand(Num: 1));
760 unsigned Opcode0 = Op0.getOpcode();
761 unsigned Opcode1 = Op1.getOpcode();
762 if (Opcode0 != Opcode1)
763 return SDValue();
764
765 if (Opcode0 != ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
766 return SDValue();
767
768 // Check if two nodes have only one use.
769 if (!Op0.hasOneUse() || !Op1.hasOneUse())
770 return SDValue();
771
772 EVT VT = N.getValueType();
773 EVT SVT0 = Op0.getValueType();
774 EVT SVT1 = Op1.getValueType();
775 // Check if two nodes have the same result type.
776 if (SVT0 != SVT1)
777 return SDValue();
778
779 // Check if two nodes have the same operand type.
780 EVT SSVT0 = Op0.getOperand(i: 0).getValueType();
781 EVT SSVT1 = Op1.getOperand(i: 0).getValueType();
782 if (SSVT0 != SSVT1)
783 return SDValue();
784
785 if (N->getOpcode() == ISD::CONCAT_VECTORS && Opcode0 == ISD::FP_ROUND) {
786 if (Subtarget.hasExtLASX() && VT.is256BitVector() && SVT0 == MVT::v4f32 &&
787 SSVT0 == MVT::v4f64) {
788 // A vector_shuffle is required in the final step, as xvfcvt instruction
789 // operates on each 128-bit segament as a lane.
790 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v8f32,
791 N1: Op1.getOperand(i: 0), N2: Op0.getOperand(i: 0));
792 SDValue Undef = DAG.getUNDEF(VT: Res.getValueType());
793 // After VFCVT, the high part of Res comes from the high parts of Op0 and
794 // Op1, and the low part comes from the low parts of Op0 and Op1. However,
795 // the desired order requires Op0 to fully occupy the lower half and Op1
796 // the upper half of Res. The Mask reorders the elements of Res to achieve
797 // this:
798 // - The first four elements (0, 1, 4, 5) come from Op0.
799 // - The next four elements (2, 3, 6, 7) come from Op1.
800 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
801 Res = DAG.getVectorShuffle(VT: Res.getValueType(), dl: DL, N1: Res, N2: Undef, Mask);
802 return DAG.getBitcast(VT, V: Res);
803 }
804 }
805
806 if ((N->getOpcode() == LoongArchISD::VPACKEV ||
807 N->getOpcode() == LoongArchISD::VPERMI) &&
808 Opcode0 == LoongArchISD::VFCVT) {
809 // For VPACKEV or VPERMI, check if the first operation of VFCVT is undef.
810 if (!Op0.getOperand(i: 0).isUndef() || !Op1.getOperand(i: 0).isUndef())
811 return SDValue();
812
813 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
814 return SDValue();
815
816 if (N->getOpcode() == LoongArchISD::VPACKEV &&
817 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
818 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32,
819 N1: Op0.getOperand(i: 1), N2: Op1.getOperand(i: 1));
820 return DAG.getBitcast(VT, V: Res);
821 }
822
823 if (N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
824 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
825 if (Imm != 68)
826 return SDValue();
827 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Op0.getOperand(i: 1),
828 N2: Op1.getOperand(i: 1));
829 }
830 }
831
832 return SDValue();
833}
834
835SDValue LoongArchTargetLowering::lowerFP_ROUND(SDValue Op,
836 SelectionDAG &DAG) const {
837 SDLoc DL(Op);
838 SDValue In = Op.getOperand(i: 0);
839 MVT VT = Op.getSimpleValueType();
840 MVT SVT = In.getSimpleValueType();
841
842 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
843 SDValue Lo, Hi;
844 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: In, DL);
845 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT, N1: Hi, N2: Lo);
846 }
847
848 return SDValue();
849}
850
851SDValue LoongArchTargetLowering::lowerFP_EXTEND(SDValue Op,
852 SelectionDAG &DAG) const {
853
854 SDLoc DL(Op);
855 EVT VT = Op.getValueType();
856 SDValue Src = Op->getOperand(Num: 0);
857 EVT SVT = Src.getValueType();
858
859 bool V2F32ToV2F64 =
860 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
861 bool V4F32ToV4F64 =
862 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
863 if (!V2F32ToV2F64 && !V4F32ToV4F64)
864 return SDValue();
865
866 // Check if Op is the high part of vector.
867 auto CheckVecHighPart = [](SDValue Op) {
868 Op = peekThroughBitcasts(V: Op);
869 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
870 SDValue SOp = Op.getOperand(i: 0);
871 EVT SVT = SOp.getValueType();
872 if (!SVT.isVector() || (SVT.getVectorNumElements() % 2 != 0))
873 return SDValue();
874
875 const uint64_t Imm = Op.getConstantOperandVal(i: 1);
876 if (Imm == SVT.getVectorNumElements() / 2)
877 return SOp;
878 return SDValue();
879 }
880 return SDValue();
881 };
882
883 unsigned Opcode;
884 SDValue VFCVTOp;
885 EVT WideOpVT = SVT.getSimpleVT().getDoubleNumVectorElementsVT();
886 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
887
888 // If the operand of ISD::FP_EXTEND comes from the high part of vector,
889 // generate LoongArchISD::VFCVTH, otherwise LoongArchISD::VFCVTL.
890 if (SDValue V = CheckVecHighPart(Src)) {
891 assert(V.getValueSizeInBits() == WideOpVT.getSizeInBits() &&
892 "Unexpected wide vector");
893 Opcode = LoongArchISD::VFCVTH;
894 VFCVTOp = DAG.getBitcast(VT: WideOpVT, V);
895 } else {
896 Opcode = LoongArchISD::VFCVTL;
897 VFCVTOp = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideOpVT,
898 N1: DAG.getUNDEF(VT: WideOpVT), N2: Src, N3: ZeroIdx);
899 }
900
901 // v2f64 = fp_extend v2f32
902 if (V2F32ToV2F64)
903 return DAG.getNode(Opcode, DL, VT, Operand: VFCVTOp);
904
905 // v4f64 = fp_extend v4f32
906 if (V4F32ToV4F64) {
907 // XVFCVT instruction operates on each 128-bit segment as a lane, so a
908 // vector_shuffle is required firstly.
909 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
910 SDValue Res = DAG.getVectorShuffle(VT: WideOpVT, dl: DL, N1: VFCVTOp,
911 N2: DAG.getUNDEF(VT: WideOpVT), Mask);
912 Res = DAG.getNode(Opcode, DL, VT, Operand: Res);
913 return Res;
914 }
915
916 return SDValue();
917}
918
919SDValue LoongArchTargetLowering::lowerConstantFP(SDValue Op,
920 SelectionDAG &DAG) const {
921 EVT VT = Op.getValueType();
922 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
923 const APFloat &FPVal = CFP->getValueAPF();
924 SDLoc DL(CFP);
925
926 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
927 (VT == MVT::f64 && Subtarget.hasBasicD()));
928
929 // If value is 0.0 or -0.0, just ignore it.
930 if (FPVal.isZero())
931 return SDValue();
932
933 // If lsx enabled, use cheaper 'vldi' instruction if possible.
934 if (isFPImmVLDILegal(Imm: FPVal, VT))
935 return SDValue();
936
937 // Construct as integer, and move to float register.
938 APInt INTVal = FPVal.bitcastToAPInt();
939
940 // If more than MaterializeFPImmInsNum instructions will be used to
941 // generate the INTVal and move it to float register, fallback to
942 // use floating point load from the constant pool.
943 auto Seq = LoongArchMatInt::generateInstSeq(Val: INTVal.getSExtValue());
944 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
945 if (InsNum > MaterializeFPImmInsNum && !FPVal.isOne())
946 return SDValue();
947
948 switch (VT.getSimpleVT().SimpleTy) {
949 default:
950 llvm_unreachable("Unexpected floating point type!");
951 break;
952 case MVT::f32: {
953 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i32);
954 if (Subtarget.is64Bit())
955 NewVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: NewVal);
956 return DAG.getNode(Opcode: Subtarget.is64Bit() ? LoongArchISD::MOVGR2FR_W_LA64
957 : LoongArchISD::MOVGR2FR_W,
958 DL, VT, Operand: NewVal);
959 }
960 case MVT::f64: {
961 if (Subtarget.is64Bit()) {
962 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i64);
963 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D, DL, VT, Operand: NewVal);
964 }
965 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
966 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
967 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D_LO_HI, DL, VT, N1: Lo, N2: Hi);
968 }
969 }
970
971 return SDValue();
972}
973
974// Ensure SETCC result and operand have the same bit width; isel does not
975// support mismatched widths.
976SDValue LoongArchTargetLowering::lowerSETCC(SDValue Op,
977 SelectionDAG &DAG) const {
978 SDLoc DL(Op);
979 EVT ResultVT = Op.getValueType();
980 EVT OperandVT = Op.getOperand(i: 0).getValueType();
981
982 EVT SetCCResultVT =
983 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT);
984
985 if (ResultVT == SetCCResultVT)
986 return Op;
987
988 assert(Op.getOperand(0).getValueType() == Op.getOperand(1).getValueType() &&
989 "SETCC operands must have the same type!");
990
991 SDValue SetCCNode =
992 DAG.getNode(Opcode: ISD::SETCC, DL, VT: SetCCResultVT, N1: Op.getOperand(i: 0),
993 N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2));
994
995 if (ResultVT.bitsGT(VT: SetCCResultVT))
996 SetCCNode = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: ResultVT, Operand: SetCCNode);
997 else if (ResultVT.bitsLT(VT: SetCCResultVT))
998 SetCCNode = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: SetCCNode);
999
1000 return SetCCNode;
1001}
1002
1003// Lower sext_invec using vslti instructions.
1004// For example:
1005// %b = sext <4 x i16> %a to <4 x i32>
1006// can be lowered to:
1007// VSLTI_H vr2, vr1, 0
1008// VILVL.H vr1, vr2, vr1
1009SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1010 SDValue Op, SelectionDAG &DAG) const {
1011 SDLoc DL(Op);
1012 SDValue Src = Op.getOperand(i: 0);
1013 MVT SrcVT = Src.getSimpleValueType();
1014 MVT DstVT = Op.getSimpleValueType();
1015
1016 if (!SrcVT.is128BitVector())
1017 return SDValue();
1018
1019 // lower to VSLTI + VILVL if extend could be done in single step.
1020 if (DstVT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits() == 2) {
1021 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1022 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Src, N2: Zero,
1023 N3: DAG.getCondCode(Cond: ISD::SETLT));
1024 SDValue LoInterleaved =
1025 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Src);
1026
1027 return DAG.getBitcast(VT: DstVT, V: LoInterleaved);
1028 }
1029
1030 return SDValue();
1031}
1032
1033// ANY_EXTEND can be replaced by ZERO_EXTEND when LASX is enabled.
1034SDValue LoongArchTargetLowering::lowerANY_EXTEND(SDValue Op,
1035 SelectionDAG &DAG) const {
1036 assert(Subtarget.hasExtLASX());
1037 // We don't have corresponding instrunction for ANY_EXTEND, lowering it to
1038 // ZERO_EXTEND won't break its semantics, while avoid scalar extract/insert.
1039 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: Op.getValueType(),
1040 Operand: Op.getOperand(i: 0));
1041}
1042
1043// Lower vecreduce_add using vhaddw instructions.
1044// For Example:
1045// call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
1046// can be lowered to:
1047// VHADDW_D_W vr0, vr0, vr0
1048// VHADDW_Q_D vr0, vr0, vr0
1049// VPICKVE2GR_D a0, vr0, 0
1050// ADDI_W a0, a0, 0
1051SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
1052 SelectionDAG &DAG) const {
1053
1054 SDLoc DL(Op);
1055 MVT OpVT = Op.getSimpleValueType();
1056 SDValue Val = Op.getOperand(i: 0);
1057
1058 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1059 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1060 unsigned ResBits = OpVT.getScalarSizeInBits();
1061
1062 unsigned LegalVecSize = 128;
1063 bool isLASX256Vector =
1064 Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
1065
1066 // Ensure operand type legal or enable it legal.
1067 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1068 Val = DAG.WidenVector(N: Val, DL);
1069 }
1070
1071 // NumEles is designed for iterations count, v4i32 for LSX
1072 // and v8i32 for LASX should have the same count.
1073 if (isLASX256Vector) {
1074 NumEles /= 2;
1075 LegalVecSize = 256;
1076 }
1077
1078 EleBits *= 2;
1079 for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1080 EleBits = std::min(a: EleBits, b: 64u);
1081 MVT IntTy = MVT::getIntegerVT(BitWidth: EleBits);
1082 MVT VecTy = MVT::getVectorVT(VT: IntTy, NumElements: LegalVecSize / EleBits);
1083 Val = DAG.getNode(Opcode: LoongArchISD::VHADDW, DL, VT: VecTy, N1: Val, N2: Val);
1084 }
1085
1086 if (isLASX256Vector) {
1087 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: Val,
1088 N2: DAG.getConstant(Val: 2, DL, VT: Subtarget.getGRLenVT()));
1089 Val = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::v4i64, N1: Tmp, N2: Val);
1090 }
1091
1092 Val = DAG.getBitcast(VT: MVT::getVectorVT(VT: OpVT, NumElements: LegalVecSize / ResBits), V: Val);
1093 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1094 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
1095}
1096
1097// Lower vecreduce_and/or/xor/[s/u]max/[s/u]min.
1098// For Example:
1099// call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
1100// can be lowered to:
1101// VBSRL_V vr1, vr0, 8
1102// VMAX_W vr0, vr1, vr0
1103// VBSRL_V vr1, vr0, 4
1104// VMAX_W vr0, vr1, vr0
1105// VPICKVE2GR_W a0, vr0, 0
1106// For 256 bit vector, it is illegal and will be spilt into
1107// two 128 bit vector by default then processed by this.
1108SDValue LoongArchTargetLowering::lowerVECREDUCE(SDValue Op,
1109 SelectionDAG &DAG) const {
1110 SDLoc DL(Op);
1111
1112 MVT OpVT = Op.getSimpleValueType();
1113 SDValue Val = Op.getOperand(i: 0);
1114
1115 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1116 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1117
1118 // Ensure operand type legal or enable it legal.
1119 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1120 Val = DAG.WidenVector(N: Val, DL);
1121 }
1122
1123 unsigned Opcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
1124 MVT VecTy = Val.getSimpleValueType();
1125 MVT GRLenVT = Subtarget.getGRLenVT();
1126
1127 for (int i = NumEles; i > 1; i /= 2) {
1128 SDValue ShiftAmt = DAG.getConstant(Val: i * EleBits / 16, DL, VT: GRLenVT);
1129 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: VecTy, N1: Val, N2: ShiftAmt);
1130 Val = DAG.getNode(Opcode, DL, VT: VecTy, N1: Tmp, N2: Val);
1131 }
1132
1133 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1134 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
1135}
1136
1137SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
1138 SelectionDAG &DAG) const {
1139 unsigned IsData = Op.getConstantOperandVal(i: 4);
1140
1141 // We don't support non-data prefetch.
1142 // Just preserve the chain.
1143 if (!IsData)
1144 return Op.getOperand(i: 0);
1145
1146 return Op;
1147}
1148
1149SDValue LoongArchTargetLowering::lowerRotate(SDValue Op,
1150 SelectionDAG &DAG) const {
1151 MVT VT = Op.getSimpleValueType();
1152 assert(VT.isVector() && "Unexpected type");
1153
1154 SDLoc DL(Op);
1155 SDValue R = Op.getOperand(i: 0);
1156 SDValue Amt = Op.getOperand(i: 1);
1157 unsigned Opcode = Op.getOpcode();
1158 unsigned EltSizeInBits = VT.getScalarSizeInBits();
1159
1160 auto checkCstSplat = [](SDValue V, APInt &CstSplatValue) {
1161 if (V.getOpcode() != ISD::BUILD_VECTOR)
1162 return false;
1163 if (SDValue SplatValue =
1164 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
1165 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
1166 CstSplatValue = C->getAPIntValue();
1167 return true;
1168 }
1169 }
1170 return false;
1171 };
1172
1173 // Check for constant splat rotation amount.
1174 APInt CstSplatValue;
1175 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1176 bool isROTL = Opcode == ISD::ROTL;
1177
1178 // Check for splat rotate by zero.
1179 if (IsCstSplat && CstSplatValue.urem(RHS: EltSizeInBits) == 0)
1180 return R;
1181
1182 // LoongArch targets always prefer ISD::ROTR.
1183 if (isROTL) {
1184 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
1185 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: R,
1186 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: Amt));
1187 }
1188
1189 // Rotate by a immediate.
1190 if (IsCstSplat) {
1191 // ISD::ROTR: Attemp to rotate by a positive immediate.
1192 SDValue Bits = DAG.getConstant(Val: EltSizeInBits, DL, VT);
1193 if (SDValue Urem =
1194 DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL, VT, Ops: {Amt, Bits}))
1195 return DAG.getNode(Opcode, DL, VT, N1: R, N2: Urem);
1196 }
1197
1198 return Op;
1199}
1200
1201// Return true if Val is equal to (setcc LHS, RHS, CC).
1202// Return false if Val is the inverse of (setcc LHS, RHS, CC).
1203// Otherwise, return std::nullopt.
1204static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
1205 ISD::CondCode CC, SDValue Val) {
1206 assert(Val->getOpcode() == ISD::SETCC);
1207 SDValue LHS2 = Val.getOperand(i: 0);
1208 SDValue RHS2 = Val.getOperand(i: 1);
1209 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
1210
1211 if (LHS == LHS2 && RHS == RHS2) {
1212 if (CC == CC2)
1213 return true;
1214 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1215 return false;
1216 } else if (LHS == RHS2 && RHS == LHS2) {
1217 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
1218 if (CC == CC2)
1219 return true;
1220 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1221 return false;
1222 }
1223
1224 return std::nullopt;
1225}
1226
1227static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
1228 const LoongArchSubtarget &Subtarget) {
1229 SDValue CondV = N->getOperand(Num: 0);
1230 SDValue TrueV = N->getOperand(Num: 1);
1231 SDValue FalseV = N->getOperand(Num: 2);
1232 MVT VT = N->getSimpleValueType(ResNo: 0);
1233 SDLoc DL(N);
1234
1235 // (select c, -1, y) -> -c | y
1236 if (isAllOnesConstant(V: TrueV)) {
1237 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1238 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1239 }
1240 // (select c, y, -1) -> (c-1) | y
1241 if (isAllOnesConstant(V: FalseV)) {
1242 SDValue Neg =
1243 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1244 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1245 }
1246
1247 // (select c, 0, y) -> (c-1) & y
1248 if (isNullConstant(V: TrueV)) {
1249 SDValue Neg =
1250 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1251 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1252 }
1253 // (select c, y, 0) -> -c & y
1254 if (isNullConstant(V: FalseV)) {
1255 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1256 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1257 }
1258
1259 // select c, ~x, x --> xor -c, x
1260 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
1261 const APInt &TrueVal = TrueV->getAsAPIntVal();
1262 const APInt &FalseVal = FalseV->getAsAPIntVal();
1263 if (~TrueVal == FalseVal) {
1264 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1265 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
1266 }
1267 }
1268
1269 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
1270 // when both truev and falsev are also setcc.
1271 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
1272 FalseV.getOpcode() == ISD::SETCC) {
1273 SDValue LHS = CondV.getOperand(i: 0);
1274 SDValue RHS = CondV.getOperand(i: 1);
1275 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1276
1277 // (select x, x, y) -> x | y
1278 // (select !x, x, y) -> x & y
1279 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
1280 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
1281 N2: DAG.getFreeze(V: FalseV));
1282 }
1283 // (select x, y, x) -> x & y
1284 // (select !x, y, x) -> x | y
1285 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
1286 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
1287 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
1288 }
1289 }
1290
1291 return SDValue();
1292}
1293
1294// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
1295// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
1296// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
1297// being `0` or `-1`. In such cases we can replace `select` with `and`.
1298// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
1299// than `c0`?
1300static SDValue
1301foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
1302 const LoongArchSubtarget &Subtarget) {
1303 unsigned SelOpNo = 0;
1304 SDValue Sel = BO->getOperand(Num: 0);
1305 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1306 SelOpNo = 1;
1307 Sel = BO->getOperand(Num: 1);
1308 }
1309
1310 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1311 return SDValue();
1312
1313 unsigned ConstSelOpNo = 1;
1314 unsigned OtherSelOpNo = 2;
1315 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
1316 ConstSelOpNo = 2;
1317 OtherSelOpNo = 1;
1318 }
1319 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
1320 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
1321 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
1322 return SDValue();
1323
1324 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
1325 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
1326 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
1327 return SDValue();
1328
1329 SDLoc DL(Sel);
1330 EVT VT = BO->getValueType(ResNo: 0);
1331
1332 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1333 if (SelOpNo == 1)
1334 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
1335
1336 SDValue NewConstOp =
1337 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
1338 if (!NewConstOp)
1339 return SDValue();
1340
1341 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
1342 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
1343 return SDValue();
1344
1345 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
1346 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1347 if (SelOpNo == 1)
1348 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
1349 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
1350
1351 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1352 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1353 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
1354}
1355
1356// Changes the condition code and swaps operands if necessary, so the SetCC
1357// operation matches one of the comparisons supported directly by branches
1358// in the LoongArch ISA. May adjust compares to favor compare with 0 over
1359// compare with 1/-1.
1360static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1361 ISD::CondCode &CC, SelectionDAG &DAG) {
1362 // If this is a single bit test that can't be handled by ANDI, shift the
1363 // bit to be tested to the MSB and perform a signed compare with 0.
1364 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
1365 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1366 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
1367 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
1368 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
1369 unsigned ShAmt = 0;
1370 if (isPowerOf2_64(Value: Mask)) {
1371 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
1372 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
1373 } else {
1374 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
1375 }
1376
1377 LHS = LHS.getOperand(i: 0);
1378 if (ShAmt != 0)
1379 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
1380 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
1381 return;
1382 }
1383 }
1384
1385 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
1386 int64_t C = RHSC->getSExtValue();
1387 switch (CC) {
1388 default:
1389 break;
1390 case ISD::SETGT:
1391 // Convert X > -1 to X >= 0.
1392 if (C == -1) {
1393 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1394 CC = ISD::SETGE;
1395 return;
1396 }
1397 break;
1398 case ISD::SETLT:
1399 // Convert X < 1 to 0 >= X.
1400 if (C == 1) {
1401 RHS = LHS;
1402 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1403 CC = ISD::SETGE;
1404 return;
1405 }
1406 break;
1407 }
1408 }
1409
1410 switch (CC) {
1411 default:
1412 break;
1413 case ISD::SETGT:
1414 case ISD::SETLE:
1415 case ISD::SETUGT:
1416 case ISD::SETULE:
1417 CC = ISD::getSetCCSwappedOperands(Operation: CC);
1418 std::swap(a&: LHS, b&: RHS);
1419 break;
1420 }
1421}
1422
1423SDValue LoongArchTargetLowering::lowerSELECT(SDValue Op,
1424 SelectionDAG &DAG) const {
1425 SDValue CondV = Op.getOperand(i: 0);
1426 SDValue TrueV = Op.getOperand(i: 1);
1427 SDValue FalseV = Op.getOperand(i: 2);
1428 SDLoc DL(Op);
1429 MVT VT = Op.getSimpleValueType();
1430 MVT GRLenVT = Subtarget.getGRLenVT();
1431
1432 if (SDValue V = combineSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
1433 return V;
1434
1435 if (Op.hasOneUse()) {
1436 unsigned UseOpc = Op->user_begin()->getOpcode();
1437 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
1438 SDNode *BinOp = *Op->user_begin();
1439 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
1440 DAG, Subtarget)) {
1441 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
1442 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
1443 // may return a constant node and cause crash in lowerSELECT.
1444 if (NewSel.getOpcode() == ISD::SELECT)
1445 return lowerSELECT(Op: NewSel, DAG);
1446 return NewSel;
1447 }
1448 }
1449 }
1450
1451 // If the condition is not an integer SETCC which operates on GRLenVT, we need
1452 // to emit a LoongArchISD::SELECT_CC comparing the condition to zero. i.e.:
1453 // (select condv, truev, falsev)
1454 // -> (loongarchisd::select_cc condv, zero, setne, truev, falsev)
1455 if (CondV.getOpcode() != ISD::SETCC ||
1456 CondV.getOperand(i: 0).getSimpleValueType() != GRLenVT) {
1457 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: GRLenVT);
1458 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
1459
1460 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
1461
1462 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1463 }
1464
1465 // If the CondV is the output of a SETCC node which operates on GRLenVT
1466 // inputs, then merge the SETCC node into the lowered LoongArchISD::SELECT_CC
1467 // to take advantage of the integer compare+branch instructions. i.e.: (select
1468 // (setcc lhs, rhs, cc), truev, falsev)
1469 // -> (loongarchisd::select_cc lhs, rhs, cc, truev, falsev)
1470 SDValue LHS = CondV.getOperand(i: 0);
1471 SDValue RHS = CondV.getOperand(i: 1);
1472 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1473
1474 // Special case for a select of 2 constants that have a difference of 1.
1475 // Normally this is done by DAGCombine, but if the select is introduced by
1476 // type legalization or op legalization, we miss it. Restricting to SETLT
1477 // case for now because that is what signed saturating add/sub need.
1478 // FIXME: We don't need the condition to be SETLT or even a SETCC,
1479 // but we would probably want to swap the true/false values if the condition
1480 // is SETGE/SETLE to avoid an XORI.
1481 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
1482 CCVal == ISD::SETLT) {
1483 const APInt &TrueVal = TrueV->getAsAPIntVal();
1484 const APInt &FalseVal = FalseV->getAsAPIntVal();
1485 if (TrueVal - 1 == FalseVal)
1486 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
1487 if (TrueVal + 1 == FalseVal)
1488 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
1489 }
1490
1491 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1492 // 1 < x ? x : 1 -> 0 < x ? x : 1
1493 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
1494 RHS == TrueV && LHS == FalseV) {
1495 LHS = DAG.getConstant(Val: 0, DL, VT);
1496 // 0 <u x is the same as x != 0.
1497 if (CCVal == ISD::SETULT) {
1498 std::swap(a&: LHS, b&: RHS);
1499 CCVal = ISD::SETNE;
1500 }
1501 }
1502
1503 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
1504 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
1505 RHS == FalseV) {
1506 RHS = DAG.getConstant(Val: 0, DL, VT);
1507 }
1508
1509 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1510
1511 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
1512 // (select (setcc lhs, rhs, CC), constant, falsev)
1513 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
1514 std::swap(a&: TrueV, b&: FalseV);
1515 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
1516 }
1517
1518 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
1519 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1520}
1521
1522SDValue LoongArchTargetLowering::lowerBRCOND(SDValue Op,
1523 SelectionDAG &DAG) const {
1524 SDValue CondV = Op.getOperand(i: 1);
1525 SDLoc DL(Op);
1526 MVT GRLenVT = Subtarget.getGRLenVT();
1527
1528 if (CondV.getOpcode() == ISD::SETCC) {
1529 if (CondV.getOperand(i: 0).getValueType() == GRLenVT) {
1530 SDValue LHS = CondV.getOperand(i: 0);
1531 SDValue RHS = CondV.getOperand(i: 1);
1532 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1533
1534 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1535
1536 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1537 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1538 N1: Op.getOperand(i: 0), N2: LHS, N3: RHS, N4: TargetCC,
1539 N5: Op.getOperand(i: 2));
1540 } else if (CondV.getOperand(i: 0).getValueType().isFloatingPoint()) {
1541 return DAG.getNode(Opcode: LoongArchISD::BRCOND, DL, VT: Op.getValueType(),
1542 N1: Op.getOperand(i: 0), N2: CondV, N3: Op.getOperand(i: 2));
1543 }
1544 }
1545
1546 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1547 N1: Op.getOperand(i: 0), N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT),
1548 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
1549}
1550
1551SDValue
1552LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
1553 SelectionDAG &DAG) const {
1554 SDLoc DL(Op);
1555 MVT OpVT = Op.getSimpleValueType();
1556
1557 SDValue Vector = DAG.getUNDEF(VT: OpVT);
1558 SDValue Val = Op.getOperand(i: 0);
1559 SDValue Idx = DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT());
1560
1561 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: OpVT, N1: Vector, N2: Val, N3: Idx);
1562}
1563
1564SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
1565 SelectionDAG &DAG) const {
1566 EVT ResTy = Op->getValueType(ResNo: 0);
1567 SDValue Src = Op->getOperand(Num: 0);
1568 SDLoc DL(Op);
1569
1570 // LoongArchISD::BITREV_8B is not supported on LA32.
1571 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1572 return SDValue();
1573
1574 EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
1575 unsigned int OrigEltNum = ResTy.getVectorNumElements();
1576 unsigned int NewEltNum = NewVT.getVectorNumElements();
1577
1578 SDValue NewSrc = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: NewVT, Operand: Src);
1579
1580 SmallVector<SDValue, 8> Ops;
1581 for (unsigned int i = 0; i < NewEltNum; i++) {
1582 SDValue Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: NewSrc,
1583 N2: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
1584 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1585 ? (unsigned)LoongArchISD::BITREV_8B
1586 : (unsigned)ISD::BITREVERSE;
1587 Ops.push_back(Elt: DAG.getNode(Opcode: RevOp, DL, VT: MVT::i64, Operand: Op));
1588 }
1589 SDValue Res =
1590 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResTy, Operand: DAG.getBuildVector(VT: NewVT, DL, Ops));
1591
1592 switch (ResTy.getSimpleVT().SimpleTy) {
1593 default:
1594 return SDValue();
1595 case MVT::v16i8:
1596 case MVT::v32i8:
1597 return Res;
1598 case MVT::v8i16:
1599 case MVT::v16i16:
1600 case MVT::v4i32:
1601 case MVT::v8i32: {
1602 SmallVector<int, 32> Mask;
1603 for (unsigned int i = 0; i < NewEltNum; i++)
1604 for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
1605 Mask.push_back(Elt: j + (OrigEltNum / NewEltNum) * i);
1606 return DAG.getVectorShuffle(VT: ResTy, dl: DL, N1: Res, N2: DAG.getUNDEF(VT: ResTy), Mask);
1607 }
1608 }
1609}
1610
1611// Widen element type to get a new mask value (if possible).
1612// For example:
1613// shufflevector <4 x i32> %a, <4 x i32> %b,
1614// <4 x i32> <i32 6, i32 7, i32 2, i32 3>
1615// is equivalent to:
1616// shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 1>
1617// can be lowered to:
1618// VPACKOD_D vr0, vr0, vr1
1619static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1620 SDValue V1, SDValue V2, SelectionDAG &DAG) {
1621 unsigned EltBits = VT.getScalarSizeInBits();
1622
1623 if (EltBits > 32 || EltBits == 1)
1624 return SDValue();
1625
1626 SmallVector<int, 8> NewMask;
1627 if (widenShuffleMaskElts(M: Mask, NewMask)) {
1628 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: EltBits * 2)
1629 : MVT::getIntegerVT(BitWidth: EltBits * 2);
1630 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
1631 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT)) {
1632 SDValue NewV1 = DAG.getBitcast(VT: NewVT, V: V1);
1633 SDValue NewV2 = DAG.getBitcast(VT: NewVT, V: V2);
1634 return DAG.getBitcast(
1635 VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: NewV1, N2: NewV2, Mask: NewMask));
1636 }
1637 }
1638
1639 return SDValue();
1640}
1641
1642/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
1643/// instruction.
1644// The funciton matches elements from one of the input vector shuffled to the
1645// left or right with zeroable elements 'shifted in'. It handles both the
1646// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
1647// lane.
1648// Mostly copied from X86.
1649static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
1650 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
1651 int MaskOffset, const APInt &Zeroable) {
1652 int Size = Mask.size();
1653 unsigned SizeInBits = Size * ScalarSizeInBits;
1654
1655 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
1656 for (int i = 0; i < Size; i += Scale)
1657 for (int j = 0; j < Shift; ++j)
1658 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
1659 return false;
1660
1661 return true;
1662 };
1663
1664 auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
1665 int Step = 1) {
1666 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
1667 if (!(Mask[i] == -1 || Mask[i] == Low))
1668 return false;
1669 return true;
1670 };
1671
1672 auto MatchShift = [&](int Shift, int Scale, bool Left) {
1673 for (int i = 0; i != Size; i += Scale) {
1674 unsigned Pos = Left ? i + Shift : i;
1675 unsigned Low = Left ? i : i + Shift;
1676 unsigned Len = Scale - Shift;
1677 if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
1678 return -1;
1679 }
1680
1681 int ShiftEltBits = ScalarSizeInBits * Scale;
1682 bool ByteShift = ShiftEltBits > 64;
1683 Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1684 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1685 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1686
1687 // Normalize the scale for byte shifts to still produce an i64 element
1688 // type.
1689 Scale = ByteShift ? Scale / 2 : Scale;
1690
1691 // We need to round trip through the appropriate type for the shift.
1692 MVT ShiftSVT = MVT::getIntegerVT(BitWidth: ScalarSizeInBits * Scale);
1693 ShiftVT = ByteShift ? MVT::getVectorVT(VT: MVT::i8, NumElements: SizeInBits / 8)
1694 : MVT::getVectorVT(VT: ShiftSVT, NumElements: Size / Scale);
1695 return (int)ShiftAmt;
1696 };
1697
1698 unsigned MaxWidth = 128;
1699 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1700 for (int Shift = 1; Shift != Scale; ++Shift)
1701 for (bool Left : {true, false})
1702 if (CheckZeros(Shift, Scale, Left)) {
1703 int ShiftAmt = MatchShift(Shift, Scale, Left);
1704 if (0 < ShiftAmt)
1705 return ShiftAmt;
1706 }
1707
1708 // no match
1709 return -1;
1710}
1711
1712/// Lower VECTOR_SHUFFLE as shift (if possible).
1713///
1714/// For example:
1715/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1716/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
1717/// is lowered to:
1718/// (VBSLL_V $v0, $v0, 4)
1719///
1720/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1721/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
1722/// is lowered to:
1723/// (VSLLI_D $v0, $v0, 32)
1724static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef<int> Mask,
1725 MVT VT, SDValue V1, SDValue V2,
1726 SelectionDAG &DAG,
1727 const LoongArchSubtarget &Subtarget,
1728 const APInt &Zeroable) {
1729 int Size = Mask.size();
1730 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
1731
1732 MVT ShiftVT;
1733 SDValue V = V1;
1734 unsigned Opcode;
1735
1736 // Try to match shuffle against V1 shift.
1737 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1738 Mask, MaskOffset: 0, Zeroable);
1739
1740 // If V1 failed, try to match shuffle against V2 shift.
1741 if (ShiftAmt < 0) {
1742 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1743 Mask, MaskOffset: Size, Zeroable);
1744 V = V2;
1745 }
1746
1747 if (ShiftAmt < 0)
1748 return SDValue();
1749
1750 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
1751 "Illegal integer vector type");
1752 V = DAG.getBitcast(VT: ShiftVT, V);
1753 V = DAG.getNode(Opcode, DL, VT: ShiftVT, N1: V,
1754 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: Subtarget.getGRLenVT()));
1755 return DAG.getBitcast(VT, V);
1756}
1757
1758/// Determine whether a range fits a regular pattern of values.
1759/// This function accounts for the possibility of jumping over the End iterator.
1760template <typename ValType>
1761static bool
1762fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,
1763 unsigned CheckStride,
1764 typename SmallVectorImpl<ValType>::const_iterator End,
1765 ValType ExpectedIndex, unsigned ExpectedIndexStride) {
1766 auto &I = Begin;
1767
1768 while (I != End) {
1769 if (*I != -1 && *I != ExpectedIndex)
1770 return false;
1771 ExpectedIndex += ExpectedIndexStride;
1772
1773 // Incrementing past End is undefined behaviour so we must increment one
1774 // step at a time and check for End at each step.
1775 for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
1776 ; // Empty loop body.
1777 }
1778 return true;
1779}
1780
1781/// Compute whether each element of a shuffle is zeroable.
1782///
1783/// A "zeroable" vector shuffle element is one which can be lowered to zero.
1784static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
1785 SDValue V2, APInt &KnownUndef,
1786 APInt &KnownZero) {
1787 int Size = Mask.size();
1788 KnownUndef = KnownZero = APInt::getZero(numBits: Size);
1789
1790 V1 = peekThroughBitcasts(V: V1);
1791 V2 = peekThroughBitcasts(V: V2);
1792
1793 bool V1IsZero = ISD::isBuildVectorAllZeros(N: V1.getNode());
1794 bool V2IsZero = ISD::isBuildVectorAllZeros(N: V2.getNode());
1795
1796 int VectorSizeInBits = V1.getValueSizeInBits();
1797 int ScalarSizeInBits = VectorSizeInBits / Size;
1798 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
1799 (void)ScalarSizeInBits;
1800
1801 for (int i = 0; i < Size; ++i) {
1802 int M = Mask[i];
1803 if (M < 0) {
1804 KnownUndef.setBit(i);
1805 continue;
1806 }
1807 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
1808 KnownZero.setBit(i);
1809 continue;
1810 }
1811 }
1812}
1813
1814/// Test whether a shuffle mask is equivalent within each sub-lane.
1815///
1816/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
1817/// non-trivial to compute in the face of undef lanes. The representation is
1818/// suitable for use with existing 128-bit shuffles as entries from the second
1819/// vector have been remapped to [LaneSize, 2*LaneSize).
1820static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
1821 ArrayRef<int> Mask,
1822 SmallVectorImpl<int> &RepeatedMask) {
1823 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
1824 RepeatedMask.assign(NumElts: LaneSize, Elt: -1);
1825 int Size = Mask.size();
1826 for (int i = 0; i < Size; ++i) {
1827 assert(Mask[i] == -1 || Mask[i] >= 0);
1828 if (Mask[i] < 0)
1829 continue;
1830 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
1831 // This entry crosses lanes, so there is no way to model this shuffle.
1832 return false;
1833
1834 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
1835 // Adjust second vector indices to start at LaneSize instead of Size.
1836 int LocalM =
1837 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1838 if (RepeatedMask[i % LaneSize] < 0)
1839 // This is the first non-undef entry in this slot of a 128-bit lane.
1840 RepeatedMask[i % LaneSize] = LocalM;
1841 else if (RepeatedMask[i % LaneSize] != LocalM)
1842 // Found a mismatch with the repeated mask.
1843 return false;
1844 }
1845 return true;
1846}
1847
1848/// Attempts to match vector shuffle as byte rotation.
1849static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
1850 ArrayRef<int> Mask) {
1851
1852 SDValue Lo, Hi;
1853 SmallVector<int, 16> RepeatedMask;
1854
1855 if (!isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask, RepeatedMask))
1856 return -1;
1857
1858 int NumElts = RepeatedMask.size();
1859 int Rotation = 0;
1860 int Scale = 16 / NumElts;
1861
1862 for (int i = 0; i < NumElts; ++i) {
1863 int M = RepeatedMask[i];
1864 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1865 "Unexpected mask index.");
1866 if (M < 0)
1867 continue;
1868
1869 // Determine where a rotated vector would have started.
1870 int StartIdx = i - (M % NumElts);
1871 if (StartIdx == 0)
1872 return -1;
1873
1874 // If we found the tail of a vector the rotation must be the missing
1875 // front. If we found the head of a vector, it must be how much of the
1876 // head.
1877 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1878
1879 if (Rotation == 0)
1880 Rotation = CandidateRotation;
1881 else if (Rotation != CandidateRotation)
1882 return -1;
1883
1884 // Compute which value this mask is pointing at.
1885 SDValue MaskV = M < NumElts ? V1 : V2;
1886
1887 // Compute which of the two target values this index should be assigned
1888 // to. This reflects whether the high elements are remaining or the low
1889 // elements are remaining.
1890 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
1891
1892 // Either set up this value if we've not encountered it before, or check
1893 // that it remains consistent.
1894 if (!TargetV)
1895 TargetV = MaskV;
1896 else if (TargetV != MaskV)
1897 return -1;
1898 }
1899
1900 // Check that we successfully analyzed the mask, and normalize the results.
1901 assert(Rotation != 0 && "Failed to locate a viable rotation!");
1902 assert((Lo || Hi) && "Failed to find a rotated input vector!");
1903 if (!Lo)
1904 Lo = Hi;
1905 else if (!Hi)
1906 Hi = Lo;
1907
1908 V1 = Lo;
1909 V2 = Hi;
1910
1911 return Rotation * Scale;
1912}
1913
1914/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
1915///
1916/// For example:
1917/// %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
1918/// <2 x i32> <i32 3, i32 0>
1919/// is lowered to:
1920/// (VBSRL_V $v1, $v1, 8)
1921/// (VBSLL_V $v0, $v0, 8)
1922/// (VOR_V $v0, $V0, $v1)
1923static SDValue
1924lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1925 SDValue V1, SDValue V2, SelectionDAG &DAG,
1926 const LoongArchSubtarget &Subtarget) {
1927
1928 SDValue Lo = V1, Hi = V2;
1929 int ByteRotation = matchShuffleAsByteRotate(VT, V1&: Lo, V2&: Hi, Mask);
1930 if (ByteRotation <= 0)
1931 return SDValue();
1932
1933 MVT ByteVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VT.getSizeInBits() / 8);
1934 Lo = DAG.getBitcast(VT: ByteVT, V: Lo);
1935 Hi = DAG.getBitcast(VT: ByteVT, V: Hi);
1936
1937 int LoByteShift = 16 - ByteRotation;
1938 int HiByteShift = ByteRotation;
1939 MVT GRLenVT = Subtarget.getGRLenVT();
1940
1941 SDValue LoShift = DAG.getNode(Opcode: LoongArchISD::VBSLL, DL, VT: ByteVT, N1: Lo,
1942 N2: DAG.getConstant(Val: LoByteShift, DL, VT: GRLenVT));
1943 SDValue HiShift = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: ByteVT, N1: Hi,
1944 N2: DAG.getConstant(Val: HiByteShift, DL, VT: GRLenVT));
1945 return DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::OR, DL, VT: ByteVT, N1: LoShift, N2: HiShift));
1946}
1947
1948/// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
1949///
1950/// For example:
1951/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1952/// <4 x i32> <i32 0, i32 4, i32 1, i32 4>
1953/// %3 = bitcast <4 x i32> %2 to <2 x i64>
1954/// is lowered to:
1955/// (VREPLI $v1, 0)
1956/// (VILVL $v0, $v1, $v0)
1957static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL,
1958 ArrayRef<int> Mask, MVT VT,
1959 SDValue V1, SDValue V2,
1960 SelectionDAG &DAG,
1961 const APInt &Zeroable) {
1962 int Bits = VT.getSizeInBits();
1963 int EltBits = VT.getScalarSizeInBits();
1964 int NumElements = VT.getVectorNumElements();
1965
1966 if (Zeroable.isAllOnes())
1967 return DAG.getConstant(Val: 0, DL, VT);
1968
1969 // Define a helper function to check a particular ext-scale and lower to it if
1970 // valid.
1971 auto Lower = [&](int Scale) -> SDValue {
1972 SDValue InputV;
1973 bool AnyExt = true;
1974 int Offset = 0;
1975 for (int i = 0; i < NumElements; i++) {
1976 int M = Mask[i];
1977 if (M < 0)
1978 continue;
1979 if (i % Scale != 0) {
1980 // Each of the extended elements need to be zeroable.
1981 if (!Zeroable[i])
1982 return SDValue();
1983
1984 AnyExt = false;
1985 continue;
1986 }
1987
1988 // Each of the base elements needs to be consecutive indices into the
1989 // same input vector.
1990 SDValue V = M < NumElements ? V1 : V2;
1991 M = M % NumElements;
1992 if (!InputV) {
1993 InputV = V;
1994 Offset = M - (i / Scale);
1995
1996 // These offset can't be handled
1997 if (Offset % (NumElements / Scale))
1998 return SDValue();
1999 } else if (InputV != V)
2000 return SDValue();
2001
2002 if (M != (Offset + (i / Scale)))
2003 return SDValue(); // Non-consecutive strided elements.
2004 }
2005
2006 // If we fail to find an input, we have a zero-shuffle which should always
2007 // have already been handled.
2008 if (!InputV)
2009 return SDValue();
2010
2011 do {
2012 unsigned VilVLoHi = LoongArchISD::VILVL;
2013 if (Offset >= (NumElements / 2)) {
2014 VilVLoHi = LoongArchISD::VILVH;
2015 Offset -= (NumElements / 2);
2016 }
2017
2018 MVT InputVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits), NumElements);
2019 SDValue Ext =
2020 AnyExt ? DAG.getFreeze(V: InputV) : DAG.getConstant(Val: 0, DL, VT: InputVT);
2021 InputV = DAG.getBitcast(VT: InputVT, V: InputV);
2022 InputV = DAG.getNode(Opcode: VilVLoHi, DL, VT: InputVT, N1: Ext, N2: InputV);
2023 Scale /= 2;
2024 EltBits *= 2;
2025 NumElements /= 2;
2026 } while (Scale > 1);
2027 return DAG.getBitcast(VT, V: InputV);
2028 };
2029
2030 // Each iteration, try extending the elements half as much, but into twice as
2031 // many elements.
2032 for (int NumExtElements = Bits / 64; NumExtElements < NumElements;
2033 NumExtElements *= 2) {
2034 if (SDValue V = Lower(NumElements / NumExtElements))
2035 return V;
2036 }
2037 return SDValue();
2038}
2039
2040/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
2041///
2042/// VREPLVEI performs vector broadcast based on an element specified by an
2043/// integer immediate, with its mask being similar to:
2044/// <x, x, x, ...>
2045/// where x is any valid index.
2046///
2047/// When undef's appear in the mask they are treated as if they were whatever
2048/// value is necessary in order to fit the above form.
2049static SDValue
2050lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2051 SDValue V1, SelectionDAG &DAG,
2052 const LoongArchSubtarget &Subtarget) {
2053 int SplatIndex = -1;
2054 for (const auto &M : Mask) {
2055 if (M != -1) {
2056 SplatIndex = M;
2057 break;
2058 }
2059 }
2060
2061 if (SplatIndex == -1)
2062 return DAG.getUNDEF(VT);
2063
2064 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2065 if (fitsRegularPattern<int>(Begin: Mask.begin(), CheckStride: 1, End: Mask.end(), ExpectedIndex: SplatIndex, ExpectedIndexStride: 0)) {
2066 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2067 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2068 }
2069
2070 return SDValue();
2071}
2072
2073/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
2074///
2075/// VSHUF4I splits the vector into blocks of four elements, then shuffles these
2076/// elements according to a <4 x i2> constant (encoded as an integer immediate).
2077///
2078/// It is therefore possible to lower into VSHUF4I when the mask takes the form:
2079/// <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2080/// When undef's appear they are treated as if they were whatever value is
2081/// necessary in order to fit the above forms.
2082///
2083/// For example:
2084/// %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2085/// <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2086/// i32 7, i32 6, i32 5, i32 4>
2087/// is lowered to:
2088/// (VSHUF4I_H $v0, $v1, 27)
2089/// where the 27 comes from:
2090/// 3 + (2 << 2) + (1 << 4) + (0 << 6)
2091static SDValue
2092lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2093 SDValue V1, SDValue V2, SelectionDAG &DAG,
2094 const LoongArchSubtarget &Subtarget) {
2095
2096 unsigned SubVecSize = 4;
2097 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2098 SubVecSize = 2;
2099
2100 int SubMask[4] = {-1, -1, -1, -1};
2101 for (unsigned i = 0; i < SubVecSize; ++i) {
2102 for (unsigned j = i; j < Mask.size(); j += SubVecSize) {
2103 int M = Mask[j];
2104
2105 // Convert from vector index to 4-element subvector index
2106 // If an index refers to an element outside of the subvector then give up
2107 if (M != -1) {
2108 M -= 4 * (j / SubVecSize);
2109 if (M < 0 || M >= 4)
2110 return SDValue();
2111 }
2112
2113 // If the mask has an undef, replace it with the current index.
2114 // Note that it might still be undef if the current index is also undef
2115 if (SubMask[i] == -1)
2116 SubMask[i] = M;
2117 // Check that non-undef values are the same as in the mask. If they
2118 // aren't then give up
2119 else if (M != -1 && M != SubMask[i])
2120 return SDValue();
2121 }
2122 }
2123
2124 // Calculate the immediate. Replace any remaining undefs with zero
2125 int Imm = 0;
2126 for (int i = SubVecSize - 1; i >= 0; --i) {
2127 int M = SubMask[i];
2128
2129 if (M == -1)
2130 M = 0;
2131
2132 Imm <<= 2;
2133 Imm |= M & 0x3;
2134 }
2135
2136 MVT GRLenVT = Subtarget.getGRLenVT();
2137
2138 // Return vshuf4i.d
2139 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2140 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2141 N3: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2142
2143 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT, N1: V1,
2144 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2145}
2146
2147/// Lower VECTOR_SHUFFLE whose result is the reversed source vector.
2148///
2149/// It is possible to do optimization for VECTOR_SHUFFLE performing vector
2150/// reverse whose mask likes:
2151/// <7, 6, 5, 4, 3, 2, 1, 0>
2152///
2153/// When undef's appear in the mask they are treated as if they were whatever
2154/// value is necessary in order to fit the above forms.
2155static SDValue
2156lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2157 SDValue V1, SelectionDAG &DAG,
2158 const LoongArchSubtarget &Subtarget) {
2159 // Only vectors with i8/i16 elements which cannot match other patterns
2160 // directly needs to do this.
2161 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2162 VT != MVT::v16i16)
2163 return SDValue();
2164
2165 if (!ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: Mask.size()))
2166 return SDValue();
2167
2168 int WidenNumElts = VT.getVectorNumElements() / 4;
2169 SmallVector<int, 16> WidenMask(WidenNumElts, -1);
2170 for (int i = 0; i < WidenNumElts; ++i)
2171 WidenMask[i] = WidenNumElts - 1 - i;
2172
2173 MVT WidenVT = MVT::getVectorVT(
2174 VT: VT.getVectorElementType() == MVT::i8 ? MVT::i32 : MVT::i64, NumElements: WidenNumElts);
2175 SDValue NewV1 = DAG.getBitcast(VT: WidenVT, V: V1);
2176 SDValue WidenRev = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: NewV1,
2177 N2: DAG.getUNDEF(VT: WidenVT), Mask: WidenMask);
2178
2179 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT,
2180 N1: DAG.getBitcast(VT, V: WidenRev),
2181 N2: DAG.getConstant(Val: 27, DL, VT: Subtarget.getGRLenVT()));
2182}
2183
2184/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).
2185///
2186/// VPACKEV interleaves the even elements from each vector.
2187///
2188/// It is possible to lower into VPACKEV when the mask consists of two of the
2189/// following forms interleaved:
2190/// <0, 2, 4, ...>
2191/// <n, n+2, n+4, ...>
2192/// where n is the number of elements in the vector.
2193/// For example:
2194/// <0, 0, 2, 2, 4, 4, ...>
2195/// <0, n, 2, n+2, 4, n+4, ...>
2196///
2197/// When undef's appear in the mask they are treated as if they were whatever
2198/// value is necessary in order to fit the above forms.
2199static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2200 MVT VT, SDValue V1, SDValue V2,
2201 SelectionDAG &DAG) {
2202
2203 const auto &Begin = Mask.begin();
2204 const auto &End = Mask.end();
2205 SDValue OriV1 = V1, OriV2 = V2;
2206
2207 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2208 V1 = OriV1;
2209 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2210 V1 = OriV2;
2211 else
2212 return SDValue();
2213
2214 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2215 V2 = OriV1;
2216 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2217 V2 = OriV2;
2218 else
2219 return SDValue();
2220
2221 return DAG.getNode(Opcode: LoongArchISD::VPACKEV, DL, VT, N1: V2, N2: V1);
2222}
2223
2224/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).
2225///
2226/// VPACKOD interleaves the odd elements from each vector.
2227///
2228/// It is possible to lower into VPACKOD when the mask consists of two of the
2229/// following forms interleaved:
2230/// <1, 3, 5, ...>
2231/// <n+1, n+3, n+5, ...>
2232/// where n is the number of elements in the vector.
2233/// For example:
2234/// <1, 1, 3, 3, 5, 5, ...>
2235/// <1, n+1, 3, n+3, 5, n+5, ...>
2236///
2237/// When undef's appear in the mask they are treated as if they were whatever
2238/// value is necessary in order to fit the above forms.
2239static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2240 MVT VT, SDValue V1, SDValue V2,
2241 SelectionDAG &DAG) {
2242
2243 const auto &Begin = Mask.begin();
2244 const auto &End = Mask.end();
2245 SDValue OriV1 = V1, OriV2 = V2;
2246
2247 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2248 V1 = OriV1;
2249 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2250 V1 = OriV2;
2251 else
2252 return SDValue();
2253
2254 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2255 V2 = OriV1;
2256 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2257 V2 = OriV2;
2258 else
2259 return SDValue();
2260
2261 return DAG.getNode(Opcode: LoongArchISD::VPACKOD, DL, VT, N1: V2, N2: V1);
2262}
2263
2264/// Lower VECTOR_SHUFFLE into VILVH (if possible).
2265///
2266/// VILVH interleaves consecutive elements from the left (highest-indexed) half
2267/// of each vector.
2268///
2269/// It is possible to lower into VILVH when the mask consists of two of the
2270/// following forms interleaved:
2271/// <x, x+1, x+2, ...>
2272/// <n+x, n+x+1, n+x+2, ...>
2273/// where n is the number of elements in the vector and x is half n.
2274/// For example:
2275/// <x, x, x+1, x+1, x+2, x+2, ...>
2276/// <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2277///
2278/// When undef's appear in the mask they are treated as if they were whatever
2279/// value is necessary in order to fit the above forms.
2280static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef<int> Mask,
2281 MVT VT, SDValue V1, SDValue V2,
2282 SelectionDAG &DAG) {
2283
2284 const auto &Begin = Mask.begin();
2285 const auto &End = Mask.end();
2286 unsigned HalfSize = Mask.size() / 2;
2287 SDValue OriV1 = V1, OriV2 = V2;
2288
2289 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2290 V1 = OriV1;
2291 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2292 V1 = OriV2;
2293 else
2294 return SDValue();
2295
2296 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2297 V2 = OriV1;
2298 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize,
2299 ExpectedIndexStride: 1))
2300 V2 = OriV2;
2301 else
2302 return SDValue();
2303
2304 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2305}
2306
2307/// Lower VECTOR_SHUFFLE into VILVL (if possible).
2308///
2309/// VILVL interleaves consecutive elements from the right (lowest-indexed) half
2310/// of each vector.
2311///
2312/// It is possible to lower into VILVL when the mask consists of two of the
2313/// following forms interleaved:
2314/// <0, 1, 2, ...>
2315/// <n, n+1, n+2, ...>
2316/// where n is the number of elements in the vector.
2317/// For example:
2318/// <0, 0, 1, 1, 2, 2, ...>
2319/// <0, n, 1, n+1, 2, n+2, ...>
2320///
2321/// When undef's appear in the mask they are treated as if they were whatever
2322/// value is necessary in order to fit the above forms.
2323static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef<int> Mask,
2324 MVT VT, SDValue V1, SDValue V2,
2325 SelectionDAG &DAG) {
2326
2327 const auto &Begin = Mask.begin();
2328 const auto &End = Mask.end();
2329 SDValue OriV1 = V1, OriV2 = V2;
2330
2331 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2332 V1 = OriV1;
2333 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2334 V1 = OriV2;
2335 else
2336 return SDValue();
2337
2338 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2339 V2 = OriV1;
2340 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2341 V2 = OriV2;
2342 else
2343 return SDValue();
2344
2345 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2346}
2347
2348/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).
2349///
2350/// VPICKEV copies the even elements of each vector into the result vector.
2351///
2352/// It is possible to lower into VPICKEV when the mask consists of two of the
2353/// following forms concatenated:
2354/// <0, 2, 4, ...>
2355/// <n, n+2, n+4, ...>
2356/// where n is the number of elements in the vector.
2357/// For example:
2358/// <0, 2, 4, ..., 0, 2, 4, ...>
2359/// <0, 2, 4, ..., n, n+2, n+4, ...>
2360///
2361/// When undef's appear in the mask they are treated as if they were whatever
2362/// value is necessary in order to fit the above forms.
2363static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2364 MVT VT, SDValue V1, SDValue V2,
2365 SelectionDAG &DAG) {
2366
2367 const auto &Begin = Mask.begin();
2368 const auto &Mid = Mask.begin() + Mask.size() / 2;
2369 const auto &End = Mask.end();
2370 SDValue OriV1 = V1, OriV2 = V2;
2371
2372 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2))
2373 V1 = OriV1;
2374 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2375 V1 = OriV2;
2376 else
2377 return SDValue();
2378
2379 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2380 V2 = OriV1;
2381 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2382 V2 = OriV2;
2383
2384 else
2385 return SDValue();
2386
2387 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2388}
2389
2390/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).
2391///
2392/// VPICKOD copies the odd elements of each vector into the result vector.
2393///
2394/// It is possible to lower into VPICKOD when the mask consists of two of the
2395/// following forms concatenated:
2396/// <1, 3, 5, ...>
2397/// <n+1, n+3, n+5, ...>
2398/// where n is the number of elements in the vector.
2399/// For example:
2400/// <1, 3, 5, ..., 1, 3, 5, ...>
2401/// <1, 3, 5, ..., n+1, n+3, n+5, ...>
2402///
2403/// When undef's appear in the mask they are treated as if they were whatever
2404/// value is necessary in order to fit the above forms.
2405static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2406 MVT VT, SDValue V1, SDValue V2,
2407 SelectionDAG &DAG) {
2408
2409 const auto &Begin = Mask.begin();
2410 const auto &Mid = Mask.begin() + Mask.size() / 2;
2411 const auto &End = Mask.end();
2412 SDValue OriV1 = V1, OriV2 = V2;
2413
2414 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2))
2415 V1 = OriV1;
2416 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2417 V1 = OriV2;
2418 else
2419 return SDValue();
2420
2421 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2422 V2 = OriV1;
2423 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2424 V2 = OriV2;
2425 else
2426 return SDValue();
2427
2428 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
2429}
2430
2431/// Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
2432///
2433/// VEXTRINS copies one element of a vector into any place of the result
2434/// vector and makes no change to the rest elements of the result vector.
2435///
2436/// It is possible to lower into VEXTRINS when the mask takes the form:
2437/// <0, 1, 2, ..., n+i, ..., n-1> or <n, n+1, n+2, ..., i, ..., 2n-1> or
2438/// <0, 1, 2, ..., i, ..., n-1> or <n, n+1, n+2, ..., n+i, ..., 2n-1>
2439/// where n is the number of elements in the vector and i is in [0, n).
2440/// For example:
2441/// <0, 1, 2, 3, 4, 5, 6, 8> , <2, 9, 10, 11, 12, 13, 14, 15> ,
2442/// <0, 1, 2, 6, 4, 5, 6, 7> , <8, 9, 10, 11, 12, 9, 14, 15>
2443///
2444/// When undef's appear in the mask they are treated as if they were whatever
2445/// value is necessary in order to fit the above forms.
2446static SDValue
2447lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2448 SDValue V1, SDValue V2, SelectionDAG &DAG,
2449 const LoongArchSubtarget &Subtarget) {
2450 unsigned NumElts = VT.getVectorNumElements();
2451 MVT EltVT = VT.getVectorElementType();
2452 MVT GRLenVT = Subtarget.getGRLenVT();
2453
2454 if (Mask.size() != NumElts)
2455 return SDValue();
2456
2457 auto tryLowerToExtrAndIns = [&](unsigned Base) -> SDValue {
2458 int DiffCount = 0;
2459 int DiffPos = -1;
2460 for (unsigned i = 0; i < NumElts; ++i) {
2461 if (Mask[i] == -1)
2462 continue;
2463 if (Mask[i] != int(Base + i)) {
2464 ++DiffCount;
2465 DiffPos = int(i);
2466 if (DiffCount > 1)
2467 return SDValue();
2468 }
2469 }
2470
2471 // Need exactly one differing element to lower into VEXTRINS.
2472 if (DiffCount != 1)
2473 return SDValue();
2474
2475 // DiffMask must be in [0, 2N).
2476 int DiffMask = Mask[DiffPos];
2477 if (DiffMask < 0 || DiffMask >= int(2 * NumElts))
2478 return SDValue();
2479
2480 // Determine source vector and source index.
2481 SDValue SrcVec;
2482 unsigned SrcIdx;
2483 if (unsigned(DiffMask) < NumElts) {
2484 SrcVec = V1;
2485 SrcIdx = unsigned(DiffMask);
2486 } else {
2487 SrcVec = V2;
2488 SrcIdx = unsigned(DiffMask) - NumElts;
2489 }
2490
2491 // Replace with EXTRACT_VECTOR_ELT + INSERT_VECTOR_ELT, it will match the
2492 // patterns of VEXTRINS in tablegen.
2493 SDValue Extracted = DAG.getNode(
2494 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT.isFloatingPoint() ? EltVT : GRLenVT,
2495 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdx, DL, VT: GRLenVT));
2496 SDValue Result =
2497 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: (Base == 0) ? V1 : V2,
2498 N2: Extracted, N3: DAG.getConstant(Val: DiffPos, DL, VT: GRLenVT));
2499
2500 return Result;
2501 };
2502
2503 // Try [0, n-1) insertion then [n, 2n-1) insertion.
2504 if (SDValue Result = tryLowerToExtrAndIns(0))
2505 return Result;
2506 return tryLowerToExtrAndIns(NumElts);
2507}
2508
2509// Check the Mask and then build SrcVec and MaskImm infos which will
2510// be used to build LoongArchISD nodes for VPERMI_W or XVPERMI_W.
2511// On success, return true. Otherwise, return false.
2512static bool buildVPERMIInfo(ArrayRef<int> Mask, SDValue V1, SDValue V2,
2513 SmallVectorImpl<SDValue> &SrcVec,
2514 unsigned &MaskImm) {
2515 unsigned MaskSize = Mask.size();
2516
2517 auto isValid = [&](int M, int Off) {
2518 return (M == -1) || (M >= Off && M < Off + 4);
2519 };
2520
2521 auto buildImm = [&](int MLo, int MHi, unsigned Off, unsigned I) {
2522 auto immPart = [&](int M, unsigned Off) {
2523 return (M == -1 ? 0 : (M - Off)) & 0x3;
2524 };
2525 MaskImm |= immPart(MLo, Off) << (I * 2);
2526 MaskImm |= immPart(MHi, Off) << ((I + 1) * 2);
2527 };
2528
2529 for (unsigned i = 0; i < 4; i += 2) {
2530 int MLo = Mask[i];
2531 int MHi = Mask[i + 1];
2532
2533 if (MaskSize == 8) { // Only v8i32/v8f32 need this check.
2534 auto isValid2 = [&](int &M, int M2) {
2535 // If high half index is undef, it's always valid.
2536 if (M2 == -1)
2537 return true;
2538 if (M == -1) {
2539 // If low half index is undef, use index from high half,
2540 // remapped to low half.
2541 if ((M2 % MaskSize) < 4)
2542 return false;
2543 M = M2 - 4;
2544 return true;
2545 }
2546 // Index in low half must be same as index in high half.
2547 return M2 == M + 4;
2548 };
2549 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2550 return false;
2551 }
2552
2553 if (isValid(MLo, 0) && isValid(MHi, 0)) {
2554 SrcVec.push_back(Elt: V1);
2555 buildImm(MLo, MHi, 0, i);
2556 } else if (isValid(MLo, MaskSize) && isValid(MHi, MaskSize)) {
2557 SrcVec.push_back(Elt: V2);
2558 buildImm(MLo, MHi, MaskSize, i);
2559 } else {
2560 return false;
2561 }
2562 }
2563
2564 return true;
2565}
2566
2567/// Lower VECTOR_SHUFFLE into VPERMI (if possible).
2568///
2569/// VPERMI selects two elements from each of the two vectors based on the
2570/// mask and places them in the corresponding positions of the result vector
2571/// in order. Only v4i32 and v4f32 types are allowed.
2572///
2573/// It is possible to lower into VPERMI when the mask consists of two of the
2574/// following forms concatenated:
2575/// <i, j, u, v>
2576/// <u, v, i, j>
2577/// where i,j are in [0,4) and u,v are in [4, 8).
2578/// For example:
2579/// <2, 3, 4, 5>
2580/// <5, 7, 0, 2>
2581///
2582/// When undef's appear in the mask they are treated as if they were whatever
2583/// value is necessary in order to fit the above forms.
2584static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef<int> Mask,
2585 MVT VT, SDValue V1, SDValue V2,
2586 SelectionDAG &DAG,
2587 const LoongArchSubtarget &Subtarget) {
2588 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2589 Mask.size() != VT.getVectorNumElements())
2590 return SDValue();
2591
2592 SmallVector<SDValue, 2> SrcVec;
2593 unsigned MaskImm = 0;
2594 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2595 return SDValue();
2596
2597 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2598 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2599}
2600
2601/// Lower VECTOR_SHUFFLE into VSHUF.
2602///
2603/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and
2604/// adding it as an operand to the resulting VSHUF.
2605static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef<int> Mask,
2606 MVT VT, SDValue V1, SDValue V2,
2607 SelectionDAG &DAG,
2608 const LoongArchSubtarget &Subtarget) {
2609
2610 SmallVector<SDValue, 16> Ops;
2611 for (auto M : Mask)
2612 Ops.push_back(Elt: DAG.getSignedConstant(Val: M, DL, VT: Subtarget.getGRLenVT()));
2613
2614 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
2615 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops);
2616
2617 // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.
2618 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>
2619 // VSHF concatenates the vectors in a bitwise fashion:
2620 // <0b00, 0b01> + <0b10, 0b11> ->
2621 // 0b0100 + 0b1110 -> 0b01001110
2622 // <0b10, 0b11, 0b00, 0b01>
2623 // We must therefore swap the operands to get the correct result.
2624 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
2625}
2626
2627/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
2628///
2629/// This routine breaks down the specific type of 128-bit shuffle and
2630/// dispatches to the lowering routines accordingly.
2631static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2632 SDValue V1, SDValue V2, SelectionDAG &DAG,
2633 const LoongArchSubtarget &Subtarget) {
2634 assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||
2635 VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||
2636 VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&
2637 "Vector type is unsupported for lsx!");
2638 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
2639 "Two operands have different types!");
2640 assert(VT.getVectorNumElements() == Mask.size() &&
2641 "Unexpected mask size for shuffle!");
2642 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
2643
2644 APInt KnownUndef, KnownZero;
2645 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
2646 APInt Zeroable = KnownUndef | KnownZero;
2647
2648 SDValue Result;
2649 // TODO: Add more comparison patterns.
2650 if (V2.isUndef()) {
2651 if ((Result =
2652 lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
2653 return Result;
2654 if ((Result =
2655 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2656 return Result;
2657 if ((Result =
2658 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
2659 return Result;
2660
2661 // TODO: This comment may be enabled in the future to better match the
2662 // pattern for instruction selection.
2663 /* V2 = V1; */
2664 }
2665
2666 // It is recommended not to change the pattern comparison order for better
2667 // performance.
2668 if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))
2669 return Result;
2670 if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))
2671 return Result;
2672 if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))
2673 return Result;
2674 if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))
2675 return Result;
2676 if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))
2677 return Result;
2678 if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
2679 return Result;
2680 if ((VT.SimpleTy == MVT::v2i64 || VT.SimpleTy == MVT::v2f64) &&
2681 (Result =
2682 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2683 return Result;
2684 if ((Result =
2685 lowerVECTOR_SHUFFLE_VEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2686 return Result;
2687 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
2688 Zeroable)))
2689 return Result;
2690 if ((Result =
2691 lowerVECTOR_SHUFFLE_VPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2692 return Result;
2693 if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
2694 Zeroable)))
2695 return Result;
2696 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
2697 Subtarget)))
2698 return Result;
2699 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
2700 return NewShuffle;
2701 if ((Result =
2702 lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2703 return Result;
2704 return SDValue();
2705}
2706
2707/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
2708///
2709/// It is a XVREPLVEI when the mask is:
2710/// <x, x, x, ..., x+n, x+n, x+n, ...>
2711/// where the number of x is equal to n and n is half the length of vector.
2712///
2713/// When undef's appear in the mask they are treated as if they were whatever
2714/// value is necessary in order to fit the above form.
2715static SDValue
2716lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2717 SDValue V1, SelectionDAG &DAG,
2718 const LoongArchSubtarget &Subtarget) {
2719 int SplatIndex = -1;
2720 for (const auto &M : Mask) {
2721 if (M != -1) {
2722 SplatIndex = M;
2723 break;
2724 }
2725 }
2726
2727 if (SplatIndex == -1)
2728 return DAG.getUNDEF(VT);
2729
2730 const auto &Begin = Mask.begin();
2731 const auto &End = Mask.end();
2732 int HalfSize = Mask.size() / 2;
2733
2734 if (SplatIndex >= HalfSize)
2735 return SDValue();
2736
2737 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2738 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: End - HalfSize, ExpectedIndex: SplatIndex, ExpectedIndexStride: 0) &&
2739 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 1, End, ExpectedIndex: SplatIndex + HalfSize,
2740 ExpectedIndexStride: 0)) {
2741 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2742 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2743 }
2744
2745 return SDValue();
2746}
2747
2748/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
2749static SDValue
2750lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2751 SDValue V1, SDValue V2, SelectionDAG &DAG,
2752 const LoongArchSubtarget &Subtarget) {
2753 // XVSHUF4I_D must be handled separately because it is different from other
2754 // types of [X]VSHUF4I instructions.
2755 if (Mask.size() == 4) {
2756 unsigned MaskImm = 0;
2757 for (int i = 1; i >= 0; --i) {
2758 int MLo = Mask[i];
2759 int MHi = Mask[i + 2];
2760 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2761 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2762 return SDValue();
2763 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2764 return SDValue();
2765
2766 MaskImm <<= 2;
2767 if (MLo != -1)
2768 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2769 else if (MHi != -1)
2770 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2771 }
2772
2773 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2774 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2775 }
2776
2777 return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget);
2778}
2779
2780/// Lower VECTOR_SHUFFLE into XVPERMI (if possible).
2781static SDValue
2782lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2783 SDValue V1, SDValue V2, SelectionDAG &DAG,
2784 const LoongArchSubtarget &Subtarget) {
2785 MVT GRLenVT = Subtarget.getGRLenVT();
2786 unsigned MaskSize = Mask.size();
2787 if (MaskSize != VT.getVectorNumElements())
2788 return SDValue();
2789
2790 // Consider XVPERMI_W.
2791 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2792 SmallVector<SDValue, 2> SrcVec;
2793 unsigned MaskImm = 0;
2794 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2795 return SDValue();
2796
2797 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2798 N3: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2799 }
2800
2801 // Consider XVPERMI_D.
2802 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2803 unsigned MaskImm = 0;
2804 for (unsigned i = 0; i < MaskSize; ++i) {
2805 if (Mask[i] == -1)
2806 continue;
2807 if (Mask[i] >= (int)MaskSize)
2808 return SDValue();
2809 MaskImm |= Mask[i] << (i * 2);
2810 }
2811
2812 return DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT, N1: V1,
2813 N2: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2814 }
2815
2816 return SDValue();
2817}
2818
2819/// Lower VECTOR_SHUFFLE into XVPERM (if possible).
2820static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef<int> Mask,
2821 MVT VT, SDValue V1, SelectionDAG &DAG,
2822 const LoongArchSubtarget &Subtarget) {
2823 // LoongArch LASX only have XVPERM_W.
2824 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2825 return SDValue();
2826
2827 unsigned NumElts = VT.getVectorNumElements();
2828 unsigned HalfSize = NumElts / 2;
2829 bool FrontLo = true, FrontHi = true;
2830 bool BackLo = true, BackHi = true;
2831
2832 auto inRange = [](int val, int low, int high) {
2833 return (val == -1) || (val >= low && val < high);
2834 };
2835
2836 for (unsigned i = 0; i < HalfSize; ++i) {
2837 int Fronti = Mask[i];
2838 int Backi = Mask[i + HalfSize];
2839
2840 FrontLo &= inRange(Fronti, 0, HalfSize);
2841 FrontHi &= inRange(Fronti, HalfSize, NumElts);
2842 BackLo &= inRange(Backi, 0, HalfSize);
2843 BackHi &= inRange(Backi, HalfSize, NumElts);
2844 }
2845
2846 // If both the lower and upper 128-bit parts access only one half of the
2847 // vector (either lower or upper), avoid using xvperm.w. The latency of
2848 // xvperm.w(3) is higher than using xvshuf(1) and xvori(1).
2849 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2850 return SDValue();
2851
2852 SmallVector<SDValue, 8> Masks;
2853 MVT GRLenVT = Subtarget.getGRLenVT();
2854 for (unsigned i = 0; i < NumElts; ++i)
2855 Masks.push_back(Elt: Mask[i] == -1 ? DAG.getUNDEF(VT: GRLenVT)
2856 : DAG.getConstant(Val: Mask[i], DL, VT: GRLenVT));
2857 SDValue MaskVec = DAG.getBuildVector(VT: MVT::v8i32, DL, Ops: Masks);
2858
2859 return DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT, N1: V1, N2: MaskVec);
2860}
2861
2862/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
2863static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2864 MVT VT, SDValue V1, SDValue V2,
2865 SelectionDAG &DAG) {
2866 return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);
2867}
2868
2869/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
2870static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2871 MVT VT, SDValue V1, SDValue V2,
2872 SelectionDAG &DAG) {
2873 return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);
2874}
2875
2876/// Lower VECTOR_SHUFFLE into XVILVH (if possible).
2877static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef<int> Mask,
2878 MVT VT, SDValue V1, SDValue V2,
2879 SelectionDAG &DAG) {
2880
2881 const auto &Begin = Mask.begin();
2882 const auto &End = Mask.end();
2883 unsigned HalfSize = Mask.size() / 2;
2884 unsigned LeftSize = HalfSize / 2;
2885 SDValue OriV1 = V1, OriV2 = V2;
2886
2887 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2888 ExpectedIndexStride: 1) &&
2889 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize, ExpectedIndexStride: 1))
2890 V1 = OriV1;
2891 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize,
2892 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2893 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2894 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2895 V1 = OriV2;
2896 else
2897 return SDValue();
2898
2899 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2900 ExpectedIndexStride: 1) &&
2901 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize,
2902 ExpectedIndexStride: 1))
2903 V2 = OriV1;
2904 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize,
2905 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2906 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2907 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2908 V2 = OriV2;
2909 else
2910 return SDValue();
2911
2912 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2913}
2914
2915/// Lower VECTOR_SHUFFLE into XVILVL (if possible).
2916static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef<int> Mask,
2917 MVT VT, SDValue V1, SDValue V2,
2918 SelectionDAG &DAG) {
2919
2920 const auto &Begin = Mask.begin();
2921 const auto &End = Mask.end();
2922 unsigned HalfSize = Mask.size() / 2;
2923 SDValue OriV1 = V1, OriV2 = V2;
2924
2925 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2926 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2927 V1 = OriV1;
2928 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1) &&
2929 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2930 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2931 V1 = OriV2;
2932 else
2933 return SDValue();
2934
2935 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2936 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2937 V2 = OriV1;
2938 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(),
2939 ExpectedIndexStride: 1) &&
2940 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2941 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2942 V2 = OriV2;
2943 else
2944 return SDValue();
2945
2946 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2947}
2948
2949/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
2950static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2951 MVT VT, SDValue V1, SDValue V2,
2952 SelectionDAG &DAG) {
2953
2954 const auto &Begin = Mask.begin();
2955 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2956 const auto &Mid = Mask.begin() + Mask.size() / 2;
2957 const auto &RightMid = Mask.end() - Mask.size() / 4;
2958 const auto &End = Mask.end();
2959 unsigned HalfSize = Mask.size() / 2;
2960 SDValue OriV1 = V1, OriV2 = V2;
2961
2962 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
2963 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
2964 V1 = OriV1;
2965 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
2966 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
2967 V1 = OriV2;
2968 else
2969 return SDValue();
2970
2971 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
2972 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
2973 V2 = OriV1;
2974 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
2975 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
2976 V2 = OriV2;
2977
2978 else
2979 return SDValue();
2980
2981 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2982}
2983
2984/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
2985static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2986 MVT VT, SDValue V1, SDValue V2,
2987 SelectionDAG &DAG) {
2988
2989 const auto &Begin = Mask.begin();
2990 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2991 const auto &Mid = Mask.begin() + Mask.size() / 2;
2992 const auto &RightMid = Mask.end() - Mask.size() / 4;
2993 const auto &End = Mask.end();
2994 unsigned HalfSize = Mask.size() / 2;
2995 SDValue OriV1 = V1, OriV2 = V2;
2996
2997 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
2998 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
2999 V1 = OriV1;
3000 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3001 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize + 1,
3002 ExpectedIndexStride: 2))
3003 V1 = OriV2;
3004 else
3005 return SDValue();
3006
3007 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3008 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3009 V2 = OriV1;
3010 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3011 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize + 1,
3012 ExpectedIndexStride: 2))
3013 V2 = OriV2;
3014 else
3015 return SDValue();
3016
3017 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
3018}
3019
3020/// Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
3021static SDValue
3022lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3023 SDValue V1, SDValue V2, SelectionDAG &DAG,
3024 const LoongArchSubtarget &Subtarget) {
3025 int NumElts = VT.getVectorNumElements();
3026 int HalfSize = NumElts / 2;
3027 MVT EltVT = VT.getVectorElementType();
3028 MVT GRLenVT = Subtarget.getGRLenVT();
3029
3030 if ((int)Mask.size() != NumElts)
3031 return SDValue();
3032
3033 auto tryLowerToExtrAndIns = [&](int Base) -> SDValue {
3034 SmallVector<int> DiffPos;
3035 for (int i = 0; i < NumElts; ++i) {
3036 if (Mask[i] == -1)
3037 continue;
3038 if (Mask[i] != Base + i) {
3039 DiffPos.push_back(Elt: i);
3040 if (DiffPos.size() > 2)
3041 return SDValue();
3042 }
3043 }
3044
3045 // Need exactly two differing element to lower into XVEXTRINS.
3046 // If only one differing element, the element at a distance of
3047 // HalfSize from it must be undef.
3048 if (DiffPos.size() == 1) {
3049 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3050 DiffPos.push_back(Elt: DiffPos[0] + HalfSize);
3051 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3052 DiffPos.insert(I: DiffPos.begin(), Elt: DiffPos[0] - HalfSize);
3053 else
3054 return SDValue();
3055 }
3056 if (DiffPos.size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3057 return SDValue();
3058
3059 // DiffMask must be in its low or high part.
3060 int DiffMaskLo = Mask[DiffPos[0]];
3061 int DiffMaskHi = Mask[DiffPos[1]];
3062 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3063 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3064 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3065 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3066 return SDValue();
3067 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3068 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3069 return SDValue();
3070 if (DiffMaskHi != DiffMaskLo + HalfSize)
3071 return SDValue();
3072
3073 // Determine source vector and source index.
3074 SDValue SrcVec = (DiffMaskLo < HalfSize) ? V1 : V2;
3075 int SrcIdxLo =
3076 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3077 bool IsEltFP = EltVT.isFloatingPoint();
3078
3079 // Replace with 2*EXTRACT_VECTOR_ELT + 2*INSERT_VECTOR_ELT, it will match
3080 // the patterns of XVEXTRINS in tablegen.
3081 SDValue BaseVec = (Base == 0) ? V1 : V2;
3082 SDValue EltLo =
3083 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3084 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo, DL, VT: GRLenVT));
3085 SDValue InsLo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: BaseVec, N2: EltLo,
3086 N3: DAG.getConstant(Val: DiffPos[0], DL, VT: GRLenVT));
3087 SDValue EltHi =
3088 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3089 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo + HalfSize, DL, VT: GRLenVT));
3090 SDValue Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: InsLo, N2: EltHi,
3091 N3: DAG.getConstant(Val: DiffPos[1], DL, VT: GRLenVT));
3092
3093 return Result;
3094 };
3095
3096 // Try [0, n-1) insertion then [n, 2n-1) insertion.
3097 if (SDValue Result = tryLowerToExtrAndIns(0))
3098 return Result;
3099 return tryLowerToExtrAndIns(NumElts);
3100}
3101
3102/// Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
3103static SDValue
3104lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3105 SDValue V1, SDValue V2, SelectionDAG &DAG,
3106 const LoongArchSubtarget &Subtarget) {
3107 // LoongArch LASX only supports xvinsve0.{w/d}.
3108 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3109 VT != MVT::v4f64)
3110 return SDValue();
3111
3112 MVT GRLenVT = Subtarget.getGRLenVT();
3113 int MaskSize = Mask.size();
3114 assert(MaskSize == (int)VT.getVectorNumElements() && "Unexpected mask size");
3115
3116 // Check if exactly one element of the Mask is replaced by 'Replaced', while
3117 // all other elements are either 'Base + i' or undef (-1). On success, return
3118 // the index of the replaced element. Otherwise, just return -1.
3119 auto checkReplaceOne = [&](int Base, int Replaced) -> int {
3120 int Idx = -1;
3121 for (int i = 0; i < MaskSize; ++i) {
3122 if (Mask[i] == Base + i || Mask[i] == -1)
3123 continue;
3124 if (Mask[i] != Replaced)
3125 return -1;
3126 if (Idx == -1)
3127 Idx = i;
3128 else
3129 return -1;
3130 }
3131 return Idx;
3132 };
3133
3134 // Case 1: the lowest element of V2 replaces one element in V1.
3135 int Idx = checkReplaceOne(0, MaskSize);
3136 if (Idx != -1)
3137 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V1, N2: V2,
3138 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3139
3140 // Case 2: the lowest element of V1 replaces one element in V2.
3141 Idx = checkReplaceOne(MaskSize, 0);
3142 if (Idx != -1)
3143 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V2, N2: V1,
3144 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3145
3146 return SDValue();
3147}
3148
3149/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).
3150static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,
3151 MVT VT, SDValue V1, SDValue V2,
3152 SelectionDAG &DAG) {
3153
3154 int MaskSize = Mask.size();
3155 int HalfSize = Mask.size() / 2;
3156 const auto &Begin = Mask.begin();
3157 const auto &Mid = Mask.begin() + HalfSize;
3158 const auto &End = Mask.end();
3159
3160 // VECTOR_SHUFFLE concatenates the vectors:
3161 // <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>
3162 // shuffling ->
3163 // <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>
3164 //
3165 // XVSHUF concatenates the vectors:
3166 // <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>
3167 // shuffling ->
3168 // <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>
3169 SmallVector<SDValue, 8> MaskAlloc;
3170 for (auto it = Begin; it < Mid; it++) {
3171 if (*it < 0) // UNDEF
3172 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3173 else if ((*it >= 0 && *it < HalfSize) ||
3174 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3175 int M = *it < HalfSize ? *it : *it - HalfSize;
3176 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3177 } else
3178 return SDValue();
3179 }
3180 assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");
3181
3182 for (auto it = Mid; it < End; it++) {
3183 if (*it < 0) // UNDEF
3184 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3185 else if ((*it >= HalfSize && *it < MaskSize) ||
3186 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3187 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3188 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3189 } else
3190 return SDValue();
3191 }
3192 assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");
3193
3194 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
3195 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops: MaskAlloc);
3196 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
3197}
3198
3199/// Shuffle vectors by lane to generate more optimized instructions.
3200/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
3201///
3202/// Therefore, except for the following four cases, other cases are regarded
3203/// as cross-lane shuffles, where optimization is relatively limited.
3204///
3205/// - Shuffle high, low lanes of two inputs vector
3206/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
3207/// - Shuffle low, high lanes of two inputs vector
3208/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
3209/// - Shuffle low, low lanes of two inputs vector
3210/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
3211/// - Shuffle high, high lanes of two inputs vector
3212/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
3213///
3214/// The first case is the closest to LoongArch instructions and the other
3215/// cases need to be converted to it for processing.
3216///
3217/// This function will return true for the last three cases above and will
3218/// modify V1, V2 and Mask. Otherwise, return false for the first case and
3219/// cross-lane shuffle cases.
3220static bool canonicalizeShuffleVectorByLane(
3221 const SDLoc &DL, MutableArrayRef<int> Mask, MVT VT, SDValue &V1,
3222 SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
3223
3224 enum HalfMaskType { HighLaneTy, LowLaneTy, None };
3225
3226 int MaskSize = Mask.size();
3227 int HalfSize = Mask.size() / 2;
3228 MVT GRLenVT = Subtarget.getGRLenVT();
3229
3230 HalfMaskType preMask = None, postMask = None;
3231
3232 if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3233 return M < 0 || (M >= 0 && M < HalfSize) ||
3234 (M >= MaskSize && M < MaskSize + HalfSize);
3235 }))
3236 preMask = HighLaneTy;
3237 else if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3238 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3239 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3240 }))
3241 preMask = LowLaneTy;
3242
3243 if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3244 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3245 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3246 }))
3247 postMask = LowLaneTy;
3248 else if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3249 return M < 0 || (M >= 0 && M < HalfSize) ||
3250 (M >= MaskSize && M < MaskSize + HalfSize);
3251 }))
3252 postMask = HighLaneTy;
3253
3254 // The pre-half of mask is high lane type, and the post-half of mask
3255 // is low lane type, which is closest to the LoongArch instructions.
3256 //
3257 // Note: In the LoongArch architecture, the high lane of mask corresponds
3258 // to the lower 128-bit of vector register, and the low lane of mask
3259 // corresponds the higher 128-bit of vector register.
3260 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3261 return false;
3262 }
3263 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3264 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3265 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3266 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3267 V1 = DAG.getBitcast(VT, V: V1);
3268
3269 if (!V2.isUndef()) {
3270 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3271 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3272 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3273 V2 = DAG.getBitcast(VT, V: V2);
3274 }
3275
3276 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3277 *it = *it < 0 ? *it : *it - HalfSize;
3278 }
3279 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3280 *it = *it < 0 ? *it : *it + HalfSize;
3281 }
3282 } else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3283 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3284 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3285 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3286 V1 = DAG.getBitcast(VT, V: V1);
3287
3288 if (!V2.isUndef()) {
3289 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3290 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3291 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3292 V2 = DAG.getBitcast(VT, V: V2);
3293 }
3294
3295 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3296 *it = *it < 0 ? *it : *it - HalfSize;
3297 }
3298 } else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3299 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3300 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3301 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3302 V1 = DAG.getBitcast(VT, V: V1);
3303
3304 if (!V2.isUndef()) {
3305 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3306 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3307 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3308 V2 = DAG.getBitcast(VT, V: V2);
3309 }
3310
3311 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3312 *it = *it < 0 ? *it : *it + HalfSize;
3313 }
3314 } else { // cross-lane
3315 return false;
3316 }
3317
3318 return true;
3319}
3320
3321/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
3322/// Only for 256-bit vector.
3323///
3324/// For example:
3325/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
3326/// <4 x i64> <i32 0, i32 3, i32 2, i32 0>
3327/// is lowerded to:
3328/// (XVPERMI $xr2, $xr0, 78)
3329/// (XVSHUF $xr1, $xr2, $xr0)
3330/// (XVORI $xr0, $xr1, 0)
3331static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL,
3332 ArrayRef<int> Mask,
3333 MVT VT, SDValue V1,
3334 SDValue V2,
3335 SelectionDAG &DAG) {
3336 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
3337 int Size = Mask.size();
3338 int LaneSize = Size / 2;
3339
3340 bool LaneCrossing[2] = {false, false};
3341 for (int i = 0; i < Size; ++i)
3342 if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
3343 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
3344
3345 // Ensure that all lanes ared involved.
3346 if (!LaneCrossing[0] && !LaneCrossing[1])
3347 return SDValue();
3348
3349 SmallVector<int> InLaneMask;
3350 InLaneMask.assign(in_start: Mask.begin(), in_end: Mask.end());
3351 for (int i = 0; i < Size; ++i) {
3352 int &M = InLaneMask[i];
3353 if (M < 0)
3354 continue;
3355 if (((M % Size) / LaneSize) != (i / LaneSize))
3356 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
3357 }
3358
3359 SDValue Flipped = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3360 Flipped = DAG.getVectorShuffle(VT: MVT::v4i64, dl: DL, N1: Flipped,
3361 N2: DAG.getUNDEF(VT: MVT::v4i64), Mask: {2, 3, 0, 1});
3362 Flipped = DAG.getBitcast(VT, V: Flipped);
3363 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: Flipped, Mask: InLaneMask);
3364}
3365
3366/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
3367///
3368/// This routine breaks down the specific type of 256-bit shuffle and
3369/// dispatches to the lowering routines accordingly.
3370static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3371 SDValue V1, SDValue V2, SelectionDAG &DAG,
3372 const LoongArchSubtarget &Subtarget) {
3373 assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||
3374 VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||
3375 VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&
3376 "Vector type is unsupported for lasx!");
3377 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
3378 "Two operands have different types!");
3379 assert(VT.getVectorNumElements() == Mask.size() &&
3380 "Unexpected mask size for shuffle!");
3381 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
3382 assert(Mask.size() >= 4 && "Mask size is less than 4.");
3383
3384 APInt KnownUndef, KnownZero;
3385 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
3386 APInt Zeroable = KnownUndef | KnownZero;
3387
3388 SDValue Result;
3389 // TODO: Add more comparison patterns.
3390 if (V2.isUndef()) {
3391 if ((Result =
3392 lowerVECTOR_SHUFFLE_XVREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
3393 return Result;
3394 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG,
3395 Subtarget)))
3396 return Result;
3397 // Try to widen vectors to gain more optimization opportunities.
3398 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
3399 return NewShuffle;
3400 if ((Result =
3401 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3402 return Result;
3403 if ((Result = lowerVECTOR_SHUFFLE_XVPERM(DL, Mask, VT, V1, DAG, Subtarget)))
3404 return Result;
3405 if ((Result =
3406 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
3407 return Result;
3408
3409 // TODO: This comment may be enabled in the future to better match the
3410 // pattern for instruction selection.
3411 /* V2 = V1; */
3412 }
3413
3414 // It is recommended not to change the pattern comparison order for better
3415 // performance.
3416 if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, Mask, VT, V1, V2, DAG)))
3417 return Result;
3418 if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, Mask, VT, V1, V2, DAG)))
3419 return Result;
3420 if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, Mask, VT, V1, V2, DAG)))
3421 return Result;
3422 if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, Mask, VT, V1, V2, DAG)))
3423 return Result;
3424 if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, Mask, VT, V1, V2, DAG)))
3425 return Result;
3426 if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, Mask, VT, V1, V2, DAG)))
3427 return Result;
3428 if ((VT.SimpleTy == MVT::v4i64 || VT.SimpleTy == MVT::v4f64) &&
3429 (Result =
3430 lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3431 return Result;
3432 if ((Result =
3433 lowerVECTOR_SHUFFLE_XVEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3434 return Result;
3435 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
3436 Zeroable)))
3437 return Result;
3438 if ((Result =
3439 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3440 return Result;
3441 if ((Result =
3442 lowerVECTOR_SHUFFLE_XVINSVE0(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3443 return Result;
3444 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
3445 Subtarget)))
3446 return Result;
3447
3448 // canonicalize non cross-lane shuffle vector
3449 SmallVector<int> NewMask(Mask);
3450 if (canonicalizeShuffleVectorByLane(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget))
3451 return lower256BitShuffle(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget);
3452
3453 // FIXME: Handling the remaining cases earlier can degrade performance
3454 // in some situations. Further analysis is required to enable more
3455 // effective optimizations.
3456 if (V2.isUndef()) {
3457 if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, Mask: NewMask, VT,
3458 V1, V2, DAG)))
3459 return Result;
3460 }
3461
3462 if (SDValue NewShuffle = widenShuffleMask(DL, Mask: NewMask, VT, V1, V2, DAG))
3463 return NewShuffle;
3464 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, Mask: NewMask, VT, V1, V2, DAG)))
3465 return Result;
3466
3467 return SDValue();
3468}
3469
3470SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3471 SelectionDAG &DAG) const {
3472 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
3473 ArrayRef<int> OrigMask = SVOp->getMask();
3474 SDValue V1 = Op.getOperand(i: 0);
3475 SDValue V2 = Op.getOperand(i: 1);
3476 MVT VT = Op.getSimpleValueType();
3477 int NumElements = VT.getVectorNumElements();
3478 SDLoc DL(Op);
3479
3480 bool V1IsUndef = V1.isUndef();
3481 bool V2IsUndef = V2.isUndef();
3482 if (V1IsUndef && V2IsUndef)
3483 return DAG.getUNDEF(VT);
3484
3485 // When we create a shuffle node we put the UNDEF node to second operand,
3486 // but in some cases the first operand may be transformed to UNDEF.
3487 // In this case we should just commute the node.
3488 if (V1IsUndef)
3489 return DAG.getCommutedVectorShuffle(SV: *SVOp);
3490
3491 // Check for non-undef masks pointing at an undef vector and make the masks
3492 // undef as well. This makes it easier to match the shuffle based solely on
3493 // the mask.
3494 if (V2IsUndef &&
3495 any_of(Range&: OrigMask, P: [NumElements](int M) { return M >= NumElements; })) {
3496 SmallVector<int, 8> NewMask(OrigMask);
3497 for (int &M : NewMask)
3498 if (M >= NumElements)
3499 M = -1;
3500 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask: NewMask);
3501 }
3502
3503 // Check for illegal shuffle mask element index values.
3504 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
3505 (void)MaskUpperLimit;
3506 assert(llvm::all_of(OrigMask,
3507 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
3508 "Out of bounds shuffle index");
3509
3510 // For each vector width, delegate to a specialized lowering routine.
3511 if (VT.is128BitVector())
3512 return lower128BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3513
3514 if (VT.is256BitVector())
3515 return lower256BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3516
3517 return SDValue();
3518}
3519
3520SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
3521 SelectionDAG &DAG) const {
3522 // Custom lower to ensure the libcall return is passed in an FPR on hard
3523 // float ABIs.
3524 SDLoc DL(Op);
3525 MakeLibCallOptions CallOptions;
3526 SDValue Op0 = Op.getOperand(i: 0);
3527 SDValue Chain = SDValue();
3528 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
3529 SDValue Res;
3530 std::tie(args&: Res, args&: Chain) =
3531 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
3532 if (Subtarget.is64Bit())
3533 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3534 return DAG.getBitcast(VT: MVT::i32, V: Res);
3535}
3536
3537SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
3538 SelectionDAG &DAG) const {
3539 // Custom lower to ensure the libcall argument is passed in an FPR on hard
3540 // float ABIs.
3541 SDLoc DL(Op);
3542 MakeLibCallOptions CallOptions;
3543 SDValue Op0 = Op.getOperand(i: 0);
3544 SDValue Chain = SDValue();
3545 SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3546 DL, VT: MVT::f32, Operand: Op0)
3547 : DAG.getBitcast(VT: MVT::f32, V: Op0);
3548 SDValue Res;
3549 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
3550 CallOptions, dl: DL, Chain);
3551 return Res;
3552}
3553
3554SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
3555 SelectionDAG &DAG) const {
3556 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3557 SDLoc DL(Op);
3558 MakeLibCallOptions CallOptions;
3559 RTLIB::Libcall LC =
3560 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
3561 SDValue Res =
3562 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
3563 if (Subtarget.is64Bit())
3564 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3565 return DAG.getBitcast(VT: MVT::i32, V: Res);
3566}
3567
3568SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
3569 SelectionDAG &DAG) const {
3570 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3571 MVT VT = Op.getSimpleValueType();
3572 SDLoc DL(Op);
3573 Op = DAG.getNode(
3574 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
3575 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
3576 SDValue Res = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3577 DL, VT: MVT::f32, Operand: Op)
3578 : DAG.getBitcast(VT: MVT::f32, V: Op);
3579 if (VT != MVT::f32)
3580 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
3581 return Res;
3582}
3583
3584// Lower BUILD_VECTOR as broadcast load (if possible).
3585// For example:
3586// %a = load i8, ptr %ptr
3587// %b = build_vector %a, %a, %a, %a
3588// is lowered to :
3589// (VLDREPL_B $a0, 0)
3590static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp,
3591 const SDLoc &DL,
3592 SelectionDAG &DAG) {
3593 MVT VT = BVOp->getSimpleValueType(ResNo: 0);
3594 int NumOps = BVOp->getNumOperands();
3595
3596 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3597 "Unsupported vector type for broadcast.");
3598
3599 SDValue IdentitySrc;
3600 bool IsIdeneity = true;
3601
3602 for (int i = 0; i != NumOps; i++) {
3603 SDValue Op = BVOp->getOperand(Num: i);
3604 if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
3605 IsIdeneity = false;
3606 break;
3607 }
3608 IdentitySrc = BVOp->getOperand(Num: 0);
3609 }
3610
3611 // make sure that this load is valid and only has one user.
3612 if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(N: IdentitySrc.getNode()))
3613 return SDValue();
3614
3615 auto *LN = cast<LoadSDNode>(Val&: IdentitySrc);
3616 auto ExtType = LN->getExtensionType();
3617
3618 if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
3619 VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) {
3620 // Indexed loads and stores are not supported on LoongArch.
3621 assert(LN->isUnindexed() && "Unexpected indexed load.");
3622
3623 SDVTList Tys = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3624 // The offset operand of unindexed load is always undefined, so there is
3625 // no need to pass it to VLDREPL.
3626 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3627 SDValue BCast = DAG.getNode(Opcode: LoongArchISD::VLDREPL, DL, VTList: Tys, Ops);
3628 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: BCast.getValue(R: 1));
3629 return BCast;
3630 }
3631 return SDValue();
3632}
3633
3634// Sequentially insert elements from Ops into Vector, from low to high indices.
3635// Note: Ops can have fewer elements than Vector.
3636static void fillVector(ArrayRef<SDValue> Ops, SelectionDAG &DAG, SDLoc DL,
3637 const LoongArchSubtarget &Subtarget, SDValue &Vector,
3638 EVT ResTy) {
3639 assert(Ops.size() <= ResTy.getVectorNumElements());
3640
3641 SDValue Op0 = Ops[0];
3642 if (!Op0.isUndef())
3643 Vector = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: ResTy, Operand: Op0);
3644 for (unsigned i = 1; i < Ops.size(); ++i) {
3645 SDValue Opi = Ops[i];
3646 if (Opi.isUndef())
3647 continue;
3648 Vector = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Vector, N2: Opi,
3649 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3650 }
3651}
3652
3653// Build a ResTy subvector from Node, taking NumElts elements starting at index
3654// 'first'.
3655static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node,
3656 SelectionDAG &DAG, SDLoc DL,
3657 const LoongArchSubtarget &Subtarget,
3658 EVT ResTy, unsigned first) {
3659 unsigned NumElts = ResTy.getVectorNumElements();
3660
3661 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3662
3663 SmallVector<SDValue, 16> Ops(Node->op_begin() + first,
3664 Node->op_begin() + first + NumElts);
3665 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3666 fillVector(Ops, DAG, DL, Subtarget, Vector, ResTy);
3667 return Vector;
3668}
3669
3670SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
3671 SelectionDAG &DAG) const {
3672 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Val&: Op);
3673 MVT VT = Node->getSimpleValueType(ResNo: 0);
3674 EVT ResTy = Op->getValueType(ResNo: 0);
3675 unsigned NumElts = ResTy.getVectorNumElements();
3676 SDLoc DL(Op);
3677 APInt SplatValue, SplatUndef;
3678 unsigned SplatBitSize;
3679 bool HasAnyUndefs;
3680 bool IsConstant = false;
3681 bool UseSameConstant = true;
3682 SDValue ConstantValue;
3683 bool Is128Vec = ResTy.is128BitVector();
3684 bool Is256Vec = ResTy.is256BitVector();
3685
3686 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3687 (!Subtarget.hasExtLASX() || !Is256Vec))
3688 return SDValue();
3689
3690 if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(BVOp: Node, DL, DAG))
3691 return Result;
3692
3693 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3694 /*MinSplatBits=*/8) &&
3695 SplatBitSize <= 64) {
3696 // We can only cope with 8, 16, 32, or 64-bit elements.
3697 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3698 SplatBitSize != 64)
3699 return SDValue();
3700
3701 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3702 // We can only handle 64-bit elements that are within
3703 // the signed 10-bit range or match vldi patterns on 32-bit targets.
3704 // See the BUILD_VECTOR case in LoongArchDAGToDAGISel::Select().
3705 if (!SplatValue.isSignedIntN(N: 10) &&
3706 !isImmVLDILegalForMode1(SplatValue, SplatBitSize).first)
3707 return SDValue();
3708 if ((Is128Vec && ResTy == MVT::v4i32) ||
3709 (Is256Vec && ResTy == MVT::v8i32))
3710 return Op;
3711 }
3712
3713 EVT ViaVecTy;
3714
3715 switch (SplatBitSize) {
3716 default:
3717 return SDValue();
3718 case 8:
3719 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3720 break;
3721 case 16:
3722 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3723 break;
3724 case 32:
3725 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3726 break;
3727 case 64:
3728 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3729 break;
3730 }
3731
3732 // SelectionDAG::getConstant will promote SplatValue appropriately.
3733 SDValue Result = DAG.getConstant(Val: SplatValue, DL, VT: ViaVecTy);
3734
3735 // Bitcast to the type we originally wanted.
3736 if (ViaVecTy != ResTy)
3737 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Node), VT: ResTy, Operand: Result);
3738
3739 return Result;
3740 }
3741
3742 if (DAG.isSplatValue(V: Op, /*AllowUndefs=*/false))
3743 return Op;
3744
3745 for (unsigned i = 0; i < NumElts; ++i) {
3746 SDValue Opi = Node->getOperand(Num: i);
3747 if (isIntOrFPConstant(V: Opi)) {
3748 IsConstant = true;
3749 if (!ConstantValue.getNode())
3750 ConstantValue = Opi;
3751 else if (ConstantValue != Opi)
3752 UseSameConstant = false;
3753 }
3754 }
3755
3756 // If the type of BUILD_VECTOR is v2f64, custom legalizing it has no benefits.
3757 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3758 SDValue Result = DAG.getSplatBuildVector(VT: ResTy, DL, Op: ConstantValue);
3759 for (unsigned i = 0; i < NumElts; ++i) {
3760 SDValue Opi = Node->getOperand(Num: i);
3761 if (!isIntOrFPConstant(V: Opi))
3762 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Result, N2: Opi,
3763 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3764 }
3765 return Result;
3766 }
3767
3768 if (!IsConstant) {
3769 // If the BUILD_VECTOR has a repeated pattern, use INSERT_VECTOR_ELT to fill
3770 // the sub-sequence of the vector and then broadcast the sub-sequence.
3771 //
3772 // TODO: If the BUILD_VECTOR contains undef elements, consider falling
3773 // back to use INSERT_VECTOR_ELT to materialize the vector, because it
3774 // generates worse code in some cases. This could be further optimized
3775 // with more consideration.
3776 SmallVector<SDValue> Sequence;
3777 BitVector UndefElements;
3778 if (Node->getRepeatedSequence(Sequence, UndefElements: &UndefElements) &&
3779 UndefElements.count() == 0) {
3780 // Using LSX instructions to fill the sub-sequence of 256-bits vector,
3781 // because the high part can be simply treated as undef.
3782 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3783 EVT FillTy = Is256Vec
3784 ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext())
3785 : ResTy;
3786 SDValue FillVec =
3787 Is256Vec ? DAG.getExtractSubvector(DL, VT: FillTy, Vec: Vector, Idx: 0) : Vector;
3788
3789 fillVector(Ops: Sequence, DAG, DL, Subtarget, Vector&: FillVec, ResTy: FillTy);
3790
3791 unsigned SeqLen = Sequence.size();
3792 unsigned SplatLen = NumElts / SeqLen;
3793 MVT SplatEltTy = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits() * SeqLen);
3794 MVT SplatTy = MVT::getVectorVT(VT: SplatEltTy, NumElements: SplatLen);
3795
3796 // If size of the sub-sequence is half of a 256-bits vector, bitcast the
3797 // vector to v4i64 type in order to match the pattern of XVREPLVE0Q.
3798 if (SplatEltTy == MVT::i128)
3799 SplatTy = MVT::v4i64;
3800
3801 SDValue SplatVec;
3802 SDValue SrcVec = DAG.getBitcast(
3803 VT: SplatTy,
3804 V: Is256Vec ? DAG.getInsertSubvector(DL, Vec: Vector, SubVec: FillVec, Idx: 0) : FillVec);
3805 if (Is256Vec) {
3806 SplatVec =
3807 DAG.getNode(Opcode: (SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
3808 : LoongArchISD::XVREPLVE0,
3809 DL, VT: SplatTy, Operand: SrcVec);
3810 } else {
3811 SplatVec = DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT: SplatTy, N1: SrcVec,
3812 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
3813 }
3814
3815 return DAG.getBitcast(VT: ResTy, V: SplatVec);
3816 }
3817
3818 // Use INSERT_VECTOR_ELT operations rather than expand to stores, because
3819 // using memory operations is much lower.
3820 //
3821 // For 256-bit vectors, normally split into two halves and concatenate.
3822 // Special case: for v8i32/v8f32/v4i64/v4f64, if the upper half has only
3823 // one non-undef element, skip spliting to avoid a worse result.
3824 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
3825 ResTy == MVT::v4f64) {
3826 unsigned NonUndefCount = 0;
3827 for (unsigned i = NumElts / 2; i < NumElts; ++i) {
3828 if (!Node->getOperand(Num: i).isUndef()) {
3829 ++NonUndefCount;
3830 if (NonUndefCount > 1)
3831 break;
3832 }
3833 }
3834 if (NonUndefCount == 1)
3835 return fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy, first: 0);
3836 }
3837
3838 EVT VecTy =
3839 Is256Vec ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext()) : ResTy;
3840 SDValue Vector =
3841 fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy: VecTy, first: 0);
3842
3843 if (Is128Vec)
3844 return Vector;
3845
3846 SDValue VectorHi = fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget,
3847 ResTy: VecTy, first: NumElts / 2);
3848
3849 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResTy, N1: Vector, N2: VectorHi);
3850 }
3851
3852 return SDValue();
3853}
3854
3855SDValue LoongArchTargetLowering::lowerCONCAT_VECTORS(SDValue Op,
3856 SelectionDAG &DAG) const {
3857 SDLoc DL(Op);
3858 MVT ResVT = Op.getSimpleValueType();
3859 assert(ResVT.is256BitVector() && Op.getNumOperands() == 2);
3860
3861 if (Op.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
3862 Op.getOperand(i: 1).getOpcode() == ISD::TRUNCATE)
3863 return Op;
3864
3865 unsigned NumOperands = Op.getNumOperands();
3866 unsigned NumFreezeUndef = 0;
3867 unsigned NumZero = 0;
3868 unsigned NumNonZero = 0;
3869 unsigned NonZeros = 0;
3870 SmallSet<SDValue, 4> Undefs;
3871 for (unsigned i = 0; i != NumOperands; ++i) {
3872 SDValue SubVec = Op.getOperand(i);
3873 if (SubVec.isUndef())
3874 continue;
3875 if (ISD::isFreezeUndef(N: SubVec.getNode())) {
3876 // If the freeze(undef) has multiple uses then we must fold to zero.
3877 if (SubVec.hasOneUse()) {
3878 ++NumFreezeUndef;
3879 } else {
3880 ++NumZero;
3881 Undefs.insert(V: SubVec);
3882 }
3883 } else if (ISD::isBuildVectorAllZeros(N: SubVec.getNode()))
3884 ++NumZero;
3885 else {
3886 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
3887 NonZeros |= 1 << i;
3888 ++NumNonZero;
3889 }
3890 }
3891
3892 // If we have more than 2 non-zeros, build each half separately.
3893 if (NumNonZero > 2) {
3894 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
3895 ArrayRef<SDUse> Ops = Op->ops();
3896 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3897 Ops: Ops.slice(N: 0, M: NumOperands / 2));
3898 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3899 Ops: Ops.slice(N: NumOperands / 2));
3900 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
3901 }
3902
3903 // Otherwise, build it up through insert_subvectors.
3904 SDValue Vec = NumZero ? DAG.getConstant(Val: 0, DL, VT: ResVT)
3905 : (NumFreezeUndef ? DAG.getFreeze(V: DAG.getUNDEF(VT: ResVT))
3906 : DAG.getUNDEF(VT: ResVT));
3907
3908 // Replace Undef operands with ZeroVector.
3909 for (SDValue U : Undefs)
3910 DAG.ReplaceAllUsesWith(From: U, To: DAG.getConstant(Val: 0, DL, VT: U.getSimpleValueType()));
3911
3912 MVT SubVT = Op.getOperand(i: 0).getSimpleValueType();
3913 unsigned NumSubElems = SubVT.getVectorNumElements();
3914 for (unsigned i = 0; i != NumOperands; ++i) {
3915 if ((NonZeros & (1 << i)) == 0)
3916 continue;
3917
3918 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ResVT, N1: Vec, N2: Op.getOperand(i),
3919 N3: DAG.getVectorIdxConstant(Val: i * NumSubElems, DL));
3920 }
3921
3922 return Vec;
3923}
3924
3925SDValue
3926LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
3927 SelectionDAG &DAG) const {
3928 MVT EltVT = Op.getSimpleValueType();
3929 SDValue Vec = Op->getOperand(Num: 0);
3930 EVT VecTy = Vec->getValueType(ResNo: 0);
3931 SDValue Idx = Op->getOperand(Num: 1);
3932 SDLoc DL(Op);
3933 MVT GRLenVT = Subtarget.getGRLenVT();
3934
3935 assert(VecTy.is256BitVector() && "Unexpected EXTRACT_VECTOR_ELT vector type");
3936
3937 if (isa<ConstantSDNode>(Val: Idx))
3938 return Op;
3939
3940 switch (VecTy.getSimpleVT().SimpleTy) {
3941 default:
3942 llvm_unreachable("Unexpected type");
3943 case MVT::v32i8:
3944 case MVT::v16i16:
3945 case MVT::v4i64:
3946 case MVT::v4f64: {
3947 // Extract the high half subvector and place it to the low half of a new
3948 // vector. It doesn't matter what the high half of the new vector is.
3949 EVT HalfTy = VecTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3950 SDValue VecHi =
3951 DAG.getExtractSubvector(DL, VT: HalfTy, Vec, Idx: HalfTy.getVectorNumElements());
3952 SDValue TmpVec =
3953 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: VecTy, N1: DAG.getUNDEF(VT: VecTy),
3954 N2: VecHi, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3955
3956 // Shuffle the origin Vec and the TmpVec using MaskVec, the lowest element
3957 // of MaskVec is Idx, the rest do not matter. ResVec[0] will hold the
3958 // desired element.
3959 SDValue IdxCp =
3960 Subtarget.is64Bit()
3961 ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Idx)
3962 : DAG.getBitcast(VT: MVT::f32, V: Idx);
3963 SDValue IdxVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v8f32, Operand: IdxCp);
3964 SDValue MaskVec =
3965 DAG.getBitcast(VT: (VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, V: IdxVec);
3966 SDValue ResVec =
3967 DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT: VecTy, N1: MaskVec, N2: TmpVec, N3: Vec);
3968
3969 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: ResVec,
3970 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3971 }
3972 case MVT::v8i32:
3973 case MVT::v8f32: {
3974 SDValue SplatIdx = DAG.getSplatBuildVector(VT: MVT::v8i32, DL, Op: Idx);
3975 SDValue SplatValue =
3976 DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT: VecTy, N1: Vec, N2: SplatIdx);
3977
3978 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: SplatValue,
3979 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3980 }
3981 }
3982}
3983
3984SDValue
3985LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
3986 SelectionDAG &DAG) const {
3987 MVT VT = Op.getSimpleValueType();
3988 MVT EltVT = VT.getVectorElementType();
3989 unsigned NumElts = VT.getVectorNumElements();
3990 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
3991 SDLoc DL(Op);
3992 SDValue Op0 = Op.getOperand(i: 0);
3993 SDValue Op1 = Op.getOperand(i: 1);
3994 SDValue Op2 = Op.getOperand(i: 2);
3995
3996 if (isa<ConstantSDNode>(Val: Op2))
3997 return Op;
3998
3999 MVT IdxTy = MVT::getIntegerVT(BitWidth: EltSizeInBits);
4000 MVT IdxVTy = MVT::getVectorVT(VT: IdxTy, NumElements: NumElts);
4001
4002 if (!isTypeLegal(VT) || !isTypeLegal(VT: IdxVTy))
4003 return SDValue();
4004
4005 SDValue SplatElt = DAG.getSplatBuildVector(VT, DL, Op: Op1);
4006 SmallVector<SDValue, 32> RawIndices;
4007 SDValue SplatIdx;
4008 SDValue Indices;
4009
4010 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4011 MVT PairVTy = MVT::getVectorVT(VT: MVT::i32, NumElements: NumElts * 2);
4012 for (unsigned i = 0; i < NumElts; ++i) {
4013 RawIndices.push_back(Elt: Op2);
4014 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4015 }
4016 SplatIdx = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4017 SplatIdx = DAG.getBitcast(VT: IdxVTy, V: SplatIdx);
4018
4019 RawIndices.clear();
4020 for (unsigned i = 0; i < NumElts; ++i) {
4021 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: MVT::i32));
4022 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4023 }
4024 Indices = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4025 Indices = DAG.getBitcast(VT: IdxVTy, V: Indices);
4026 } else {
4027 SplatIdx = DAG.getSplatBuildVector(VT: IdxVTy, DL, Op: Op2);
4028
4029 for (unsigned i = 0; i < NumElts; ++i)
4030 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
4031 Indices = DAG.getBuildVector(VT: IdxVTy, DL, Ops: RawIndices);
4032 }
4033
4034 // insert vec, elt, idx
4035 // =>
4036 // select (splatidx == {0,1,2...}) ? splatelt : vec
4037 SDValue SelectCC =
4038 DAG.getSetCC(DL, VT: IdxVTy, LHS: SplatIdx, RHS: Indices, Cond: ISD::CondCode::SETEQ);
4039 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectCC, N2: SplatElt, N3: Op0);
4040}
4041
4042SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4043 SelectionDAG &DAG) const {
4044 SDLoc DL(Op);
4045 SyncScope::ID FenceSSID =
4046 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4047
4048 // singlethread fences only synchronize with signal handlers on the same
4049 // thread and thus only need to preserve instruction order, not actually
4050 // enforce memory ordering.
4051 if (FenceSSID == SyncScope::SingleThread)
4052 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4053 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
4054
4055 return Op;
4056}
4057
4058static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL,
4059 MVT GRLenVT, SDValue RMValue) {
4060 // LLVM rounding mode encoding differs from LoongArch FCSR encoding:
4061 // LLVM: 0=RTZ, 1=RNE, 2=RUP, 3=RDN
4062 // FCSR: 0=RNE, 1=RZ, 2=RP, 3=RN
4063 //
4064 // The conversion swaps encodings 0 and 1 while preserving 2 and 3.
4065 // Since the transformation is self-inverse, it applies in both directions:
4066 // LLVM RM <-> LoongArch FCSR RM
4067 //
4068 // Transformation: RM ^ (~(RM >> 1) & 1)
4069 SDValue ShiftRight1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4070 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4071
4072 SDValue SwapMask = DAG.getNode(Opcode: ISD::AND, DL, VT: GRLenVT,
4073 N1: DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: ShiftRight1,
4074 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT)),
4075 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4076
4077 return DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: RMValue, N2: SwapMask);
4078}
4079
4080SDValue LoongArchTargetLowering::lowerSET_ROUNDING(SDValue Op,
4081 SelectionDAG &DAG) const {
4082 MVT GRLenVT = Subtarget.getGRLenVT();
4083 SDLoc DL(Op);
4084 SDValue Chain = Op.getOperand(i: 0);
4085 SDValue RMValue = Op.getOperand(i: 1);
4086
4087 if (auto *CVal = dyn_cast<ConstantSDNode>(Val&: RMValue)) {
4088 uint64_t RM = CVal->getZExtValue();
4089 if (RM > 3) {
4090 MachineFunction &MF = DAG.getMachineFunction();
4091 LLVMContext &C = MF.getFunction().getContext();
4092 C.diagnose(DI: DiagnosticInfoUnsupported(
4093 MF.getFunction(),
4094 "rounding mode is not supported by LoongArch hardware",
4095 DiagnosticLocation(DL.getDebugLoc()), DS_Error));
4096 return Chain;
4097 }
4098 }
4099
4100 RMValue = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: RMValue);
4101 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4102
4103 // The RM field in FCSR is at bits [9:8]. Shift the rounding mode value
4104 // into position before writing via WRFCSR.
4105 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: RMValue,
4106 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4107
4108 // FCSR3 is an alias of the RM field; writing it avoids clobbering
4109 // unrelated fields in FCSR0.
4110 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4111 MachineSDNode *RN = DAG.getMachineNode(Opcode: LoongArch::WRFCSR, dl: DL, VT: MVT::Other,
4112 Op1: FCSRNo, Op2: RMValue, Op3: Chain);
4113 return SDValue(RN, 0);
4114}
4115
4116SDValue LoongArchTargetLowering::lowerGET_ROUNDING(SDValue Op,
4117 SelectionDAG &DAG) const {
4118 MVT GRLenVT = Subtarget.getGRLenVT();
4119 SDLoc DL(Op);
4120 SDValue Chain = Op->getOperand(Num: 0);
4121
4122 // FCSR3 is an alias of the RM field.
4123 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4124 MachineSDNode *FCSR = DAG.getMachineNode(Opcode: LoongArch::RDFCSR, dl: DL, VT1: GRLenVT,
4125 VT2: MVT::Other, Op1: FCSRNo, Op2: Chain);
4126 SDValue RMValue = SDValue(FCSR, 0);
4127 Chain = SDValue(FCSR, 1);
4128
4129 // The RM field in FCSR is at bits [9:8].
4130 RMValue = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4131 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4132 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4133
4134 SDValue RetVal = DAG.getZExtOrTrunc(Op: RMValue, DL, VT: Op.getValueType());
4135 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl: DL);
4136}
4137
4138SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
4139 SelectionDAG &DAG) const {
4140
4141 if (Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i32) {
4142 DAG.getContext()->emitError(
4143 ErrorStr: "On LA64, only 64-bit registers can be written.");
4144 return Op.getOperand(i: 0);
4145 }
4146
4147 if (!Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i64) {
4148 DAG.getContext()->emitError(
4149 ErrorStr: "On LA32, only 32-bit registers can be written.");
4150 return Op.getOperand(i: 0);
4151 }
4152
4153 return Op;
4154}
4155
4156SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
4157 SelectionDAG &DAG) const {
4158 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 0))) {
4159 DAG.getContext()->emitError(ErrorStr: "argument to '__builtin_frame_address' must "
4160 "be a constant integer");
4161 return SDValue();
4162 }
4163
4164 MachineFunction &MF = DAG.getMachineFunction();
4165 MF.getFrameInfo().setFrameAddressIsTaken(true);
4166 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4167 EVT VT = Op.getValueType();
4168 SDLoc DL(Op);
4169 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
4170 unsigned Depth = Op.getConstantOperandVal(i: 0);
4171 int GRLenInBytes = Subtarget.getGRLen() / 8;
4172
4173 while (Depth--) {
4174 int Offset = -(GRLenInBytes * 2);
4175 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
4176 N2: DAG.getSignedConstant(Val: Offset, DL, VT));
4177 FrameAddr =
4178 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
4179 }
4180 return FrameAddr;
4181}
4182
4183SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
4184 SelectionDAG &DAG) const {
4185 // Currently only support lowering return address for current frame.
4186 if (Op.getConstantOperandVal(i: 0) != 0) {
4187 DAG.getContext()->emitError(
4188 ErrorStr: "return address can only be determined for the current frame");
4189 return SDValue();
4190 }
4191
4192 MachineFunction &MF = DAG.getMachineFunction();
4193 MF.getFrameInfo().setReturnAddressIsTaken(true);
4194 MVT GRLenVT = Subtarget.getGRLenVT();
4195
4196 // Return the value of the return address register, marking it an implicit
4197 // live-in.
4198 Register Reg = MF.addLiveIn(PReg: Subtarget.getRegisterInfo()->getRARegister(),
4199 RC: getRegClassFor(VT: GRLenVT));
4200 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg, VT: GRLenVT);
4201}
4202
4203SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
4204 SelectionDAG &DAG) const {
4205 MachineFunction &MF = DAG.getMachineFunction();
4206 auto Size = Subtarget.getGRLen() / 8;
4207 auto FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: 0, IsImmutable: false);
4208 return DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4209}
4210
4211SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,
4212 SelectionDAG &DAG) const {
4213 MachineFunction &MF = DAG.getMachineFunction();
4214 auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();
4215
4216 SDLoc DL(Op);
4217 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4218 VT: getPointerTy(DL: MF.getDataLayout()));
4219
4220 // vastart just stores the address of the VarArgsFrameIndex slot into the
4221 // memory location argument.
4222 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4223 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
4224 PtrInfo: MachinePointerInfo(SV));
4225}
4226
4227SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
4228 SelectionDAG &DAG) const {
4229 SDLoc DL(Op);
4230 SDValue Op0 = Op.getOperand(i: 0);
4231 EVT VT = Op.getValueType();
4232 EVT Op0VT = Op0.getValueType();
4233
4234 if (VT.isVector()) {
4235 if (VT.getScalarSizeInBits() != Op0VT.getScalarSizeInBits())
4236 return SDValue();
4237 return Op;
4238 }
4239
4240 if ((DAG.SignBitIsZero(Op: Op0) || Op->getFlags().hasNonNeg()) &&
4241 !isOperationLegal(Op: ISD::UINT_TO_FP, VT: Op0VT) &&
4242 isOperationLegal(Op: ISD::SINT_TO_FP, VT: Op0VT))
4243 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: Op0);
4244
4245 // We can't do uint64 -> double -> float because of double-rounding issue.
4246 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4247 Op0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i64, Operand: Op0);
4248 SDValue Conv = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::v2f64, Operand: Op0);
4249 Conv = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f64, N1: Conv,
4250 N2: DAG.getIntPtrConstant(Val: 0, DL));
4251 return Conv;
4252 }
4253
4254 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4255 return SDValue();
4256
4257 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4258 !Subtarget.hasBasicD() && "unexpected target features");
4259
4260 if (Op0->getOpcode() == ISD::AND) {
4261 auto *C = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
4262 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4263 return Op;
4264 }
4265
4266 if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&
4267 Op0.getConstantOperandVal(i: 1) < UINT64_C(0X1F) &&
4268 Op0.getConstantOperandVal(i: 2) == UINT64_C(0))
4269 return Op;
4270
4271 if (Op0.getOpcode() == ISD::AssertZext &&
4272 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLT(VT: MVT::i32))
4273 return Op;
4274
4275 EVT OpVT = Op0.getValueType();
4276 EVT RetVT = Op.getValueType();
4277 RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
4278 MakeLibCallOptions CallOptions;
4279 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4280 SDValue Chain = SDValue();
4281 SDValue Result;
4282 std::tie(args&: Result, args&: Chain) =
4283 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4284 return Result;
4285}
4286
4287SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
4288 SelectionDAG &DAG) const {
4289 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4290 !Subtarget.hasBasicD() && "unexpected target features");
4291
4292 SDLoc DL(Op);
4293 SDValue Op0 = Op.getOperand(i: 0);
4294
4295 if ((Op0.getOpcode() == ISD::AssertSext ||
4296 Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&
4297 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLE(VT: MVT::i32))
4298 return Op;
4299
4300 EVT OpVT = Op0.getValueType();
4301 EVT RetVT = Op.getValueType();
4302 RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
4303 MakeLibCallOptions CallOptions;
4304 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4305 SDValue Chain = SDValue();
4306 SDValue Result;
4307 std::tie(args&: Result, args&: Chain) =
4308 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4309 return Result;
4310}
4311
4312SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
4313 SelectionDAG &DAG) const {
4314
4315 SDLoc DL(Op);
4316 EVT VT = Op.getValueType();
4317 SDValue Op0 = Op.getOperand(i: 0);
4318 EVT Op0VT = Op0.getValueType();
4319
4320 if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4321 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4322 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4323 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: NewOp0);
4324 }
4325 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4326 SDValue Lo, Hi;
4327 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
4328 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
4329 }
4330 return Op;
4331}
4332
4333SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
4334 SelectionDAG &DAG) const {
4335
4336 SDLoc DL(Op);
4337 SDValue Op0 = Op.getOperand(i: 0);
4338
4339 if (Op0.getValueType() == MVT::f16)
4340 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
4341
4342 if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4343 !Subtarget.hasBasicD()) {
4344 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f32, Operand: Op0);
4345 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Dst);
4346 }
4347
4348 EVT FPTy = EVT::getFloatingPointVT(BitWidth: Op.getValueSizeInBits());
4349 SDValue Trunc = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FPTy, Operand: Op0);
4350 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op.getValueType(), Operand: Trunc);
4351}
4352
4353SDValue LoongArchTargetLowering::lowerFP_TO_UINT(SDValue Op,
4354 SelectionDAG &DAG) const {
4355 if (!Subtarget.hasExtLSX())
4356 return SDValue();
4357
4358 SDLoc DL(Op);
4359 SDValue Src = Op.getOperand(i: 0);
4360 EVT VT = Op.getValueType();
4361 EVT SrcVT = Src.getValueType();
4362
4363 if (VT != MVT::i64)
4364 return SDValue();
4365
4366 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4367 return SDValue();
4368
4369 if (SrcVT == MVT::f32)
4370 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Src);
4371 Src = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2f64, Operand: Src);
4372 SDValue Conv = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: MVT::v2i64, Operand: Src);
4373 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Conv,
4374 N2: DAG.getIntPtrConstant(Val: 0, DL));
4375}
4376
4377static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
4378 SelectionDAG &DAG, unsigned Flags) {
4379 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
4380}
4381
4382static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
4383 SelectionDAG &DAG, unsigned Flags) {
4384 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
4385 TargetFlags: Flags);
4386}
4387
4388static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
4389 SelectionDAG &DAG, unsigned Flags) {
4390 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
4391 Offset: N->getOffset(), TargetFlags: Flags);
4392}
4393
4394static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
4395 SelectionDAG &DAG, unsigned Flags) {
4396 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
4397}
4398
4399template <class NodeTy>
4400SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4401 CodeModel::Model M,
4402 bool IsLocal) const {
4403 SDLoc DL(N);
4404 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4405 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
4406 SDValue Load;
4407
4408 switch (M) {
4409 default:
4410 report_fatal_error(reason: "Unsupported code model");
4411
4412 case CodeModel::Large: {
4413 assert(Subtarget.is64Bit() && "Large code model requires LA64");
4414
4415 // This is not actually used, but is necessary for successfully matching
4416 // the PseudoLA_*_LARGE nodes.
4417 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4418 if (IsLocal) {
4419 // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that
4420 // eventually becomes the desired 5-insn code sequence.
4421 Load = SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL_LARGE, dl: DL, VT: Ty,
4422 Op1: Tmp, Op2: Addr),
4423 0);
4424 } else {
4425 // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that
4426 // eventually becomes the desired 5-insn code sequence.
4427 Load = SDValue(
4428 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT_LARGE, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr),
4429 0);
4430 }
4431 break;
4432 }
4433
4434 case CodeModel::Small:
4435 case CodeModel::Medium:
4436 if (IsLocal) {
4437 // This generates the pattern (PseudoLA_PCREL sym), which
4438 //
4439 // for la32r expands to:
4440 // (addi.w (pcaddu12i %pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4441 //
4442 // for la32s and la64 expands to:
4443 // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).
4444 Load = SDValue(
4445 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL, dl: DL, VT: Ty, Op1: Addr), 0);
4446 } else {
4447 // This generates the pattern (PseudoLA_GOT sym), which
4448 //
4449 // for la32r expands to:
4450 // (ld.w (pcaddu12i %got_pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4451 //
4452 // for la32s and la64 expands to:
4453 // (ld.w/d (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).
4454 Load =
4455 SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT, dl: DL, VT: Ty, Op1: Addr), 0);
4456 }
4457 }
4458
4459 if (!IsLocal) {
4460 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4461 MachineFunction &MF = DAG.getMachineFunction();
4462 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4463 PtrInfo: MachinePointerInfo::getGOT(MF),
4464 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4465 MachineMemOperand::MOInvariant,
4466 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4467 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
4468 }
4469
4470 return Load;
4471}
4472
4473SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,
4474 SelectionDAG &DAG) const {
4475 return getAddr(N: cast<BlockAddressSDNode>(Val&: Op), DAG,
4476 M: DAG.getTarget().getCodeModel());
4477}
4478
4479SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,
4480 SelectionDAG &DAG) const {
4481 return getAddr(N: cast<JumpTableSDNode>(Val&: Op), DAG,
4482 M: DAG.getTarget().getCodeModel());
4483}
4484
4485SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,
4486 SelectionDAG &DAG) const {
4487 return getAddr(N: cast<ConstantPoolSDNode>(Val&: Op), DAG,
4488 M: DAG.getTarget().getCodeModel());
4489}
4490
4491SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,
4492 SelectionDAG &DAG) const {
4493 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4494 assert(N->getOffset() == 0 && "unexpected offset in global node");
4495 auto CM = DAG.getTarget().getCodeModel();
4496 const GlobalValue *GV = N->getGlobal();
4497
4498 if (GV->isDSOLocal() && isa<GlobalVariable>(Val: GV)) {
4499 if (auto GCM = dyn_cast<GlobalVariable>(Val: GV)->getCodeModel())
4500 CM = *GCM;
4501 }
4502
4503 return getAddr(N, DAG, M: CM, IsLocal: GV->isDSOLocal());
4504}
4505
4506SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
4507 SelectionDAG &DAG,
4508 unsigned Opc, bool UseGOT,
4509 bool Large) const {
4510 SDLoc DL(N);
4511 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4512 MVT GRLenVT = Subtarget.getGRLenVT();
4513
4514 // This is not actually used, but is necessary for successfully matching the
4515 // PseudoLA_*_LARGE nodes.
4516 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4517 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4518
4519 // Only IE needs an extra argument for large code model.
4520 SDValue Offset = Opc == LoongArch::PseudoLA_TLS_IE_LARGE
4521 ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4522 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4523
4524 // If it is LE for normal/medium code model, the add tp operation will occur
4525 // during the pseudo-instruction expansion.
4526 if (Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4527 return Offset;
4528
4529 if (UseGOT) {
4530 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4531 MachineFunction &MF = DAG.getMachineFunction();
4532 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4533 PtrInfo: MachinePointerInfo::getGOT(MF),
4534 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4535 MachineMemOperand::MOInvariant,
4536 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4537 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Offset.getNode()), NewMemRefs: {MemOp});
4538 }
4539
4540 // Add the thread pointer.
4541 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Offset,
4542 N2: DAG.getRegister(Reg: LoongArch::R2, VT: GRLenVT));
4543}
4544
4545SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
4546 SelectionDAG &DAG,
4547 unsigned Opc,
4548 bool Large) const {
4549 SDLoc DL(N);
4550 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4551 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
4552
4553 // This is not actually used, but is necessary for successfully matching the
4554 // PseudoLA_*_LARGE nodes.
4555 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4556
4557 // Use a PC-relative addressing mode to access the dynamic GOT address.
4558 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4559 SDValue Load = Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4560 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4561
4562 // Prepare argument list to generate call.
4563 ArgListTy Args;
4564 Args.emplace_back(args&: Load, args&: CallTy);
4565
4566 // Setup call to __tls_get_addr.
4567 TargetLowering::CallLoweringInfo CLI(DAG);
4568 CLI.setDebugLoc(DL)
4569 .setChain(DAG.getEntryNode())
4570 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
4571 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
4572 ArgsList: std::move(Args));
4573
4574 return LowerCallTo(CLI).first;
4575}
4576
4577SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
4578 SelectionDAG &DAG, unsigned Opc,
4579 bool Large) const {
4580 SDLoc DL(N);
4581 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4582 const GlobalValue *GV = N->getGlobal();
4583
4584 // This is not actually used, but is necessary for successfully matching the
4585 // PseudoLA_*_LARGE nodes.
4586 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4587
4588 // Use a PC-relative addressing mode to access the global dynamic GOT address.
4589 // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).
4590 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
4591 return Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4592 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4593}
4594
4595SDValue
4596LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,
4597 SelectionDAG &DAG) const {
4598 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4599 CallingConv::GHC)
4600 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4601
4602 bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;
4603 assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");
4604
4605 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4606 assert(N->getOffset() == 0 && "unexpected offset in global node");
4607
4608 if (DAG.getTarget().useEmulatedTLS())
4609 reportFatalUsageError(reason: "the emulated TLS is prohibited");
4610
4611 bool IsDesc = DAG.getTarget().useTLSDESC();
4612
4613 switch (getTargetMachine().getTLSModel(GV: N->getGlobal())) {
4614 case TLSModel::GeneralDynamic:
4615 // In this model, application code calls the dynamic linker function
4616 // __tls_get_addr to locate TLS offsets into the dynamic thread vector at
4617 // runtime.
4618 if (!IsDesc)
4619 return getDynamicTLSAddr(N, DAG,
4620 Opc: Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4621 : LoongArch::PseudoLA_TLS_GD,
4622 Large);
4623 break;
4624 case TLSModel::LocalDynamic:
4625 // Same as GeneralDynamic, except for assembly modifiers and relocation
4626 // records.
4627 if (!IsDesc)
4628 return getDynamicTLSAddr(N, DAG,
4629 Opc: Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4630 : LoongArch::PseudoLA_TLS_LD,
4631 Large);
4632 break;
4633 case TLSModel::InitialExec:
4634 // This model uses the GOT to resolve TLS offsets.
4635 return getStaticTLSAddr(N, DAG,
4636 Opc: Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4637 : LoongArch::PseudoLA_TLS_IE,
4638 /*UseGOT=*/true, Large);
4639 case TLSModel::LocalExec:
4640 // This model is used when static linking as the TLS offsets are resolved
4641 // during program linking.
4642 //
4643 // This node doesn't need an extra argument for the large code model.
4644 return getStaticTLSAddr(N, DAG, Opc: LoongArch::PseudoLA_TLS_LE,
4645 /*UseGOT=*/false, Large);
4646 }
4647
4648 return getTLSDescAddr(N, DAG,
4649 Opc: Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
4650 : LoongArch::PseudoLA_TLS_DESC,
4651 Large);
4652}
4653
4654template <unsigned N>
4655static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp,
4656 SelectionDAG &DAG, bool IsSigned = false) {
4657 auto *CImm = cast<ConstantSDNode>(Val: Op->getOperand(Num: ImmOp));
4658 // Check the ImmArg.
4659 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
4660 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
4661 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) +
4662 ": argument out of range.");
4663 return DAG.getNode(Opcode: ISD::UNDEF, DL: SDLoc(Op), VT: Op.getValueType());
4664 }
4665 return SDValue();
4666}
4667
4668SDValue
4669LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
4670 SelectionDAG &DAG) const {
4671 switch (Op.getConstantOperandVal(i: 0)) {
4672 default:
4673 return SDValue(); // Don't custom lower most intrinsics.
4674 case Intrinsic::thread_pointer: {
4675 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4676 return DAG.getRegister(Reg: LoongArch::R2, VT: PtrVT);
4677 }
4678 case Intrinsic::loongarch_lsx_vpickve2gr_d:
4679 case Intrinsic::loongarch_lsx_vpickve2gr_du:
4680 case Intrinsic::loongarch_lsx_vreplvei_d:
4681 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
4682 return checkIntrinsicImmArg<1>(Op, ImmOp: 2, DAG);
4683 case Intrinsic::loongarch_lsx_vreplvei_w:
4684 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
4685 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
4686 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
4687 case Intrinsic::loongarch_lasx_xvpickve_d:
4688 case Intrinsic::loongarch_lasx_xvpickve_d_f:
4689 return checkIntrinsicImmArg<2>(Op, ImmOp: 2, DAG);
4690 case Intrinsic::loongarch_lasx_xvinsve0_d:
4691 return checkIntrinsicImmArg<2>(Op, ImmOp: 3, DAG);
4692 case Intrinsic::loongarch_lsx_vsat_b:
4693 case Intrinsic::loongarch_lsx_vsat_bu:
4694 case Intrinsic::loongarch_lsx_vrotri_b:
4695 case Intrinsic::loongarch_lsx_vsllwil_h_b:
4696 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
4697 case Intrinsic::loongarch_lsx_vsrlri_b:
4698 case Intrinsic::loongarch_lsx_vsrari_b:
4699 case Intrinsic::loongarch_lsx_vreplvei_h:
4700 case Intrinsic::loongarch_lasx_xvsat_b:
4701 case Intrinsic::loongarch_lasx_xvsat_bu:
4702 case Intrinsic::loongarch_lasx_xvrotri_b:
4703 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
4704 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
4705 case Intrinsic::loongarch_lasx_xvsrlri_b:
4706 case Intrinsic::loongarch_lasx_xvsrari_b:
4707 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
4708 case Intrinsic::loongarch_lasx_xvpickve_w:
4709 case Intrinsic::loongarch_lasx_xvpickve_w_f:
4710 return checkIntrinsicImmArg<3>(Op, ImmOp: 2, DAG);
4711 case Intrinsic::loongarch_lasx_xvinsve0_w:
4712 return checkIntrinsicImmArg<3>(Op, ImmOp: 3, DAG);
4713 case Intrinsic::loongarch_lsx_vsat_h:
4714 case Intrinsic::loongarch_lsx_vsat_hu:
4715 case Intrinsic::loongarch_lsx_vrotri_h:
4716 case Intrinsic::loongarch_lsx_vsllwil_w_h:
4717 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
4718 case Intrinsic::loongarch_lsx_vsrlri_h:
4719 case Intrinsic::loongarch_lsx_vsrari_h:
4720 case Intrinsic::loongarch_lsx_vreplvei_b:
4721 case Intrinsic::loongarch_lasx_xvsat_h:
4722 case Intrinsic::loongarch_lasx_xvsat_hu:
4723 case Intrinsic::loongarch_lasx_xvrotri_h:
4724 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
4725 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
4726 case Intrinsic::loongarch_lasx_xvsrlri_h:
4727 case Intrinsic::loongarch_lasx_xvsrari_h:
4728 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
4729 return checkIntrinsicImmArg<4>(Op, ImmOp: 2, DAG);
4730 case Intrinsic::loongarch_lsx_vsrlni_b_h:
4731 case Intrinsic::loongarch_lsx_vsrani_b_h:
4732 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
4733 case Intrinsic::loongarch_lsx_vsrarni_b_h:
4734 case Intrinsic::loongarch_lsx_vssrlni_b_h:
4735 case Intrinsic::loongarch_lsx_vssrani_b_h:
4736 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
4737 case Intrinsic::loongarch_lsx_vssrani_bu_h:
4738 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
4739 case Intrinsic::loongarch_lsx_vssrarni_b_h:
4740 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
4741 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
4742 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
4743 case Intrinsic::loongarch_lasx_xvsrani_b_h:
4744 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
4745 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
4746 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
4747 case Intrinsic::loongarch_lasx_xvssrani_b_h:
4748 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
4749 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
4750 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
4751 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
4752 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
4753 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
4754 return checkIntrinsicImmArg<4>(Op, ImmOp: 3, DAG);
4755 case Intrinsic::loongarch_lsx_vsat_w:
4756 case Intrinsic::loongarch_lsx_vsat_wu:
4757 case Intrinsic::loongarch_lsx_vrotri_w:
4758 case Intrinsic::loongarch_lsx_vsllwil_d_w:
4759 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
4760 case Intrinsic::loongarch_lsx_vsrlri_w:
4761 case Intrinsic::loongarch_lsx_vsrari_w:
4762 case Intrinsic::loongarch_lsx_vslei_bu:
4763 case Intrinsic::loongarch_lsx_vslei_hu:
4764 case Intrinsic::loongarch_lsx_vslei_wu:
4765 case Intrinsic::loongarch_lsx_vslei_du:
4766 case Intrinsic::loongarch_lsx_vslti_bu:
4767 case Intrinsic::loongarch_lsx_vslti_hu:
4768 case Intrinsic::loongarch_lsx_vslti_wu:
4769 case Intrinsic::loongarch_lsx_vslti_du:
4770 case Intrinsic::loongarch_lsx_vbsll_v:
4771 case Intrinsic::loongarch_lsx_vbsrl_v:
4772 case Intrinsic::loongarch_lasx_xvsat_w:
4773 case Intrinsic::loongarch_lasx_xvsat_wu:
4774 case Intrinsic::loongarch_lasx_xvrotri_w:
4775 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
4776 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
4777 case Intrinsic::loongarch_lasx_xvsrlri_w:
4778 case Intrinsic::loongarch_lasx_xvsrari_w:
4779 case Intrinsic::loongarch_lasx_xvslei_bu:
4780 case Intrinsic::loongarch_lasx_xvslei_hu:
4781 case Intrinsic::loongarch_lasx_xvslei_wu:
4782 case Intrinsic::loongarch_lasx_xvslei_du:
4783 case Intrinsic::loongarch_lasx_xvslti_bu:
4784 case Intrinsic::loongarch_lasx_xvslti_hu:
4785 case Intrinsic::loongarch_lasx_xvslti_wu:
4786 case Intrinsic::loongarch_lasx_xvslti_du:
4787 case Intrinsic::loongarch_lasx_xvbsll_v:
4788 case Intrinsic::loongarch_lasx_xvbsrl_v:
4789 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG);
4790 case Intrinsic::loongarch_lsx_vseqi_b:
4791 case Intrinsic::loongarch_lsx_vseqi_h:
4792 case Intrinsic::loongarch_lsx_vseqi_w:
4793 case Intrinsic::loongarch_lsx_vseqi_d:
4794 case Intrinsic::loongarch_lsx_vslei_b:
4795 case Intrinsic::loongarch_lsx_vslei_h:
4796 case Intrinsic::loongarch_lsx_vslei_w:
4797 case Intrinsic::loongarch_lsx_vslei_d:
4798 case Intrinsic::loongarch_lsx_vslti_b:
4799 case Intrinsic::loongarch_lsx_vslti_h:
4800 case Intrinsic::loongarch_lsx_vslti_w:
4801 case Intrinsic::loongarch_lsx_vslti_d:
4802 case Intrinsic::loongarch_lasx_xvseqi_b:
4803 case Intrinsic::loongarch_lasx_xvseqi_h:
4804 case Intrinsic::loongarch_lasx_xvseqi_w:
4805 case Intrinsic::loongarch_lasx_xvseqi_d:
4806 case Intrinsic::loongarch_lasx_xvslei_b:
4807 case Intrinsic::loongarch_lasx_xvslei_h:
4808 case Intrinsic::loongarch_lasx_xvslei_w:
4809 case Intrinsic::loongarch_lasx_xvslei_d:
4810 case Intrinsic::loongarch_lasx_xvslti_b:
4811 case Intrinsic::loongarch_lasx_xvslti_h:
4812 case Intrinsic::loongarch_lasx_xvslti_w:
4813 case Intrinsic::loongarch_lasx_xvslti_d:
4814 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG, /*IsSigned=*/true);
4815 case Intrinsic::loongarch_lsx_vsrlni_h_w:
4816 case Intrinsic::loongarch_lsx_vsrani_h_w:
4817 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
4818 case Intrinsic::loongarch_lsx_vsrarni_h_w:
4819 case Intrinsic::loongarch_lsx_vssrlni_h_w:
4820 case Intrinsic::loongarch_lsx_vssrani_h_w:
4821 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
4822 case Intrinsic::loongarch_lsx_vssrani_hu_w:
4823 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
4824 case Intrinsic::loongarch_lsx_vssrarni_h_w:
4825 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
4826 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
4827 case Intrinsic::loongarch_lsx_vfrstpi_b:
4828 case Intrinsic::loongarch_lsx_vfrstpi_h:
4829 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
4830 case Intrinsic::loongarch_lasx_xvsrani_h_w:
4831 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
4832 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
4833 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
4834 case Intrinsic::loongarch_lasx_xvssrani_h_w:
4835 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
4836 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
4837 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
4838 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
4839 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
4840 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
4841 case Intrinsic::loongarch_lasx_xvfrstpi_b:
4842 case Intrinsic::loongarch_lasx_xvfrstpi_h:
4843 return checkIntrinsicImmArg<5>(Op, ImmOp: 3, DAG);
4844 case Intrinsic::loongarch_lsx_vsat_d:
4845 case Intrinsic::loongarch_lsx_vsat_du:
4846 case Intrinsic::loongarch_lsx_vrotri_d:
4847 case Intrinsic::loongarch_lsx_vsrlri_d:
4848 case Intrinsic::loongarch_lsx_vsrari_d:
4849 case Intrinsic::loongarch_lasx_xvsat_d:
4850 case Intrinsic::loongarch_lasx_xvsat_du:
4851 case Intrinsic::loongarch_lasx_xvrotri_d:
4852 case Intrinsic::loongarch_lasx_xvsrlri_d:
4853 case Intrinsic::loongarch_lasx_xvsrari_d:
4854 return checkIntrinsicImmArg<6>(Op, ImmOp: 2, DAG);
4855 case Intrinsic::loongarch_lsx_vsrlni_w_d:
4856 case Intrinsic::loongarch_lsx_vsrani_w_d:
4857 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
4858 case Intrinsic::loongarch_lsx_vsrarni_w_d:
4859 case Intrinsic::loongarch_lsx_vssrlni_w_d:
4860 case Intrinsic::loongarch_lsx_vssrani_w_d:
4861 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
4862 case Intrinsic::loongarch_lsx_vssrani_wu_d:
4863 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
4864 case Intrinsic::loongarch_lsx_vssrarni_w_d:
4865 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
4866 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
4867 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
4868 case Intrinsic::loongarch_lasx_xvsrani_w_d:
4869 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
4870 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
4871 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
4872 case Intrinsic::loongarch_lasx_xvssrani_w_d:
4873 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
4874 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
4875 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
4876 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
4877 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
4878 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
4879 return checkIntrinsicImmArg<6>(Op, ImmOp: 3, DAG);
4880 case Intrinsic::loongarch_lsx_vsrlni_d_q:
4881 case Intrinsic::loongarch_lsx_vsrani_d_q:
4882 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
4883 case Intrinsic::loongarch_lsx_vsrarni_d_q:
4884 case Intrinsic::loongarch_lsx_vssrlni_d_q:
4885 case Intrinsic::loongarch_lsx_vssrani_d_q:
4886 case Intrinsic::loongarch_lsx_vssrlni_du_q:
4887 case Intrinsic::loongarch_lsx_vssrani_du_q:
4888 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
4889 case Intrinsic::loongarch_lsx_vssrarni_d_q:
4890 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
4891 case Intrinsic::loongarch_lsx_vssrarni_du_q:
4892 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
4893 case Intrinsic::loongarch_lasx_xvsrani_d_q:
4894 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
4895 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
4896 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
4897 case Intrinsic::loongarch_lasx_xvssrani_d_q:
4898 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
4899 case Intrinsic::loongarch_lasx_xvssrani_du_q:
4900 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
4901 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
4902 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
4903 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
4904 return checkIntrinsicImmArg<7>(Op, ImmOp: 3, DAG);
4905 case Intrinsic::loongarch_lsx_vnori_b:
4906 case Intrinsic::loongarch_lsx_vshuf4i_b:
4907 case Intrinsic::loongarch_lsx_vshuf4i_h:
4908 case Intrinsic::loongarch_lsx_vshuf4i_w:
4909 case Intrinsic::loongarch_lasx_xvnori_b:
4910 case Intrinsic::loongarch_lasx_xvshuf4i_b:
4911 case Intrinsic::loongarch_lasx_xvshuf4i_h:
4912 case Intrinsic::loongarch_lasx_xvshuf4i_w:
4913 case Intrinsic::loongarch_lasx_xvpermi_d:
4914 return checkIntrinsicImmArg<8>(Op, ImmOp: 2, DAG);
4915 case Intrinsic::loongarch_lsx_vshuf4i_d:
4916 case Intrinsic::loongarch_lsx_vpermi_w:
4917 case Intrinsic::loongarch_lsx_vbitseli_b:
4918 case Intrinsic::loongarch_lsx_vextrins_b:
4919 case Intrinsic::loongarch_lsx_vextrins_h:
4920 case Intrinsic::loongarch_lsx_vextrins_w:
4921 case Intrinsic::loongarch_lsx_vextrins_d:
4922 case Intrinsic::loongarch_lasx_xvshuf4i_d:
4923 case Intrinsic::loongarch_lasx_xvpermi_w:
4924 case Intrinsic::loongarch_lasx_xvpermi_q:
4925 case Intrinsic::loongarch_lasx_xvbitseli_b:
4926 case Intrinsic::loongarch_lasx_xvextrins_b:
4927 case Intrinsic::loongarch_lasx_xvextrins_h:
4928 case Intrinsic::loongarch_lasx_xvextrins_w:
4929 case Intrinsic::loongarch_lasx_xvextrins_d:
4930 return checkIntrinsicImmArg<8>(Op, ImmOp: 3, DAG);
4931 case Intrinsic::loongarch_lsx_vrepli_b:
4932 case Intrinsic::loongarch_lsx_vrepli_h:
4933 case Intrinsic::loongarch_lsx_vrepli_w:
4934 case Intrinsic::loongarch_lsx_vrepli_d:
4935 case Intrinsic::loongarch_lasx_xvrepli_b:
4936 case Intrinsic::loongarch_lasx_xvrepli_h:
4937 case Intrinsic::loongarch_lasx_xvrepli_w:
4938 case Intrinsic::loongarch_lasx_xvrepli_d:
4939 return checkIntrinsicImmArg<10>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4940 case Intrinsic::loongarch_lsx_vldi:
4941 case Intrinsic::loongarch_lasx_xvldi:
4942 return checkIntrinsicImmArg<13>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4943 }
4944}
4945
4946// Helper function that emits error message for intrinsics with chain and return
4947// merge values of a UNDEF and the chain.
4948static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op,
4949 StringRef ErrorMsg,
4950 SelectionDAG &DAG) {
4951 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
4952 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: Op.getValueType()), Op.getOperand(i: 0)},
4953 dl: SDLoc(Op));
4954}
4955
4956SDValue
4957LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
4958 SelectionDAG &DAG) const {
4959 SDLoc DL(Op);
4960 MVT GRLenVT = Subtarget.getGRLenVT();
4961 EVT VT = Op.getValueType();
4962 SDValue Chain = Op.getOperand(i: 0);
4963 const StringRef ErrorMsgOOR = "argument out of range";
4964 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
4965 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
4966
4967 switch (Op.getConstantOperandVal(i: 1)) {
4968 default:
4969 return Op;
4970 case Intrinsic::loongarch_crc_w_b_w:
4971 case Intrinsic::loongarch_crc_w_h_w:
4972 case Intrinsic::loongarch_crc_w_w_w:
4973 case Intrinsic::loongarch_crc_w_d_w:
4974 case Intrinsic::loongarch_crcc_w_b_w:
4975 case Intrinsic::loongarch_crcc_w_h_w:
4976 case Intrinsic::loongarch_crcc_w_w_w:
4977 case Intrinsic::loongarch_crcc_w_d_w:
4978 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
4979 case Intrinsic::loongarch_csrrd_w:
4980 case Intrinsic::loongarch_csrrd_d: {
4981 unsigned Imm = Op.getConstantOperandVal(i: 2);
4982 return !isUInt<14>(x: Imm)
4983 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
4984 : DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
4985 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
4986 }
4987 case Intrinsic::loongarch_csrwr_w:
4988 case Intrinsic::loongarch_csrwr_d: {
4989 unsigned Imm = Op.getConstantOperandVal(i: 3);
4990 return !isUInt<14>(x: Imm)
4991 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
4992 : DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
4993 Ops: {Chain, Op.getOperand(i: 2),
4994 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
4995 }
4996 case Intrinsic::loongarch_csrxchg_w:
4997 case Intrinsic::loongarch_csrxchg_d: {
4998 unsigned Imm = Op.getConstantOperandVal(i: 4);
4999 return !isUInt<14>(x: Imm)
5000 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5001 : DAG.getNode(Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5002 Ops: {Chain, Op.getOperand(i: 2), Op.getOperand(i: 3),
5003 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5004 }
5005 case Intrinsic::loongarch_iocsrrd_d: {
5006 return DAG.getNode(
5007 Opcode: LoongArchISD::IOCSRRD_D, DL, ResultTys: {GRLenVT, MVT::Other},
5008 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op.getOperand(i: 2))});
5009 }
5010#define IOCSRRD_CASE(NAME, NODE) \
5011 case Intrinsic::loongarch_##NAME: { \
5012 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5013 {Chain, Op.getOperand(2)}); \
5014 }
5015 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5016 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5017 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5018#undef IOCSRRD_CASE
5019 case Intrinsic::loongarch_cpucfg: {
5020 return DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5021 Ops: {Chain, Op.getOperand(i: 2)});
5022 }
5023 case Intrinsic::loongarch_lddir_d: {
5024 unsigned Imm = Op.getConstantOperandVal(i: 3);
5025 return !isUInt<8>(x: Imm)
5026 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5027 : Op;
5028 }
5029 case Intrinsic::loongarch_movfcsr2gr: {
5030 if (!Subtarget.hasBasicF())
5031 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5032 unsigned Imm = Op.getConstantOperandVal(i: 2);
5033 return !isUInt<2>(x: Imm)
5034 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5035 : DAG.getNode(Opcode: LoongArchISD::MOVFCSR2GR, DL, ResultTys: {VT, MVT::Other},
5036 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5037 }
5038 case Intrinsic::loongarch_lsx_vld:
5039 case Intrinsic::loongarch_lsx_vldrepl_b:
5040 case Intrinsic::loongarch_lasx_xvld:
5041 case Intrinsic::loongarch_lasx_xvldrepl_b:
5042 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5043 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5044 : SDValue();
5045 case Intrinsic::loongarch_lsx_vldrepl_h:
5046 case Intrinsic::loongarch_lasx_xvldrepl_h:
5047 return !isShiftedInt<11, 1>(
5048 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5049 ? emitIntrinsicWithChainErrorMessage(
5050 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5051 : SDValue();
5052 case Intrinsic::loongarch_lsx_vldrepl_w:
5053 case Intrinsic::loongarch_lasx_xvldrepl_w:
5054 return !isShiftedInt<10, 2>(
5055 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5056 ? emitIntrinsicWithChainErrorMessage(
5057 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5058 : SDValue();
5059 case Intrinsic::loongarch_lsx_vldrepl_d:
5060 case Intrinsic::loongarch_lasx_xvldrepl_d:
5061 return !isShiftedInt<9, 3>(
5062 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5063 ? emitIntrinsicWithChainErrorMessage(
5064 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5065 : SDValue();
5066 }
5067}
5068
5069// Helper function that emits error message for intrinsics with void return
5070// value and return the chain.
5071static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg,
5072 SelectionDAG &DAG) {
5073
5074 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5075 return Op.getOperand(i: 0);
5076}
5077
5078SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
5079 SelectionDAG &DAG) const {
5080 SDLoc DL(Op);
5081 MVT GRLenVT = Subtarget.getGRLenVT();
5082 SDValue Chain = Op.getOperand(i: 0);
5083 uint64_t IntrinsicEnum = Op.getConstantOperandVal(i: 1);
5084 SDValue Op2 = Op.getOperand(i: 2);
5085 const StringRef ErrorMsgOOR = "argument out of range";
5086 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5087 const StringRef ErrorMsgReqLA32 = "requires loongarch32";
5088 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5089
5090 switch (IntrinsicEnum) {
5091 default:
5092 // TODO: Add more Intrinsics.
5093 return SDValue();
5094 case Intrinsic::loongarch_cacop_d:
5095 case Intrinsic::loongarch_cacop_w: {
5096 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5097 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5098 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5099 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA32, DAG);
5100 // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)
5101 unsigned Imm1 = Op2->getAsZExtVal();
5102 int Imm2 = cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue();
5103 if (!isUInt<5>(x: Imm1) || !isInt<12>(x: Imm2))
5104 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG);
5105 return Op;
5106 }
5107 case Intrinsic::loongarch_dbar: {
5108 unsigned Imm = Op2->getAsZExtVal();
5109 return !isUInt<15>(x: Imm)
5110 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5111 : DAG.getNode(Opcode: LoongArchISD::DBAR, DL, VT: MVT::Other, N1: Chain,
5112 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5113 }
5114 case Intrinsic::loongarch_ibar: {
5115 unsigned Imm = Op2->getAsZExtVal();
5116 return !isUInt<15>(x: Imm)
5117 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5118 : DAG.getNode(Opcode: LoongArchISD::IBAR, DL, VT: MVT::Other, N1: Chain,
5119 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5120 }
5121 case Intrinsic::loongarch_break: {
5122 unsigned Imm = Op2->getAsZExtVal();
5123 return !isUInt<15>(x: Imm)
5124 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5125 : DAG.getNode(Opcode: LoongArchISD::BREAK, DL, VT: MVT::Other, N1: Chain,
5126 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5127 }
5128 case Intrinsic::loongarch_movgr2fcsr: {
5129 if (!Subtarget.hasBasicF())
5130 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5131 unsigned Imm = Op2->getAsZExtVal();
5132 return !isUInt<2>(x: Imm)
5133 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5134 : DAG.getNode(Opcode: LoongArchISD::MOVGR2FCSR, DL, VT: MVT::Other, N1: Chain,
5135 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT),
5136 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT,
5137 Operand: Op.getOperand(i: 3)));
5138 }
5139 case Intrinsic::loongarch_syscall: {
5140 unsigned Imm = Op2->getAsZExtVal();
5141 return !isUInt<15>(x: Imm)
5142 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5143 : DAG.getNode(Opcode: LoongArchISD::SYSCALL, DL, VT: MVT::Other, N1: Chain,
5144 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5145 }
5146#define IOCSRWR_CASE(NAME, NODE) \
5147 case Intrinsic::loongarch_##NAME: { \
5148 SDValue Op3 = Op.getOperand(3); \
5149 return Subtarget.is64Bit() \
5150 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5151 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5152 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5153 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5154 Op3); \
5155 }
5156 IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);
5157 IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);
5158 IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);
5159#undef IOCSRWR_CASE
5160 case Intrinsic::loongarch_iocsrwr_d: {
5161 return !Subtarget.is64Bit()
5162 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5163 : DAG.getNode(Opcode: LoongArchISD::IOCSRWR_D, DL, VT: MVT::Other, N1: Chain,
5164 N2: Op2,
5165 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64,
5166 Operand: Op.getOperand(i: 3)));
5167 }
5168#define ASRT_LE_GT_CASE(NAME) \
5169 case Intrinsic::loongarch_##NAME: { \
5170 return !Subtarget.is64Bit() \
5171 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5172 : Op; \
5173 }
5174 ASRT_LE_GT_CASE(asrtle_d)
5175 ASRT_LE_GT_CASE(asrtgt_d)
5176#undef ASRT_LE_GT_CASE
5177 case Intrinsic::loongarch_ldpte_d: {
5178 unsigned Imm = Op.getConstantOperandVal(i: 3);
5179 return !Subtarget.is64Bit()
5180 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5181 : !isUInt<8>(x: Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5182 : Op;
5183 }
5184 case Intrinsic::loongarch_lsx_vst:
5185 case Intrinsic::loongarch_lasx_xvst:
5186 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue())
5187 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5188 : SDValue();
5189 case Intrinsic::loongarch_lasx_xvstelm_b:
5190 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5191 !isUInt<5>(x: Op.getConstantOperandVal(i: 5)))
5192 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5193 : SDValue();
5194 case Intrinsic::loongarch_lsx_vstelm_b:
5195 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5196 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5197 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5198 : SDValue();
5199 case Intrinsic::loongarch_lasx_xvstelm_h:
5200 return (!isShiftedInt<8, 1>(
5201 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5202 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5203 ? emitIntrinsicErrorMessage(
5204 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5205 : SDValue();
5206 case Intrinsic::loongarch_lsx_vstelm_h:
5207 return (!isShiftedInt<8, 1>(
5208 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5209 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5210 ? emitIntrinsicErrorMessage(
5211 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5212 : SDValue();
5213 case Intrinsic::loongarch_lasx_xvstelm_w:
5214 return (!isShiftedInt<8, 2>(
5215 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5216 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5217 ? emitIntrinsicErrorMessage(
5218 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5219 : SDValue();
5220 case Intrinsic::loongarch_lsx_vstelm_w:
5221 return (!isShiftedInt<8, 2>(
5222 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5223 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5224 ? emitIntrinsicErrorMessage(
5225 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5226 : SDValue();
5227 case Intrinsic::loongarch_lasx_xvstelm_d:
5228 return (!isShiftedInt<8, 3>(
5229 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5230 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5231 ? emitIntrinsicErrorMessage(
5232 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5233 : SDValue();
5234 case Intrinsic::loongarch_lsx_vstelm_d:
5235 return (!isShiftedInt<8, 3>(
5236 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5237 !isUInt<1>(x: Op.getConstantOperandVal(i: 5)))
5238 ? emitIntrinsicErrorMessage(
5239 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5240 : SDValue();
5241 }
5242}
5243
5244SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,
5245 SelectionDAG &DAG) const {
5246 SDLoc DL(Op);
5247 SDValue Lo = Op.getOperand(i: 0);
5248 SDValue Hi = Op.getOperand(i: 1);
5249 SDValue Shamt = Op.getOperand(i: 2);
5250 EVT VT = Lo.getValueType();
5251
5252 // if Shamt-GRLen < 0: // Shamt < GRLen
5253 // Lo = Lo << Shamt
5254 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))
5255 // else:
5256 // Lo = 0
5257 // Hi = Lo << (Shamt-GRLen)
5258
5259 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5260 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5261 SDValue MinusGRLen =
5262 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5263 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5264 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5265 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5266
5267 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
5268 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
5269 SDValue ShiftRightLo =
5270 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: GRLenMinus1Shamt);
5271 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
5272 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
5273 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusGRLen);
5274
5275 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5276
5277 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
5278 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5279
5280 SDValue Parts[2] = {Lo, Hi};
5281 return DAG.getMergeValues(Ops: Parts, dl: DL);
5282}
5283
5284SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,
5285 SelectionDAG &DAG,
5286 bool IsSRA) const {
5287 SDLoc DL(Op);
5288 SDValue Lo = Op.getOperand(i: 0);
5289 SDValue Hi = Op.getOperand(i: 1);
5290 SDValue Shamt = Op.getOperand(i: 2);
5291 EVT VT = Lo.getValueType();
5292
5293 // SRA expansion:
5294 // if Shamt-GRLen < 0: // Shamt < GRLen
5295 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5296 // Hi = Hi >>s Shamt
5297 // else:
5298 // Lo = Hi >>s (Shamt-GRLen);
5299 // Hi = Hi >>s (GRLen-1)
5300 //
5301 // SRL expansion:
5302 // if Shamt-GRLen < 0: // Shamt < GRLen
5303 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5304 // Hi = Hi >>u Shamt
5305 // else:
5306 // Lo = Hi >>u (Shamt-GRLen);
5307 // Hi = 0;
5308
5309 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
5310
5311 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5312 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5313 SDValue MinusGRLen =
5314 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5315 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5316 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5317 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5318
5319 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
5320 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
5321 SDValue ShiftLeftHi =
5322 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: GRLenMinus1Shamt);
5323 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
5324 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
5325 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusGRLen);
5326 SDValue HiFalse =
5327 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: GRLenMinus1) : Zero;
5328
5329 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5330
5331 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
5332 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5333
5334 SDValue Parts[2] = {Lo, Hi};
5335 return DAG.getMergeValues(Ops: Parts, dl: DL);
5336}
5337
5338// Returns the opcode of the target-specific SDNode that implements the 32-bit
5339// form of the given Opcode.
5340static unsigned getLoongArchWOpcode(unsigned Opcode) {
5341 switch (Opcode) {
5342 default:
5343 llvm_unreachable("Unexpected opcode");
5344 case ISD::SDIV:
5345 return LoongArchISD::DIV_W;
5346 case ISD::UDIV:
5347 return LoongArchISD::DIV_WU;
5348 case ISD::SREM:
5349 return LoongArchISD::MOD_W;
5350 case ISD::UREM:
5351 return LoongArchISD::MOD_WU;
5352 case ISD::SHL:
5353 return LoongArchISD::SLL_W;
5354 case ISD::SRA:
5355 return LoongArchISD::SRA_W;
5356 case ISD::SRL:
5357 return LoongArchISD::SRL_W;
5358 case ISD::ROTL:
5359 case ISD::ROTR:
5360 return LoongArchISD::ROTR_W;
5361 case ISD::CTTZ:
5362 return LoongArchISD::CTZ_W;
5363 case ISD::CTLZ:
5364 return LoongArchISD::CLZ_W;
5365 }
5366}
5367
5368// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5369// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
5370// otherwise be promoted to i64, making it difficult to select the
5371// SLL_W/.../*W later one because the fact the operation was originally of
5372// type i8/i16/i32 is lost.
5373static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
5374 unsigned ExtOpc = ISD::ANY_EXTEND) {
5375 SDLoc DL(N);
5376 unsigned WOpcode = getLoongArchWOpcode(Opcode: N->getOpcode());
5377 SDValue NewOp0, NewRes;
5378
5379 switch (NumOp) {
5380 default:
5381 llvm_unreachable("Unexpected NumOp");
5382 case 1: {
5383 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5384 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, Operand: NewOp0);
5385 break;
5386 }
5387 case 2: {
5388 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5389 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5390 if (N->getOpcode() == ISD::ROTL) {
5391 SDValue TmpOp = DAG.getConstant(Val: 32, DL, VT: MVT::i64);
5392 NewOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: TmpOp, N2: NewOp1);
5393 }
5394 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5395 break;
5396 }
5397 // TODO:Handle more NumOp.
5398 }
5399
5400 // ReplaceNodeResults requires we maintain the same type for the return
5401 // value.
5402 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
5403}
5404
5405// Converts the given 32-bit operation to a i64 operation with signed extension
5406// semantic to reduce the signed extension instructions.
5407static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
5408 SDLoc DL(N);
5409 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5410 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5411 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5412 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
5413 N2: DAG.getValueType(MVT::i32));
5414 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
5415}
5416
5417// Helper function that emits error message for intrinsics with/without chain
5418// and return a UNDEF or and the chain as the results.
5419static void emitErrorAndReplaceIntrinsicResults(
5420 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
5421 StringRef ErrorMsg, bool WithChain = true) {
5422 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5423 Results.push_back(Elt: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)));
5424 if (!WithChain)
5425 return;
5426 Results.push_back(Elt: N->getOperand(Num: 0));
5427}
5428
5429template <unsigned N>
5430static void
5431replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl<SDValue> &Results,
5432 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget,
5433 unsigned ResOp) {
5434 const StringRef ErrorMsgOOR = "argument out of range";
5435 unsigned Imm = Node->getConstantOperandVal(Num: 2);
5436 if (!isUInt<N>(Imm)) {
5437 emitErrorAndReplaceIntrinsicResults(N: Node, Results, DAG, ErrorMsg: ErrorMsgOOR,
5438 /*WithChain=*/false);
5439 return;
5440 }
5441 SDLoc DL(Node);
5442 SDValue Vec = Node->getOperand(Num: 1);
5443
5444 SDValue PickElt =
5445 DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), N1: Vec,
5446 N2: DAG.getConstant(Val: Imm, DL, VT: Subtarget.getGRLenVT()),
5447 N3: DAG.getValueType(Vec.getValueType().getVectorElementType()));
5448 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Node->getValueType(ResNo: 0),
5449 Operand: PickElt.getValue(R: 0)));
5450}
5451
5452static void replaceVecCondBranchResults(SDNode *N,
5453 SmallVectorImpl<SDValue> &Results,
5454 SelectionDAG &DAG,
5455 const LoongArchSubtarget &Subtarget,
5456 unsigned ResOp) {
5457 SDLoc DL(N);
5458 SDValue Vec = N->getOperand(Num: 1);
5459
5460 SDValue CB = DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), Operand: Vec);
5461 Results.push_back(
5462 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: CB.getValue(R: 0)));
5463}
5464
5465static void
5466replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
5467 SelectionDAG &DAG,
5468 const LoongArchSubtarget &Subtarget) {
5469 switch (N->getConstantOperandVal(Num: 0)) {
5470 default:
5471 llvm_unreachable("Unexpected Intrinsic.");
5472 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5473 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5474 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5475 break;
5476 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5477 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5478 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5479 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5480 break;
5481 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5482 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5483 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5484 break;
5485 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5486 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5487 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5488 break;
5489 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5490 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5491 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5492 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5493 break;
5494 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5495 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5496 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5497 break;
5498 case Intrinsic::loongarch_lsx_bz_b:
5499 case Intrinsic::loongarch_lsx_bz_h:
5500 case Intrinsic::loongarch_lsx_bz_w:
5501 case Intrinsic::loongarch_lsx_bz_d:
5502 case Intrinsic::loongarch_lasx_xbz_b:
5503 case Intrinsic::loongarch_lasx_xbz_h:
5504 case Intrinsic::loongarch_lasx_xbz_w:
5505 case Intrinsic::loongarch_lasx_xbz_d:
5506 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5507 ResOp: LoongArchISD::VALL_ZERO);
5508 break;
5509 case Intrinsic::loongarch_lsx_bz_v:
5510 case Intrinsic::loongarch_lasx_xbz_v:
5511 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5512 ResOp: LoongArchISD::VANY_ZERO);
5513 break;
5514 case Intrinsic::loongarch_lsx_bnz_b:
5515 case Intrinsic::loongarch_lsx_bnz_h:
5516 case Intrinsic::loongarch_lsx_bnz_w:
5517 case Intrinsic::loongarch_lsx_bnz_d:
5518 case Intrinsic::loongarch_lasx_xbnz_b:
5519 case Intrinsic::loongarch_lasx_xbnz_h:
5520 case Intrinsic::loongarch_lasx_xbnz_w:
5521 case Intrinsic::loongarch_lasx_xbnz_d:
5522 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5523 ResOp: LoongArchISD::VALL_NONZERO);
5524 break;
5525 case Intrinsic::loongarch_lsx_bnz_v:
5526 case Intrinsic::loongarch_lasx_xbnz_v:
5527 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5528 ResOp: LoongArchISD::VANY_NONZERO);
5529 break;
5530 }
5531}
5532
5533static void replaceCMP_XCHG_128Results(SDNode *N,
5534 SmallVectorImpl<SDValue> &Results,
5535 SelectionDAG &DAG) {
5536 assert(N->getValueType(0) == MVT::i128 &&
5537 "AtomicCmpSwap on types less than 128 should be legal");
5538 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
5539
5540 unsigned Opcode;
5541 switch (MemOp->getMergedOrdering()) {
5542 case AtomicOrdering::Acquire:
5543 case AtomicOrdering::AcquireRelease:
5544 case AtomicOrdering::SequentiallyConsistent:
5545 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5546 break;
5547 case AtomicOrdering::Monotonic:
5548 case AtomicOrdering::Release:
5549 Opcode = LoongArch::PseudoCmpXchg128;
5550 break;
5551 default:
5552 llvm_unreachable("Unexpected ordering!");
5553 }
5554
5555 SDLoc DL(N);
5556 auto CmpVal = DAG.SplitScalar(N: N->getOperand(Num: 2), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5557 auto NewVal = DAG.SplitScalar(N: N->getOperand(Num: 3), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5558 SDValue Ops[] = {N->getOperand(Num: 1), CmpVal.first, CmpVal.second,
5559 NewVal.first, NewVal.second, N->getOperand(Num: 0)};
5560
5561 SDNode *CmpSwap = DAG.getMachineNode(
5562 Opcode, dl: SDLoc(N), VTs: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::i64, VT4: MVT::Other),
5563 Ops);
5564 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
5565 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
5566 N1: SDValue(CmpSwap, 0), N2: SDValue(CmpSwap, 1)));
5567 Results.push_back(Elt: SDValue(CmpSwap, 3));
5568}
5569
5570void LoongArchTargetLowering::ReplaceNodeResults(
5571 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
5572 SDLoc DL(N);
5573 EVT VT = N->getValueType(ResNo: 0);
5574 switch (N->getOpcode()) {
5575 default:
5576 llvm_unreachable("Don't know how to legalize this operation");
5577 case ISD::ADD:
5578 case ISD::SUB:
5579 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5580 "Unexpected custom legalisation");
5581 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
5582 break;
5583 case ISD::SDIV:
5584 case ISD::UDIV:
5585 case ISD::SREM:
5586 case ISD::UREM:
5587 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5588 "Unexpected custom legalisation");
5589 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2,
5590 ExtOpc: Subtarget.hasDiv32() && VT == MVT::i32
5591 ? ISD::ANY_EXTEND
5592 : ISD::SIGN_EXTEND));
5593 break;
5594 case ISD::SHL:
5595 case ISD::SRA:
5596 case ISD::SRL:
5597 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5598 "Unexpected custom legalisation");
5599 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
5600 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5601 break;
5602 }
5603 break;
5604 case ISD::ROTL:
5605 case ISD::ROTR:
5606 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5607 "Unexpected custom legalisation");
5608 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5609 break;
5610 case ISD::LOAD: {
5611 // Use an f64 load and a scalar_to_vector for v2f32 loads. This avoids
5612 // scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
5613 // cast since type legalization will try to use an i64 load.
5614 MVT VT = N->getSimpleValueType(ResNo: 0);
5615 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5616 "Unexpected custom legalisation");
5617 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
5618 "Unexpected type action!");
5619 if (!ISD::isNON_EXTLoad(N))
5620 return;
5621 auto *Ld = cast<LoadSDNode>(Val: N);
5622 SDValue Res = DAG.getLoad(VT: MVT::f64, dl: DL, Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5623 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getBaseAlign(),
5624 MMOFlags: Ld->getMemOperand()->getFlags());
5625 SDValue Chain = Res.getValue(R: 1);
5626 MVT VecVT = MVT::getVectorVT(VT: MVT::f64, NumElements: 2);
5627 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecVT, Operand: Res);
5628 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5629 Res = DAG.getBitcast(VT: WideVT, V: Res);
5630 Results.push_back(Elt: Res);
5631 Results.push_back(Elt: Chain);
5632 break;
5633 }
5634 case ISD::FP_TO_SINT: {
5635 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5636 "Unexpected custom legalisation");
5637 SDValue Src = N->getOperand(Num: 0);
5638 EVT FVT = EVT::getFloatingPointVT(BitWidth: N->getValueSizeInBits(ResNo: 0));
5639 if (getTypeAction(Context&: *DAG.getContext(), VT: Src.getValueType()) !=
5640 TargetLowering::TypeSoftenFloat) {
5641 if (!isTypeLegal(VT: Src.getValueType()))
5642 return;
5643 if (Src.getValueType() == MVT::f16)
5644 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
5645 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FVT, Operand: Src);
5646 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Dst));
5647 return;
5648 }
5649 // If the FP type needs to be softened, emit a library call using the 'si'
5650 // version. If we left it to default legalization we'd end up with 'di'.
5651 RTLIB::Libcall LC;
5652 LC = RTLIB::getFPTOSINT(OpVT: Src.getValueType(), RetVT: VT);
5653 MakeLibCallOptions CallOptions;
5654 EVT OpVT = Src.getValueType();
5655 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: VT);
5656 SDValue Chain = SDValue();
5657 SDValue Result;
5658 std::tie(args&: Result, args&: Chain) =
5659 makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl: DL, Chain);
5660 Results.push_back(Elt: Result);
5661 break;
5662 }
5663 case ISD::BITCAST: {
5664 SDValue Src = N->getOperand(Num: 0);
5665 EVT SrcVT = Src.getValueType();
5666 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
5667 Subtarget.hasBasicF()) {
5668 SDValue Dst =
5669 DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Src);
5670 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Dst));
5671 } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
5672 SDValue NewReg = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
5673 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Src);
5674 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64,
5675 N1: NewReg.getValue(R: 0), N2: NewReg.getValue(R: 1));
5676 Results.push_back(Elt: RetReg);
5677 }
5678 break;
5679 }
5680 case ISD::FP_TO_UINT: {
5681 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5682 "Unexpected custom legalisation");
5683 auto &TLI = DAG.getTargetLoweringInfo();
5684 SDValue Tmp1, Tmp2;
5685 TLI.expandFP_TO_UINT(N, Result&: Tmp1, Chain&: Tmp2, DAG);
5686 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Tmp1));
5687 break;
5688 }
5689 case ISD::FP_ROUND: {
5690 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5691 "Unexpected custom legalisation");
5692 // On LSX platforms, rounding from v2f64 to v4f32 (after legalization from
5693 // v2f32) is scalarized. Add a customized v2f32 widening to convert it into
5694 // a target-specific LoongArchISD::VFCVT to optimize it.
5695 SDValue Op0 = N->getOperand(Num: 0);
5696 EVT OpVT = Op0.getValueType();
5697 if (OpVT == MVT::v2f64) {
5698 SDValue Undef = DAG.getUNDEF(VT: OpVT);
5699 SDValue Dst =
5700 DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Undef, N2: Op0);
5701 Results.push_back(Elt: Dst);
5702 }
5703 break;
5704 }
5705 case ISD::BSWAP: {
5706 SDValue Src = N->getOperand(Num: 0);
5707 assert((VT == MVT::i16 || VT == MVT::i32) &&
5708 "Unexpected custom legalization");
5709 MVT GRLenVT = Subtarget.getGRLenVT();
5710 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5711 SDValue Tmp;
5712 switch (VT.getSizeInBits()) {
5713 default:
5714 llvm_unreachable("Unexpected operand width");
5715 case 16:
5716 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2H, DL, VT: GRLenVT, Operand: NewSrc);
5717 break;
5718 case 32:
5719 // Only LA64 will get to here due to the size mismatch between VT and
5720 // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
5721 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2W, DL, VT: GRLenVT, Operand: NewSrc);
5722 break;
5723 }
5724 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5725 break;
5726 }
5727 case ISD::BITREVERSE: {
5728 SDValue Src = N->getOperand(Num: 0);
5729 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
5730 "Unexpected custom legalization");
5731 MVT GRLenVT = Subtarget.getGRLenVT();
5732 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5733 SDValue Tmp;
5734 switch (VT.getSizeInBits()) {
5735 default:
5736 llvm_unreachable("Unexpected operand width");
5737 case 8:
5738 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL, VT: GRLenVT, Operand: NewSrc);
5739 break;
5740 case 32:
5741 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_W, DL, VT: GRLenVT, Operand: NewSrc);
5742 break;
5743 }
5744 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5745 break;
5746 }
5747 case ISD::CTLZ:
5748 case ISD::CTTZ: {
5749 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5750 "Unexpected custom legalisation");
5751 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 1));
5752 break;
5753 }
5754 case ISD::INTRINSIC_W_CHAIN: {
5755 SDValue Chain = N->getOperand(Num: 0);
5756 SDValue Op2 = N->getOperand(Num: 2);
5757 MVT GRLenVT = Subtarget.getGRLenVT();
5758 const StringRef ErrorMsgOOR = "argument out of range";
5759 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5760 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5761
5762 switch (N->getConstantOperandVal(Num: 1)) {
5763 default:
5764 llvm_unreachable("Unexpected Intrinsic.");
5765 case Intrinsic::loongarch_movfcsr2gr: {
5766 if (!Subtarget.hasBasicF()) {
5767 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqF);
5768 return;
5769 }
5770 unsigned Imm = Op2->getAsZExtVal();
5771 if (!isUInt<2>(x: Imm)) {
5772 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5773 return;
5774 }
5775 SDValue MOVFCSR2GRResults = DAG.getNode(
5776 Opcode: LoongArchISD::MOVFCSR2GR, DL: SDLoc(N), ResultTys: {MVT::i64, MVT::Other},
5777 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5778 Results.push_back(
5779 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: MOVFCSR2GRResults.getValue(R: 0)));
5780 Results.push_back(Elt: MOVFCSR2GRResults.getValue(R: 1));
5781 break;
5782 }
5783#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
5784 case Intrinsic::loongarch_##NAME: { \
5785 SDValue NODE = DAG.getNode( \
5786 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5787 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5788 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5789 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5790 Results.push_back(NODE.getValue(1)); \
5791 break; \
5792 }
5793 CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
5794 CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)
5795 CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)
5796 CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)
5797 CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)
5798 CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)
5799#undef CRC_CASE_EXT_BINARYOP
5800
5801#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
5802 case Intrinsic::loongarch_##NAME: { \
5803 SDValue NODE = DAG.getNode( \
5804 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5805 {Chain, Op2, \
5806 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5807 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5808 Results.push_back(NODE.getValue(1)); \
5809 break; \
5810 }
5811 CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
5812 CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)
5813#undef CRC_CASE_EXT_UNARYOP
5814#define CSR_CASE(ID) \
5815 case Intrinsic::loongarch_##ID: { \
5816 if (!Subtarget.is64Bit()) \
5817 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
5818 break; \
5819 }
5820 CSR_CASE(csrrd_d);
5821 CSR_CASE(csrwr_d);
5822 CSR_CASE(csrxchg_d);
5823 CSR_CASE(iocsrrd_d);
5824#undef CSR_CASE
5825 case Intrinsic::loongarch_csrrd_w: {
5826 unsigned Imm = Op2->getAsZExtVal();
5827 if (!isUInt<14>(x: Imm)) {
5828 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5829 return;
5830 }
5831 SDValue CSRRDResults =
5832 DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
5833 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5834 Results.push_back(
5835 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRRDResults.getValue(R: 0)));
5836 Results.push_back(Elt: CSRRDResults.getValue(R: 1));
5837 break;
5838 }
5839 case Intrinsic::loongarch_csrwr_w: {
5840 unsigned Imm = N->getConstantOperandVal(Num: 3);
5841 if (!isUInt<14>(x: Imm)) {
5842 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5843 return;
5844 }
5845 SDValue CSRWRResults =
5846 DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5847 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5848 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5849 Results.push_back(
5850 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRWRResults.getValue(R: 0)));
5851 Results.push_back(Elt: CSRWRResults.getValue(R: 1));
5852 break;
5853 }
5854 case Intrinsic::loongarch_csrxchg_w: {
5855 unsigned Imm = N->getConstantOperandVal(Num: 4);
5856 if (!isUInt<14>(x: Imm)) {
5857 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5858 return;
5859 }
5860 SDValue CSRXCHGResults = DAG.getNode(
5861 Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5862 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5863 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 3)),
5864 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5865 Results.push_back(
5866 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRXCHGResults.getValue(R: 0)));
5867 Results.push_back(Elt: CSRXCHGResults.getValue(R: 1));
5868 break;
5869 }
5870#define IOCSRRD_CASE(NAME, NODE) \
5871 case Intrinsic::loongarch_##NAME: { \
5872 SDValue IOCSRRDResults = \
5873 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5874 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
5875 Results.push_back( \
5876 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
5877 Results.push_back(IOCSRRDResults.getValue(1)); \
5878 break; \
5879 }
5880 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5881 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5882 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5883#undef IOCSRRD_CASE
5884 case Intrinsic::loongarch_cpucfg: {
5885 SDValue CPUCFGResults =
5886 DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5887 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2)});
5888 Results.push_back(
5889 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CPUCFGResults.getValue(R: 0)));
5890 Results.push_back(Elt: CPUCFGResults.getValue(R: 1));
5891 break;
5892 }
5893 case Intrinsic::loongarch_lddir_d: {
5894 if (!Subtarget.is64Bit()) {
5895 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqLA64);
5896 return;
5897 }
5898 break;
5899 }
5900 }
5901 break;
5902 }
5903 case ISD::READ_REGISTER: {
5904 if (Subtarget.is64Bit())
5905 DAG.getContext()->emitError(
5906 ErrorStr: "On LA64, only 64-bit registers can be read.");
5907 else
5908 DAG.getContext()->emitError(
5909 ErrorStr: "On LA32, only 32-bit registers can be read.");
5910 Results.push_back(Elt: DAG.getUNDEF(VT));
5911 Results.push_back(Elt: N->getOperand(Num: 0));
5912 break;
5913 }
5914 case ISD::INTRINSIC_WO_CHAIN: {
5915 replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);
5916 break;
5917 }
5918 case ISD::LROUND: {
5919 SDValue Op0 = N->getOperand(Num: 0);
5920 EVT OpVT = Op0.getValueType();
5921 RTLIB::Libcall LC =
5922 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
5923 MakeLibCallOptions CallOptions;
5924 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
5925 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
5926 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
5927 Results.push_back(Elt: Result);
5928 break;
5929 }
5930 case ISD::ATOMIC_CMP_SWAP: {
5931 replaceCMP_XCHG_128Results(N, Results, DAG);
5932 break;
5933 }
5934 case ISD::TRUNCATE: {
5935 MVT VT = N->getSimpleValueType(ResNo: 0);
5936 if (getTypeAction(Context&: *DAG.getContext(), VT) != TypeWidenVector)
5937 return;
5938
5939 MVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT).getSimpleVT();
5940 SDValue In = N->getOperand(Num: 0);
5941 EVT InVT = In.getValueType();
5942 EVT InEltVT = InVT.getVectorElementType();
5943 EVT EltVT = VT.getVectorElementType();
5944 unsigned MinElts = VT.getVectorNumElements();
5945 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5946 unsigned InBits = InVT.getSizeInBits();
5947
5948 // v8i64 -> (v8i32) -> v8i8
5949 if (InVT == MVT::v8i64 && WidenVT.is128BitVector()) {
5950 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 256 / MinElts), NumElements: MinElts);
5951 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5952 InBits = 256;
5953 }
5954
5955 // v8i32 -> v8i8 / v4i64 -> v4i16 / v4i64 -> v4i8
5956 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
5957 WidenVT.is128BitVector()) {
5958 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / MinElts), NumElements: MinElts);
5959 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5960 InBits = 128;
5961 InEltVT = InVT.getVectorElementType();
5962 }
5963
5964 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
5965 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
5966 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
5967 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
5968 for (unsigned I = 0; I < MinElts; ++I)
5969 TruncMask[I] = Scale * I;
5970
5971 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
5972 MVT SVT = In.getSimpleValueType().getScalarType();
5973 MVT VT = MVT::getVectorVT(VT: SVT, NumElements: WidenNumElts);
5974 SDValue WidenIn =
5975 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT), N2: In,
5976 N3: DAG.getVectorIdxConstant(Val: 0, DL));
5977 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
5978 "Illegal vector type in truncation");
5979 WidenIn = DAG.getBitcast(VT: WidenVT, V: WidenIn);
5980 Results.push_back(
5981 Elt: DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: WidenIn, N2: WidenIn, Mask: TruncMask));
5982 return;
5983 }
5984 }
5985
5986 break;
5987 }
5988 case ISD::SIGN_EXTEND: {
5989 // LASX has native VEXT2XV_* for sign extension.
5990 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
5991 return;
5992
5993 EVT DstVT = N->getValueType(ResNo: 0);
5994 SDValue Src = N->getOperand(Num: 0);
5995 MVT SrcVT = Src.getSimpleValueType();
5996
5997 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5998 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5999 unsigned NumElts = DstVT.getVectorNumElements();
6000
6001 if (SrcVT.getSizeInBits() > 128)
6002 return;
6003
6004 if (!DstVT.isVector() || DstVT.getSizeInBits() <= 128)
6005 return;
6006
6007 // Legalize and extend the src to 128-bit first.
6008 if (SrcVT.getSizeInBits() < 128) {
6009 unsigned WidenSrcElts = 128 / SrcEltBits;
6010 MVT WidenSrcVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: WidenSrcElts);
6011 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WidenSrcVT,
6012 N1: DAG.getUNDEF(VT: WidenSrcVT), N2: Src,
6013 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6014 SrcVT = WidenSrcVT;
6015
6016 unsigned FirstStageEltBits = 128 / NumElts;
6017 MVT FirstStageEltVT = MVT::getIntegerVT(BitWidth: FirstStageEltBits);
6018 MVT FirstStageVT = MVT::getVectorVT(VT: FirstStageEltVT, NumElements: NumElts);
6019 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: FirstStageVT, Operand: Src);
6020 SrcVT = FirstStageVT;
6021 SrcEltBits = FirstStageEltBits;
6022 }
6023
6024 SmallVector<SDValue, 8> Blocks;
6025 Blocks.push_back(Elt: Src);
6026
6027 // Sign-extend the src by using SLTI + VILVL + VILVH recursively.
6028 while (SrcEltBits < DstEltBits) {
6029 unsigned NextEltBits = SrcEltBits * 2;
6030 MVT NextEltVT = MVT::getIntegerVT(BitWidth: NextEltBits);
6031 unsigned CurEltsPerBlock = SrcVT.getVectorNumElements();
6032 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6033 MVT NextBlockVT = MVT::getVectorVT(VT: NextEltVT, NumElements: NextEltsPerBlock);
6034
6035 SmallVector<SDValue, 8> NextBlocks;
6036 NextBlocks.reserve(N: Blocks.size() * 2);
6037 for (SDValue Block : Blocks) {
6038 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
6039 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Block, N2: Zero,
6040 N3: DAG.getCondCode(Cond: ISD::SETLT));
6041 SDValue LoInterleaved =
6042 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Block);
6043 SDValue HiInterleaved =
6044 DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT: SrcVT, N1: Mask, N2: Block);
6045
6046 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: LoInterleaved));
6047 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: HiInterleaved));
6048 }
6049
6050 Blocks = std::move(NextBlocks);
6051 SrcVT = NextBlockVT;
6052 SrcEltBits = NextEltBits;
6053 }
6054
6055 Results.push_back(Elt: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Blocks));
6056 break;
6057 }
6058 case ISD::FP_EXTEND:
6059 // FP_EXTEND may reach here due to the Custom action for v2f32 results, but
6060 // no target-specific lowering is required. Leave it unchanged and rely on
6061 // the default type legalization.
6062 break;
6063 }
6064}
6065
6066/// Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
6067static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL,
6068 SelectionDAG &DAG) {
6069 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDN");
6070
6071 MVT VT = N->getSimpleValueType(ResNo: 0);
6072 if (!VT.is128BitVector() && !VT.is256BitVector())
6073 return SDValue();
6074
6075 SDValue X, Y;
6076 SDValue N0 = N->getOperand(Num: 0);
6077 SDValue N1 = N->getOperand(Num: 1);
6078
6079 if (SDValue Not = isNOT(V: N0, DAG)) {
6080 X = Not;
6081 Y = N1;
6082 } else if (SDValue Not = isNOT(V: N1, DAG)) {
6083 X = Not;
6084 Y = N0;
6085 } else
6086 return SDValue();
6087
6088 X = DAG.getBitcast(VT, V: X);
6089 Y = DAG.getBitcast(VT, V: Y);
6090 return DAG.getNode(Opcode: LoongArchISD::VANDN, DL, VT, N1: X, N2: Y);
6091}
6092
6093static bool isConstantSplatVector(SDValue N, APInt &SplatValue,
6094 unsigned MinSizeInBits) {
6095 N = peekThroughBitcasts(V: N);
6096 BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(Val&: N);
6097
6098 if (!Node)
6099 return false;
6100
6101 APInt SplatUndef;
6102 unsigned SplatBitSize;
6103 bool HasAnyUndefs;
6104
6105 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6106 HasAnyUndefs, MinSplatBits: MinSizeInBits,
6107 /*IsBigEndian=*/isBigEndian: false);
6108}
6109
6110static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex) {
6111 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6112 if (!BV)
6113 return SDValue();
6114
6115 SDValue Src;
6116 int Start = -1;
6117
6118 for (unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6119 SDValue Op = BV->getOperand(Num: i);
6120 if (Op.isUndef())
6121 continue;
6122 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6123 return SDValue();
6124
6125 auto *IdxC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6126 if (!IdxC)
6127 return SDValue();
6128
6129 unsigned EltIdx = IdxC->getZExtValue();
6130 if (Start < 0)
6131 Start = (int)EltIdx - (int)(i * 2);
6132 if (Start < 0 || Start > 1 || EltIdx != (unsigned)(Start + (int)(i * 2)))
6133 return SDValue();
6134
6135 SDValue CurSrc = Op.getOperand(i: 0);
6136 if (!Src)
6137 Src = CurSrc;
6138 else if (Src != CurSrc)
6139 return SDValue();
6140 }
6141
6142 if (!Src || Start < 0)
6143 return SDValue();
6144
6145 StartIndex = (unsigned)Start;
6146 return Src;
6147}
6148
6149static SDValue
6150performHorizWideningCombine(SDNode *N, SelectionDAG &DAG,
6151 const LoongArchSubtarget &Subtarget) {
6152 if (!Subtarget.hasExtLSX())
6153 return SDValue();
6154
6155 unsigned Opc = N->getOpcode();
6156 assert((Opc == ISD::ADD || Opc == ISD::SUB) && "Unexpected opcode");
6157
6158 EVT VT = N->getValueType(ResNo: 0);
6159 SDLoc DL(N);
6160
6161 SDValue LHS = N->getOperand(Num: 0);
6162 SDValue RHS = N->getOperand(Num: 1);
6163
6164 bool isSigned;
6165 unsigned ExtOpc = LHS.getOpcode();
6166 if (ExtOpc == ISD::SIGN_EXTEND)
6167 isSigned = true;
6168 else if (ExtOpc == ISD::ZERO_EXTEND)
6169 isSigned = false;
6170 else
6171 return SDValue();
6172
6173 if (ExtOpc != RHS.getOpcode())
6174 return SDValue();
6175
6176 if (!LHS.hasOneUse() || !RHS.hasOneUse())
6177 return SDValue();
6178
6179 unsigned OddIdx, EvenIdx;
6180 SDValue LHSVec = matchDeinterleaveBuildVector(N: LHS.getOperand(i: 0), StartIndex&: OddIdx);
6181 SDValue RHSVec = matchDeinterleaveBuildVector(N: RHS.getOperand(i: 0), StartIndex&: EvenIdx);
6182
6183 if (!LHSVec || !RHSVec)
6184 return SDValue();
6185 if (OddIdx != 1 || EvenIdx != 0)
6186 return SDValue();
6187 if (LHSVec.getValueType() != RHSVec.getValueType())
6188 return SDValue();
6189
6190 EVT SrcVT = LHSVec.getValueType();
6191 EVT SrcEltVT = SrcVT.getVectorElementType();
6192 EVT DstEltVT = VT.getVectorElementType();
6193 auto &TLI = DAG.getTargetLoweringInfo();
6194
6195 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
6196 return SDValue();
6197 if (!SrcVT.isVector() || !VT.isVector())
6198 return SDValue();
6199 if (SrcVT.getSizeInBits() != VT.getSizeInBits())
6200 return SDValue();
6201 if (DstEltVT.getSizeInBits() != SrcEltVT.getSizeInBits() * 2)
6202 return SDValue();
6203 if (!SrcEltVT.isInteger() || SrcEltVT.getSizeInBits() > 32)
6204 return SDValue();
6205
6206 unsigned TargetOpc;
6207 if (Opc == ISD::ADD)
6208 TargetOpc = isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6209 else
6210 TargetOpc = isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6211
6212 return DAG.getNode(Opcode: TargetOpc, DL, VT, N1: LHSVec, N2: RHSVec);
6213}
6214
6215static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
6216 TargetLowering::DAGCombinerInfo &DCI,
6217 const LoongArchSubtarget &Subtarget) {
6218 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6219 return V;
6220
6221 if (DCI.isBeforeLegalizeOps())
6222 return SDValue();
6223
6224 EVT VT = N->getValueType(ResNo: 0);
6225 if (!VT.isVector())
6226 return SDValue();
6227
6228 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6229 return SDValue();
6230
6231 EVT EltVT = VT.getVectorElementType();
6232 if (!EltVT.isInteger())
6233 return SDValue();
6234
6235 // match:
6236 //
6237 // add
6238 // (and
6239 // (srl X, shift-1) / X
6240 // 1)
6241 // (srl/sra X, shift)
6242
6243 SDValue Add0 = N->getOperand(Num: 0);
6244 SDValue Add1 = N->getOperand(Num: 1);
6245 SDValue And;
6246 SDValue Shr;
6247
6248 if (Add0.getOpcode() == ISD::AND) {
6249 And = Add0;
6250 Shr = Add1;
6251 } else if (Add1.getOpcode() == ISD::AND) {
6252 And = Add1;
6253 Shr = Add0;
6254 } else {
6255 return SDValue();
6256 }
6257
6258 // match:
6259 //
6260 // srl/sra X, shift
6261
6262 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
6263 return SDValue();
6264
6265 SDValue X = Shr.getOperand(i: 0);
6266 SDValue Shift = Shr.getOperand(i: 1);
6267 APInt ShiftVal;
6268
6269 if (!isConstantSplatVector(N: Shift, SplatValue&: ShiftVal, MinSizeInBits: EltVT.getSizeInBits()))
6270 return SDValue();
6271
6272 if (ShiftVal == 0)
6273 return SDValue();
6274
6275 // match:
6276 //
6277 // and
6278 // (srl X, shift-1) / X
6279 // 1
6280
6281 SDValue One = And.getOperand(i: 1);
6282 APInt SplatVal;
6283
6284 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6285 return SDValue();
6286
6287 if (SplatVal != 1)
6288 return SDValue();
6289
6290 if (And.getOperand(i: 0) == X) {
6291 // match:
6292 //
6293 // shift == 1
6294
6295 if (ShiftVal != 1)
6296 return SDValue();
6297 } else {
6298 // match:
6299 //
6300 // srl X, shift-1
6301
6302 SDValue Srl = And.getOperand(i: 0);
6303
6304 if (Srl.getOpcode() != ISD::SRL)
6305 return SDValue();
6306
6307 if (Srl.getOperand(i: 0) != X)
6308 return SDValue();
6309
6310 // match:
6311 //
6312 // shift-1
6313
6314 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
6315
6316 if (!isConstantSplatVector(N: ShiftMinus1, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6317 return SDValue();
6318
6319 if (ShiftVal != (SplatVal + 1))
6320 return SDValue();
6321 }
6322
6323 // We matched a rounded right shift pattern and can lower it
6324 // to a single vector rounded shift instruction.
6325
6326 SDLoc DL(N);
6327 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
6328 : LoongArchISD::VSRAR,
6329 DL, VT, N1: X, N2: Shift);
6330}
6331
6332static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
6333 TargetLowering::DAGCombinerInfo &DCI,
6334 const LoongArchSubtarget &Subtarget) {
6335 if (DCI.isBeforeLegalizeOps())
6336 return SDValue();
6337
6338 SDValue FirstOperand = N->getOperand(Num: 0);
6339 SDValue SecondOperand = N->getOperand(Num: 1);
6340 unsigned FirstOperandOpc = FirstOperand.getOpcode();
6341 EVT ValTy = N->getValueType(ResNo: 0);
6342 SDLoc DL(N);
6343 uint64_t lsb, msb;
6344 unsigned SMIdx, SMLen;
6345 ConstantSDNode *CN;
6346 SDValue NewOperand;
6347 MVT GRLenVT = Subtarget.getGRLenVT();
6348
6349 if (SDValue R = combineAndNotIntoVANDN(N, DL, DAG))
6350 return R;
6351
6352 // BSTRPICK requires the 32S feature.
6353 if (!Subtarget.has32S())
6354 return SDValue();
6355
6356 // Op's second operand must be a shifted mask.
6357 if (!(CN = dyn_cast<ConstantSDNode>(Val&: SecondOperand)) ||
6358 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx&: SMIdx, MaskLen&: SMLen))
6359 return SDValue();
6360
6361 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
6362 // Pattern match BSTRPICK.
6363 // $dst = and ((sra or srl) $src , lsb), (2**len - 1)
6364 // => BSTRPICK $dst, $src, msb, lsb
6365 // where msb = lsb + len - 1
6366
6367 // The second operand of the shift must be an immediate.
6368 if (!(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))))
6369 return SDValue();
6370
6371 lsb = CN->getZExtValue();
6372
6373 // Return if the shifted mask does not start at bit 0 or the sum of its
6374 // length and lsb exceeds the word's size.
6375 if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())
6376 return SDValue();
6377
6378 NewOperand = FirstOperand.getOperand(i: 0);
6379 } else {
6380 // Pattern match BSTRPICK.
6381 // $dst = and $src, (2**len- 1) , if len > 12
6382 // => BSTRPICK $dst, $src, msb, lsb
6383 // where lsb = 0 and msb = len - 1
6384
6385 // If the mask is <= 0xfff, andi can be used instead.
6386 if (CN->getZExtValue() <= 0xfff)
6387 return SDValue();
6388
6389 // Return if the MSB exceeds.
6390 if (SMIdx + SMLen > ValTy.getSizeInBits())
6391 return SDValue();
6392
6393 if (SMIdx > 0) {
6394 // Omit if the constant has more than 2 uses. This a conservative
6395 // decision. Whether it is a win depends on the HW microarchitecture.
6396 // However it should always be better for 1 and 2 uses.
6397 if (CN->use_size() > 2)
6398 return SDValue();
6399 // Return if the constant can be composed by a single LU12I.W.
6400 if ((CN->getZExtValue() & 0xfff) == 0)
6401 return SDValue();
6402 // Return if the constand can be composed by a single ADDI with
6403 // the zero register.
6404 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)
6405 return SDValue();
6406 }
6407
6408 lsb = SMIdx;
6409 NewOperand = FirstOperand;
6410 }
6411
6412 msb = lsb + SMLen - 1;
6413 SDValue NR0 = DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy, N1: NewOperand,
6414 N2: DAG.getConstant(Val: msb, DL, VT: GRLenVT),
6415 N3: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6416 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)
6417 return NR0;
6418 // Try to optimize to
6419 // bstrpick $Rd, $Rs, msb, lsb
6420 // slli $Rd, $Rd, lsb
6421 return DAG.getNode(Opcode: ISD::SHL, DL, VT: ValTy, N1: NR0,
6422 N2: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6423}
6424
6425// Return the original source vector if N consists of the half
6426// of each 128-bit lane.
6427static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow) {
6428 N = peekThroughBitcasts(V: N);
6429
6430 EVT DstVT = N.getValueType();
6431 if (!DstVT.isVector())
6432 return SDValue();
6433
6434 unsigned NumElts = DstVT.getVectorNumElements();
6435
6436 // LSX canonical form:
6437 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6438 SDValue Src = N.getOperand(i: 0);
6439 EVT SrcVT = Src.getValueType();
6440
6441 if (!SrcVT.isVector() || !SrcVT.is128BitVector())
6442 return SDValue();
6443 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6444 return SDValue();
6445 if (SrcVT.getVectorNumElements() != NumElts * 2)
6446 return SDValue();
6447 if (N.getConstantOperandVal(i: 1) != (isLow ? 0 : NumElts))
6448 return SDValue();
6449
6450 return Src;
6451 }
6452
6453 // LASX canonical form:
6454 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6455 if (!BV)
6456 return SDValue();
6457
6458 if (NumElts % 2 != 0)
6459 return SDValue();
6460
6461 SDValue Src;
6462 EVT SrcVT;
6463
6464 for (unsigned I = 0; I != NumElts; ++I) {
6465 SDValue Elt = BV->getOperand(Num: I);
6466 if (Elt.isUndef())
6467 continue;
6468 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6469 return SDValue();
6470
6471 SDValue ThisSrc = Elt.getOperand(i: 0);
6472 SDValue Idx = Elt.getOperand(i: 1);
6473 auto *CI = dyn_cast<ConstantSDNode>(Val&: Idx);
6474 if (!CI)
6475 return SDValue();
6476
6477 if (!Src) {
6478 Src = ThisSrc;
6479 SrcVT = Src.getValueType();
6480 if (!SrcVT.isVector())
6481 return SDValue();
6482
6483 if (!SrcVT.is256BitVector())
6484 return SDValue();
6485 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6486 return SDValue();
6487 if (SrcVT.getVectorNumElements() != NumElts * 2)
6488 return SDValue();
6489 } else if (ThisSrc != Src) {
6490 return SDValue();
6491 }
6492
6493 unsigned Half = NumElts / 2;
6494 unsigned ExpectedIdx = (I < Half) ? I : (I + Half);
6495 ExpectedIdx += isLow ? 0 : Half;
6496
6497 if (CI->getZExtValue() != ExpectedIdx)
6498 return SDValue();
6499 }
6500
6501 return Src;
6502}
6503
6504static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
6505 TargetLowering::DAGCombinerInfo &DCI,
6506 const LoongArchSubtarget &Subtarget) {
6507 assert(N->getOpcode() == ISD::SHL && "Unexpected opcode");
6508
6509 EVT VT = N->getValueType(ResNo: 0);
6510 SDLoc DL(N);
6511
6512 SDValue LHS = N->getOperand(Num: 0);
6513 SDValue RHS = N->getOperand(Num: 1);
6514
6515 bool isSigned;
6516 unsigned ExtOpc = LHS.getOpcode();
6517 if (ExtOpc == ISD::SIGN_EXTEND)
6518 isSigned = true;
6519 else if (ExtOpc == ISD::ZERO_EXTEND)
6520 isSigned = false;
6521 else
6522 return SDValue();
6523
6524 if (!LHS.hasOneUse())
6525 return SDValue();
6526
6527 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
6528 N->getValueSizeInBits(ResNo: 0) != LHS->getOperand(Num: 0).getValueSizeInBits() * 2)
6529 return SDValue();
6530
6531 SDValue Vec = matchHalfOf128BitLanes(N: LHS.getOperand(i: 0), /*isLow=*/true);
6532 if (!Vec)
6533 return SDValue();
6534
6535 EVT SrcVT = Vec.getValueType();
6536 EVT SrcEltVT = SrcVT.getVectorElementType();
6537 EVT DstEltVT = VT.getVectorElementType();
6538 APInt Imm;
6539 if (!isConstantSplatVector(N: RHS, SplatValue&: Imm, MinSizeInBits: DstEltVT.getSizeInBits()))
6540 return SDValue();
6541 if (!Imm.ult(RHS: SrcEltVT.getSizeInBits()))
6542 return SDValue();
6543
6544 unsigned Opc = isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
6545 SDValue Sht = DAG.getConstant(Val: Imm.getZExtValue(), DL, VT: Subtarget.getGRLenVT());
6546 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Sht);
6547}
6548
6549static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
6550 TargetLowering::DAGCombinerInfo &DCI,
6551 const LoongArchSubtarget &Subtarget) {
6552 // BSTRPICK requires the 32S feature.
6553 if (!Subtarget.has32S())
6554 return SDValue();
6555
6556 if (DCI.isBeforeLegalizeOps())
6557 return SDValue();
6558
6559 // $dst = srl (and $src, Mask), Shamt
6560 // =>
6561 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt
6562 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1
6563 //
6564
6565 SDValue FirstOperand = N->getOperand(Num: 0);
6566 ConstantSDNode *CN;
6567 EVT ValTy = N->getValueType(ResNo: 0);
6568 SDLoc DL(N);
6569 MVT GRLenVT = Subtarget.getGRLenVT();
6570 unsigned MaskIdx, MaskLen;
6571 uint64_t Shamt;
6572
6573 // The first operand must be an AND and the second operand of the AND must be
6574 // a shifted mask.
6575 if (FirstOperand.getOpcode() != ISD::AND ||
6576 !(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))) ||
6577 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx, MaskLen))
6578 return SDValue();
6579
6580 // The second operand (shift amount) must be an immediate.
6581 if (!(CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))))
6582 return SDValue();
6583
6584 Shamt = CN->getZExtValue();
6585 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
6586 return DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy,
6587 N1: FirstOperand->getOperand(Num: 0),
6588 N2: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
6589 N3: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6590
6591 return SDValue();
6592}
6593
6594static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
6595 TargetLowering::DAGCombinerInfo &DCI,
6596 const LoongArchSubtarget &Subtarget) {
6597 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6598 return V;
6599
6600 return SDValue();
6601}
6602
6603// Helper to peek through bitops/trunc/setcc to determine size of source vector.
6604// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
6605static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
6606 unsigned Depth) {
6607 // Limit recursion.
6608 if (Depth >= SelectionDAG::MaxRecursionDepth)
6609 return false;
6610 switch (Src.getOpcode()) {
6611 case ISD::SETCC:
6612 case ISD::TRUNCATE:
6613 return Src.getOperand(i: 0).getValueSizeInBits() == Size;
6614 case ISD::FREEZE:
6615 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1);
6616 case ISD::AND:
6617 case ISD::XOR:
6618 case ISD::OR:
6619 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1) &&
6620 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1);
6621 case ISD::SELECT:
6622 case ISD::VSELECT:
6623 return Src.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
6624 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1) &&
6625 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 2), Size, Depth: Depth + 1);
6626 case ISD::BUILD_VECTOR:
6627 return ISD::isBuildVectorAllZeros(N: Src.getNode()) ||
6628 ISD::isBuildVectorAllOnes(N: Src.getNode());
6629 }
6630 return false;
6631}
6632
6633// Helper to push sign extension of vXi1 SETCC result through bitops.
6634static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
6635 SDValue Src, const SDLoc &DL) {
6636 switch (Src.getOpcode()) {
6637 case ISD::SETCC:
6638 case ISD::FREEZE:
6639 case ISD::TRUNCATE:
6640 case ISD::BUILD_VECTOR:
6641 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6642 case ISD::AND:
6643 case ISD::XOR:
6644 case ISD::OR:
6645 return DAG.getNode(
6646 Opcode: Src.getOpcode(), DL, VT: SExtVT,
6647 N1: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 0), DL),
6648 N2: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL));
6649 case ISD::SELECT:
6650 case ISD::VSELECT:
6651 return DAG.getSelect(
6652 DL, VT: SExtVT, Cond: Src.getOperand(i: 0),
6653 LHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL),
6654 RHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 2), DL));
6655 }
6656 llvm_unreachable("Unexpected node type for vXi1 sign extension");
6657}
6658
6659static SDValue
6660performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
6661 TargetLowering::DAGCombinerInfo &DCI,
6662 const LoongArchSubtarget &Subtarget) {
6663 SDLoc DL(N);
6664 EVT VT = N->getValueType(ResNo: 0);
6665 SDValue Src = N->getOperand(Num: 0);
6666 EVT SrcVT = Src.getValueType();
6667
6668 if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse())
6669 return SDValue();
6670
6671 bool UseLASX;
6672 unsigned Opc = ISD::DELETED_NODE;
6673 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6674 EVT EltVT = CmpVT.getVectorElementType();
6675
6676 if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128)
6677 UseLASX = false;
6678 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
6679 CmpVT.getSizeInBits() == 256)
6680 UseLASX = true;
6681 else
6682 return SDValue();
6683
6684 SDValue SrcN1 = Src.getOperand(i: 1);
6685 switch (cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get()) {
6686 default:
6687 break;
6688 case ISD::SETEQ:
6689 // x == 0 => not (vmsknez.b x)
6690 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6691 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
6692 break;
6693 case ISD::SETGT:
6694 // x > -1 => vmskgez.b x
6695 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) && EltVT == MVT::i8)
6696 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6697 break;
6698 case ISD::SETGE:
6699 // x >= 0 => vmskgez.b x
6700 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6701 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6702 break;
6703 case ISD::SETLT:
6704 // x < 0 => vmskltz.{b,h,w,d} x
6705 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) &&
6706 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6707 EltVT == MVT::i64))
6708 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6709 break;
6710 case ISD::SETLE:
6711 // x <= -1 => vmskltz.{b,h,w,d} x
6712 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) &&
6713 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6714 EltVT == MVT::i64))
6715 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6716 break;
6717 case ISD::SETNE:
6718 // x != 0 => vmsknez.b x
6719 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6720 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
6721 break;
6722 }
6723
6724 if (Opc == ISD::DELETED_NODE)
6725 return SDValue();
6726
6727 SDValue V = DAG.getNode(Opcode: Opc, DL, VT: Subtarget.getGRLenVT(), Operand: Src.getOperand(i: 0));
6728 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6729 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6730 return DAG.getBitcast(VT, V);
6731}
6732
6733static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG,
6734 TargetLowering::DAGCombinerInfo &DCI,
6735 const LoongArchSubtarget &Subtarget) {
6736 SDLoc DL(N);
6737 EVT VT = N->getValueType(ResNo: 0);
6738 SDValue Src = N->getOperand(Num: 0);
6739 EVT SrcVT = Src.getValueType();
6740 MVT GRLenVT = Subtarget.getGRLenVT();
6741
6742 if (!DCI.isBeforeLegalizeOps())
6743 return SDValue();
6744
6745 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
6746 return SDValue();
6747
6748 // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
6749 SDValue Res = performSETCC_BITCASTCombine(N, DAG, DCI, Subtarget);
6750 if (Res)
6751 return Res;
6752
6753 // Generate vXi1 using [X]VMSKLTZ
6754 MVT SExtVT;
6755 unsigned Opc;
6756 bool UseLASX = false;
6757 bool PropagateSExt = false;
6758
6759 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
6760 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6761 if (CmpVT.getSizeInBits() > 256)
6762 return SDValue();
6763 }
6764
6765 switch (SrcVT.getSimpleVT().SimpleTy) {
6766 default:
6767 return SDValue();
6768 case MVT::v2i1:
6769 SExtVT = MVT::v2i64;
6770 break;
6771 case MVT::v4i1:
6772 SExtVT = MVT::v4i32;
6773 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6774 SExtVT = MVT::v4i64;
6775 UseLASX = true;
6776 PropagateSExt = true;
6777 }
6778 break;
6779 case MVT::v8i1:
6780 SExtVT = MVT::v8i16;
6781 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6782 SExtVT = MVT::v8i32;
6783 UseLASX = true;
6784 PropagateSExt = true;
6785 }
6786 break;
6787 case MVT::v16i1:
6788 SExtVT = MVT::v16i8;
6789 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6790 SExtVT = MVT::v16i16;
6791 UseLASX = true;
6792 PropagateSExt = true;
6793 }
6794 break;
6795 case MVT::v32i1:
6796 SExtVT = MVT::v32i8;
6797 UseLASX = true;
6798 break;
6799 };
6800 Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
6801 : DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6802
6803 SDValue V;
6804 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
6805 if (Src.getSimpleValueType() == MVT::v32i8) {
6806 SDValue Lo, Hi;
6807 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Src, DL);
6808 Lo = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Lo);
6809 Hi = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Hi);
6810 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: Hi,
6811 N2: DAG.getShiftAmountConstant(Val: 16, VT: GRLenVT, DL));
6812 V = DAG.getNode(Opcode: ISD::OR, DL, VT: GRLenVT, N1: Lo, N2: Hi);
6813 } else if (UseLASX) {
6814 return SDValue();
6815 }
6816 }
6817
6818 if (!V) {
6819 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6820 V = DAG.getNode(Opcode: Opc, DL, VT: GRLenVT, Operand: Src);
6821 }
6822
6823 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6824 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6825 return DAG.getBitcast(VT, V);
6826}
6827
6828static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
6829 TargetLowering::DAGCombinerInfo &DCI,
6830 const LoongArchSubtarget &Subtarget) {
6831 MVT GRLenVT = Subtarget.getGRLenVT();
6832 EVT ValTy = N->getValueType(ResNo: 0);
6833 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
6834 ConstantSDNode *CN0, *CN1;
6835 SDLoc DL(N);
6836 unsigned ValBits = ValTy.getSizeInBits();
6837 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
6838 unsigned Shamt;
6839 bool SwapAndRetried = false;
6840
6841 // BSTRPICK requires the 32S feature.
6842 if (!Subtarget.has32S())
6843 return SDValue();
6844
6845 if (DCI.isBeforeLegalizeOps())
6846 return SDValue();
6847
6848 if (ValBits != 32 && ValBits != 64)
6849 return SDValue();
6850
6851Retry:
6852 // 1st pattern to match BSTRINS:
6853 // R = or (and X, mask0), (and (shl Y, lsb), mask1)
6854 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1
6855 // =>
6856 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6857 if (N0.getOpcode() == ISD::AND &&
6858 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6859 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6860 N1.getOpcode() == ISD::AND && N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
6861 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6862 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6863 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
6864 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6865 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6866 (MaskIdx0 + MaskLen0 <= ValBits)) {
6867 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");
6868 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6869 N2: N1.getOperand(i: 0).getOperand(i: 0),
6870 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6871 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6872 }
6873
6874 // 2nd pattern to match BSTRINS:
6875 // R = or (and X, mask0), (shl (and Y, mask1), lsb)
6876 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)
6877 // =>
6878 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6879 if (N0.getOpcode() == ISD::AND &&
6880 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6881 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6882 N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
6883 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6884 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6885 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6886 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6887 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
6888 (MaskIdx0 + MaskLen0 <= ValBits)) {
6889 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");
6890 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6891 N2: N1.getOperand(i: 0).getOperand(i: 0),
6892 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6893 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6894 }
6895
6896 // 3rd pattern to match BSTRINS:
6897 // R = or (and X, mask0), (and Y, mask1)
6898 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0
6899 // =>
6900 // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb
6901 // where msb = lsb + size - 1
6902 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
6903 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6904 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6905 (MaskIdx0 + MaskLen0 <= 64) &&
6906 (CN1 = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) &&
6907 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6908 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");
6909 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6910 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0), N1,
6911 N2: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT)),
6912 N3: DAG.getConstant(Val: ValBits == 32
6913 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6914 : (MaskIdx0 + MaskLen0 - 1),
6915 DL, VT: GRLenVT),
6916 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6917 }
6918
6919 // 4th pattern to match BSTRINS:
6920 // R = or (and X, mask), (shl Y, shamt)
6921 // where mask = (2**shamt - 1)
6922 // =>
6923 // R = BSTRINS X, Y, ValBits - 1, shamt
6924 // where ValBits = 32 or 64
6925 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&
6926 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6927 isShiftedMask_64(Value: CN0->getZExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6928 MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6929 (Shamt = CN1->getZExtValue()) == MaskLen0 &&
6930 (MaskIdx0 + MaskLen0 <= ValBits)) {
6931 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");
6932 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6933 N2: N1.getOperand(i: 0),
6934 N3: DAG.getConstant(Val: (ValBits - 1), DL, VT: GRLenVT),
6935 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6936 }
6937
6938 // 5th pattern to match BSTRINS:
6939 // R = or (and X, mask), const
6940 // where ~mask = (2**size - 1) << lsb, mask & const = 0
6941 // =>
6942 // R = BSTRINS X, (const >> lsb), msb, lsb
6943 // where msb = lsb + size - 1
6944 if (N0.getOpcode() == ISD::AND &&
6945 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6946 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6947 (CN1 = dyn_cast<ConstantSDNode>(Val&: N1)) &&
6948 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6949 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");
6950 return DAG.getNode(
6951 Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6952 N2: DAG.getSignedConstant(Val: CN1->getSExtValue() >> MaskIdx0, DL, VT: ValTy),
6953 N3: DAG.getConstant(Val: ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6954 : (MaskIdx0 + MaskLen0 - 1),
6955 DL, VT: GRLenVT),
6956 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6957 }
6958
6959 // 6th pattern.
6960 // a = b | ((c & mask) << shamt), where all positions in b to be overwritten
6961 // by the incoming bits are known to be zero.
6962 // =>
6963 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt
6964 //
6965 // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th
6966 // pattern is more common than the 1st. So we put the 1st before the 6th in
6967 // order to match as many nodes as possible.
6968 ConstantSDNode *CNMask, *CNShamt;
6969 unsigned MaskIdx, MaskLen;
6970 if (N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
6971 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6972 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
6973 MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6974 CNShamt->getZExtValue() + MaskLen <= ValBits) {
6975 Shamt = CNShamt->getZExtValue();
6976 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);
6977 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
6978 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");
6979 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
6980 N2: N1.getOperand(i: 0).getOperand(i: 0),
6981 N3: DAG.getConstant(Val: Shamt + MaskLen - 1, DL, VT: GRLenVT),
6982 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6983 }
6984 }
6985
6986 // 7th pattern.
6987 // a = b | ((c << shamt) & shifted_mask), where all positions in b to be
6988 // overwritten by the incoming bits are known to be zero.
6989 // =>
6990 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx
6991 //
6992 // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd
6993 // before the 7th in order to match as many nodes as possible.
6994 if (N1.getOpcode() == ISD::AND &&
6995 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6996 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
6997 N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
6998 (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6999 CNShamt->getZExtValue() == MaskIdx) {
7000 APInt ShMask(ValBits, CNMask->getZExtValue());
7001 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7002 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");
7003 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7004 N2: N1.getOperand(i: 0).getOperand(i: 0),
7005 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7006 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7007 }
7008 }
7009
7010 // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.
7011 if (!SwapAndRetried) {
7012 std::swap(a&: N0, b&: N1);
7013 SwapAndRetried = true;
7014 goto Retry;
7015 }
7016
7017 SwapAndRetried = false;
7018Retry2:
7019 // 8th pattern.
7020 // a = b | (c & shifted_mask), where all positions in b to be overwritten by
7021 // the incoming bits are known to be zero.
7022 // =>
7023 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx
7024 //
7025 // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So
7026 // we put it here in order to match as many nodes as possible or generate less
7027 // instructions.
7028 if (N1.getOpcode() == ISD::AND &&
7029 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7030 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen)) {
7031 APInt ShMask(ValBits, CNMask->getZExtValue());
7032 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7033 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");
7034 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7035 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0),
7036 N1: N1->getOperand(Num: 0),
7037 N2: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT)),
7038 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7039 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7040 }
7041 }
7042 // Swap N0/N1 and retry.
7043 if (!SwapAndRetried) {
7044 std::swap(a&: N0, b&: N1);
7045 SwapAndRetried = true;
7046 goto Retry2;
7047 }
7048
7049 return SDValue();
7050}
7051
7052static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
7053 ExtType = ISD::NON_EXTLOAD;
7054
7055 switch (V.getNode()->getOpcode()) {
7056 case ISD::LOAD: {
7057 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: V.getNode());
7058 if ((LoadNode->getMemoryVT() == MVT::i8) ||
7059 (LoadNode->getMemoryVT() == MVT::i16)) {
7060 ExtType = LoadNode->getExtensionType();
7061 return true;
7062 }
7063 return false;
7064 }
7065 case ISD::AssertSext: {
7066 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7067 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7068 ExtType = ISD::SEXTLOAD;
7069 return true;
7070 }
7071 return false;
7072 }
7073 case ISD::AssertZext: {
7074 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7075 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7076 ExtType = ISD::ZEXTLOAD;
7077 return true;
7078 }
7079 return false;
7080 }
7081 default:
7082 return false;
7083 }
7084
7085 return false;
7086}
7087
7088// Eliminate redundant truncation and zero-extension nodes.
7089// * Case 1:
7090// +------------+ +------------+ +------------+
7091// | Input1 | | Input2 | | CC |
7092// +------------+ +------------+ +------------+
7093// | | |
7094// V V +----+
7095// +------------+ +------------+ |
7096// | TRUNCATE | | TRUNCATE | |
7097// +------------+ +------------+ |
7098// | | |
7099// V V |
7100// +------------+ +------------+ |
7101// | ZERO_EXT | | ZERO_EXT | |
7102// +------------+ +------------+ |
7103// | | |
7104// | +-------------+ |
7105// V V | |
7106// +----------------+ | |
7107// | AND | | |
7108// +----------------+ | |
7109// | | |
7110// +---------------+ | |
7111// | | |
7112// V V V
7113// +-------------+
7114// | CMP |
7115// +-------------+
7116// * Case 2:
7117// +------------+ +------------+ +-------------+ +------------+ +------------+
7118// | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
7119// +------------+ +------------+ +-------------+ +------------+ +------------+
7120// | | | | |
7121// V | | | |
7122// +------------+ | | | |
7123// | XOR |<---------------------+ | |
7124// +------------+ | | |
7125// | | | |
7126// V V +---------------+ |
7127// +------------+ +------------+ | |
7128// | TRUNCATE | | TRUNCATE | | +-------------------------+
7129// +------------+ +------------+ | |
7130// | | | |
7131// V V | |
7132// +------------+ +------------+ | |
7133// | ZERO_EXT | | ZERO_EXT | | |
7134// +------------+ +------------+ | |
7135// | | | |
7136// V V | |
7137// +----------------+ | |
7138// | AND | | |
7139// +----------------+ | |
7140// | | |
7141// +---------------+ | |
7142// | | |
7143// V V V
7144// +-------------+
7145// | CMP |
7146// +-------------+
7147static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
7148 TargetLowering::DAGCombinerInfo &DCI,
7149 const LoongArchSubtarget &Subtarget) {
7150 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
7151
7152 SDNode *AndNode = N->getOperand(Num: 0).getNode();
7153 if (AndNode->getOpcode() != ISD::AND)
7154 return SDValue();
7155
7156 SDValue AndInputValue2 = AndNode->getOperand(Num: 1);
7157 if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
7158 return SDValue();
7159
7160 SDValue CmpInputValue = N->getOperand(Num: 1);
7161 SDValue AndInputValue1 = AndNode->getOperand(Num: 0);
7162 if (AndInputValue1.getOpcode() == ISD::XOR) {
7163 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7164 return SDValue();
7165 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: AndInputValue1.getOperand(i: 1));
7166 if (!CN || !CN->isAllOnes())
7167 return SDValue();
7168 CN = dyn_cast<ConstantSDNode>(Val&: CmpInputValue);
7169 if (!CN || !CN->isZero())
7170 return SDValue();
7171 AndInputValue1 = AndInputValue1.getOperand(i: 0);
7172 if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
7173 return SDValue();
7174 } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
7175 if (AndInputValue2 != CmpInputValue)
7176 return SDValue();
7177 } else {
7178 return SDValue();
7179 }
7180
7181 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(Num: 0);
7182 if (TruncValue1.getOpcode() != ISD::TRUNCATE)
7183 return SDValue();
7184
7185 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(Num: 0);
7186 if (TruncValue2.getOpcode() != ISD::TRUNCATE)
7187 return SDValue();
7188
7189 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(Num: 0);
7190 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(Num: 0);
7191 ISD::LoadExtType ExtType1;
7192 ISD::LoadExtType ExtType2;
7193
7194 if (!checkValueWidth(V: TruncInputValue1, ExtType&: ExtType1) ||
7195 !checkValueWidth(V: TruncInputValue2, ExtType&: ExtType2))
7196 return SDValue();
7197
7198 if (TruncInputValue1->getValueType(ResNo: 0) != TruncInputValue2->getValueType(ResNo: 0) ||
7199 AndNode->getValueType(ResNo: 0) != TruncInputValue1->getValueType(ResNo: 0))
7200 return SDValue();
7201
7202 if ((ExtType2 != ISD::ZEXTLOAD) &&
7203 ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
7204 return SDValue();
7205
7206 // These truncation and zero-extension nodes are not necessary, remove them.
7207 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: AndNode->getValueType(ResNo: 0),
7208 N1: TruncInputValue1, N2: TruncInputValue2);
7209 SDValue NewSetCC =
7210 DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: NewAnd, RHS: TruncInputValue2, Cond: CC);
7211 DAG.ReplaceAllUsesWith(From: N, To: NewSetCC.getNode());
7212 return SDValue(N, 0);
7213}
7214
7215// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
7216static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
7217 TargetLowering::DAGCombinerInfo &DCI,
7218 const LoongArchSubtarget &Subtarget) {
7219 if (DCI.isBeforeLegalizeOps())
7220 return SDValue();
7221
7222 SDValue Src = N->getOperand(Num: 0);
7223 if (Src.getOpcode() != LoongArchISD::REVB_2W)
7224 return SDValue();
7225
7226 return DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7227 Operand: Src.getOperand(i: 0));
7228}
7229
7230// Perform common combines for BR_CC and SELECT_CC conditions.
7231static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
7232 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
7233 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7234
7235 // As far as arithmetic right shift always saves the sign,
7236 // shift can be omitted.
7237 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
7238 // setge (sra X, N), 0 -> setge X, 0
7239 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
7240 LHS.getOpcode() == ISD::SRA) {
7241 LHS = LHS.getOperand(i: 0);
7242 return true;
7243 }
7244
7245 if (!ISD::isIntEqualitySetCC(Code: CCVal))
7246 return false;
7247
7248 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
7249 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
7250 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
7251 LHS.getOperand(i: 0).getValueType() == Subtarget.getGRLenVT()) {
7252 // If we're looking for eq 0 instead of ne 0, we need to invert the
7253 // condition.
7254 bool Invert = CCVal == ISD::SETEQ;
7255 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
7256 if (Invert)
7257 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7258
7259 RHS = LHS.getOperand(i: 1);
7260 LHS = LHS.getOperand(i: 0);
7261 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
7262
7263 CC = DAG.getCondCode(Cond: CCVal);
7264 return true;
7265 }
7266
7267 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, GRLen-1-C), 0, ge/lt)
7268 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
7269 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
7270 SDValue LHS0 = LHS.getOperand(i: 0);
7271 if (LHS0.getOpcode() == ISD::AND &&
7272 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7273 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
7274 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
7275 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
7276 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
7277 CC = DAG.getCondCode(Cond: CCVal);
7278
7279 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
7280 LHS = LHS0.getOperand(i: 0);
7281 if (ShAmt != 0)
7282 LHS =
7283 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
7284 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
7285 return true;
7286 }
7287 }
7288 }
7289
7290 // (X, 1, setne) -> (X, 0, seteq) if we can prove X is 0/1.
7291 // This can occur when legalizing some floating point comparisons.
7292 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
7293 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
7294 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7295 CC = DAG.getCondCode(Cond: CCVal);
7296 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
7297 return true;
7298 }
7299
7300 // Fold ((shl (extract_vector_elt X, I), GRLen - EleBits)), 0, eq/ne) ->
7301 // ((extract_vector_elt X, I), 0, eq/ne)
7302 if (isNullConstant(V: RHS) && (CCVal == ISD::SETEQ || CCVal == ISD::SETNE) &&
7303 LHS.getOpcode() == ISD::SHL && LHS.hasOneUse() &&
7304 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
7305 SDValue Ext = LHS.getOperand(i: 0);
7306 unsigned Sht = LHS.getConstantOperandVal(i: 1);
7307 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
7308 SDValue Vec = Ext.getOperand(i: 0);
7309 unsigned EleBits = Vec.getScalarValueSizeInBits();
7310 if ((EleBits + Sht) == Subtarget.getGRLen()) {
7311 LHS = Ext;
7312 return true;
7313 }
7314 }
7315 }
7316
7317 return false;
7318}
7319
7320static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG,
7321 TargetLowering::DAGCombinerInfo &DCI,
7322 const LoongArchSubtarget &Subtarget) {
7323 SDValue LHS = N->getOperand(Num: 1);
7324 SDValue RHS = N->getOperand(Num: 2);
7325 SDValue CC = N->getOperand(Num: 3);
7326 SDLoc DL(N);
7327
7328 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7329 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
7330 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
7331
7332 return SDValue();
7333}
7334
7335static SDValue performSELECT_CCCombine(SDNode *N, SelectionDAG &DAG,
7336 TargetLowering::DAGCombinerInfo &DCI,
7337 const LoongArchSubtarget &Subtarget) {
7338 // Transform
7339 SDValue LHS = N->getOperand(Num: 0);
7340 SDValue RHS = N->getOperand(Num: 1);
7341 SDValue CC = N->getOperand(Num: 2);
7342 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7343 SDValue TrueV = N->getOperand(Num: 3);
7344 SDValue FalseV = N->getOperand(Num: 4);
7345 SDLoc DL(N);
7346 EVT VT = N->getValueType(ResNo: 0);
7347
7348 // If the True and False values are the same, we don't need a select_cc.
7349 if (TrueV == FalseV)
7350 return TrueV;
7351
7352 // (select (x < 0), y, z) -> x >> (GRLEN - 1) & (y - z) + z
7353 // (select (x >= 0), y, z) -> x >> (GRLEN - 1) & (z - y) + y
7354 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
7355 isNullConstant(V: RHS) &&
7356 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
7357 if (CCVal == ISD::CondCode::SETGE)
7358 std::swap(a&: TrueV, b&: FalseV);
7359
7360 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
7361 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
7362 // Only handle simm12, if it is not in this range, it can be considered as
7363 // register.
7364 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
7365 isInt<12>(x: TrueSImm - FalseSImm)) {
7366 SDValue SRA =
7367 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
7368 N2: DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT));
7369 SDValue AND =
7370 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
7371 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
7372 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
7373 }
7374
7375 if (CCVal == ISD::CondCode::SETGE)
7376 std::swap(a&: TrueV, b&: FalseV);
7377 }
7378
7379 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7380 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
7381 Ops: {LHS, RHS, CC, TrueV, FalseV});
7382
7383 return SDValue();
7384}
7385
7386template <unsigned N>
7387static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp,
7388 SelectionDAG &DAG,
7389 const LoongArchSubtarget &Subtarget,
7390 bool IsSigned = false) {
7391 SDLoc DL(Node);
7392 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7393 // Check the ImmArg.
7394 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7395 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7396 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7397 ": argument out of range.");
7398 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: Subtarget.getGRLenVT());
7399 }
7400 return DAG.getConstant(Val: CImm->getZExtValue(), DL, VT: Subtarget.getGRLenVT());
7401}
7402
7403template <unsigned N>
7404static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,
7405 SelectionDAG &DAG, bool IsSigned = false) {
7406 SDLoc DL(Node);
7407 EVT ResTy = Node->getValueType(ResNo: 0);
7408 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7409
7410 // Check the ImmArg.
7411 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7412 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7413 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7414 ": argument out of range.");
7415 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7416 }
7417 return DAG.getConstant(
7418 Val: APInt(ResTy.getScalarType().getSizeInBits(),
7419 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
7420 DL, VT: ResTy);
7421}
7422
7423static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG) {
7424 SDLoc DL(Node);
7425 EVT ResTy = Node->getValueType(ResNo: 0);
7426 SDValue Vec = Node->getOperand(Num: 2);
7427 SDValue Mask = DAG.getConstant(Val: Vec.getScalarValueSizeInBits() - 1, DL, VT: ResTy);
7428 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Vec, N2: Mask);
7429}
7430
7431static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG) {
7432 SDLoc DL(Node);
7433 EVT ResTy = Node->getValueType(ResNo: 0);
7434 SDValue One = DAG.getConstant(Val: 1, DL, VT: ResTy);
7435 SDValue Bit =
7436 DAG.getNode(Opcode: ISD::SHL, DL, VT: ResTy, N1: One, N2: truncateVecElts(Node, DAG));
7437
7438 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1),
7439 N2: DAG.getNOT(DL, Val: Bit, VT: ResTy));
7440}
7441
7442template <unsigned N>
7443static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG) {
7444 SDLoc DL(Node);
7445 EVT ResTy = Node->getValueType(ResNo: 0);
7446 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7447 // Check the unsigned ImmArg.
7448 if (!isUInt<N>(CImm->getZExtValue())) {
7449 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7450 ": argument out of range.");
7451 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7452 }
7453
7454 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7455 SDValue Mask = DAG.getConstant(Val: ~BitImm, DL, VT: ResTy);
7456
7457 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: Mask);
7458}
7459
7460template <unsigned N>
7461static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG) {
7462 SDLoc DL(Node);
7463 EVT ResTy = Node->getValueType(ResNo: 0);
7464 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7465 // Check the unsigned ImmArg.
7466 if (!isUInt<N>(CImm->getZExtValue())) {
7467 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7468 ": argument out of range.");
7469 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7470 }
7471
7472 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7473 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7474 return DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7475}
7476
7477template <unsigned N>
7478static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG) {
7479 SDLoc DL(Node);
7480 EVT ResTy = Node->getValueType(ResNo: 0);
7481 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7482 // Check the unsigned ImmArg.
7483 if (!isUInt<N>(CImm->getZExtValue())) {
7484 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7485 ": argument out of range.");
7486 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7487 }
7488
7489 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7490 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7491 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7492}
7493
7494template <unsigned W>
7495static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG,
7496 unsigned ResOp) {
7497 unsigned Imm = N->getConstantOperandVal(Num: 2);
7498 if (!isUInt<W>(Imm)) {
7499 const StringRef ErrorMsg = "argument out of range";
7500 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
7501 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
7502 }
7503 SDLoc DL(N);
7504 SDValue Vec = N->getOperand(Num: 1);
7505 SDValue Idx = DAG.getConstant(Val: Imm, DL, VT: MVT::i32);
7506 SDValue EltVT = DAG.getValueType(Vec.getValueType().getVectorElementType());
7507 return DAG.getNode(Opcode: ResOp, DL, VT: N->getValueType(ResNo: 0), N1: Vec, N2: Idx, N3: EltVT);
7508}
7509
7510static SDValue
7511performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
7512 TargetLowering::DAGCombinerInfo &DCI,
7513 const LoongArchSubtarget &Subtarget) {
7514 SDLoc DL(N);
7515 switch (N->getConstantOperandVal(Num: 0)) {
7516 default:
7517 break;
7518 case Intrinsic::loongarch_lsx_vadd_b:
7519 case Intrinsic::loongarch_lsx_vadd_h:
7520 case Intrinsic::loongarch_lsx_vadd_w:
7521 case Intrinsic::loongarch_lsx_vadd_d:
7522 case Intrinsic::loongarch_lasx_xvadd_b:
7523 case Intrinsic::loongarch_lasx_xvadd_h:
7524 case Intrinsic::loongarch_lasx_xvadd_w:
7525 case Intrinsic::loongarch_lasx_xvadd_d:
7526 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7527 N2: N->getOperand(Num: 2));
7528 case Intrinsic::loongarch_lsx_vaddi_bu:
7529 case Intrinsic::loongarch_lsx_vaddi_hu:
7530 case Intrinsic::loongarch_lsx_vaddi_wu:
7531 case Intrinsic::loongarch_lsx_vaddi_du:
7532 case Intrinsic::loongarch_lasx_xvaddi_bu:
7533 case Intrinsic::loongarch_lasx_xvaddi_hu:
7534 case Intrinsic::loongarch_lasx_xvaddi_wu:
7535 case Intrinsic::loongarch_lasx_xvaddi_du:
7536 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7537 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7538 case Intrinsic::loongarch_lsx_vsub_b:
7539 case Intrinsic::loongarch_lsx_vsub_h:
7540 case Intrinsic::loongarch_lsx_vsub_w:
7541 case Intrinsic::loongarch_lsx_vsub_d:
7542 case Intrinsic::loongarch_lasx_xvsub_b:
7543 case Intrinsic::loongarch_lasx_xvsub_h:
7544 case Intrinsic::loongarch_lasx_xvsub_w:
7545 case Intrinsic::loongarch_lasx_xvsub_d:
7546 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7547 N2: N->getOperand(Num: 2));
7548 case Intrinsic::loongarch_lsx_vsubi_bu:
7549 case Intrinsic::loongarch_lsx_vsubi_hu:
7550 case Intrinsic::loongarch_lsx_vsubi_wu:
7551 case Intrinsic::loongarch_lsx_vsubi_du:
7552 case Intrinsic::loongarch_lasx_xvsubi_bu:
7553 case Intrinsic::loongarch_lasx_xvsubi_hu:
7554 case Intrinsic::loongarch_lasx_xvsubi_wu:
7555 case Intrinsic::loongarch_lasx_xvsubi_du:
7556 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7557 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7558 case Intrinsic::loongarch_lsx_vneg_b:
7559 case Intrinsic::loongarch_lsx_vneg_h:
7560 case Intrinsic::loongarch_lsx_vneg_w:
7561 case Intrinsic::loongarch_lsx_vneg_d:
7562 case Intrinsic::loongarch_lasx_xvneg_b:
7563 case Intrinsic::loongarch_lasx_xvneg_h:
7564 case Intrinsic::loongarch_lasx_xvneg_w:
7565 case Intrinsic::loongarch_lasx_xvneg_d:
7566 return DAG.getNode(
7567 Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0),
7568 N1: DAG.getConstant(
7569 Val: APInt(N->getValueType(ResNo: 0).getScalarType().getSizeInBits(), 0,
7570 /*isSigned=*/true),
7571 DL: SDLoc(N), VT: N->getValueType(ResNo: 0)),
7572 N2: N->getOperand(Num: 1));
7573 case Intrinsic::loongarch_lsx_vmax_b:
7574 case Intrinsic::loongarch_lsx_vmax_h:
7575 case Intrinsic::loongarch_lsx_vmax_w:
7576 case Intrinsic::loongarch_lsx_vmax_d:
7577 case Intrinsic::loongarch_lasx_xvmax_b:
7578 case Intrinsic::loongarch_lasx_xvmax_h:
7579 case Intrinsic::loongarch_lasx_xvmax_w:
7580 case Intrinsic::loongarch_lasx_xvmax_d:
7581 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7582 N2: N->getOperand(Num: 2));
7583 case Intrinsic::loongarch_lsx_vmax_bu:
7584 case Intrinsic::loongarch_lsx_vmax_hu:
7585 case Intrinsic::loongarch_lsx_vmax_wu:
7586 case Intrinsic::loongarch_lsx_vmax_du:
7587 case Intrinsic::loongarch_lasx_xvmax_bu:
7588 case Intrinsic::loongarch_lasx_xvmax_hu:
7589 case Intrinsic::loongarch_lasx_xvmax_wu:
7590 case Intrinsic::loongarch_lasx_xvmax_du:
7591 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7592 N2: N->getOperand(Num: 2));
7593 case Intrinsic::loongarch_lsx_vmaxi_b:
7594 case Intrinsic::loongarch_lsx_vmaxi_h:
7595 case Intrinsic::loongarch_lsx_vmaxi_w:
7596 case Intrinsic::loongarch_lsx_vmaxi_d:
7597 case Intrinsic::loongarch_lasx_xvmaxi_b:
7598 case Intrinsic::loongarch_lasx_xvmaxi_h:
7599 case Intrinsic::loongarch_lasx_xvmaxi_w:
7600 case Intrinsic::loongarch_lasx_xvmaxi_d:
7601 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7602 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7603 case Intrinsic::loongarch_lsx_vmaxi_bu:
7604 case Intrinsic::loongarch_lsx_vmaxi_hu:
7605 case Intrinsic::loongarch_lsx_vmaxi_wu:
7606 case Intrinsic::loongarch_lsx_vmaxi_du:
7607 case Intrinsic::loongarch_lasx_xvmaxi_bu:
7608 case Intrinsic::loongarch_lasx_xvmaxi_hu:
7609 case Intrinsic::loongarch_lasx_xvmaxi_wu:
7610 case Intrinsic::loongarch_lasx_xvmaxi_du:
7611 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7612 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7613 case Intrinsic::loongarch_lsx_vmin_b:
7614 case Intrinsic::loongarch_lsx_vmin_h:
7615 case Intrinsic::loongarch_lsx_vmin_w:
7616 case Intrinsic::loongarch_lsx_vmin_d:
7617 case Intrinsic::loongarch_lasx_xvmin_b:
7618 case Intrinsic::loongarch_lasx_xvmin_h:
7619 case Intrinsic::loongarch_lasx_xvmin_w:
7620 case Intrinsic::loongarch_lasx_xvmin_d:
7621 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7622 N2: N->getOperand(Num: 2));
7623 case Intrinsic::loongarch_lsx_vmin_bu:
7624 case Intrinsic::loongarch_lsx_vmin_hu:
7625 case Intrinsic::loongarch_lsx_vmin_wu:
7626 case Intrinsic::loongarch_lsx_vmin_du:
7627 case Intrinsic::loongarch_lasx_xvmin_bu:
7628 case Intrinsic::loongarch_lasx_xvmin_hu:
7629 case Intrinsic::loongarch_lasx_xvmin_wu:
7630 case Intrinsic::loongarch_lasx_xvmin_du:
7631 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7632 N2: N->getOperand(Num: 2));
7633 case Intrinsic::loongarch_lsx_vmini_b:
7634 case Intrinsic::loongarch_lsx_vmini_h:
7635 case Intrinsic::loongarch_lsx_vmini_w:
7636 case Intrinsic::loongarch_lsx_vmini_d:
7637 case Intrinsic::loongarch_lasx_xvmini_b:
7638 case Intrinsic::loongarch_lasx_xvmini_h:
7639 case Intrinsic::loongarch_lasx_xvmini_w:
7640 case Intrinsic::loongarch_lasx_xvmini_d:
7641 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7642 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7643 case Intrinsic::loongarch_lsx_vmini_bu:
7644 case Intrinsic::loongarch_lsx_vmini_hu:
7645 case Intrinsic::loongarch_lsx_vmini_wu:
7646 case Intrinsic::loongarch_lsx_vmini_du:
7647 case Intrinsic::loongarch_lasx_xvmini_bu:
7648 case Intrinsic::loongarch_lasx_xvmini_hu:
7649 case Intrinsic::loongarch_lasx_xvmini_wu:
7650 case Intrinsic::loongarch_lasx_xvmini_du:
7651 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7652 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7653 case Intrinsic::loongarch_lsx_vmul_b:
7654 case Intrinsic::loongarch_lsx_vmul_h:
7655 case Intrinsic::loongarch_lsx_vmul_w:
7656 case Intrinsic::loongarch_lsx_vmul_d:
7657 case Intrinsic::loongarch_lasx_xvmul_b:
7658 case Intrinsic::loongarch_lasx_xvmul_h:
7659 case Intrinsic::loongarch_lasx_xvmul_w:
7660 case Intrinsic::loongarch_lasx_xvmul_d:
7661 return DAG.getNode(Opcode: ISD::MUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7662 N2: N->getOperand(Num: 2));
7663 case Intrinsic::loongarch_lsx_vmadd_b:
7664 case Intrinsic::loongarch_lsx_vmadd_h:
7665 case Intrinsic::loongarch_lsx_vmadd_w:
7666 case Intrinsic::loongarch_lsx_vmadd_d:
7667 case Intrinsic::loongarch_lasx_xvmadd_b:
7668 case Intrinsic::loongarch_lasx_xvmadd_h:
7669 case Intrinsic::loongarch_lasx_xvmadd_w:
7670 case Intrinsic::loongarch_lasx_xvmadd_d: {
7671 EVT ResTy = N->getValueType(ResNo: 0);
7672 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7673 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7674 N2: N->getOperand(Num: 3)));
7675 }
7676 case Intrinsic::loongarch_lsx_vmsub_b:
7677 case Intrinsic::loongarch_lsx_vmsub_h:
7678 case Intrinsic::loongarch_lsx_vmsub_w:
7679 case Intrinsic::loongarch_lsx_vmsub_d:
7680 case Intrinsic::loongarch_lasx_xvmsub_b:
7681 case Intrinsic::loongarch_lasx_xvmsub_h:
7682 case Intrinsic::loongarch_lasx_xvmsub_w:
7683 case Intrinsic::loongarch_lasx_xvmsub_d: {
7684 EVT ResTy = N->getValueType(ResNo: 0);
7685 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7686 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7687 N2: N->getOperand(Num: 3)));
7688 }
7689 case Intrinsic::loongarch_lsx_vdiv_b:
7690 case Intrinsic::loongarch_lsx_vdiv_h:
7691 case Intrinsic::loongarch_lsx_vdiv_w:
7692 case Intrinsic::loongarch_lsx_vdiv_d:
7693 case Intrinsic::loongarch_lasx_xvdiv_b:
7694 case Intrinsic::loongarch_lasx_xvdiv_h:
7695 case Intrinsic::loongarch_lasx_xvdiv_w:
7696 case Intrinsic::loongarch_lasx_xvdiv_d:
7697 return DAG.getNode(Opcode: ISD::SDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7698 N2: N->getOperand(Num: 2));
7699 case Intrinsic::loongarch_lsx_vdiv_bu:
7700 case Intrinsic::loongarch_lsx_vdiv_hu:
7701 case Intrinsic::loongarch_lsx_vdiv_wu:
7702 case Intrinsic::loongarch_lsx_vdiv_du:
7703 case Intrinsic::loongarch_lasx_xvdiv_bu:
7704 case Intrinsic::loongarch_lasx_xvdiv_hu:
7705 case Intrinsic::loongarch_lasx_xvdiv_wu:
7706 case Intrinsic::loongarch_lasx_xvdiv_du:
7707 return DAG.getNode(Opcode: ISD::UDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7708 N2: N->getOperand(Num: 2));
7709 case Intrinsic::loongarch_lsx_vmod_b:
7710 case Intrinsic::loongarch_lsx_vmod_h:
7711 case Intrinsic::loongarch_lsx_vmod_w:
7712 case Intrinsic::loongarch_lsx_vmod_d:
7713 case Intrinsic::loongarch_lasx_xvmod_b:
7714 case Intrinsic::loongarch_lasx_xvmod_h:
7715 case Intrinsic::loongarch_lasx_xvmod_w:
7716 case Intrinsic::loongarch_lasx_xvmod_d:
7717 return DAG.getNode(Opcode: ISD::SREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7718 N2: N->getOperand(Num: 2));
7719 case Intrinsic::loongarch_lsx_vmod_bu:
7720 case Intrinsic::loongarch_lsx_vmod_hu:
7721 case Intrinsic::loongarch_lsx_vmod_wu:
7722 case Intrinsic::loongarch_lsx_vmod_du:
7723 case Intrinsic::loongarch_lasx_xvmod_bu:
7724 case Intrinsic::loongarch_lasx_xvmod_hu:
7725 case Intrinsic::loongarch_lasx_xvmod_wu:
7726 case Intrinsic::loongarch_lasx_xvmod_du:
7727 return DAG.getNode(Opcode: ISD::UREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7728 N2: N->getOperand(Num: 2));
7729 case Intrinsic::loongarch_lsx_vand_v:
7730 case Intrinsic::loongarch_lasx_xvand_v:
7731 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7732 N2: N->getOperand(Num: 2));
7733 case Intrinsic::loongarch_lsx_vor_v:
7734 case Intrinsic::loongarch_lasx_xvor_v:
7735 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7736 N2: N->getOperand(Num: 2));
7737 case Intrinsic::loongarch_lsx_vxor_v:
7738 case Intrinsic::loongarch_lasx_xvxor_v:
7739 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7740 N2: N->getOperand(Num: 2));
7741 case Intrinsic::loongarch_lsx_vnor_v:
7742 case Intrinsic::loongarch_lasx_xvnor_v: {
7743 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7744 N2: N->getOperand(Num: 2));
7745 return DAG.getNOT(DL, Val: Res, VT: Res->getValueType(ResNo: 0));
7746 }
7747 case Intrinsic::loongarch_lsx_vandi_b:
7748 case Intrinsic::loongarch_lasx_xvandi_b:
7749 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7750 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7751 case Intrinsic::loongarch_lsx_vori_b:
7752 case Intrinsic::loongarch_lasx_xvori_b:
7753 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7754 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7755 case Intrinsic::loongarch_lsx_vxori_b:
7756 case Intrinsic::loongarch_lasx_xvxori_b:
7757 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7758 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7759 case Intrinsic::loongarch_lsx_vsll_b:
7760 case Intrinsic::loongarch_lsx_vsll_h:
7761 case Intrinsic::loongarch_lsx_vsll_w:
7762 case Intrinsic::loongarch_lsx_vsll_d:
7763 case Intrinsic::loongarch_lasx_xvsll_b:
7764 case Intrinsic::loongarch_lasx_xvsll_h:
7765 case Intrinsic::loongarch_lasx_xvsll_w:
7766 case Intrinsic::loongarch_lasx_xvsll_d:
7767 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7768 N2: truncateVecElts(Node: N, DAG));
7769 case Intrinsic::loongarch_lsx_vslli_b:
7770 case Intrinsic::loongarch_lasx_xvslli_b:
7771 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7772 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7773 case Intrinsic::loongarch_lsx_vslli_h:
7774 case Intrinsic::loongarch_lasx_xvslli_h:
7775 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7776 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7777 case Intrinsic::loongarch_lsx_vslli_w:
7778 case Intrinsic::loongarch_lasx_xvslli_w:
7779 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7780 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7781 case Intrinsic::loongarch_lsx_vslli_d:
7782 case Intrinsic::loongarch_lasx_xvslli_d:
7783 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7784 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7785 case Intrinsic::loongarch_lsx_vsrl_b:
7786 case Intrinsic::loongarch_lsx_vsrl_h:
7787 case Intrinsic::loongarch_lsx_vsrl_w:
7788 case Intrinsic::loongarch_lsx_vsrl_d:
7789 case Intrinsic::loongarch_lasx_xvsrl_b:
7790 case Intrinsic::loongarch_lasx_xvsrl_h:
7791 case Intrinsic::loongarch_lasx_xvsrl_w:
7792 case Intrinsic::loongarch_lasx_xvsrl_d:
7793 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7794 N2: truncateVecElts(Node: N, DAG));
7795 case Intrinsic::loongarch_lsx_vsrli_b:
7796 case Intrinsic::loongarch_lasx_xvsrli_b:
7797 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7798 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7799 case Intrinsic::loongarch_lsx_vsrli_h:
7800 case Intrinsic::loongarch_lasx_xvsrli_h:
7801 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7802 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7803 case Intrinsic::loongarch_lsx_vsrli_w:
7804 case Intrinsic::loongarch_lasx_xvsrli_w:
7805 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7806 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7807 case Intrinsic::loongarch_lsx_vsrli_d:
7808 case Intrinsic::loongarch_lasx_xvsrli_d:
7809 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7810 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7811 case Intrinsic::loongarch_lsx_vsra_b:
7812 case Intrinsic::loongarch_lsx_vsra_h:
7813 case Intrinsic::loongarch_lsx_vsra_w:
7814 case Intrinsic::loongarch_lsx_vsra_d:
7815 case Intrinsic::loongarch_lasx_xvsra_b:
7816 case Intrinsic::loongarch_lasx_xvsra_h:
7817 case Intrinsic::loongarch_lasx_xvsra_w:
7818 case Intrinsic::loongarch_lasx_xvsra_d:
7819 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7820 N2: truncateVecElts(Node: N, DAG));
7821 case Intrinsic::loongarch_lsx_vsrai_b:
7822 case Intrinsic::loongarch_lasx_xvsrai_b:
7823 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7824 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7825 case Intrinsic::loongarch_lsx_vsrai_h:
7826 case Intrinsic::loongarch_lasx_xvsrai_h:
7827 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7828 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7829 case Intrinsic::loongarch_lsx_vsrai_w:
7830 case Intrinsic::loongarch_lasx_xvsrai_w:
7831 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7832 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7833 case Intrinsic::loongarch_lsx_vsrai_d:
7834 case Intrinsic::loongarch_lasx_xvsrai_d:
7835 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7836 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7837 case Intrinsic::loongarch_lsx_vclz_b:
7838 case Intrinsic::loongarch_lsx_vclz_h:
7839 case Intrinsic::loongarch_lsx_vclz_w:
7840 case Intrinsic::loongarch_lsx_vclz_d:
7841 case Intrinsic::loongarch_lasx_xvclz_b:
7842 case Intrinsic::loongarch_lasx_xvclz_h:
7843 case Intrinsic::loongarch_lasx_xvclz_w:
7844 case Intrinsic::loongarch_lasx_xvclz_d:
7845 return DAG.getNode(Opcode: ISD::CTLZ, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
7846 case Intrinsic::loongarch_lsx_vpcnt_b:
7847 case Intrinsic::loongarch_lsx_vpcnt_h:
7848 case Intrinsic::loongarch_lsx_vpcnt_w:
7849 case Intrinsic::loongarch_lsx_vpcnt_d:
7850 case Intrinsic::loongarch_lasx_xvpcnt_b:
7851 case Intrinsic::loongarch_lasx_xvpcnt_h:
7852 case Intrinsic::loongarch_lasx_xvpcnt_w:
7853 case Intrinsic::loongarch_lasx_xvpcnt_d:
7854 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
7855 case Intrinsic::loongarch_lsx_vbitclr_b:
7856 case Intrinsic::loongarch_lsx_vbitclr_h:
7857 case Intrinsic::loongarch_lsx_vbitclr_w:
7858 case Intrinsic::loongarch_lsx_vbitclr_d:
7859 case Intrinsic::loongarch_lasx_xvbitclr_b:
7860 case Intrinsic::loongarch_lasx_xvbitclr_h:
7861 case Intrinsic::loongarch_lasx_xvbitclr_w:
7862 case Intrinsic::loongarch_lasx_xvbitclr_d:
7863 return lowerVectorBitClear(Node: N, DAG);
7864 case Intrinsic::loongarch_lsx_vbitclri_b:
7865 case Intrinsic::loongarch_lasx_xvbitclri_b:
7866 return lowerVectorBitClearImm<3>(Node: N, DAG);
7867 case Intrinsic::loongarch_lsx_vbitclri_h:
7868 case Intrinsic::loongarch_lasx_xvbitclri_h:
7869 return lowerVectorBitClearImm<4>(Node: N, DAG);
7870 case Intrinsic::loongarch_lsx_vbitclri_w:
7871 case Intrinsic::loongarch_lasx_xvbitclri_w:
7872 return lowerVectorBitClearImm<5>(Node: N, DAG);
7873 case Intrinsic::loongarch_lsx_vbitclri_d:
7874 case Intrinsic::loongarch_lasx_xvbitclri_d:
7875 return lowerVectorBitClearImm<6>(Node: N, DAG);
7876 case Intrinsic::loongarch_lsx_vbitset_b:
7877 case Intrinsic::loongarch_lsx_vbitset_h:
7878 case Intrinsic::loongarch_lsx_vbitset_w:
7879 case Intrinsic::loongarch_lsx_vbitset_d:
7880 case Intrinsic::loongarch_lasx_xvbitset_b:
7881 case Intrinsic::loongarch_lasx_xvbitset_h:
7882 case Intrinsic::loongarch_lasx_xvbitset_w:
7883 case Intrinsic::loongarch_lasx_xvbitset_d: {
7884 EVT VecTy = N->getValueType(ResNo: 0);
7885 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
7886 return DAG.getNode(
7887 Opcode: ISD::OR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
7888 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
7889 }
7890 case Intrinsic::loongarch_lsx_vbitseti_b:
7891 case Intrinsic::loongarch_lasx_xvbitseti_b:
7892 return lowerVectorBitSetImm<3>(Node: N, DAG);
7893 case Intrinsic::loongarch_lsx_vbitseti_h:
7894 case Intrinsic::loongarch_lasx_xvbitseti_h:
7895 return lowerVectorBitSetImm<4>(Node: N, DAG);
7896 case Intrinsic::loongarch_lsx_vbitseti_w:
7897 case Intrinsic::loongarch_lasx_xvbitseti_w:
7898 return lowerVectorBitSetImm<5>(Node: N, DAG);
7899 case Intrinsic::loongarch_lsx_vbitseti_d:
7900 case Intrinsic::loongarch_lasx_xvbitseti_d:
7901 return lowerVectorBitSetImm<6>(Node: N, DAG);
7902 case Intrinsic::loongarch_lsx_vbitrev_b:
7903 case Intrinsic::loongarch_lsx_vbitrev_h:
7904 case Intrinsic::loongarch_lsx_vbitrev_w:
7905 case Intrinsic::loongarch_lsx_vbitrev_d:
7906 case Intrinsic::loongarch_lasx_xvbitrev_b:
7907 case Intrinsic::loongarch_lasx_xvbitrev_h:
7908 case Intrinsic::loongarch_lasx_xvbitrev_w:
7909 case Intrinsic::loongarch_lasx_xvbitrev_d: {
7910 EVT VecTy = N->getValueType(ResNo: 0);
7911 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
7912 return DAG.getNode(
7913 Opcode: ISD::XOR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
7914 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
7915 }
7916 case Intrinsic::loongarch_lsx_vbitrevi_b:
7917 case Intrinsic::loongarch_lasx_xvbitrevi_b:
7918 return lowerVectorBitRevImm<3>(Node: N, DAG);
7919 case Intrinsic::loongarch_lsx_vbitrevi_h:
7920 case Intrinsic::loongarch_lasx_xvbitrevi_h:
7921 return lowerVectorBitRevImm<4>(Node: N, DAG);
7922 case Intrinsic::loongarch_lsx_vbitrevi_w:
7923 case Intrinsic::loongarch_lasx_xvbitrevi_w:
7924 return lowerVectorBitRevImm<5>(Node: N, DAG);
7925 case Intrinsic::loongarch_lsx_vbitrevi_d:
7926 case Intrinsic::loongarch_lasx_xvbitrevi_d:
7927 return lowerVectorBitRevImm<6>(Node: N, DAG);
7928 case Intrinsic::loongarch_lsx_vfadd_s:
7929 case Intrinsic::loongarch_lsx_vfadd_d:
7930 case Intrinsic::loongarch_lasx_xvfadd_s:
7931 case Intrinsic::loongarch_lasx_xvfadd_d:
7932 return DAG.getNode(Opcode: ISD::FADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7933 N2: N->getOperand(Num: 2));
7934 case Intrinsic::loongarch_lsx_vfsub_s:
7935 case Intrinsic::loongarch_lsx_vfsub_d:
7936 case Intrinsic::loongarch_lasx_xvfsub_s:
7937 case Intrinsic::loongarch_lasx_xvfsub_d:
7938 return DAG.getNode(Opcode: ISD::FSUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7939 N2: N->getOperand(Num: 2));
7940 case Intrinsic::loongarch_lsx_vfmul_s:
7941 case Intrinsic::loongarch_lsx_vfmul_d:
7942 case Intrinsic::loongarch_lasx_xvfmul_s:
7943 case Intrinsic::loongarch_lasx_xvfmul_d:
7944 return DAG.getNode(Opcode: ISD::FMUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7945 N2: N->getOperand(Num: 2));
7946 case Intrinsic::loongarch_lsx_vfdiv_s:
7947 case Intrinsic::loongarch_lsx_vfdiv_d:
7948 case Intrinsic::loongarch_lasx_xvfdiv_s:
7949 case Intrinsic::loongarch_lasx_xvfdiv_d:
7950 return DAG.getNode(Opcode: ISD::FDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7951 N2: N->getOperand(Num: 2));
7952 case Intrinsic::loongarch_lsx_vfmadd_s:
7953 case Intrinsic::loongarch_lsx_vfmadd_d:
7954 case Intrinsic::loongarch_lasx_xvfmadd_s:
7955 case Intrinsic::loongarch_lasx_xvfmadd_d:
7956 return DAG.getNode(Opcode: ISD::FMA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7957 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
7958 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
7959 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7960 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
7961 N3: legalizeIntrinsicImmArg<4>(Node: N, ImmOp: 3, DAG, Subtarget));
7962 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
7963 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
7964 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7965 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
7966 N3: legalizeIntrinsicImmArg<3>(Node: N, ImmOp: 3, DAG, Subtarget));
7967 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
7968 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
7969 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7970 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
7971 N3: legalizeIntrinsicImmArg<2>(Node: N, ImmOp: 3, DAG, Subtarget));
7972 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
7973 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7974 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
7975 N3: legalizeIntrinsicImmArg<1>(Node: N, ImmOp: 3, DAG, Subtarget));
7976 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
7977 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
7978 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
7979 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
7980 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
7981 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
7982 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
7983 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
7984 return DAG.getNode(Opcode: LoongArchISD::VREPLGR2VR, DL, VT: N->getValueType(ResNo: 0),
7985 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
7986 Operand: N->getOperand(Num: 1)));
7987 case Intrinsic::loongarch_lsx_vreplve_b:
7988 case Intrinsic::loongarch_lsx_vreplve_h:
7989 case Intrinsic::loongarch_lsx_vreplve_w:
7990 case Intrinsic::loongarch_lsx_vreplve_d:
7991 case Intrinsic::loongarch_lasx_xvreplve_b:
7992 case Intrinsic::loongarch_lasx_xvreplve_h:
7993 case Intrinsic::loongarch_lasx_xvreplve_w:
7994 case Intrinsic::loongarch_lasx_xvreplve_d:
7995 return DAG.getNode(Opcode: LoongArchISD::VREPLVE, DL, VT: N->getValueType(ResNo: 0),
7996 N1: N->getOperand(Num: 1),
7997 N2: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
7998 Operand: N->getOperand(Num: 2)));
7999 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8000 if (!Subtarget.is64Bit())
8001 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8002 break;
8003 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8004 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8005 if (!Subtarget.is64Bit())
8006 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8007 break;
8008 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8009 if (!Subtarget.is64Bit())
8010 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8011 break;
8012 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8013 if (!Subtarget.is64Bit())
8014 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8015 break;
8016 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8017 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8018 if (!Subtarget.is64Bit())
8019 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8020 break;
8021 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8022 if (!Subtarget.is64Bit())
8023 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8024 break;
8025 case Intrinsic::loongarch_lsx_bz_b:
8026 case Intrinsic::loongarch_lsx_bz_h:
8027 case Intrinsic::loongarch_lsx_bz_w:
8028 case Intrinsic::loongarch_lsx_bz_d:
8029 case Intrinsic::loongarch_lasx_xbz_b:
8030 case Intrinsic::loongarch_lasx_xbz_h:
8031 case Intrinsic::loongarch_lasx_xbz_w:
8032 case Intrinsic::loongarch_lasx_xbz_d:
8033 if (!Subtarget.is64Bit())
8034 return DAG.getNode(Opcode: LoongArchISD::VALL_ZERO, DL, VT: N->getValueType(ResNo: 0),
8035 Operand: N->getOperand(Num: 1));
8036 break;
8037 case Intrinsic::loongarch_lsx_bz_v:
8038 case Intrinsic::loongarch_lasx_xbz_v:
8039 if (!Subtarget.is64Bit())
8040 return DAG.getNode(Opcode: LoongArchISD::VANY_ZERO, DL, VT: N->getValueType(ResNo: 0),
8041 Operand: N->getOperand(Num: 1));
8042 break;
8043 case Intrinsic::loongarch_lsx_bnz_b:
8044 case Intrinsic::loongarch_lsx_bnz_h:
8045 case Intrinsic::loongarch_lsx_bnz_w:
8046 case Intrinsic::loongarch_lsx_bnz_d:
8047 case Intrinsic::loongarch_lasx_xbnz_b:
8048 case Intrinsic::loongarch_lasx_xbnz_h:
8049 case Intrinsic::loongarch_lasx_xbnz_w:
8050 case Intrinsic::loongarch_lasx_xbnz_d:
8051 if (!Subtarget.is64Bit())
8052 return DAG.getNode(Opcode: LoongArchISD::VALL_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8053 Operand: N->getOperand(Num: 1));
8054 break;
8055 case Intrinsic::loongarch_lsx_bnz_v:
8056 case Intrinsic::loongarch_lasx_xbnz_v:
8057 if (!Subtarget.is64Bit())
8058 return DAG.getNode(Opcode: LoongArchISD::VANY_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8059 Operand: N->getOperand(Num: 1));
8060 break;
8061 case Intrinsic::loongarch_lasx_concat_128_s:
8062 case Intrinsic::loongarch_lasx_concat_128_d:
8063 case Intrinsic::loongarch_lasx_concat_128:
8064 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
8065 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2));
8066 }
8067 return SDValue();
8068}
8069
8070static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG,
8071 TargetLowering::DAGCombinerInfo &DCI,
8072 const LoongArchSubtarget &Subtarget) {
8073 // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
8074 // conversion is unnecessary and can be replaced with the
8075 // MOVFR2GR_S_LA64 operand.
8076 SDValue Op0 = N->getOperand(Num: 0);
8077 if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8078 return Op0.getOperand(i: 0);
8079 return SDValue();
8080}
8081
8082static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG,
8083 TargetLowering::DAGCombinerInfo &DCI,
8084 const LoongArchSubtarget &Subtarget) {
8085 // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
8086 // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
8087 // operand.
8088 SDValue Op0 = N->getOperand(Num: 0);
8089 if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8090 assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
8091 "Unexpected value type!");
8092 return Op0.getOperand(i: 0);
8093 }
8094 return SDValue();
8095}
8096
8097static SDValue
8098performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG,
8099 TargetLowering::DAGCombinerInfo &DCI) {
8100 MVT VT = N->getSimpleValueType(ResNo: 0);
8101 unsigned NumBits = VT.getScalarSizeInBits();
8102
8103 // Simplify the inputs.
8104 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8105 APInt DemandedMask(APInt::getAllOnes(numBits: NumBits));
8106 if (TLI.SimplifyDemandedBits(Op: SDValue(N, 0), DemandedBits: DemandedMask, DCI))
8107 return SDValue(N, 0);
8108
8109 return SDValue();
8110}
8111
8112static SDValue
8113performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG,
8114 TargetLowering::DAGCombinerInfo &DCI,
8115 const LoongArchSubtarget &Subtarget) {
8116 SDValue Op0 = N->getOperand(Num: 0);
8117 SDLoc DL(N);
8118
8119 // If the input to SplitPairF64 is just BuildPairF64 then the operation is
8120 // redundant. Instead, use BuildPairF64's operands directly.
8121 if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8122 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
8123
8124 if (Op0->isUndef()) {
8125 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
8126 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
8127 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8128 }
8129
8130 // It's cheaper to materialise two 32-bit integers than to load a double
8131 // from the constant pool and transfer it to integer registers through the
8132 // stack.
8133 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
8134 APInt V = C->getValueAPF().bitcastToAPInt();
8135 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
8136 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
8137 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8138 }
8139
8140 return SDValue();
8141}
8142
8143/// Do target-specific dag combines on LoongArchISD::VANDN nodes.
8144static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG,
8145 TargetLowering::DAGCombinerInfo &DCI,
8146 const LoongArchSubtarget &Subtarget) {
8147 SDValue N0 = N->getOperand(Num: 0);
8148 SDValue N1 = N->getOperand(Num: 1);
8149 MVT VT = N->getSimpleValueType(ResNo: 0);
8150 SDLoc DL(N);
8151
8152 // VANDN(undef, x) -> 0
8153 // VANDN(x, undef) -> 0
8154 if (N0.isUndef() || N1.isUndef())
8155 return DAG.getConstant(Val: 0, DL, VT);
8156
8157 // VANDN(0, x) -> x
8158 if (ISD::isBuildVectorAllZeros(N: N0.getNode()))
8159 return N1;
8160
8161 // VANDN(x, 0) -> 0
8162 if (ISD::isBuildVectorAllZeros(N: N1.getNode()))
8163 return DAG.getConstant(Val: 0, DL, VT);
8164
8165 // VANDN(x, -1) -> NOT(x) -> XOR(x, -1)
8166 if (ISD::isBuildVectorAllOnes(N: N1.getNode()))
8167 return DAG.getNOT(DL, Val: N0, VT);
8168
8169 // Turn VANDN back to AND if input is inverted.
8170 if (SDValue Not = isNOT(V: N0, DAG))
8171 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: DAG.getBitcast(VT, V: Not), N2: N1);
8172
8173 // Folds for better commutativity:
8174 if (N1->hasOneUse()) {
8175 // VANDN(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
8176 if (SDValue Not = isNOT(V: N1, DAG))
8177 return DAG.getNOT(
8178 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)), VT);
8179
8180 // VANDN(x, SplatVector(Imm)) -> AND(NOT(x), NOT(SplatVector(~Imm)))
8181 // -> NOT(OR(x, SplatVector(-Imm))
8182 // Combination is performed only when VT is v16i8/v32i8, using `vnori.b` to
8183 // gain benefits.
8184 if (!DCI.isBeforeLegalizeOps() && (VT == MVT::v16i8 || VT == MVT::v32i8) &&
8185 N1.getOpcode() == ISD::BUILD_VECTOR) {
8186 if (SDValue SplatValue =
8187 cast<BuildVectorSDNode>(Val: N1.getNode())->getSplatValue()) {
8188 if (!N1->isOnlyUserOf(N: SplatValue.getNode()))
8189 return SDValue();
8190
8191 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
8192 uint8_t NCVal = static_cast<uint8_t>(~(C->getSExtValue()));
8193 SDValue Not =
8194 DAG.getSplat(VT, DL, Op: DAG.getTargetConstant(Val: NCVal, DL, VT: MVT::i8));
8195 return DAG.getNOT(
8196 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)),
8197 VT);
8198 }
8199 }
8200 }
8201 }
8202
8203 return SDValue();
8204}
8205
8206static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp) {
8207 SDLoc DL(N);
8208 EVT VT = N->getValueType(ResNo: 0);
8209 SDValue Src = N->getOperand(Num: 0);
8210 EVT SrcVT = Src.getValueType();
8211
8212 unsigned DstElts = VT.getVectorNumElements();
8213 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8214 unsigned DstEltBits = VT.getScalarSizeInBits();
8215
8216 if (SrcEltBits >= DstEltBits)
8217 return SDValue();
8218
8219 MVT WidenEltVT = MVT::getIntegerVT(BitWidth: DstEltBits);
8220 MVT WidenSrcVT = MVT::getVectorVT(VT: WidenEltVT, NumElements: DstElts);
8221
8222 SDValue Extend = DAG.getNode(Opcode: ExtendOp, DL, VT: WidenSrcVT, Operand: Src);
8223 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Extend);
8224}
8225
8226// Merge two 64 to 32 convert instructions into one,
8227// e.g.
8228// vffint.s.l $vr0, $vr1, $vr2
8229// will convert 4 si64 into 4 float at once.
8230// or
8231// vftintrz.w.d $vr0, $vr1, $vr2
8232// which will convert 4 double into 4 si32 at once.
8233// also deal with their 256-bits LASX version.
8234static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode,
8235 unsigned BlockBits) {
8236 SDLoc DL(N);
8237 MVT DstVT = N->getSimpleValueType(ResNo: 0);
8238 SDValue Src = N->getOperand(Num: 0);
8239 MVT SrcVT = Src.getSimpleValueType();
8240 unsigned SrcBits = SrcVT.getSizeInBits();
8241
8242 SmallVector<SDValue, 4> Blocks;
8243 unsigned BlockNumElts = BlockBits / SrcVT.getScalarSizeInBits();
8244 MVT BlockVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: BlockNumElts);
8245 if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
8246 Src.getOperand(i: 0).getValueType() == BlockVT) {
8247 for (unsigned i = 0; i < Src.getNumOperands(); ++i)
8248 Blocks.push_back(Elt: Src.getOperand(i));
8249 } else if (SrcBits > BlockBits) {
8250 // Wider than one register: extract each BlockBits-wide sub-vector.
8251 for (unsigned i = 0; i < SrcBits / BlockBits; ++i)
8252 Blocks.push_back(
8253 Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: BlockVT, N1: Src,
8254 N2: DAG.getVectorIdxConstant(Val: i * BlockNumElts, DL)));
8255 } else {
8256 BlockBits = SrcBits;
8257 Blocks.push_back(Elt: Src);
8258 }
8259
8260 MVT NativeVecVT = MVT::getVectorVT(VT: DstVT.getScalarType(),
8261 NumElements: BlockBits / DstVT.getScalarSizeInBits());
8262 SmallVector<SDValue, 4> Parts;
8263 for (unsigned i = 0; i < Blocks.size(); i += 2) {
8264 SDValue Lo = Blocks[i];
8265 SDValue Hi = Blocks.size() > 1 ? Blocks[i + 1] : Lo;
8266 SDValue Res = DAG.getNode(Opcode, DL, VT: NativeVecVT, N1: Hi, N2: Lo);
8267
8268 if (BlockBits == 256) {
8269 SDValue Undef = DAG.getUNDEF(VT: NativeVecVT);
8270 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
8271 Res = DAG.getVectorShuffle(VT: NativeVecVT, dl: DL, N1: Res, N2: Undef, Mask);
8272 Res = DAG.getBitcast(VT: NativeVecVT, V: Res);
8273 }
8274
8275 Parts.push_back(Elt: Res);
8276 }
8277
8278 if (Blocks.size() == 1)
8279 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: DstVT, N1: Parts[0],
8280 N2: DAG.getVectorIdxConstant(Val: 0, DL));
8281 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Parts);
8282}
8283
8284static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8285 TargetLowering::DAGCombinerInfo &DCI,
8286 const LoongArchSubtarget &Subtarget) {
8287 SDLoc DL(N);
8288 EVT VT = N->getValueType(ResNo: 0);
8289 SDValue Src = N->getOperand(Num: 0);
8290 EVT SrcVT = Src.getValueType();
8291
8292 if (VT.isVector()) {
8293 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8294 unsigned DstEltBits = VT.getScalarSizeInBits();
8295 unsigned NumElts = VT.getVectorNumElements();
8296 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8297
8298 // Sign-extend src to avoid scalarization.
8299 if (SrcEltBits <= DstEltBits)
8300 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::SIGN_EXTEND);
8301
8302 if (SrcEltBits != 64 || DstEltBits != 32 || !isPowerOf2_32(Value: NumElts))
8303 return SDValue();
8304
8305 if (!SrcVT.isSimple() || !VT.isSimple())
8306 return SDValue();
8307
8308 // Combine [x]vffint.s.l for vector si64 to float conversion.
8309 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFFINT, BlockBits);
8310 }
8311
8312 if (VT != MVT::f32 && VT != MVT::f64)
8313 return SDValue();
8314 if (VT == MVT::f32 && !Subtarget.hasBasicF())
8315 return SDValue();
8316 if (VT == MVT::f64 && !Subtarget.hasBasicD())
8317 return SDValue();
8318
8319 // Only optimize when the source and destination types have the same width.
8320 if (VT.getSizeInBits() != N->getOperand(Num: 0).getValueSizeInBits())
8321 return SDValue();
8322
8323 // If the result of an integer load is only used by an integer-to-float
8324 // conversion, use a fp load instead. This eliminates an integer-to-float-move
8325 // (movgr2fr) instruction.
8326 if (ISD::isNormalLoad(N: Src.getNode()) && Src.hasOneUse() &&
8327 // Do not change the width of a volatile load. This condition check is
8328 // inspired by AArch64.
8329 !cast<LoadSDNode>(Val&: Src)->isVolatile()) {
8330 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: Src);
8331 SDValue Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
8332 PtrInfo: LN0->getPointerInfo(), Alignment: LN0->getAlign(),
8333 MMOFlags: LN0->getMemOperand()->getFlags());
8334
8335 // Make sure successors of the original load stay after it by updating them
8336 // to use the new Chain.
8337 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: Load.getValue(R: 1));
8338 return DAG.getNode(Opcode: LoongArchISD::SITOF, DL: SDLoc(N), VT, Operand: Load);
8339 }
8340
8341 return SDValue();
8342}
8343
8344static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8345 TargetLowering::DAGCombinerInfo &DCI,
8346 const LoongArchSubtarget &Subtarget) {
8347 SDLoc DL(N);
8348 EVT VT = N->getValueType(ResNo: 0);
8349
8350 // Zero-extend src to avoid scalarization.
8351 if (VT.isVector())
8352 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::ZERO_EXTEND);
8353
8354 return SDValue();
8355}
8356
8357// Using [X]VFTINTRZ_W_D for double to signed 32-bit integer conversion.
8358// For example:
8359// v4i32 = fp_to_sint (concat_vectors v2f64, v2f64)
8360// Can be combined into:
8361// v4i32 = VFTINTRZ_W_D v2f64. v2f64
8362static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG,
8363 TargetLowering::DAGCombinerInfo &DCI,
8364 const LoongArchSubtarget &Subtarget) {
8365 if (!Subtarget.hasExtLSX())
8366 return SDValue();
8367
8368 SDLoc DL(N);
8369 EVT DstVT = N->getValueType(ResNo: 0);
8370 SDValue Src = N->getOperand(Num: 0);
8371 EVT SrcVT = Src.getValueType();
8372 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
8373
8374 if (!DstVT.isVector() || !DstVT.isSimple() || !SrcVT.isSimple())
8375 return SDValue();
8376
8377 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8378 unsigned SrcBits = SrcVT.getSizeInBits();
8379 unsigned DstEltBits = DstVT.getScalarSizeInBits();
8380 unsigned NumElts = DstVT.getVectorNumElements();
8381 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8382
8383 if (!isPowerOf2_32(Value: NumElts) || !isPowerOf2_32(Value: DstEltBits))
8384 return SDValue();
8385
8386 if (SrcBits % BlockBits != 0 && SrcBits != 128)
8387 return SDValue();
8388
8389 if (DstEltBits < 32) {
8390 MVT PromoteVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 32), NumElements: NumElts);
8391 SDValue Conv = DAG.getNode(Opcode: N->getOpcode(), DL, VT: PromoteVT, Operand: Src);
8392 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Conv);
8393 }
8394
8395 if (SrcEltBits != 64 || DstEltBits != 32)
8396 return SDValue();
8397
8398 if (!IsSigned) {
8399 // LASX already has pattern for double convert to uint32.
8400 if (Subtarget.hasExtLASX())
8401 return SDValue();
8402 MVT TmpVT = MVT::getVectorVT(VT: MVT::i64, NumElements: NumElts);
8403 SDValue Tmp = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: TmpVT, Operand: Src);
8404 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Tmp);
8405 }
8406
8407 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFTINTRZ, BlockBits);
8408}
8409
8410// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
8411// logical operations, like in the example below.
8412// or (and (truncate x, truncate y)),
8413// (xor (truncate z, build_vector (constants)))
8414// Given a target type \p VT, we generate
8415// or (and x, y), (xor z, zext(build_vector (constants)))
8416// given x, y and z are of type \p VT. We can do so, if operands are either
8417// truncates from VT types, the second operand is a vector of constants, can
8418// be recursively promoted or is an existing extension we can extend further.
8419static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
8420 SelectionDAG &DAG,
8421 const LoongArchSubtarget &Subtarget,
8422 unsigned Depth) {
8423 // Limit recursion to avoid excessive compile times.
8424 if (Depth >= SelectionDAG::MaxRecursionDepth)
8425 return SDValue();
8426
8427 if (!ISD::isBitwiseLogicOp(Opcode: N.getOpcode()))
8428 return SDValue();
8429
8430 SDValue N0 = N.getOperand(i: 0);
8431 SDValue N1 = N.getOperand(i: 1);
8432
8433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8434 if (!TLI.isOperationLegalOrPromote(Op: N.getOpcode(), VT))
8435 return SDValue();
8436
8437 if (SDValue NN0 =
8438 PromoteMaskArithmetic(N: N0, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8439 N0 = NN0;
8440 else {
8441 // The left side has to be a 'trunc'.
8442 bool LHSTrunc = N0.getOpcode() == ISD::TRUNCATE &&
8443 N0.getOperand(i: 0).getValueType() == VT;
8444 if (LHSTrunc)
8445 N0 = N0.getOperand(i: 0);
8446 else
8447 return SDValue();
8448 }
8449
8450 if (SDValue NN1 =
8451 PromoteMaskArithmetic(N: N1, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8452 N1 = NN1;
8453 else {
8454 // The right side has to be a 'trunc', a (foldable) constant or an
8455 // existing extension we can extend further.
8456 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
8457 N1.getOperand(i: 0).getValueType() == VT;
8458 if (RHSTrunc)
8459 N1 = N1.getOperand(i: 0);
8460 else if (ISD::isExtVecInRegOpcode(Opcode: N1.getOpcode()) && VT.is256BitVector() &&
8461 Subtarget.hasExtLASX() && N1.hasOneUse())
8462 N1 = DAG.getNode(Opcode: N1.getOpcode(), DL, VT, Operand: N1.getOperand(i: 0));
8463 // On 32-bit platform, i64 is an illegal integer scalar type, and
8464 // FoldConstantArithmetic will fail for v4i64. This may be optimized in the
8465 // future.
8466 else if (SDValue Cst =
8467 DAG.FoldConstantArithmetic(Opcode: ISD::ZERO_EXTEND, DL, VT, Ops: {N1}))
8468 N1 = Cst;
8469 else
8470 return SDValue();
8471 }
8472
8473 return DAG.getNode(Opcode: N.getOpcode(), DL, VT, N1: N0, N2: N1);
8474}
8475
8476// On LASX the type v4i1/v8i1/v16i1 may be legalized to v4i32/v8i16/v16i8, which
8477// is LSX-sized register. In most cases we actually compare or select LASX-sized
8478// registers and mixing the two types creates horrible code. This method
8479// optimizes some of the transition sequences.
8480static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
8481 SelectionDAG &DAG,
8482 const LoongArchSubtarget &Subtarget) {
8483 EVT VT = N.getValueType();
8484 assert(VT.isVector() && "Expected vector type");
8485 assert((N.getOpcode() == ISD::ANY_EXTEND ||
8486 N.getOpcode() == ISD::ZERO_EXTEND ||
8487 N.getOpcode() == ISD::SIGN_EXTEND) &&
8488 "Invalid Node");
8489
8490 if (!Subtarget.hasExtLASX() || !VT.is256BitVector())
8491 return SDValue();
8492
8493 SDValue Narrow = N.getOperand(i: 0);
8494 EVT NarrowVT = Narrow.getValueType();
8495
8496 // Generate the wide operation.
8497 SDValue Op = PromoteMaskArithmetic(N: Narrow, DL, VT, DAG, Subtarget, Depth: 0);
8498 if (!Op)
8499 return SDValue();
8500 switch (N.getOpcode()) {
8501 default:
8502 llvm_unreachable("Unexpected opcode");
8503 case ISD::ANY_EXTEND:
8504 return Op;
8505 case ISD::ZERO_EXTEND:
8506 return DAG.getZeroExtendInReg(Op, DL, VT: NarrowVT);
8507 case ISD::SIGN_EXTEND:
8508 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
8509 N2: DAG.getValueType(NarrowVT));
8510 }
8511}
8512
8513static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG,
8514 TargetLowering::DAGCombinerInfo &DCI,
8515 const LoongArchSubtarget &Subtarget) {
8516 EVT VT = N->getValueType(ResNo: 0);
8517 SDLoc DL(N);
8518
8519 if (VT.isVector()) {
8520 if (SDValue R = PromoteMaskArithmetic(N: SDValue(N, 0), DL, DAG, Subtarget))
8521 return R;
8522
8523 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
8524 N->getValueSizeInBits(ResNo: 0) != N->getOperand(Num: 0).getValueSizeInBits() * 2)
8525 return SDValue();
8526
8527 if (SDValue R = matchHalfOf128BitLanes(N: N->getOperand(Num: 0), /*isLow=*/false)) {
8528 if (N->getOpcode() == ISD::SIGN_EXTEND)
8529 return DAG.getNode(Opcode: LoongArchISD::VEXTH, DL, VT, Operand: R);
8530 if (N->getOpcode() == ISD::ZERO_EXTEND)
8531 return DAG.getNode(Opcode: LoongArchISD::VEXTH_U, DL, VT, Operand: R);
8532 }
8533 }
8534
8535 return SDValue();
8536}
8537
8538static SDValue
8539performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
8540 TargetLowering::DAGCombinerInfo &DCI,
8541 const LoongArchSubtarget &Subtarget) {
8542 SDLoc DL(N);
8543 EVT VT = N->getValueType(ResNo: 0);
8544
8545 if (VT.isVector() && N->getNumOperands() == 2)
8546 if (SDValue R = combineFP_ROUND(N: SDValue(N, 0), DL, DAG, Subtarget))
8547 return R;
8548
8549 return SDValue();
8550}
8551
8552static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
8553 TargetLowering::DAGCombinerInfo &DCI,
8554 const LoongArchSubtarget &Subtarget) {
8555 if (DCI.isBeforeLegalizeOps())
8556 return SDValue();
8557
8558 EVT VT = N->getValueType(ResNo: 0);
8559 if (!VT.isVector())
8560 return SDValue();
8561
8562 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
8563 return SDValue();
8564
8565 EVT EltVT = VT.getVectorElementType();
8566 if (!EltVT.isInteger())
8567 return SDValue();
8568
8569 SDValue Cond = N->getOperand(Num: 0);
8570 SDValue TrueVal = N->getOperand(Num: 1);
8571 SDValue FalseVal = N->getOperand(Num: 2);
8572
8573 // match:
8574 //
8575 // vselect (setcc shift, 0, seteq),
8576 // x,
8577 // rounded_shift
8578
8579 if (Cond.getOpcode() != ISD::SETCC)
8580 return SDValue();
8581
8582 if (!ISD::isConstantSplatVectorAllZeros(N: Cond.getOperand(i: 1).getNode()))
8583 return SDValue();
8584
8585 auto *CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2));
8586 if (CC->get() != ISD::SETEQ)
8587 return SDValue();
8588
8589 SDValue Shift = Cond.getOperand(i: 0);
8590
8591 // True branch must be original value:
8592 //
8593 // vselect cond, x, ...
8594
8595 SDValue X = TrueVal;
8596
8597 // Now match rounded shift pattern:
8598 //
8599 // add
8600 // (and
8601 // (srl X, shift-1)
8602 // 1)
8603 // (srl/sra X, shift)
8604
8605 if (FalseVal.getOpcode() != ISD::ADD)
8606 return SDValue();
8607
8608 SDValue Add0 = FalseVal.getOperand(i: 0);
8609 SDValue Add1 = FalseVal.getOperand(i: 1);
8610 SDValue And;
8611 SDValue Shr;
8612
8613 if (Add0.getOpcode() == ISD::AND) {
8614 And = Add0;
8615 Shr = Add1;
8616 } else if (Add1.getOpcode() == ISD::AND) {
8617 And = Add1;
8618 Shr = Add0;
8619 } else {
8620 return SDValue();
8621 }
8622
8623 // match:
8624 //
8625 // srl/sra X, shift
8626
8627 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
8628 return SDValue();
8629
8630 if (Shr.getOperand(i: 0) != X)
8631 return SDValue();
8632
8633 if (Shr.getOperand(i: 1) != Shift)
8634 return SDValue();
8635
8636 // match:
8637 //
8638 // and
8639 // (srl X, shift-1)
8640 // 1
8641
8642 SDValue Srl = And.getOperand(i: 0);
8643 SDValue One = And.getOperand(i: 1);
8644 APInt SplatVal;
8645
8646 if (Srl.getOpcode() != ISD::SRL)
8647 return SDValue();
8648
8649 One = peekThroughBitcasts(V: One);
8650 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
8651 return SDValue();
8652
8653 if (SplatVal != 1)
8654 return SDValue();
8655
8656 if (Srl.getOperand(i: 0) != X)
8657 return SDValue();
8658
8659 // match:
8660 //
8661 // shift-1
8662
8663 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
8664
8665 if (ShiftMinus1.getOpcode() != ISD::ADD)
8666 return SDValue();
8667
8668 if (ShiftMinus1.getOperand(i: 0) != Shift)
8669 return SDValue();
8670
8671 if (!ISD::isConstantSplatVectorAllOnes(N: ShiftMinus1.getOperand(i: 1).getNode()))
8672 return SDValue();
8673
8674 // We matched a rounded right shift pattern and can lower it
8675 // to a single vector rounded shift instruction.
8676
8677 SDLoc DL(N);
8678 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
8679 : LoongArchISD::VSRAR,
8680 DL, VT, N1: X, N2: Shift);
8681}
8682
8683SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
8684 DAGCombinerInfo &DCI) const {
8685 SelectionDAG &DAG = DCI.DAG;
8686 switch (N->getOpcode()) {
8687 default:
8688 break;
8689 case ISD::ADD:
8690 return performADDCombine(N, DAG, DCI, Subtarget);
8691 case ISD::AND:
8692 return performANDCombine(N, DAG, DCI, Subtarget);
8693 case ISD::OR:
8694 return performORCombine(N, DAG, DCI, Subtarget);
8695 case ISD::SETCC:
8696 return performSETCCCombine(N, DAG, DCI, Subtarget);
8697 case ISD::SHL:
8698 return performSHLCombine(N, DAG, DCI, Subtarget);
8699 case ISD::SRL:
8700 return performSRLCombine(N, DAG, DCI, Subtarget);
8701 case ISD::SUB:
8702 return performSUBCombine(N, DAG, DCI, Subtarget);
8703 case ISD::BITCAST:
8704 return performBITCASTCombine(N, DAG, DCI, Subtarget);
8705 case ISD::ANY_EXTEND:
8706 case ISD::ZERO_EXTEND:
8707 case ISD::SIGN_EXTEND:
8708 return performEXTENDCombine(N, DAG, DCI, Subtarget);
8709 case ISD::SINT_TO_FP:
8710 return performSINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8711 case ISD::UINT_TO_FP:
8712 return performUINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8713 case ISD::FP_TO_SINT:
8714 case ISD::FP_TO_UINT:
8715 return performFP_TO_INTCombine(N, DAG, DCI, Subtarget);
8716 case LoongArchISD::BITREV_W:
8717 return performBITREV_WCombine(N, DAG, DCI, Subtarget);
8718 case LoongArchISD::BR_CC:
8719 return performBR_CCCombine(N, DAG, DCI, Subtarget);
8720 case LoongArchISD::SELECT_CC:
8721 return performSELECT_CCCombine(N, DAG, DCI, Subtarget);
8722 case ISD::INTRINSIC_WO_CHAIN:
8723 return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
8724 case LoongArchISD::MOVGR2FR_W_LA64:
8725 return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
8726 case LoongArchISD::MOVFR2GR_S_LA64:
8727 return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
8728 case LoongArchISD::CRC_W_B_W:
8729 case LoongArchISD::CRC_W_H_W:
8730 case LoongArchISD::CRCC_W_B_W:
8731 case LoongArchISD::CRCC_W_H_W:
8732 case LoongArchISD::VMSKLTZ:
8733 case LoongArchISD::XVMSKLTZ:
8734 return performDemandedBitsCombine(N, DAG, DCI);
8735 case LoongArchISD::SPLIT_PAIR_F64:
8736 return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
8737 case LoongArchISD::VANDN:
8738 return performVANDNCombine(N, DAG, DCI, Subtarget);
8739 case ISD::CONCAT_VECTORS:
8740 return performCONCAT_VECTORSCombine(N, DAG, DCI, Subtarget);
8741 case ISD::VSELECT:
8742 return performVSELECTCombine(N, DAG, DCI, Subtarget);
8743 case LoongArchISD::VPACKEV:
8744 case LoongArchISD::VPERMI:
8745 if (SDValue Result =
8746 combineFP_ROUND(N: SDValue(N, 0), DL: SDLoc(N), DAG, Subtarget))
8747 return Result;
8748 }
8749 return SDValue();
8750}
8751
8752static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI,
8753 MachineBasicBlock *MBB) {
8754 if (!ZeroDivCheck)
8755 return MBB;
8756
8757 // Build instructions:
8758 // MBB:
8759 // div(or mod) $dst, $dividend, $divisor
8760 // bne $divisor, $zero, SinkMBB
8761 // BreakMBB:
8762 // break 7 // BRK_DIVZERO
8763 // SinkMBB:
8764 // fallthrough
8765 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8766 MachineFunction::iterator It = ++MBB->getIterator();
8767 MachineFunction *MF = MBB->getParent();
8768 auto BreakMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8769 auto SinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8770 MF->insert(MBBI: It, MBB: BreakMBB);
8771 MF->insert(MBBI: It, MBB: SinkMBB);
8772
8773 // Transfer the remainder of MBB and its successor edges to SinkMBB.
8774 SinkMBB->splice(Where: SinkMBB->end(), Other: MBB, From: std::next(x: MI.getIterator()), To: MBB->end());
8775 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
8776
8777 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
8778 DebugLoc DL = MI.getDebugLoc();
8779 MachineOperand &Divisor = MI.getOperand(i: 2);
8780 Register DivisorReg = Divisor.getReg();
8781
8782 // MBB:
8783 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BNE))
8784 .addReg(RegNo: DivisorReg, Flags: getKillRegState(B: Divisor.isKill()))
8785 .addReg(RegNo: LoongArch::R0)
8786 .addMBB(MBB: SinkMBB);
8787 MBB->addSuccessor(Succ: BreakMBB);
8788 MBB->addSuccessor(Succ: SinkMBB);
8789
8790 // BreakMBB:
8791 // See linux header file arch/loongarch/include/uapi/asm/break.h for the
8792 // definition of BRK_DIVZERO.
8793 BuildMI(BB: BreakMBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BREAK)).addImm(Val: 7 /*BRK_DIVZERO*/);
8794 BreakMBB->addSuccessor(Succ: SinkMBB);
8795
8796 // Clear Divisor's kill flag.
8797 Divisor.setIsKill(false);
8798
8799 return SinkMBB;
8800}
8801
8802static MachineBasicBlock *
8803emitVecCondBranchPseudo(MachineInstr &MI, MachineBasicBlock *BB,
8804 const LoongArchSubtarget &Subtarget) {
8805 unsigned CondOpc;
8806 switch (MI.getOpcode()) {
8807 default:
8808 llvm_unreachable("Unexpected opcode");
8809 case LoongArch::PseudoVBZ:
8810 CondOpc = LoongArch::VSETEQZ_V;
8811 break;
8812 case LoongArch::PseudoVBZ_B:
8813 CondOpc = LoongArch::VSETANYEQZ_B;
8814 break;
8815 case LoongArch::PseudoVBZ_H:
8816 CondOpc = LoongArch::VSETANYEQZ_H;
8817 break;
8818 case LoongArch::PseudoVBZ_W:
8819 CondOpc = LoongArch::VSETANYEQZ_W;
8820 break;
8821 case LoongArch::PseudoVBZ_D:
8822 CondOpc = LoongArch::VSETANYEQZ_D;
8823 break;
8824 case LoongArch::PseudoVBNZ:
8825 CondOpc = LoongArch::VSETNEZ_V;
8826 break;
8827 case LoongArch::PseudoVBNZ_B:
8828 CondOpc = LoongArch::VSETALLNEZ_B;
8829 break;
8830 case LoongArch::PseudoVBNZ_H:
8831 CondOpc = LoongArch::VSETALLNEZ_H;
8832 break;
8833 case LoongArch::PseudoVBNZ_W:
8834 CondOpc = LoongArch::VSETALLNEZ_W;
8835 break;
8836 case LoongArch::PseudoVBNZ_D:
8837 CondOpc = LoongArch::VSETALLNEZ_D;
8838 break;
8839 case LoongArch::PseudoXVBZ:
8840 CondOpc = LoongArch::XVSETEQZ_V;
8841 break;
8842 case LoongArch::PseudoXVBZ_B:
8843 CondOpc = LoongArch::XVSETANYEQZ_B;
8844 break;
8845 case LoongArch::PseudoXVBZ_H:
8846 CondOpc = LoongArch::XVSETANYEQZ_H;
8847 break;
8848 case LoongArch::PseudoXVBZ_W:
8849 CondOpc = LoongArch::XVSETANYEQZ_W;
8850 break;
8851 case LoongArch::PseudoXVBZ_D:
8852 CondOpc = LoongArch::XVSETANYEQZ_D;
8853 break;
8854 case LoongArch::PseudoXVBNZ:
8855 CondOpc = LoongArch::XVSETNEZ_V;
8856 break;
8857 case LoongArch::PseudoXVBNZ_B:
8858 CondOpc = LoongArch::XVSETALLNEZ_B;
8859 break;
8860 case LoongArch::PseudoXVBNZ_H:
8861 CondOpc = LoongArch::XVSETALLNEZ_H;
8862 break;
8863 case LoongArch::PseudoXVBNZ_W:
8864 CondOpc = LoongArch::XVSETALLNEZ_W;
8865 break;
8866 case LoongArch::PseudoXVBNZ_D:
8867 CondOpc = LoongArch::XVSETALLNEZ_D;
8868 break;
8869 }
8870
8871 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
8872 const BasicBlock *LLVM_BB = BB->getBasicBlock();
8873 DebugLoc DL = MI.getDebugLoc();
8874 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8875 MachineFunction::iterator It = ++BB->getIterator();
8876
8877 MachineFunction *F = BB->getParent();
8878 MachineBasicBlock *FalseBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
8879 MachineBasicBlock *TrueBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
8880 MachineBasicBlock *SinkBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
8881
8882 F->insert(MBBI: It, MBB: FalseBB);
8883 F->insert(MBBI: It, MBB: TrueBB);
8884 F->insert(MBBI: It, MBB: SinkBB);
8885
8886 // Transfer the remainder of MBB and its successor edges to Sink.
8887 SinkBB->splice(Where: SinkBB->end(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
8888 SinkBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
8889
8890 // Insert the real instruction to BB.
8891 Register FCC = MRI.createVirtualRegister(RegClass: &LoongArch::CFRRegClass);
8892 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: CondOpc), DestReg: FCC).addReg(RegNo: MI.getOperand(i: 1).getReg());
8893
8894 // Insert branch.
8895 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::BCNEZ)).addReg(RegNo: FCC).addMBB(MBB: TrueBB);
8896 BB->addSuccessor(Succ: FalseBB);
8897 BB->addSuccessor(Succ: TrueBB);
8898
8899 // FalseBB.
8900 Register RD1 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
8901 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD1)
8902 .addReg(RegNo: LoongArch::R0)
8903 .addImm(Val: 0);
8904 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::PseudoBR)).addMBB(MBB: SinkBB);
8905 FalseBB->addSuccessor(Succ: SinkBB);
8906
8907 // TrueBB.
8908 Register RD2 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
8909 BuildMI(BB: TrueBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD2)
8910 .addReg(RegNo: LoongArch::R0)
8911 .addImm(Val: 1);
8912 TrueBB->addSuccessor(Succ: SinkBB);
8913
8914 // SinkBB: merge the results.
8915 BuildMI(BB&: *SinkBB, I: SinkBB->begin(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::PHI),
8916 DestReg: MI.getOperand(i: 0).getReg())
8917 .addReg(RegNo: RD1)
8918 .addMBB(MBB: FalseBB)
8919 .addReg(RegNo: RD2)
8920 .addMBB(MBB: TrueBB);
8921
8922 // The pseudo instruction is gone now.
8923 MI.eraseFromParent();
8924 return SinkBB;
8925}
8926
8927static MachineBasicBlock *
8928emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
8929 const LoongArchSubtarget &Subtarget) {
8930 unsigned InsOp;
8931 unsigned BroadcastOp;
8932 unsigned HalfSize;
8933 switch (MI.getOpcode()) {
8934 default:
8935 llvm_unreachable("Unexpected opcode");
8936 case LoongArch::PseudoXVINSGR2VR_B:
8937 HalfSize = 16;
8938 BroadcastOp = LoongArch::XVREPLGR2VR_B;
8939 InsOp = LoongArch::XVEXTRINS_B;
8940 break;
8941 case LoongArch::PseudoXVINSGR2VR_H:
8942 HalfSize = 8;
8943 BroadcastOp = LoongArch::XVREPLGR2VR_H;
8944 InsOp = LoongArch::XVEXTRINS_H;
8945 break;
8946 }
8947 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
8948 const TargetRegisterClass *RC = &LoongArch::LASX256RegClass;
8949 const TargetRegisterClass *SubRC = &LoongArch::LSX128RegClass;
8950 DebugLoc DL = MI.getDebugLoc();
8951 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8952 // XDst = vector_insert XSrc, Elt, Idx
8953 Register XDst = MI.getOperand(i: 0).getReg();
8954 Register XSrc = MI.getOperand(i: 1).getReg();
8955 Register Elt = MI.getOperand(i: 2).getReg();
8956 unsigned Idx = MI.getOperand(i: 3).getImm();
8957
8958 if (XSrc.isVirtual() && MRI.getVRegDef(Reg: XSrc)->isImplicitDef() &&
8959 Idx < HalfSize) {
8960 Register ScratchSubReg1 = MRI.createVirtualRegister(RegClass: SubRC);
8961 Register ScratchSubReg2 = MRI.createVirtualRegister(RegClass: SubRC);
8962
8963 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::COPY), DestReg: ScratchSubReg1)
8964 .addReg(RegNo: XSrc, Flags: {}, SubReg: LoongArch::sub_128);
8965 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
8966 MCID: TII->get(Opcode: HalfSize == 8 ? LoongArch::VINSGR2VR_H
8967 : LoongArch::VINSGR2VR_B),
8968 DestReg: ScratchSubReg2)
8969 .addReg(RegNo: ScratchSubReg1)
8970 .addReg(RegNo: Elt)
8971 .addImm(Val: Idx);
8972
8973 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::SUBREG_TO_REG), DestReg: XDst)
8974 .addReg(RegNo: ScratchSubReg2)
8975 .addImm(Val: LoongArch::sub_128);
8976 } else {
8977 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
8978 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
8979
8980 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Elt);
8981
8982 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPERMI_Q), DestReg: ScratchReg2)
8983 .addReg(RegNo: ScratchReg1)
8984 .addReg(RegNo: XSrc)
8985 .addImm(Val: Idx >= HalfSize ? 48 : 18);
8986
8987 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InsOp), DestReg: XDst)
8988 .addReg(RegNo: XSrc)
8989 .addReg(RegNo: ScratchReg2)
8990 .addImm(Val: (Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
8991 }
8992
8993 MI.eraseFromParent();
8994 return BB;
8995}
8996
8997static MachineBasicBlock *emitPseudoCTPOP(MachineInstr &MI,
8998 MachineBasicBlock *BB,
8999 const LoongArchSubtarget &Subtarget) {
9000 assert(Subtarget.hasExtLSX());
9001 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9002 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9003 DebugLoc DL = MI.getDebugLoc();
9004 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9005 Register Dst = MI.getOperand(i: 0).getReg();
9006 Register Src = MI.getOperand(i: 1).getReg();
9007
9008 unsigned BroadcastOp, CTOp, PickOp;
9009 switch (MI.getOpcode()) {
9010 default:
9011 llvm_unreachable("Unexpected opcode");
9012 case LoongArch::PseudoCTPOP_B:
9013 BroadcastOp = LoongArch::VREPLGR2VR_B;
9014 CTOp = LoongArch::VPCNT_B;
9015 PickOp = LoongArch::VPICKVE2GR_B;
9016 break;
9017 case LoongArch::PseudoCTPOP_H:
9018 case LoongArch::PseudoCTPOP_H_LA32:
9019 BroadcastOp = LoongArch::VREPLGR2VR_H;
9020 CTOp = LoongArch::VPCNT_H;
9021 PickOp = LoongArch::VPICKVE2GR_H;
9022 break;
9023 case LoongArch::PseudoCTPOP_W:
9024 case LoongArch::PseudoCTPOP_W_LA32:
9025 BroadcastOp = LoongArch::VREPLGR2VR_W;
9026 CTOp = LoongArch::VPCNT_W;
9027 PickOp = LoongArch::VPICKVE2GR_W;
9028 break;
9029 case LoongArch::PseudoCTPOP_D:
9030 BroadcastOp = LoongArch::VREPLGR2VR_D;
9031 CTOp = LoongArch::VPCNT_D;
9032 PickOp = LoongArch::VPICKVE2GR_D;
9033 break;
9034 }
9035
9036 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9037 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9038 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Src);
9039 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: CTOp), DestReg: ScratchReg2).addReg(RegNo: ScratchReg1);
9040 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PickOp), DestReg: Dst).addReg(RegNo: ScratchReg2).addImm(Val: 0);
9041
9042 MI.eraseFromParent();
9043 return BB;
9044}
9045
9046static MachineBasicBlock *
9047emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
9048 const LoongArchSubtarget &Subtarget) {
9049 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9050 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9051 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
9052 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9053 Register Dst = MI.getOperand(i: 0).getReg();
9054 Register Src = MI.getOperand(i: 1).getReg();
9055 DebugLoc DL = MI.getDebugLoc();
9056 unsigned EleBits = 8;
9057 unsigned NotOpc = 0;
9058 unsigned MskOpc;
9059
9060 switch (MI.getOpcode()) {
9061 default:
9062 llvm_unreachable("Unexpected opcode");
9063 case LoongArch::PseudoVMSKLTZ_B:
9064 MskOpc = LoongArch::VMSKLTZ_B;
9065 break;
9066 case LoongArch::PseudoVMSKLTZ_H:
9067 MskOpc = LoongArch::VMSKLTZ_H;
9068 EleBits = 16;
9069 break;
9070 case LoongArch::PseudoVMSKLTZ_W:
9071 MskOpc = LoongArch::VMSKLTZ_W;
9072 EleBits = 32;
9073 break;
9074 case LoongArch::PseudoVMSKLTZ_D:
9075 MskOpc = LoongArch::VMSKLTZ_D;
9076 EleBits = 64;
9077 break;
9078 case LoongArch::PseudoVMSKGEZ_B:
9079 MskOpc = LoongArch::VMSKGEZ_B;
9080 break;
9081 case LoongArch::PseudoVMSKEQZ_B:
9082 MskOpc = LoongArch::VMSKNZ_B;
9083 NotOpc = LoongArch::VNOR_V;
9084 break;
9085 case LoongArch::PseudoVMSKNEZ_B:
9086 MskOpc = LoongArch::VMSKNZ_B;
9087 break;
9088 case LoongArch::PseudoXVMSKLTZ_B:
9089 MskOpc = LoongArch::XVMSKLTZ_B;
9090 RC = &LoongArch::LASX256RegClass;
9091 break;
9092 case LoongArch::PseudoXVMSKLTZ_H:
9093 MskOpc = LoongArch::XVMSKLTZ_H;
9094 RC = &LoongArch::LASX256RegClass;
9095 EleBits = 16;
9096 break;
9097 case LoongArch::PseudoXVMSKLTZ_W:
9098 MskOpc = LoongArch::XVMSKLTZ_W;
9099 RC = &LoongArch::LASX256RegClass;
9100 EleBits = 32;
9101 break;
9102 case LoongArch::PseudoXVMSKLTZ_D:
9103 MskOpc = LoongArch::XVMSKLTZ_D;
9104 RC = &LoongArch::LASX256RegClass;
9105 EleBits = 64;
9106 break;
9107 case LoongArch::PseudoXVMSKGEZ_B:
9108 MskOpc = LoongArch::XVMSKGEZ_B;
9109 RC = &LoongArch::LASX256RegClass;
9110 break;
9111 case LoongArch::PseudoXVMSKEQZ_B:
9112 MskOpc = LoongArch::XVMSKNZ_B;
9113 NotOpc = LoongArch::XVNOR_V;
9114 RC = &LoongArch::LASX256RegClass;
9115 break;
9116 case LoongArch::PseudoXVMSKNEZ_B:
9117 MskOpc = LoongArch::XVMSKNZ_B;
9118 RC = &LoongArch::LASX256RegClass;
9119 break;
9120 }
9121
9122 Register Msk = MRI.createVirtualRegister(RegClass: RC);
9123 if (NotOpc) {
9124 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
9125 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Tmp).addReg(RegNo: Src);
9126 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: NotOpc), DestReg: Msk)
9127 .addReg(RegNo: Tmp, Flags: RegState::Kill)
9128 .addReg(RegNo: Tmp, Flags: RegState::Kill);
9129 } else {
9130 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Msk).addReg(RegNo: Src);
9131 }
9132
9133 if (TRI->getRegSizeInBits(RC: *RC) > 128) {
9134 Register Lo = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9135 Register Hi = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9136 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Lo)
9137 .addReg(RegNo: Msk)
9138 .addImm(Val: 0);
9139 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Hi)
9140 .addReg(RegNo: Msk, Flags: RegState::Kill)
9141 .addImm(Val: 4);
9142 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9143 MCID: TII->get(Opcode: Subtarget.is64Bit() ? LoongArch::BSTRINS_D
9144 : LoongArch::BSTRINS_W),
9145 DestReg: Dst)
9146 .addReg(RegNo: Lo, Flags: RegState::Kill)
9147 .addReg(RegNo: Hi, Flags: RegState::Kill)
9148 .addImm(Val: 256 / EleBits - 1)
9149 .addImm(Val: 128 / EleBits);
9150 } else {
9151 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::VPICKVE2GR_HU), DestReg: Dst)
9152 .addReg(RegNo: Msk, Flags: RegState::Kill)
9153 .addImm(Val: 0);
9154 }
9155
9156 MI.eraseFromParent();
9157 return BB;
9158}
9159
9160static MachineBasicBlock *
9161emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9162 const LoongArchSubtarget &Subtarget) {
9163 assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
9164 "Unexpected instruction");
9165
9166 MachineFunction &MF = *BB->getParent();
9167 DebugLoc DL = MI.getDebugLoc();
9168 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9169 Register LoReg = MI.getOperand(i: 0).getReg();
9170 Register HiReg = MI.getOperand(i: 1).getReg();
9171 Register SrcReg = MI.getOperand(i: 2).getReg();
9172
9173 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFR2GR_S_64), DestReg: LoReg).addReg(RegNo: SrcReg);
9174 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFRH2GR_S), DestReg: HiReg)
9175 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9176 MI.eraseFromParent(); // The pseudo instruction is gone now.
9177 return BB;
9178}
9179
9180static MachineBasicBlock *
9181emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9182 const LoongArchSubtarget &Subtarget) {
9183 assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
9184 "Unexpected instruction");
9185
9186 MachineFunction &MF = *BB->getParent();
9187 DebugLoc DL = MI.getDebugLoc();
9188 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9189 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9190 Register TmpReg = MRI.createVirtualRegister(RegClass: &LoongArch::FPR64RegClass);
9191 Register DstReg = MI.getOperand(i: 0).getReg();
9192 Register LoReg = MI.getOperand(i: 1).getReg();
9193 Register HiReg = MI.getOperand(i: 2).getReg();
9194
9195 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FR_W_64), DestReg: TmpReg)
9196 .addReg(RegNo: LoReg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()));
9197 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FRH_W), DestReg: DstReg)
9198 .addReg(RegNo: TmpReg, Flags: RegState::Kill)
9199 .addReg(RegNo: HiReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9200 MI.eraseFromParent(); // The pseudo instruction is gone now.
9201 return BB;
9202}
9203
9204static bool isSelectPseudo(MachineInstr &MI) {
9205 switch (MI.getOpcode()) {
9206 default:
9207 return false;
9208 case LoongArch::Select_GPR_Using_CC_GPR:
9209 return true;
9210 }
9211}
9212
9213static MachineBasicBlock *
9214emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB,
9215 const LoongArchSubtarget &Subtarget) {
9216 // To "insert" Select_* instructions, we actually have to insert the triangle
9217 // control-flow pattern. The incoming instructions know the destination vreg
9218 // to set, the condition code register to branch on, the true/false values to
9219 // select between, and the condcode to use to select the appropriate branch.
9220 //
9221 // We produce the following control flow:
9222 // HeadMBB
9223 // | \
9224 // | IfFalseMBB
9225 // | /
9226 // TailMBB
9227 //
9228 // When we find a sequence of selects we attempt to optimize their emission
9229 // by sharing the control flow. Currently we only handle cases where we have
9230 // multiple selects with the exact same condition (same LHS, RHS and CC).
9231 // The selects may be interleaved with other instructions if the other
9232 // instructions meet some requirements we deem safe:
9233 // - They are not pseudo instructions.
9234 // - They are debug instructions. Otherwise,
9235 // - They do not have side-effects, do not access memory and their inputs do
9236 // not depend on the results of the select pseudo-instructions.
9237 // The TrueV/FalseV operands of the selects cannot depend on the result of
9238 // previous selects in the sequence.
9239 // These conditions could be further relaxed. See the X86 target for a
9240 // related approach and more information.
9241
9242 Register LHS = MI.getOperand(i: 1).getReg();
9243 Register RHS;
9244 if (MI.getOperand(i: 2).isReg())
9245 RHS = MI.getOperand(i: 2).getReg();
9246 auto CC = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
9247
9248 SmallVector<MachineInstr *, 4> SelectDebugValues;
9249 SmallSet<Register, 4> SelectDests;
9250 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
9251
9252 MachineInstr *LastSelectPseudo = &MI;
9253 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
9254 SequenceMBBI != E; ++SequenceMBBI) {
9255 if (SequenceMBBI->isDebugInstr())
9256 continue;
9257 if (isSelectPseudo(MI&: *SequenceMBBI)) {
9258 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
9259 !SequenceMBBI->getOperand(i: 2).isReg() ||
9260 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
9261 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
9262 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
9263 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
9264 break;
9265 LastSelectPseudo = &*SequenceMBBI;
9266 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
9267 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
9268 continue;
9269 }
9270 if (SequenceMBBI->hasUnmodeledSideEffects() ||
9271 SequenceMBBI->mayLoadOrStore() ||
9272 SequenceMBBI->usesCustomInsertionHook())
9273 break;
9274 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
9275 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
9276 }))
9277 break;
9278 }
9279
9280 const LoongArchInstrInfo &TII = *Subtarget.getInstrInfo();
9281 const BasicBlock *LLVM_BB = BB->getBasicBlock();
9282 DebugLoc DL = MI.getDebugLoc();
9283 MachineFunction::iterator I = ++BB->getIterator();
9284
9285 MachineBasicBlock *HeadMBB = BB;
9286 MachineFunction *F = BB->getParent();
9287 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9288 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9289
9290 F->insert(MBBI: I, MBB: IfFalseMBB);
9291 F->insert(MBBI: I, MBB: TailMBB);
9292
9293 // Set the call frame size on entry to the new basic blocks.
9294 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
9295 IfFalseMBB->setCallFrameSize(CallFrameSize);
9296 TailMBB->setCallFrameSize(CallFrameSize);
9297
9298 // Transfer debug instructions associated with the selects to TailMBB.
9299 for (MachineInstr *DebugInstr : SelectDebugValues) {
9300 TailMBB->push_back(MI: DebugInstr->removeFromParent());
9301 }
9302
9303 // Move all instructions after the sequence to TailMBB.
9304 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
9305 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
9306 // Update machine-CFG edges by transferring all successors of the current
9307 // block to the new block which will contain the Phi nodes for the selects.
9308 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
9309 // Set the successors for HeadMBB.
9310 HeadMBB->addSuccessor(Succ: IfFalseMBB);
9311 HeadMBB->addSuccessor(Succ: TailMBB);
9312
9313 // Insert appropriate branch.
9314 if (MI.getOperand(i: 2).isImm())
9315 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC))
9316 .addReg(RegNo: LHS)
9317 .addImm(Val: MI.getOperand(i: 2).getImm())
9318 .addMBB(MBB: TailMBB);
9319 else
9320 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: TailMBB);
9321
9322 // IfFalseMBB just falls through to TailMBB.
9323 IfFalseMBB->addSuccessor(Succ: TailMBB);
9324
9325 // Create PHIs for all of the select pseudo-instructions.
9326 auto SelectMBBI = MI.getIterator();
9327 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
9328 auto InsertionPoint = TailMBB->begin();
9329 while (SelectMBBI != SelectEnd) {
9330 auto Next = std::next(x: SelectMBBI);
9331 if (isSelectPseudo(MI&: *SelectMBBI)) {
9332 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
9333 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
9334 MCID: TII.get(Opcode: LoongArch::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
9335 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
9336 .addMBB(MBB: HeadMBB)
9337 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
9338 .addMBB(MBB: IfFalseMBB);
9339 SelectMBBI->eraseFromParent();
9340 }
9341 SelectMBBI = Next;
9342 }
9343
9344 F->getProperties().resetNoPHIs();
9345 return TailMBB;
9346}
9347
9348MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
9349 MachineInstr &MI, MachineBasicBlock *BB) const {
9350 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9351 DebugLoc DL = MI.getDebugLoc();
9352
9353 switch (MI.getOpcode()) {
9354 default:
9355 llvm_unreachable("Unexpected instr type to insert");
9356 case LoongArch::DIV_W:
9357 case LoongArch::DIV_WU:
9358 case LoongArch::MOD_W:
9359 case LoongArch::MOD_WU:
9360 case LoongArch::DIV_D:
9361 case LoongArch::DIV_DU:
9362 case LoongArch::MOD_D:
9363 case LoongArch::MOD_DU:
9364 return insertDivByZeroTrap(MI, MBB: BB);
9365 break;
9366 case LoongArch::WRFCSR: {
9367 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVGR2FCSR),
9368 DestReg: LoongArch::FCSR0 + MI.getOperand(i: 0).getImm())
9369 .addReg(RegNo: MI.getOperand(i: 1).getReg());
9370 MI.eraseFromParent();
9371 return BB;
9372 }
9373 case LoongArch::RDFCSR: {
9374 MachineInstr *ReadFCSR =
9375 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVFCSR2GR),
9376 DestReg: MI.getOperand(i: 0).getReg())
9377 .addReg(RegNo: LoongArch::FCSR0 + MI.getOperand(i: 1).getImm());
9378 ReadFCSR->getOperand(i: 1).setIsUndef();
9379 MI.eraseFromParent();
9380 return BB;
9381 }
9382 case LoongArch::Select_GPR_Using_CC_GPR:
9383 return emitSelectPseudo(MI, BB, Subtarget);
9384 case LoongArch::BuildPairF64Pseudo:
9385 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
9386 case LoongArch::SplitPairF64Pseudo:
9387 return emitSplitPairF64Pseudo(MI, BB, Subtarget);
9388 case LoongArch::PseudoVBZ:
9389 case LoongArch::PseudoVBZ_B:
9390 case LoongArch::PseudoVBZ_H:
9391 case LoongArch::PseudoVBZ_W:
9392 case LoongArch::PseudoVBZ_D:
9393 case LoongArch::PseudoVBNZ:
9394 case LoongArch::PseudoVBNZ_B:
9395 case LoongArch::PseudoVBNZ_H:
9396 case LoongArch::PseudoVBNZ_W:
9397 case LoongArch::PseudoVBNZ_D:
9398 case LoongArch::PseudoXVBZ:
9399 case LoongArch::PseudoXVBZ_B:
9400 case LoongArch::PseudoXVBZ_H:
9401 case LoongArch::PseudoXVBZ_W:
9402 case LoongArch::PseudoXVBZ_D:
9403 case LoongArch::PseudoXVBNZ:
9404 case LoongArch::PseudoXVBNZ_B:
9405 case LoongArch::PseudoXVBNZ_H:
9406 case LoongArch::PseudoXVBNZ_W:
9407 case LoongArch::PseudoXVBNZ_D:
9408 return emitVecCondBranchPseudo(MI, BB, Subtarget);
9409 case LoongArch::PseudoXVINSGR2VR_B:
9410 case LoongArch::PseudoXVINSGR2VR_H:
9411 return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
9412 case LoongArch::PseudoCTPOP_B:
9413 case LoongArch::PseudoCTPOP_H:
9414 case LoongArch::PseudoCTPOP_W:
9415 case LoongArch::PseudoCTPOP_D:
9416 case LoongArch::PseudoCTPOP_H_LA32:
9417 case LoongArch::PseudoCTPOP_W_LA32:
9418 return emitPseudoCTPOP(MI, BB, Subtarget);
9419 case LoongArch::PseudoVMSKLTZ_B:
9420 case LoongArch::PseudoVMSKLTZ_H:
9421 case LoongArch::PseudoVMSKLTZ_W:
9422 case LoongArch::PseudoVMSKLTZ_D:
9423 case LoongArch::PseudoVMSKGEZ_B:
9424 case LoongArch::PseudoVMSKEQZ_B:
9425 case LoongArch::PseudoVMSKNEZ_B:
9426 case LoongArch::PseudoXVMSKLTZ_B:
9427 case LoongArch::PseudoXVMSKLTZ_H:
9428 case LoongArch::PseudoXVMSKLTZ_W:
9429 case LoongArch::PseudoXVMSKLTZ_D:
9430 case LoongArch::PseudoXVMSKGEZ_B:
9431 case LoongArch::PseudoXVMSKEQZ_B:
9432 case LoongArch::PseudoXVMSKNEZ_B:
9433 return emitPseudoVMSKCOND(MI, BB, Subtarget);
9434 case TargetOpcode::STATEPOINT:
9435 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
9436 // while bl call instruction (where statepoint will be lowered at the
9437 // end) has implicit def. This def is early-clobber as it will be set at
9438 // the moment of the call and earlier than any use is read.
9439 // Add this implicit dead def here as a workaround.
9440 MI.addOperand(MF&: *MI.getMF(),
9441 Op: MachineOperand::CreateReg(
9442 Reg: LoongArch::R1, /*isDef*/ true,
9443 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
9444 /*isUndef*/ false, /*isEarlyClobber*/ true));
9445 if (!Subtarget.is64Bit())
9446 report_fatal_error(reason: "STATEPOINT is only supported on 64-bit targets");
9447 return emitPatchPoint(MI, MBB: BB);
9448 case LoongArch::PROBED_STACKALLOC_DYN:
9449 return emitDynamicProbedAlloc(MI, MBB: BB);
9450 }
9451}
9452
9453bool LoongArchTargetLowering::allowsMisalignedMemoryAccesses(
9454 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
9455 unsigned *Fast) const {
9456 if (!Subtarget.hasUAL())
9457 return false;
9458
9459 // TODO: set reasonable speed number.
9460 if (Fast)
9461 *Fast = 1;
9462 return true;
9463}
9464
9465//===----------------------------------------------------------------------===//
9466// Calling Convention Implementation
9467//===----------------------------------------------------------------------===//
9468
9469// Eight general-purpose registers a0-a7 used for passing integer arguments,
9470// with a0-a1 reused to return values. Generally, the GPRs are used to pass
9471// fixed-point arguments, and floating-point arguments when no FPR is available
9472// or with soft float ABI.
9473const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6,
9474 LoongArch::R7, LoongArch::R8, LoongArch::R9,
9475 LoongArch::R10, LoongArch::R11};
9476
9477// PreserveNone calling convention:
9478// Arguments may be passed in any general-purpose registers except:
9479// - R1 : return address register
9480// - R22 : frame pointer
9481// - R31 : base pointer
9482//
9483// All general-purpose registers are treated as caller-saved,
9484// except R1 (RA) and R22 (FP).
9485//
9486// Non-volatile registers are allocated first so that a function
9487// can call normal functions without having to spill and reload
9488// argument registers.
9489const MCPhysReg PreserveNoneArgGPRs[] = {
9490 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
9491 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
9492 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
9493 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
9494 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
9495 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
9496 LoongArch::R20};
9497
9498// Eight floating-point registers fa0-fa7 used for passing floating-point
9499// arguments, and fa0-fa1 are also used to return values.
9500const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2,
9501 LoongArch::F3, LoongArch::F4, LoongArch::F5,
9502 LoongArch::F6, LoongArch::F7};
9503// FPR32 and FPR64 alias each other.
9504const MCPhysReg ArgFPR64s[] = {
9505 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
9506 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
9507
9508const MCPhysReg ArgVRs[] = {LoongArch::VR0, LoongArch::VR1, LoongArch::VR2,
9509 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
9510 LoongArch::VR6, LoongArch::VR7};
9511
9512const MCPhysReg ArgXRs[] = {LoongArch::XR0, LoongArch::XR1, LoongArch::XR2,
9513 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
9514 LoongArch::XR6, LoongArch::XR7};
9515
9516static Register allocateArgGPR(CCState &State) {
9517 switch (State.getCallingConv()) {
9518 case CallingConv::PreserveNone:
9519 if (!State.isVarArg())
9520 return State.AllocateReg(Regs: PreserveNoneArgGPRs);
9521 [[fallthrough]];
9522 default:
9523 return State.AllocateReg(Regs: ArgGPRs);
9524 }
9525}
9526
9527// Pass a 2*GRLen argument that has been split into two GRLen values through
9528// registers or the stack as necessary.
9529static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
9530 CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1,
9531 unsigned ValNo2, MVT ValVT2, MVT LocVT2,
9532 ISD::ArgFlagsTy ArgFlags2) {
9533 unsigned GRLenInBytes = GRLen / 8;
9534 if (Register Reg = allocateArgGPR(State)) {
9535 // At least one half can be passed via register.
9536 State.addLoc(V: CCValAssign::getReg(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(), Reg,
9537 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9538 } else {
9539 // Both halves must be passed on the stack, with proper alignment.
9540 Align StackAlign =
9541 std::max(a: Align(GRLenInBytes), b: ArgFlags1.getNonZeroOrigAlign());
9542 State.addLoc(
9543 V: CCValAssign::getMem(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(),
9544 Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: StackAlign),
9545 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9546 State.addLoc(V: CCValAssign::getMem(
9547 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9548 LocVT: LocVT2, HTP: CCValAssign::Full));
9549 return false;
9550 }
9551 if (Register Reg = allocateArgGPR(State)) {
9552 // The second half can also be passed via register.
9553 State.addLoc(
9554 V: CCValAssign::getReg(ValNo: ValNo2, ValVT: ValVT2, Reg, LocVT: LocVT2, HTP: CCValAssign::Full));
9555 } else {
9556 // The second half is passed via the stack, without additional alignment.
9557 State.addLoc(V: CCValAssign::getMem(
9558 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9559 LocVT: LocVT2, HTP: CCValAssign::Full));
9560 }
9561 return false;
9562}
9563
9564// Implements the LoongArch calling convention. Returns true upon failure.
9565static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
9566 unsigned ValNo, MVT ValVT,
9567 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
9568 CCState &State, bool IsRet, Type *OrigTy) {
9569 unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
9570 assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
9571 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
9572 MVT LocVT = ValVT;
9573
9574 // Any return value split into more than two values can't be returned
9575 // directly.
9576 if (IsRet && ValNo > 1)
9577 return true;
9578
9579 // If passing a variadic argument, or if no FPR is available.
9580 bool UseGPRForFloat = true;
9581
9582 switch (ABI) {
9583 default:
9584 llvm_unreachable("Unexpected ABI");
9585 break;
9586 case LoongArchABI::ABI_ILP32F:
9587 case LoongArchABI::ABI_LP64F:
9588 case LoongArchABI::ABI_ILP32D:
9589 case LoongArchABI::ABI_LP64D:
9590 UseGPRForFloat = ArgFlags.isVarArg();
9591 break;
9592 case LoongArchABI::ABI_ILP32S:
9593 case LoongArchABI::ABI_LP64S:
9594 break;
9595 }
9596
9597 // If this is a variadic argument, the LoongArch calling convention requires
9598 // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
9599 // byte alignment. An aligned register should be used regardless of whether
9600 // the original argument was split during legalisation or not. The argument
9601 // will not be passed by registers if the original type is larger than
9602 // 2*GRLen, so the register alignment rule does not apply.
9603 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
9604 if (ArgFlags.isVarArg() &&
9605 ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
9606 DL.getTypeAllocSize(Ty: OrigTy) == TwoGRLenInBytes) {
9607 unsigned RegIdx = State.getFirstUnallocated(Regs: ArgGPRs);
9608 // Skip 'odd' register if necessary.
9609 if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1)
9610 State.AllocateReg(Regs: ArgGPRs);
9611 }
9612
9613 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
9614 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
9615 State.getPendingArgFlags();
9616
9617 assert(PendingLocs.size() == PendingArgFlags.size() &&
9618 "PendingLocs and PendingArgFlags out of sync");
9619
9620 // FPR32 and FPR64 alias each other.
9621 if (State.getFirstUnallocated(Regs: ArgFPR32s) == std::size(ArgFPR32s))
9622 UseGPRForFloat = true;
9623
9624 if (UseGPRForFloat && ValVT == MVT::f32) {
9625 LocVT = GRLenVT;
9626 LocInfo = CCValAssign::BCvt;
9627 } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
9628 LocVT = MVT::i64;
9629 LocInfo = CCValAssign::BCvt;
9630 } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
9631 // Handle passing f64 on LA32D with a soft float ABI or when floating point
9632 // registers are exhausted.
9633 assert(PendingLocs.empty() && "Can't lower f64 if it is split");
9634 // Depending on available argument GPRS, f64 may be passed in a pair of
9635 // GPRs, split between a GPR and the stack, or passed completely on the
9636 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
9637 // cases.
9638 MCRegister Reg = allocateArgGPR(State);
9639 if (!Reg) {
9640 int64_t StackOffset = State.AllocateStack(Size: 8, Alignment: Align(8));
9641 State.addLoc(
9642 V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9643 return false;
9644 }
9645 LocVT = MVT::i32;
9646 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9647 MCRegister HiReg = allocateArgGPR(State);
9648 if (HiReg) {
9649 State.addLoc(
9650 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: HiReg, LocVT, HTP: LocInfo));
9651 } else {
9652 int64_t StackOffset = State.AllocateStack(Size: 4, Alignment: Align(4));
9653 State.addLoc(
9654 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9655 }
9656 return false;
9657 }
9658
9659 // Split arguments might be passed indirectly, so keep track of the pending
9660 // values.
9661 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
9662 LocVT = GRLenVT;
9663 LocInfo = CCValAssign::Indirect;
9664 PendingLocs.push_back(
9665 Elt: CCValAssign::getPending(ValNo, ValVT, LocVT, HTP: LocInfo));
9666 PendingArgFlags.push_back(Elt: ArgFlags);
9667 if (!ArgFlags.isSplitEnd()) {
9668 return false;
9669 }
9670 }
9671
9672 // If the split argument only had two elements, it should be passed directly
9673 // in registers or on the stack.
9674 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
9675 PendingLocs.size() <= 2) {
9676 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
9677 // Apply the normal calling convention rules to the first half of the
9678 // split argument.
9679 CCValAssign VA = PendingLocs[0];
9680 ISD::ArgFlagsTy AF = PendingArgFlags[0];
9681 PendingLocs.clear();
9682 PendingArgFlags.clear();
9683 return CC_LoongArchAssign2GRLen(GRLen, State, VA1: VA, ArgFlags1: AF, ValNo2: ValNo, ValVT2: ValVT, LocVT2: LocVT,
9684 ArgFlags2: ArgFlags);
9685 }
9686
9687 // Allocate to a register if possible, or else a stack slot.
9688 Register Reg;
9689 unsigned StoreSizeBytes = GRLen / 8;
9690 Align StackAlign = Align(GRLen / 8);
9691
9692 if (ValVT == MVT::f32 && !UseGPRForFloat) {
9693 Reg = State.AllocateReg(Regs: ArgFPR32s);
9694 } else if (ValVT == MVT::f64 && !UseGPRForFloat) {
9695 Reg = State.AllocateReg(Regs: ArgFPR64s);
9696 } else if (ValVT.is128BitVector()) {
9697 Reg = State.AllocateReg(Regs: ArgVRs);
9698 UseGPRForFloat = false;
9699 StoreSizeBytes = 16;
9700 StackAlign = Align(16);
9701 } else if (ValVT.is256BitVector()) {
9702 Reg = State.AllocateReg(Regs: ArgXRs);
9703 UseGPRForFloat = false;
9704 StoreSizeBytes = 32;
9705 StackAlign = Align(32);
9706 } else {
9707 Reg = allocateArgGPR(State);
9708 }
9709
9710 unsigned StackOffset =
9711 Reg ? 0 : State.AllocateStack(Size: StoreSizeBytes, Alignment: StackAlign);
9712
9713 // If we reach this point and PendingLocs is non-empty, we must be at the
9714 // end of a split argument that must be passed indirectly.
9715 if (!PendingLocs.empty()) {
9716 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
9717 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
9718 for (auto &It : PendingLocs) {
9719 if (Reg)
9720 It.convertToReg(Reg);
9721 else
9722 It.convertToMem(Offset: StackOffset);
9723 State.addLoc(V: It);
9724 }
9725 PendingLocs.clear();
9726 PendingArgFlags.clear();
9727 return false;
9728 }
9729 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
9730 "Expected an GRLenVT at this stage");
9731
9732 if (Reg) {
9733 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9734 return false;
9735 }
9736
9737 // When a floating-point value is passed on the stack, no bit-cast is needed.
9738 if (ValVT.isFloatingPoint()) {
9739 LocVT = ValVT;
9740 LocInfo = CCValAssign::Full;
9741 }
9742
9743 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9744 return false;
9745}
9746
9747void LoongArchTargetLowering::analyzeInputArgs(
9748 MachineFunction &MF, CCState &CCInfo,
9749 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9750 LoongArchCCAssignFn Fn) const {
9751 FunctionType *FType = MF.getFunction().getFunctionType();
9752 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
9753 MVT ArgVT = Ins[i].VT;
9754 Type *ArgTy = nullptr;
9755 if (IsRet)
9756 ArgTy = FType->getReturnType();
9757 else if (Ins[i].isOrigArg())
9758 ArgTy = FType->getParamType(i: Ins[i].getOrigArgIndex());
9759 LoongArchABI::ABI ABI =
9760 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9761 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
9762 CCInfo, IsRet, ArgTy)) {
9763 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
9764 << '\n');
9765 llvm_unreachable("");
9766 }
9767 }
9768}
9769
9770void LoongArchTargetLowering::analyzeOutputArgs(
9771 MachineFunction &MF, CCState &CCInfo,
9772 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9773 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const {
9774 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9775 MVT ArgVT = Outs[i].VT;
9776 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9777 LoongArchABI::ABI ABI =
9778 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9779 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
9780 CCInfo, IsRet, OrigTy)) {
9781 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
9782 << "\n");
9783 llvm_unreachable("");
9784 }
9785 }
9786}
9787
9788// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9789// values.
9790static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9791 const CCValAssign &VA, const SDLoc &DL) {
9792 switch (VA.getLocInfo()) {
9793 default:
9794 llvm_unreachable("Unexpected CCValAssign::LocInfo");
9795 case CCValAssign::Full:
9796 case CCValAssign::Indirect:
9797 break;
9798 case CCValAssign::BCvt:
9799 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9800 Val = DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Val);
9801 else
9802 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
9803 break;
9804 }
9805 return Val;
9806}
9807
9808static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9809 const CCValAssign &VA, const SDLoc &DL,
9810 const ISD::InputArg &In,
9811 const LoongArchTargetLowering &TLI) {
9812 MachineFunction &MF = DAG.getMachineFunction();
9813 MachineRegisterInfo &RegInfo = MF.getRegInfo();
9814 EVT LocVT = VA.getLocVT();
9815 SDValue Val;
9816 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
9817 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
9818 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
9819 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
9820
9821 // If input is sign extended from 32 bits, note it for the OptW pass.
9822 if (In.isOrigArg()) {
9823 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
9824 if (OrigArg->getType()->isIntegerTy()) {
9825 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
9826 // An input zero extended from i31 can also be considered sign extended.
9827 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
9828 (BitWidth < 32 && In.Flags.isZExt())) {
9829 LoongArchMachineFunctionInfo *LAFI =
9830 MF.getInfo<LoongArchMachineFunctionInfo>();
9831 LAFI->addSExt32Register(Reg: VReg);
9832 }
9833 }
9834 }
9835
9836 return convertLocVTToValVT(DAG, Val, VA, DL);
9837}
9838
9839// The caller is responsible for loading the full value if the argument is
9840// passed with CCValAssign::Indirect.
9841static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9842 const CCValAssign &VA, const SDLoc &DL) {
9843 MachineFunction &MF = DAG.getMachineFunction();
9844 MachineFrameInfo &MFI = MF.getFrameInfo();
9845 EVT ValVT = VA.getValVT();
9846 int FI = MFI.CreateFixedObject(Size: ValVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
9847 /*IsImmutable=*/true);
9848 SDValue FIN = DAG.getFrameIndex(
9849 FI, VT: MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0)));
9850
9851 ISD::LoadExtType ExtType;
9852 switch (VA.getLocInfo()) {
9853 default:
9854 llvm_unreachable("Unexpected CCValAssign::LocInfo");
9855 case CCValAssign::Full:
9856 case CCValAssign::Indirect:
9857 case CCValAssign::BCvt:
9858 ExtType = ISD::NON_EXTLOAD;
9859 break;
9860 }
9861 return DAG.getExtLoad(
9862 ExtType, dl: DL, VT: VA.getLocVT(), Chain, Ptr: FIN,
9863 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT: ValVT);
9864}
9865
9866static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9867 const CCValAssign &VA,
9868 const CCValAssign &HiVA,
9869 const SDLoc &DL) {
9870 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9871 "Unexpected VA");
9872 MachineFunction &MF = DAG.getMachineFunction();
9873 MachineFrameInfo &MFI = MF.getFrameInfo();
9874 MachineRegisterInfo &RegInfo = MF.getRegInfo();
9875
9876 assert(VA.isRegLoc() && "Expected register VA assignment");
9877
9878 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9879 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
9880 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
9881 SDValue Hi;
9882 if (HiVA.isMemLoc()) {
9883 // Second half of f64 is passed on the stack.
9884 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
9885 /*IsImmutable=*/true);
9886 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
9887 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
9888 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
9889 } else {
9890 // Second half of f64 is passed in another GPR.
9891 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9892 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
9893 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
9894 }
9895 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
9896}
9897
9898static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9899 const CCValAssign &VA, const SDLoc &DL) {
9900 EVT LocVT = VA.getLocVT();
9901
9902 switch (VA.getLocInfo()) {
9903 default:
9904 llvm_unreachable("Unexpected CCValAssign::LocInfo");
9905 case CCValAssign::Full:
9906 break;
9907 case CCValAssign::BCvt:
9908 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9909 Val = DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Val);
9910 else
9911 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
9912 break;
9913 }
9914 return Val;
9915}
9916
9917static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9918 CCValAssign::LocInfo LocInfo,
9919 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
9920 CCState &State) {
9921 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9922 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
9923 // s0 s1 s2 s3 s4 s5 s6 s7 s8
9924 static const MCPhysReg GPRList[] = {
9925 LoongArch::R23, LoongArch::R24, LoongArch::R25,
9926 LoongArch::R26, LoongArch::R27, LoongArch::R28,
9927 LoongArch::R29, LoongArch::R30, LoongArch::R31};
9928 if (MCRegister Reg = State.AllocateReg(Regs: GPRList)) {
9929 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9930 return false;
9931 }
9932 }
9933
9934 if (LocVT == MVT::f32) {
9935 // Pass in STG registers: F1, F2, F3, F4
9936 // fs0,fs1,fs2,fs3
9937 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
9938 LoongArch::F26, LoongArch::F27};
9939 if (MCRegister Reg = State.AllocateReg(Regs: FPR32List)) {
9940 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9941 return false;
9942 }
9943 }
9944
9945 if (LocVT == MVT::f64) {
9946 // Pass in STG registers: D1, D2, D3, D4
9947 // fs4,fs5,fs6,fs7
9948 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
9949 LoongArch::F30_64, LoongArch::F31_64};
9950 if (MCRegister Reg = State.AllocateReg(Regs: FPR64List)) {
9951 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9952 return false;
9953 }
9954 }
9955
9956 report_fatal_error(reason: "No registers left in GHC calling convention");
9957 return true;
9958}
9959
9960// Transform physical registers into virtual registers.
9961SDValue LoongArchTargetLowering::LowerFormalArguments(
9962 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9963 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9964 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9965
9966 MachineFunction &MF = DAG.getMachineFunction();
9967
9968 switch (CallConv) {
9969 default:
9970 llvm_unreachable("Unsupported calling convention");
9971 case CallingConv::C:
9972 case CallingConv::Fast:
9973 case CallingConv::PreserveNone:
9974 case CallingConv::PreserveMost:
9975 break;
9976 case CallingConv::GHC:
9977 if (!MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicF) ||
9978 !MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicD))
9979 report_fatal_error(
9980 reason: "GHC calling convention requires the F and D extensions");
9981 }
9982
9983 const Function &Func = MF.getFunction();
9984 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
9985 MVT GRLenVT = Subtarget.getGRLenVT();
9986 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
9987
9988 // Check if this function has any musttail calls. If so, incoming indirect
9989 // arg pointers must be saved in virtual registers so they survive across
9990 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
9991 // when needed to avoid adding register pressure to non-musttail functions.
9992 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
9993 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
9994 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
9995 return CI->isMustTailCall();
9996 return false;
9997 });
9998 });
9999 // Used with varargs to acumulate store chains.
10000 std::vector<SDValue> OutChains;
10001
10002 // Assign locations to all of the incoming arguments.
10003 SmallVector<CCValAssign> ArgLocs;
10004 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10005
10006 if (CallConv == CallingConv::GHC)
10007 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_LoongArch_GHC);
10008 else
10009 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, Fn: CC_LoongArch);
10010
10011 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10012 CCValAssign &VA = ArgLocs[i];
10013 SDValue ArgValue;
10014 // Passing f64 on LA32D with a soft float ABI must be handled as a special
10015 // case.
10016 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10017 assert(VA.needsCustom());
10018 ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
10019 } else if (VA.isRegLoc())
10020 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
10021 else
10022 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
10023 if (VA.getLocInfo() == CCValAssign::Indirect) {
10024 // If the original argument was split and passed by reference, we need to
10025 // load all parts of it here (using the same address).
10026 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
10027 PtrInfo: MachinePointerInfo()));
10028 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
10029 if (HasMusttail) {
10030 LoongArchMachineFunctionInfo *LAFI =
10031 MF.getInfo<LoongArchMachineFunctionInfo>();
10032 Register VReg =
10033 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10034 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
10035 LAFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
10036 }
10037 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
10038 assert(ArgPartOffset == 0);
10039 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
10040 CCValAssign &PartVA = ArgLocs[i + 1];
10041 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
10042 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10043 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
10044 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
10045 PtrInfo: MachinePointerInfo()));
10046 ++i;
10047 ++InsIdx;
10048 }
10049 continue;
10050 }
10051 InVals.push_back(Elt: ArgValue);
10052 }
10053
10054 if (IsVarArg) {
10055 ArrayRef<MCPhysReg> ArgRegs = ArrayRef(ArgGPRs);
10056 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
10057 const TargetRegisterClass *RC = &LoongArch::GPRRegClass;
10058 MachineFrameInfo &MFI = MF.getFrameInfo();
10059 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10060 auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>();
10061
10062 // Offset of the first variable argument from stack pointer, and size of
10063 // the vararg save area. For now, the varargs save area is either zero or
10064 // large enough to hold a0-a7.
10065 int VaArgOffset, VarArgsSaveSize;
10066
10067 // If all registers are allocated, then all varargs must be passed on the
10068 // stack and we don't need to save any argregs.
10069 if (ArgRegs.size() == Idx) {
10070 VaArgOffset = CCInfo.getStackSize();
10071 VarArgsSaveSize = 0;
10072 } else {
10073 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx);
10074 VaArgOffset = -VarArgsSaveSize;
10075 }
10076
10077 // Record the frame index of the first variable argument
10078 // which is a value necessary to VASTART.
10079 int FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10080 LoongArchFI->setVarArgsFrameIndex(FI);
10081
10082 // If saving an odd number of registers then create an extra stack slot to
10083 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures
10084 // offsets to even-numbered registered remain 2*GRLen-aligned.
10085 if (Idx % 2) {
10086 MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset - (int)GRLenInBytes,
10087 IsImmutable: true);
10088 VarArgsSaveSize += GRLenInBytes;
10089 }
10090
10091 // Copy the integer registers that may have been used for passing varargs
10092 // to the vararg save area.
10093 for (unsigned I = Idx; I < ArgRegs.size();
10094 ++I, VaArgOffset += GRLenInBytes) {
10095 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
10096 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
10097 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: GRLenVT);
10098 FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10099 SDValue PtrOff = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10100 SDValue Store = DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: PtrOff,
10101 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10102 cast<StoreSDNode>(Val: Store.getNode())
10103 ->getMemOperand()
10104 ->setValue((Value *)nullptr);
10105 OutChains.push_back(x: Store);
10106 }
10107 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
10108 }
10109
10110 // All stores are grouped in one node to allow the matching between
10111 // the size of Ins and InVals. This only happens for vararg functions.
10112 if (!OutChains.empty()) {
10113 OutChains.push_back(x: Chain);
10114 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
10115 }
10116
10117 return Chain;
10118}
10119
10120bool LoongArchTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10121 return CI->isTailCall();
10122}
10123
10124// Check if the return value is used as only a return value, as otherwise
10125// we can't perform a tail-call.
10126bool LoongArchTargetLowering::isUsedByReturnOnly(SDNode *N,
10127 SDValue &Chain) const {
10128 if (N->getNumValues() != 1)
10129 return false;
10130 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
10131 return false;
10132
10133 SDNode *Copy = *N->user_begin();
10134 if (Copy->getOpcode() != ISD::CopyToReg)
10135 return false;
10136
10137 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
10138 // isn't safe to perform a tail call.
10139 if (Copy->getGluedNode())
10140 return false;
10141
10142 // The copy must be used by a LoongArchISD::RET, and nothing else.
10143 bool HasRet = false;
10144 for (SDNode *Node : Copy->users()) {
10145 if (Node->getOpcode() != LoongArchISD::RET)
10146 return false;
10147 HasRet = true;
10148 }
10149
10150 if (!HasRet)
10151 return false;
10152
10153 Chain = Copy->getOperand(Num: 0);
10154 return true;
10155}
10156
10157// Check whether the call is eligible for tail call optimization.
10158bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
10159 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
10160 const SmallVectorImpl<CCValAssign> &ArgLocs) const {
10161
10162 auto CalleeCC = CLI.CallConv;
10163 auto &Outs = CLI.Outs;
10164 auto &Caller = MF.getFunction();
10165 auto CallerCC = Caller.getCallingConv();
10166
10167 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
10168
10169 // Byval parameters hand the function a pointer directly into the stack area
10170 // we want to reuse during a tail call. Working around this *is* possible
10171 // but less efficient and uglier in LowerCall. For musttail, there is no
10172 // workaround today: a byval arg requires a local copy that becomes invalid
10173 // after the tail call deallocates the caller's frame, so rejecting here
10174 // (and triggering reportFatalInternalError in LowerCall) is safer than
10175 // miscompiling.
10176 for (auto &Arg : Outs)
10177 if (Arg.Flags.isByVal())
10178 return false;
10179
10180 // musttail bypasses the remaining checks: the checks either reject cases
10181 // we handle specially (indirect args are forwarded via incoming pointers,
10182 // stack-passed args reuse the matching incoming layout, sret is forwarded
10183 // like any other pointer arg) or are optimizations not applicable to
10184 // mandatory tail calls.
10185 if (IsMustTail)
10186 return true;
10187
10188 // Do not tail call opt if the stack is used to pass parameters.
10189 if (CCInfo.getStackSize() != 0)
10190 return false;
10191
10192 // Do not tail call opt if any parameters need to be passed indirectly.
10193 for (auto &VA : ArgLocs)
10194 if (VA.getLocInfo() == CCValAssign::Indirect)
10195 return false;
10196
10197 // Do not tail call opt if either caller or callee uses struct return
10198 // semantics.
10199 auto IsCallerStructRet = Caller.hasStructRetAttr();
10200 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
10201 if (IsCallerStructRet || IsCalleeStructRet)
10202 return false;
10203
10204 // The callee has to preserve all registers the caller needs to preserve.
10205 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
10206 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
10207 if (CalleeCC != CallerCC) {
10208 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
10209 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
10210 return false;
10211 }
10212 return true;
10213}
10214
10215static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
10216 return DAG.getDataLayout().getPrefTypeAlign(
10217 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
10218}
10219
10220// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
10221// and output parameter nodes.
10222SDValue
10223LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
10224 SmallVectorImpl<SDValue> &InVals) const {
10225 SelectionDAG &DAG = CLI.DAG;
10226 SDLoc &DL = CLI.DL;
10227 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
10228 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
10229 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
10230 SDValue Chain = CLI.Chain;
10231 SDValue Callee = CLI.Callee;
10232 CallingConv::ID CallConv = CLI.CallConv;
10233 bool IsVarArg = CLI.IsVarArg;
10234 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10235 MVT GRLenVT = Subtarget.getGRLenVT();
10236 bool &IsTailCall = CLI.IsTailCall;
10237
10238 MachineFunction &MF = DAG.getMachineFunction();
10239
10240 // Analyze the operands of the call, assigning locations to each operand.
10241 SmallVector<CCValAssign> ArgLocs;
10242 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10243
10244 if (CallConv == CallingConv::GHC)
10245 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_LoongArch_GHC);
10246 else
10247 analyzeOutputArgs(MF, CCInfo&: ArgCCInfo, Outs, /*IsRet=*/false, CLI: &CLI, Fn: CC_LoongArch);
10248
10249 // Check if it's really possible to do a tail call.
10250 if (IsTailCall)
10251 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
10252
10253 if (IsTailCall)
10254 ++NumTailCalls;
10255 else if (CLI.CB && CLI.CB->isMustTailCall())
10256 report_fatal_error(reason: "failed to perform tail call elimination on a call "
10257 "site marked musttail");
10258
10259 // Get a count of how many bytes are to be pushed on the stack.
10260 unsigned NumBytes = ArgCCInfo.getStackSize();
10261
10262 // Create local copies for byval args.
10263 SmallVector<SDValue> ByValArgs;
10264 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10265 ISD::ArgFlagsTy Flags = Outs[i].Flags;
10266 if (!Flags.isByVal())
10267 continue;
10268
10269 SDValue Arg = OutVals[i];
10270 unsigned Size = Flags.getByValSize();
10271 Align Alignment = Flags.getNonZeroByValAlign();
10272
10273 int FI =
10274 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
10275 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10276 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: GRLenVT);
10277
10278 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
10279 /*IsVolatile=*/isVol: false,
10280 /*AlwaysInline=*/false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
10281 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
10282 ByValArgs.push_back(Elt: FIPtr);
10283 }
10284
10285 if (!IsTailCall)
10286 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
10287
10288 // Copy argument values to their designated locations.
10289 SmallVector<std::pair<Register, SDValue>> RegsToPass;
10290 SmallVector<SDValue> MemOpChains;
10291 SDValue StackPtr;
10292 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
10293 ++i, ++OutIdx) {
10294 CCValAssign &VA = ArgLocs[i];
10295 SDValue ArgValue = OutVals[OutIdx];
10296 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
10297
10298 // Handle passing f64 on LA32D with a soft float ABI as a special case.
10299 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10300 assert(VA.isRegLoc() && "Expected register VA assignment");
10301 assert(VA.needsCustom());
10302 SDValue SplitF64 =
10303 DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10304 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
10305 SDValue Lo = SplitF64.getValue(R: 0);
10306 SDValue Hi = SplitF64.getValue(R: 1);
10307
10308 Register RegLo = VA.getLocReg();
10309 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
10310
10311 // Get the CCValAssign for the Hi part.
10312 CCValAssign &HiVA = ArgLocs[++i];
10313
10314 if (HiVA.isMemLoc()) {
10315 // Second half of f64 is passed on the stack.
10316 if (!StackPtr.getNode())
10317 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10318 SDValue Address =
10319 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10320 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
10321 // Emit the store.
10322 MemOpChains.push_back(Elt: DAG.getStore(
10323 Chain, dl: DL, Val: Hi, Ptr: Address,
10324 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
10325 } else {
10326 // Second half of f64 is passed in another GPR.
10327 Register RegHigh = HiVA.getLocReg();
10328 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
10329 }
10330 continue;
10331 }
10332
10333 // Promote the value if needed.
10334 // For now, only handle fully promoted and indirect arguments.
10335 if (VA.getLocInfo() == CCValAssign::Indirect) {
10336 // For musttail calls, reuse incoming indirect pointers instead of
10337 // creating new stack temporaries. The incoming pointers point to the
10338 // caller's caller's frame, which remains valid after a tail call.
10339 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
10340 LoongArchMachineFunctionInfo *LAFI =
10341 MF.getInfo<LoongArchMachineFunctionInfo>();
10342 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
10343
10344 // Resolve which formal parameter is being passed at this call
10345 // position.
10346 //
10347 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
10348 // but Outs[].OrigArgIndex is an index into a filtered arg list
10349 // (empty types removed, via CallLoweringInfo in the target-
10350 // independent layer). IncomingIndirectArgs is keyed by the
10351 // caller's unfiltered Argument::getArgNo(), so we have to walk
10352 // the caller's formals (same filter) to translate the index.
10353 // This target-independent asymmetry should be normalized so
10354 // backends do not need to re-derive the mapping.
10355 //
10356 // Steps:
10357 // 1. Find the call operand at filtered position CallArgIdx.
10358 // 2. If it is an Argument, use getArgNo() directly (same filter
10359 // for caller formals and call operands).
10360 // 3. Otherwise (computed value), walk the caller's formals and
10361 // skip empty types to map the filtered index to getArgNo().
10362 const Argument *FormalArg = nullptr;
10363 unsigned FilteredIdx = 0;
10364 for (const auto &CallArg : CLI.CB->args()) {
10365 if (CallArg->getType()->isEmptyTy())
10366 continue;
10367 if (FilteredIdx == CallArgIdx) {
10368 FormalArg = dyn_cast<Argument>(Val: CallArg);
10369 break;
10370 }
10371 ++FilteredIdx;
10372 }
10373
10374 // For forwarded args, getArgNo() gives the unfiltered index directly.
10375 // For computed args, walk the caller's formals to resolve it.
10376 unsigned FormalArgIdx = CallArgIdx;
10377 if (FormalArg) {
10378 FormalArgIdx = FormalArg->getArgNo();
10379 } else {
10380 FilteredIdx = 0;
10381 for (const auto &Arg : MF.getFunction().args()) {
10382 if (Arg.getType()->isEmptyTy())
10383 continue;
10384 if (FilteredIdx == CallArgIdx) {
10385 FormalArgIdx = Arg.getArgNo();
10386 break;
10387 }
10388 ++FilteredIdx;
10389 }
10390 }
10391
10392 Register VReg = LAFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
10393 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
10394 // Thread the CopyFromReg output chain through MemOpChains so the
10395 // TokenFactor below sequences the copy with any stores we emit
10396 // for this argument.
10397 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
10398 SDValue IncomingPtr = CopyOp;
10399
10400 if (!FormalArg) {
10401 // Computed value: store into the incoming indirect pointer for the
10402 // same-position formal parameter (musttail guarantees matching
10403 // prototypes, so types match). The pointer survives the tail call
10404 // since it points to the caller's caller's frame.
10405 //
10406 // The data-flow edge through IncomingPtr already prevents the
10407 // store from being scheduled before the CopyFromReg. Threading
10408 // CopyOp.getValue(1) (the copy's output chain) into the store
10409 // makes that ordering explicit on the chain edge as well, which
10410 // is the convention for memory ops chaining off their producers.
10411 MemOpChains.push_back(
10412 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
10413 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10414 // Store any split parts at their respective offsets.
10415 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10416 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10417 SDValue PartValue = OutVals[OutIdx + 1];
10418 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10419 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10420 SDValue Addr =
10421 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
10422 MemOpChains.push_back(
10423 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
10424 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10425 ++i;
10426 ++OutIdx;
10427 }
10428 }
10429 ArgValue = IncomingPtr;
10430
10431 // Skip any remaining split parts (for forwarded args, they are
10432 // covered by the forwarded pointer).
10433 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10434 ++i;
10435 ++OutIdx;
10436 }
10437 } else {
10438 // Store the argument in a stack slot and pass its address.
10439 Align StackAlign =
10440 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
10441 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
10442 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
10443 // If the original argument was split and passed by reference, we need
10444 // to store the required parts of it here (and pass just one address).
10445 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
10446 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10447 assert(ArgPartOffset == 0);
10448 // Calculate the total size to store. We don't have access to what we're
10449 // actually storing other than performing the loop and collecting the
10450 // info.
10451 SmallVector<std::pair<SDValue, SDValue>> Parts;
10452 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
10453 SDValue PartValue = OutVals[OutIdx + 1];
10454 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10455 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10456 EVT PartVT = PartValue.getValueType();
10457 StoredSize += PartVT.getStoreSize();
10458 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
10459 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
10460 ++i;
10461 ++OutIdx;
10462 }
10463 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
10464 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
10465 MemOpChains.push_back(
10466 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
10467 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10468 for (const auto &Part : Parts) {
10469 SDValue PartValue = Part.first;
10470 SDValue PartOffset = Part.second;
10471 SDValue Address =
10472 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
10473 MemOpChains.push_back(
10474 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
10475 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10476 }
10477 ArgValue = SpillSlot;
10478 }
10479 } else {
10480 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL);
10481 }
10482
10483 // Use local copy if it is a byval arg.
10484 if (Flags.isByVal())
10485 ArgValue = ByValArgs[j++];
10486
10487 if (VA.isRegLoc()) {
10488 // Queue up the argument copies and emit them at the end.
10489 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
10490 } else {
10491 assert(VA.isMemLoc() && "Argument not register or memory");
10492 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
10493 "Tail call not allowed if stack is used for passing parameters");
10494
10495 // Work out the address of the stack slot.
10496 if (!StackPtr.getNode())
10497 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10498 SDValue Address =
10499 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10500 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
10501
10502 // Emit the store.
10503 MemOpChains.push_back(
10504 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
10505 }
10506 }
10507
10508 // Join the stores, which are independent of one another.
10509 if (!MemOpChains.empty())
10510 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
10511
10512 SDValue Glue;
10513
10514 // Build a sequence of copy-to-reg nodes, chained and glued together.
10515 for (auto &Reg : RegsToPass) {
10516 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
10517 Glue = Chain.getValue(R: 1);
10518 }
10519
10520 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
10521 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
10522 // split it and then direct call can be matched by PseudoCALL_SMALL.
10523 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
10524 const GlobalValue *GV = S->getGlobal();
10525 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV)
10526 ? LoongArchII::MO_CALL
10527 : LoongArchII::MO_CALL_PLT;
10528 Callee = DAG.getTargetGlobalAddress(GV: S->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: OpFlags);
10529 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
10530 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV: nullptr)
10531 ? LoongArchII::MO_CALL
10532 : LoongArchII::MO_CALL_PLT;
10533 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: OpFlags);
10534 }
10535
10536 // The first call operand is the chain and the second is the target address.
10537 SmallVector<SDValue> Ops;
10538 Ops.push_back(Elt: Chain);
10539 Ops.push_back(Elt: Callee);
10540
10541 // Add argument registers to the end of the list so that they are
10542 // known live into the call.
10543 for (auto &Reg : RegsToPass)
10544 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
10545
10546 if (!IsTailCall) {
10547 // Add a register mask operand representing the call-preserved registers.
10548 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
10549 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
10550 assert(Mask && "Missing call preserved mask for calling convention");
10551 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
10552 }
10553
10554 // Glue the call to the argument copies, if any.
10555 if (Glue.getNode())
10556 Ops.push_back(Elt: Glue);
10557
10558 // Emit the call.
10559 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
10560 unsigned Op;
10561 switch (DAG.getTarget().getCodeModel()) {
10562 default:
10563 report_fatal_error(reason: "Unsupported code model");
10564 case CodeModel::Small:
10565 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
10566 break;
10567 case CodeModel::Medium:
10568 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
10569 break;
10570 case CodeModel::Large:
10571 assert(Subtarget.is64Bit() && "Large code model requires LA64");
10572 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
10573 break;
10574 }
10575
10576 if (IsTailCall) {
10577 MF.getFrameInfo().setHasTailCall();
10578 SDValue Ret = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10579 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
10580 return Ret;
10581 }
10582
10583 Chain = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10584 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
10585 Glue = Chain.getValue(R: 1);
10586
10587 // Mark the end of the call, which is glued to the call itself.
10588 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
10589 Glue = Chain.getValue(R: 1);
10590
10591 // Assign locations to each value returned by this call.
10592 SmallVector<CCValAssign> RVLocs;
10593 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
10594 analyzeInputArgs(MF, CCInfo&: RetCCInfo, Ins, /*IsRet=*/true, Fn: CC_LoongArch);
10595
10596 // Copy all of the result registers out of their specified physreg.
10597 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
10598 auto &VA = RVLocs[i];
10599 // Copy the value out.
10600 SDValue RetValue =
10601 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
10602 // Glue the RetValue to the end of the call sequence.
10603 Chain = RetValue.getValue(R: 1);
10604 Glue = RetValue.getValue(R: 2);
10605
10606 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10607 assert(VA.needsCustom());
10608 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
10609 VT: MVT::i32, Glue);
10610 Chain = RetValue2.getValue(R: 1);
10611 Glue = RetValue2.getValue(R: 2);
10612 RetValue = DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64,
10613 N1: RetValue, N2: RetValue2);
10614 } else
10615 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL);
10616
10617 InVals.push_back(Elt: RetValue);
10618 }
10619
10620 return Chain;
10621}
10622
10623bool LoongArchTargetLowering::CanLowerReturn(
10624 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
10625 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
10626 const Type *RetTy) const {
10627 SmallVector<CCValAssign> RVLocs;
10628 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
10629
10630 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10631 LoongArchABI::ABI ABI =
10632 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10633 if (CC_LoongArch(DL: MF.getDataLayout(), ABI, ValNo: i, ValVT: Outs[i].VT, LocInfo: CCValAssign::Full,
10634 ArgFlags: Outs[i].Flags, State&: CCInfo, /*IsRet=*/true, OrigTy: nullptr))
10635 return false;
10636 }
10637 return true;
10638}
10639
10640SDValue LoongArchTargetLowering::LowerReturn(
10641 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10642 const SmallVectorImpl<ISD::OutputArg> &Outs,
10643 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
10644 SelectionDAG &DAG) const {
10645 // Stores the assignment of the return value to a location.
10646 SmallVector<CCValAssign> RVLocs;
10647
10648 // Info about the registers and stack slot.
10649 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
10650 *DAG.getContext());
10651
10652 analyzeOutputArgs(MF&: DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
10653 CLI: nullptr, Fn: CC_LoongArch);
10654 if (CallConv == CallingConv::GHC && !RVLocs.empty())
10655 report_fatal_error(reason: "GHC functions return void only");
10656 SDValue Glue;
10657 SmallVector<SDValue, 4> RetOps(1, Chain);
10658
10659 // Copy the result values into the output registers.
10660 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
10661 SDValue Val = OutVals[OutIdx];
10662 CCValAssign &VA = RVLocs[i];
10663 assert(VA.isRegLoc() && "Can only return in registers!");
10664
10665 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10666 // Handle returning f64 on LA32D with a soft float ABI.
10667 assert(VA.isRegLoc() && "Expected return via registers");
10668 assert(VA.needsCustom());
10669 SDValue SplitF64 = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10670 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
10671 SDValue Lo = SplitF64.getValue(R: 0);
10672 SDValue Hi = SplitF64.getValue(R: 1);
10673 Register RegLo = VA.getLocReg();
10674 Register RegHi = RVLocs[++i].getLocReg();
10675
10676 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
10677 Glue = Chain.getValue(R: 1);
10678 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
10679 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
10680 Glue = Chain.getValue(R: 1);
10681 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
10682 } else {
10683 // Handle a 'normal' return.
10684 Val = convertValVTToLocVT(DAG, Val, VA, DL);
10685 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
10686
10687 // Guarantee that all emitted copies are stuck together.
10688 Glue = Chain.getValue(R: 1);
10689 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
10690 }
10691 }
10692
10693 RetOps[0] = Chain; // Update chain.
10694
10695 // Add the glue node if we have it.
10696 if (Glue.getNode())
10697 RetOps.push_back(Elt: Glue);
10698
10699 return DAG.getNode(Opcode: LoongArchISD::RET, DL, VT: MVT::Other, Ops: RetOps);
10700}
10701
10702// Check if a constant splat can be generated using [x]vldi, where imm[12] == 1.
10703// Note: The following prefixes are excluded:
10704// imm[11:8] == 4'b0000, 4'b0100, 4'b1000
10705// as they can be represented using [x]vrepli.[whb]
10706std::pair<bool, uint64_t> LoongArchTargetLowering::isImmVLDILegalForMode1(
10707 const APInt &SplatValue, const unsigned SplatBitSize) const {
10708 uint64_t RequiredImm = 0;
10709 uint64_t V = SplatValue.getZExtValue();
10710 if (SplatBitSize == 16 && !(V & 0x00FF)) {
10711 // 4'b0101
10712 RequiredImm = (0b10101 << 8) | (V >> 8);
10713 return {true, RequiredImm};
10714 } else if (SplatBitSize == 32) {
10715 // 4'b0001
10716 if (!(V & 0xFFFF00FF)) {
10717 RequiredImm = (0b10001 << 8) | (V >> 8);
10718 return {true, RequiredImm};
10719 }
10720 // 4'b0010
10721 if (!(V & 0xFF00FFFF)) {
10722 RequiredImm = (0b10010 << 8) | (V >> 16);
10723 return {true, RequiredImm};
10724 }
10725 // 4'b0011
10726 if (!(V & 0x00FFFFFF)) {
10727 RequiredImm = (0b10011 << 8) | (V >> 24);
10728 return {true, RequiredImm};
10729 }
10730 // 4'b0110
10731 if ((V & 0xFFFF00FF) == 0xFF) {
10732 RequiredImm = (0b10110 << 8) | (V >> 8);
10733 return {true, RequiredImm};
10734 }
10735 // 4'b0111
10736 if ((V & 0xFF00FFFF) == 0xFFFF) {
10737 RequiredImm = (0b10111 << 8) | (V >> 16);
10738 return {true, RequiredImm};
10739 }
10740 // 4'b1010
10741 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
10742 RequiredImm =
10743 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10744 return {true, RequiredImm};
10745 }
10746 } else if (SplatBitSize == 64) {
10747 // 4'b1011
10748 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
10749 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
10750 RequiredImm =
10751 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10752 return {true, RequiredImm};
10753 }
10754 // 4'b1100
10755 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
10756 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
10757 RequiredImm =
10758 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
10759 return {true, RequiredImm};
10760 }
10761 // 4'b1001
10762 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
10763 uint8_t res = 0;
10764 for (int i = 0; i < 8; ++i) {
10765 uint8_t byte = x & 0xFF;
10766 if (byte == 0 || byte == 0xFF)
10767 res |= ((byte & 1) << i);
10768 else
10769 return {false, 0};
10770 x >>= 8;
10771 }
10772 return {true, res};
10773 };
10774 auto [IsSame, Suffix] = sameBitsPreByte(V);
10775 if (IsSame) {
10776 RequiredImm = (0b11001 << 8) | Suffix;
10777 return {true, RequiredImm};
10778 }
10779 }
10780 return {false, RequiredImm};
10781}
10782
10783bool LoongArchTargetLowering::isFPImmVLDILegal(const APFloat &Imm,
10784 EVT VT) const {
10785 if (!Subtarget.hasExtLSX())
10786 return false;
10787
10788 if (VT == MVT::f32) {
10789 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
10790 return (masked == 0x3e000000 || masked == 0x40000000);
10791 }
10792
10793 if (VT == MVT::f64) {
10794 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
10795 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
10796 }
10797
10798 return false;
10799}
10800
10801bool LoongArchTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
10802 bool ForCodeSize) const {
10803 // TODO: Maybe need more checks here after vector extension is supported.
10804 if (VT == MVT::f32 && !Subtarget.hasBasicF())
10805 return false;
10806 if (VT == MVT::f64 && !Subtarget.hasBasicD())
10807 return false;
10808 return (Imm.isZero() || Imm.isOne() || isFPImmVLDILegal(Imm, VT));
10809}
10810
10811bool LoongArchTargetLowering::isCheapToSpeculateCttz(Type *) const {
10812 return true;
10813}
10814
10815bool LoongArchTargetLowering::isCheapToSpeculateCtlz(Type *) const {
10816 return true;
10817}
10818
10819bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
10820 const Instruction *I) const {
10821 if (!Subtarget.is64Bit())
10822 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
10823
10824 if (isa<LoadInst>(Val: I))
10825 return true;
10826
10827 // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
10828 // require fences beacuse we can use amswap_db.[w/d].
10829 Type *Ty = I->getOperand(i: 0)->getType();
10830 if (isa<StoreInst>(Val: I) && Ty->isIntegerTy()) {
10831 unsigned Size = Ty->getIntegerBitWidth();
10832 return (Size == 8 || Size == 16);
10833 }
10834
10835 return false;
10836}
10837
10838EVT LoongArchTargetLowering::getSetCCResultType(const DataLayout &DL,
10839 LLVMContext &Context,
10840 EVT VT) const {
10841 if (!VT.isVector())
10842 return getPointerTy(DL);
10843 return VT.changeVectorElementTypeToInteger();
10844}
10845
10846bool LoongArchTargetLowering::canMergeStoresTo(
10847 unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const {
10848 // Do not merge to float value size (128 or 256 bits) if no implicit
10849 // float attribute is set.
10850 bool NoFloat = MF.getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat);
10851 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
10852 if (NoFloat)
10853 return MemVT.getSizeInBits() <= MaxIntSize;
10854
10855 // Make sure we don't merge greater than our maximum supported vector width.
10856 if (Subtarget.hasExtLASX())
10857 MaxIntSize = 256;
10858 else if (Subtarget.hasExtLSX())
10859 MaxIntSize = 128;
10860
10861 return MemVT.getSizeInBits() <= MaxIntSize;
10862}
10863
10864bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
10865 EVT VT = Y.getValueType();
10866
10867 if (VT.isVector())
10868 return Subtarget.hasExtLSX() && VT.isInteger();
10869
10870 return VT.isScalarInteger() && !isa<ConstantSDNode>(Val: Y);
10871}
10872
10873void LoongArchTargetLowering::getTgtMemIntrinsic(
10874 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
10875 MachineFunction &MF, unsigned Intrinsic) const {
10876 switch (Intrinsic) {
10877 default:
10878 return;
10879 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
10880 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
10881 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
10882 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
10883 IntrinsicInfo Info;
10884 Info.opc = ISD::INTRINSIC_W_CHAIN;
10885 Info.memVT = MVT::i32;
10886 Info.ptrVal = I.getArgOperand(i: 0);
10887 Info.offset = 0;
10888 Info.align = Align(4);
10889 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
10890 MachineMemOperand::MOVolatile;
10891 Infos.push_back(Elt: Info);
10892 return;
10893 // TODO: Add more Intrinsics later.
10894 }
10895 }
10896}
10897
10898// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
10899// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
10900// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
10901// regression, we need to implement it manually.
10902void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
10903 AtomicRMWInst::BinOp Op = AI->getOperation();
10904
10905 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
10906 Op == AtomicRMWInst::And) &&
10907 "Unable to expand");
10908 unsigned MinWordSize = 4;
10909
10910 IRBuilder<> Builder(AI);
10911 LLVMContext &Ctx = Builder.getContext();
10912 const DataLayout &DL = AI->getDataLayout();
10913 Type *ValueType = AI->getType();
10914 Type *WordType = Type::getIntNTy(C&: Ctx, N: MinWordSize * 8);
10915
10916 Value *Addr = AI->getPointerOperand();
10917 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
10918 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
10919
10920 Value *AlignedAddr = Builder.CreateIntrinsic(
10921 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
10922 Args: {Addr, ConstantInt::get(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))}, FMFSource: nullptr,
10923 Name: "AlignedAddr");
10924
10925 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
10926 Value *PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
10927 Value *ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
10928 ShiftAmt = Builder.CreateTrunc(V: ShiftAmt, DestTy: WordType, Name: "ShiftAmt");
10929 Value *Mask = Builder.CreateShl(
10930 LHS: ConstantInt::get(Ty: WordType,
10931 V: (1 << (DL.getTypeStoreSize(Ty: ValueType) * 8)) - 1),
10932 RHS: ShiftAmt, Name: "Mask");
10933 Value *Inv_Mask = Builder.CreateNot(V: Mask, Name: "Inv_Mask");
10934 Value *ValOperand_Shifted =
10935 Builder.CreateShl(LHS: Builder.CreateZExt(V: AI->getValOperand(), DestTy: WordType),
10936 RHS: ShiftAmt, Name: "ValOperand_Shifted");
10937 Value *NewOperand;
10938 if (Op == AtomicRMWInst::And)
10939 NewOperand = Builder.CreateOr(LHS: ValOperand_Shifted, RHS: Inv_Mask, Name: "AndOperand");
10940 else
10941 NewOperand = ValOperand_Shifted;
10942
10943 AtomicRMWInst *NewAI =
10944 Builder.CreateAtomicRMW(Op, Ptr: AlignedAddr, Val: NewOperand, Align: Align(MinWordSize),
10945 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
10946
10947 Value *Shift = Builder.CreateLShr(LHS: NewAI, RHS: ShiftAmt, Name: "shifted");
10948 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: ValueType, Name: "extracted");
10949 Value *FinalOldResult = Builder.CreateBitCast(V: Trunc, DestTy: ValueType);
10950 AI->replaceAllUsesWith(V: FinalOldResult);
10951 AI->eraseFromParent();
10952}
10953
10954TargetLowering::AtomicExpansionKind
10955LoongArchTargetLowering::shouldExpandAtomicRMWInIR(
10956 const AtomicRMWInst *AI) const {
10957 // TODO: Add more AtomicRMWInst that needs to be extended.
10958
10959 // Since floating-point operation requires a non-trivial set of data
10960 // operations, use CmpXChg to expand.
10961 if (AI->isFloatingPointOperation() ||
10962 AI->getOperation() == AtomicRMWInst::UIncWrap ||
10963 AI->getOperation() == AtomicRMWInst::UDecWrap ||
10964 AI->getOperation() == AtomicRMWInst::USubCond ||
10965 AI->getOperation() == AtomicRMWInst::USubSat)
10966 return AtomicExpansionKind::CmpXChg;
10967
10968 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
10969 (AI->getOperation() == AtomicRMWInst::Xchg ||
10970 AI->getOperation() == AtomicRMWInst::Add ||
10971 AI->getOperation() == AtomicRMWInst::Sub)) {
10972 return AtomicExpansionKind::None;
10973 }
10974
10975 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10976 if (Subtarget.hasLAMCAS()) {
10977 if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
10978 AI->getOperation() == AtomicRMWInst::Or ||
10979 AI->getOperation() == AtomicRMWInst::Xor))
10980 return AtomicExpansionKind::CustomExpand;
10981 if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
10982 return AtomicExpansionKind::CmpXChg;
10983 }
10984
10985 if (Size == 8 || Size == 16)
10986 return AtomicExpansionKind::MaskedIntrinsic;
10987 return AtomicExpansionKind::None;
10988}
10989
10990static Intrinsic::ID
10991getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
10992 AtomicRMWInst::BinOp BinOp) {
10993 if (GRLen == 64) {
10994 switch (BinOp) {
10995 default:
10996 llvm_unreachable("Unexpected AtomicRMW BinOp");
10997 case AtomicRMWInst::Xchg:
10998 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
10999 case AtomicRMWInst::Add:
11000 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
11001 case AtomicRMWInst::Sub:
11002 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
11003 case AtomicRMWInst::Nand:
11004 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
11005 case AtomicRMWInst::UMax:
11006 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
11007 case AtomicRMWInst::UMin:
11008 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
11009 case AtomicRMWInst::Max:
11010 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
11011 case AtomicRMWInst::Min:
11012 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
11013 // TODO: support other AtomicRMWInst.
11014 }
11015 }
11016
11017 if (GRLen == 32) {
11018 switch (BinOp) {
11019 default:
11020 llvm_unreachable("Unexpected AtomicRMW BinOp");
11021 case AtomicRMWInst::Xchg:
11022 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
11023 case AtomicRMWInst::Add:
11024 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
11025 case AtomicRMWInst::Sub:
11026 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
11027 case AtomicRMWInst::Nand:
11028 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
11029 case AtomicRMWInst::UMax:
11030 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
11031 case AtomicRMWInst::UMin:
11032 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
11033 case AtomicRMWInst::Max:
11034 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
11035 case AtomicRMWInst::Min:
11036 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
11037 // TODO: support other AtomicRMWInst.
11038 }
11039 }
11040
11041 llvm_unreachable("Unexpected GRLen\n");
11042}
11043
11044TargetLowering::AtomicExpansionKind
11045LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
11046 const AtomicCmpXchgInst *CI) const {
11047
11048 if (Subtarget.hasLAMCAS())
11049 return AtomicExpansionKind::None;
11050
11051 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
11052 if (Size == 8 || Size == 16)
11053 return AtomicExpansionKind::MaskedIntrinsic;
11054 return AtomicExpansionKind::None;
11055}
11056
11057Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
11058 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
11059 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
11060 unsigned GRLen = Subtarget.getGRLen();
11061 AtomicOrdering FailOrd = CI->getFailureOrdering();
11062 Value *FailureOrdering =
11063 Builder.getIntN(N: Subtarget.getGRLen(), C: static_cast<uint64_t>(FailOrd));
11064 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
11065 if (GRLen == 64) {
11066 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
11067 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
11068 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
11069 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11070 }
11071 Type *Tys[] = {AlignedAddr->getType()};
11072 Value *Result = Builder.CreateIntrinsic(
11073 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
11074 if (GRLen == 64)
11075 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11076 return Result;
11077}
11078
11079Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic(
11080 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
11081 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
11082 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
11083 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
11084 // mask, as this produces better code than the LL/SC loop emitted by
11085 // int_loongarch_masked_atomicrmw_xchg.
11086 if (AI->getOperation() == AtomicRMWInst::Xchg &&
11087 isa<ConstantInt>(Val: AI->getValOperand())) {
11088 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
11089 if (CVal->isZero())
11090 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
11091 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
11092 Align: AI->getAlign(), Ordering: Ord);
11093 if (CVal->isMinusOne())
11094 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
11095 Align: AI->getAlign(), Ordering: Ord);
11096 }
11097
11098 unsigned GRLen = Subtarget.getGRLen();
11099 Value *Ordering =
11100 Builder.getIntN(N: GRLen, C: static_cast<uint64_t>(AI->getOrdering()));
11101 Type *Tys[] = {AlignedAddr->getType()};
11102 Function *LlwOpScwLoop = Intrinsic::getOrInsertDeclaration(
11103 M: AI->getModule(),
11104 id: getIntrinsicForMaskedAtomicRMWBinOp(GRLen, BinOp: AI->getOperation()), OverloadTys: Tys);
11105
11106 if (GRLen == 64) {
11107 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
11108 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11109 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
11110 }
11111
11112 Value *Result;
11113
11114 // Must pass the shift amount needed to sign extend the loaded value prior
11115 // to performing a signed comparison for min/max. ShiftAmt is the number of
11116 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which
11117 // is the number of bits to left+right shift the value in order to
11118 // sign-extend.
11119 if (AI->getOperation() == AtomicRMWInst::Min ||
11120 AI->getOperation() == AtomicRMWInst::Max) {
11121 const DataLayout &DL = AI->getDataLayout();
11122 unsigned ValWidth =
11123 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
11124 Value *SextShamt =
11125 Builder.CreateSub(LHS: Builder.getIntN(N: GRLen, C: GRLen - ValWidth), RHS: ShiftAmt);
11126 Result = Builder.CreateCall(Callee: LlwOpScwLoop,
11127 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
11128 } else {
11129 Result =
11130 Builder.CreateCall(Callee: LlwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
11131 }
11132
11133 if (GRLen == 64)
11134 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11135 return Result;
11136}
11137
11138bool LoongArchTargetLowering::isFMAFasterThanFMulAndFAdd(
11139 const MachineFunction &MF, EVT VT) const {
11140 VT = VT.getScalarType();
11141
11142 if (!VT.isSimple())
11143 return false;
11144
11145 switch (VT.getSimpleVT().SimpleTy) {
11146 case MVT::f32:
11147 case MVT::f64:
11148 return true;
11149 default:
11150 break;
11151 }
11152
11153 return false;
11154}
11155
11156Register LoongArchTargetLowering::getExceptionPointerRegister(
11157 ExceptionHandling EH, const Constant *PersonalityFn) const {
11158 return LoongArch::R4;
11159}
11160
11161Register LoongArchTargetLowering::getExceptionSelectorRegister(
11162 ExceptionHandling EH, const Constant *PersonalityFn) const {
11163 return LoongArch::R5;
11164}
11165
11166//===----------------------------------------------------------------------===//
11167// Target Optimization Hooks
11168//===----------------------------------------------------------------------===//
11169
11170static int getEstimateRefinementSteps(EVT VT,
11171 const LoongArchSubtarget &Subtarget) {
11172 // Feature FRECIPE instrucions relative accuracy is 2^-14.
11173 // IEEE float has 23 digits and double has 52 digits.
11174 int RefinementSteps = VT.getScalarType() == MVT::f64 ? 2 : 1;
11175 return RefinementSteps;
11176}
11177
11178static bool
11179isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget) {
11180 assert(Subtarget.hasFrecipe() &&
11181 "Reciprocal estimate queried on unsupported target");
11182
11183 if (!VT.isSimple())
11184 return false;
11185
11186 switch (VT.getSimpleVT().SimpleTy) {
11187 case MVT::f32:
11188 // f32 is the base type for reciprocal estimate instructions.
11189 return true;
11190
11191 case MVT::f64:
11192 return Subtarget.hasBasicD();
11193
11194 case MVT::v4f32:
11195 case MVT::v2f64:
11196 return Subtarget.hasExtLSX();
11197
11198 case MVT::v8f32:
11199 case MVT::v4f64:
11200 return Subtarget.hasExtLASX();
11201
11202 default:
11203 return false;
11204 }
11205}
11206
11207SDValue LoongArchTargetLowering::getSqrtEstimate(SDValue Operand,
11208 SelectionDAG &DAG, int Enabled,
11209 int &RefinementSteps,
11210 bool &UseOneConstNR,
11211 bool Reciprocal) const {
11212 assert(Enabled != ReciprocalEstimate::Disabled &&
11213 "Enabled should never be Disabled here");
11214
11215 if (!Subtarget.hasFrecipe())
11216 return SDValue();
11217
11218 SDLoc DL(Operand);
11219 EVT VT = Operand.getValueType();
11220
11221 // Check supported types.
11222 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11223 return SDValue();
11224
11225 // Handle refinement steps.
11226 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11227 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11228
11229 // LoongArch only has FRSQRTE which is 1.0 / sqrt(x).
11230 UseOneConstNR = false;
11231 SDValue Rsqrt = DAG.getNode(Opcode: LoongArchISD::FRSQRTE, DL, VT, Operand);
11232
11233 // If the caller wants 1.0 / sqrt(x), or if further refinement steps
11234 // are needed (which rely on the reciprocal form), return the raw reciprocal
11235 // estimate.
11236 if (Reciprocal || RefinementSteps > 0)
11237 return Rsqrt;
11238
11239 // Otherwise, return sqrt(x) by multiplying with the operand.
11240 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Operand, N2: Rsqrt);
11241}
11242
11243SDValue LoongArchTargetLowering::getRecipEstimate(SDValue Operand,
11244 SelectionDAG &DAG,
11245 int Enabled,
11246 int &RefinementSteps) const {
11247 assert(Enabled != ReciprocalEstimate::Disabled &&
11248 "Enabled should never be Disabled here");
11249
11250 if (!Subtarget.hasFrecipe())
11251 return SDValue();
11252
11253 SDLoc DL(Operand);
11254 EVT VT = Operand.getValueType();
11255
11256 // Check supported types.
11257 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11258 return SDValue();
11259
11260 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11261 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11262
11263 // FRECIPE computes 1.0 / x.
11264 return DAG.getNode(Opcode: LoongArchISD::FRECIPE, DL, VT, Operand);
11265}
11266
11267//===----------------------------------------------------------------------===//
11268// LoongArch Inline Assembly Support
11269//===----------------------------------------------------------------------===//
11270
11271LoongArchTargetLowering::ConstraintType
11272LoongArchTargetLowering::getConstraintType(StringRef Constraint) const {
11273 // LoongArch specific constraints in GCC: config/loongarch/constraints.md
11274 //
11275 // 'f': A floating-point register (if available).
11276 // 'k': A memory operand whose address is formed by a base register and
11277 // (optionally scaled) index register.
11278 // 'l': A signed 16-bit constant.
11279 // 'm': A memory operand whose address is formed by a base register and
11280 // offset that is suitable for use in instructions with the same
11281 // addressing mode as st.w and ld.w.
11282 // 'q': A general-purpose register except for $r0 and $r1 (for the csrxchg
11283 // instruction)
11284 // 'I': A signed 12-bit constant (for arithmetic instructions).
11285 // 'J': Integer zero.
11286 // 'K': An unsigned 12-bit constant (for logic instructions).
11287 // "ZB": An address that is held in a general-purpose register. The offset is
11288 // zero.
11289 // "ZC": A memory operand whose address is formed by a base register and
11290 // offset that is suitable for use in instructions with the same
11291 // addressing mode as ll.w and sc.w.
11292 if (Constraint.size() == 1) {
11293 switch (Constraint[0]) {
11294 default:
11295 break;
11296 case 'f':
11297 case 'q':
11298 return C_RegisterClass;
11299 case 'l':
11300 case 'I':
11301 case 'J':
11302 case 'K':
11303 return C_Immediate;
11304 case 'k':
11305 return C_Memory;
11306 }
11307 }
11308
11309 if (Constraint == "ZC" || Constraint == "ZB")
11310 return C_Memory;
11311
11312 // 'm' is handled here.
11313 return TargetLowering::getConstraintType(Constraint);
11314}
11315
11316InlineAsm::ConstraintCode LoongArchTargetLowering::getInlineAsmMemConstraint(
11317 StringRef ConstraintCode) const {
11318 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
11319 .Case(S: "k", Value: InlineAsm::ConstraintCode::k)
11320 .Case(S: "ZB", Value: InlineAsm::ConstraintCode::ZB)
11321 .Case(S: "ZC", Value: InlineAsm::ConstraintCode::ZC)
11322 .Default(Value: TargetLowering::getInlineAsmMemConstraint(ConstraintCode));
11323}
11324
11325std::pair<unsigned, const TargetRegisterClass *>
11326LoongArchTargetLowering::getRegForInlineAsmConstraint(
11327 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
11328 // First, see if this is a constraint that directly corresponds to a LoongArch
11329 // register class.
11330 if (Constraint.size() == 1) {
11331 switch (Constraint[0]) {
11332 case 'r':
11333 // TODO: Support fixed vectors up to GRLen?
11334 if (VT.isVector())
11335 break;
11336 return std::make_pair(x: 0U, y: &LoongArch::GPRRegClass);
11337 case 'q':
11338 return std::make_pair(x: 0U, y: &LoongArch::GPRNoR0R1RegClass);
11339 case 'f':
11340 if (Subtarget.hasBasicF() && VT == MVT::f32)
11341 return std::make_pair(x: 0U, y: &LoongArch::FPR32RegClass);
11342 if (Subtarget.hasBasicD() && VT == MVT::f64)
11343 return std::make_pair(x: 0U, y: &LoongArch::FPR64RegClass);
11344 if (Subtarget.hasExtLSX() &&
11345 TRI->isTypeLegalForClass(RC: LoongArch::LSX128RegClass, T: VT))
11346 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11347 if (Subtarget.hasExtLSX() && VT == MVT::i128)
11348 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11349 if (Subtarget.hasExtLASX() &&
11350 TRI->isTypeLegalForClass(RC: LoongArch::LASX256RegClass, T: VT))
11351 return std::make_pair(x: 0U, y: &LoongArch::LASX256RegClass);
11352 break;
11353 default:
11354 break;
11355 }
11356 }
11357
11358 // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen
11359 // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm
11360 // constraints while the official register name is prefixed with a '$'. So we
11361 // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.)
11362 // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is
11363 // case insensitive, so no need to convert the constraint to upper case here.
11364 //
11365 // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly
11366 // decode the usage of register name aliases into their official names. And
11367 // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use
11368 // official register names.
11369 if (Constraint.starts_with(Prefix: "{$r") || Constraint.starts_with(Prefix: "{$f") ||
11370 Constraint.starts_with(Prefix: "{$vr") || Constraint.starts_with(Prefix: "{$xr")) {
11371 bool IsFP = Constraint[2] == 'f';
11372 std::pair<StringRef, StringRef> Temp = Constraint.split(Separator: '$');
11373 std::pair<unsigned, const TargetRegisterClass *> R;
11374 R = TargetLowering::getRegForInlineAsmConstraint(
11375 TRI, Constraint: join_items(Separator: "", Items&: Temp.first, Items&: Temp.second), VT);
11376 // Match those names to the widest floating point register type available.
11377 if (IsFP) {
11378 unsigned RegNo = R.first;
11379 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
11380 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
11381 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
11382 return std::make_pair(x&: DReg, y: &LoongArch::FPR64RegClass);
11383 }
11384 }
11385 }
11386 return R;
11387 }
11388
11389 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
11390}
11391
11392void LoongArchTargetLowering::LowerAsmOperandForConstraint(
11393 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
11394 SelectionDAG &DAG) const {
11395 // Currently only support length 1 constraints.
11396 if (Constraint.size() == 1) {
11397 switch (Constraint[0]) {
11398 case 'l':
11399 // Validate & create a 16-bit signed immediate operand.
11400 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11401 uint64_t CVal = C->getSExtValue();
11402 if (isInt<16>(x: CVal))
11403 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11404 VT: Subtarget.getGRLenVT()));
11405 }
11406 return;
11407 case 'I':
11408 // Validate & create a 12-bit signed immediate operand.
11409 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11410 uint64_t CVal = C->getSExtValue();
11411 if (isInt<12>(x: CVal))
11412 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11413 VT: Subtarget.getGRLenVT()));
11414 }
11415 return;
11416 case 'J':
11417 // Validate & create an integer zero operand.
11418 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
11419 if (C->getZExtValue() == 0)
11420 Ops.push_back(
11421 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11422 return;
11423 case 'K':
11424 // Validate & create a 12-bit unsigned immediate operand.
11425 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11426 uint64_t CVal = C->getZExtValue();
11427 if (isUInt<12>(x: CVal))
11428 Ops.push_back(
11429 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11430 }
11431 return;
11432 default:
11433 break;
11434 }
11435 }
11436 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
11437}
11438
11439#define GET_REGISTER_MATCHER
11440#include "LoongArchGenAsmMatcher.inc"
11441
11442Register
11443LoongArchTargetLowering::getRegisterByName(const char *RegName, LLT VT,
11444 const MachineFunction &MF) const {
11445 std::pair<StringRef, StringRef> Name = StringRef(RegName).split(Separator: '$');
11446 std::string NewRegName = Name.second.str();
11447 Register Reg = MatchRegisterAltName(Name: NewRegName);
11448 if (!Reg)
11449 Reg = MatchRegisterName(Name: NewRegName);
11450 if (!Reg)
11451 return Reg;
11452 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11453 if (!ReservedRegs.test(Idx: Reg))
11454 report_fatal_error(reason: Twine("Trying to obtain non-reserved register \"" +
11455 StringRef(RegName) + "\"."));
11456 return Reg;
11457}
11458
11459bool LoongArchTargetLowering::decomposeMulByConstant(LLVMContext &Context,
11460 EVT VT, SDValue C) const {
11461 // TODO: Support vectors.
11462 if (!VT.isScalarInteger())
11463 return false;
11464
11465 // Omit the optimization if the data size exceeds GRLen.
11466 if (VT.getSizeInBits() > Subtarget.getGRLen())
11467 return false;
11468
11469 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
11470 const APInt &Imm = ConstNode->getAPIntValue();
11471 // Break MUL into (SLLI + ADD/SUB) or ALSL.
11472 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
11473 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
11474 return true;
11475 // Break MUL into (ALSL x, (SLLI x, imm0), imm1).
11476 if (ConstNode->hasOneUse() &&
11477 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
11478 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
11479 return true;
11480 // Break (MUL x, imm) into (ADD (SLLI x, s0), (SLLI x, s1)),
11481 // in which the immediate has two set bits. Or Break (MUL x, imm)
11482 // into (SUB (SLLI x, s0), (SLLI x, s1)), in which the immediate
11483 // equals to (1 << s0) - (1 << s1).
11484 if (ConstNode->hasOneUse() && !(Imm.sge(RHS: -2048) && Imm.sle(RHS: 4095))) {
11485 unsigned Shifts = Imm.countr_zero();
11486 // Reject immediates which can be composed via a single LUI.
11487 if (Shifts >= 12)
11488 return false;
11489 // Reject multiplications can be optimized to
11490 // (SLLI (ALSL x, x, 1/2/3/4), s).
11491 APInt ImmPop = Imm.ashr(ShiftAmt: Shifts);
11492 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
11493 return false;
11494 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`,
11495 // since it needs one more instruction than other 3 cases.
11496 APInt ImmSmall = APInt(Imm.getBitWidth(), 1ULL << Shifts, true);
11497 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
11498 (ImmSmall - Imm).isPowerOf2())
11499 return true;
11500 }
11501 }
11502
11503 return false;
11504}
11505
11506bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
11507 const AddrMode &AM,
11508 Type *Ty, unsigned AS,
11509 Instruction *I) const {
11510 // LoongArch has four basic addressing modes:
11511 // 1. reg
11512 // 2. reg + 12-bit signed offset
11513 // 3. reg + 14-bit signed offset left-shifted by 2
11514 // 4. reg1 + reg2
11515 // TODO: Add more checks after support vector extension.
11516
11517 // No global is ever allowed as a base.
11518 if (AM.BaseGV)
11519 return false;
11520
11521 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
11522 // with `UAL` feature.
11523 if (!isInt<12>(x: AM.BaseOffs) &&
11524 !(isShiftedInt<14, 2>(x: AM.BaseOffs) && Subtarget.hasUAL()))
11525 return false;
11526
11527 switch (AM.Scale) {
11528 case 0:
11529 // "r+i" or just "i", depending on HasBaseReg.
11530 break;
11531 case 1:
11532 // "r+r+i" is not allowed.
11533 if (AM.HasBaseReg && AM.BaseOffs)
11534 return false;
11535 // Otherwise we have "r+r" or "r+i".
11536 break;
11537 case 2:
11538 // "2*r+r" or "2*r+i" is not allowed.
11539 if (AM.HasBaseReg || AM.BaseOffs)
11540 return false;
11541 // Allow "2*r" as "r+r".
11542 break;
11543 default:
11544 return false;
11545 }
11546
11547 return true;
11548}
11549
11550bool LoongArchTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
11551 return isInt<12>(x: Imm);
11552}
11553
11554bool LoongArchTargetLowering::isLegalAddImmediate(int64_t Imm) const {
11555 return isInt<12>(x: Imm);
11556}
11557
11558bool LoongArchTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11559 // Zexts are free if they can be combined with a load.
11560 // Don't advertise i32->i64 zextload as being free for LA64. It interacts
11561 // poorly with type legalization of compares preferring sext.
11562 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
11563 EVT MemVT = LD->getMemoryVT();
11564 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11565 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
11566 LD->getExtensionType() == ISD::ZEXTLOAD))
11567 return true;
11568 }
11569
11570 return TargetLowering::isZExtFree(Val, VT2);
11571}
11572
11573bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT,
11574 EVT DstVT) const {
11575 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
11576}
11577
11578bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const {
11579 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
11580}
11581
11582bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
11583 // TODO: Support vectors.
11584 if (Y.getValueType().isVector())
11585 return false;
11586
11587 return !isa<ConstantSDNode>(Val: Y);
11588}
11589
11590ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
11591 // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
11592 return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
11593}
11594
11595bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
11596 Type *Ty, bool IsSigned) const {
11597 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
11598 return true;
11599
11600 return IsSigned;
11601}
11602
11603bool LoongArchTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
11604 // Return false to suppress the unnecessary extensions if the LibCall
11605 // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
11606 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
11607 Type.getSizeInBits() < Subtarget.getGRLen()))
11608 return false;
11609 return true;
11610}
11611
11612// memcpy, and other memory intrinsics, typically tries to use wider load/store
11613// if the source/dest is aligned and the copy size is large enough. We therefore
11614// want to align such objects passed to memory intrinsics.
11615bool LoongArchTargetLowering::shouldAlignPointerArgs(CallInst *CI,
11616 unsigned &MinSize,
11617 Align &PrefAlign) const {
11618 if (!isa<MemIntrinsic>(Val: CI))
11619 return false;
11620
11621 if (Subtarget.is64Bit()) {
11622 MinSize = 8;
11623 PrefAlign = Align(8);
11624 } else {
11625 MinSize = 4;
11626 PrefAlign = Align(4);
11627 }
11628
11629 return true;
11630}
11631
11632TargetLoweringBase::LegalizeTypeAction
11633LoongArchTargetLowering::getPreferredVectorAction(MVT VT) const {
11634 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
11635 VT.getVectorElementType() != MVT::i1)
11636 return TypeWidenVector;
11637
11638 return TargetLoweringBase::getPreferredVectorAction(VT);
11639}
11640
11641bool LoongArchTargetLowering::splitValueIntoRegisterParts(
11642 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
11643 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
11644 bool IsABIRegCopy = CC.has_value();
11645 EVT ValueVT = Val.getValueType();
11646
11647 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11648 PartVT == MVT::f32) {
11649 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
11650 // nan, and cast to f32.
11651 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
11652 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
11653 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
11654 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
11655 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Val);
11656 Parts[0] = Val;
11657 return true;
11658 }
11659
11660 return false;
11661}
11662
11663SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
11664 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
11665 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
11666 bool IsABIRegCopy = CC.has_value();
11667
11668 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11669 PartVT == MVT::f32) {
11670 SDValue Val = Parts[0];
11671
11672 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
11673 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
11674 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
11675 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
11676 return Val;
11677 }
11678
11679 return SDValue();
11680}
11681
11682MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
11683 CallingConv::ID CC,
11684 EVT VT) const {
11685 // Use f32 to pass f16.
11686 if (VT == MVT::f16 && Subtarget.hasBasicF())
11687 return MVT::f32;
11688
11689 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
11690}
11691
11692unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
11693 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
11694 // Use f32 to pass f16.
11695 if (VT == MVT::f16 && Subtarget.hasBasicF())
11696 return 1;
11697
11698 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
11699}
11700
11701void LoongArchTargetLowering::computeKnownBitsForTargetNode(
11702 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
11703 const SelectionDAG &DAG, unsigned Depth) const {
11704 unsigned Opc = Op.getOpcode();
11705 Known.resetAll();
11706 switch (Opc) {
11707 default:
11708 break;
11709 case LoongArchISD::VPICK_ZEXT_ELT: {
11710 assert(isa<VTSDNode>(Op->getOperand(2)) && "Unexpected operand!");
11711 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 2))->getVT();
11712 unsigned VTBits = VT.getScalarSizeInBits();
11713 assert(Known.getBitWidth() >= VTBits && "Unexpected width!");
11714 Known.Zero.setBitsFrom(VTBits);
11715 break;
11716 }
11717 }
11718}
11719
11720bool LoongArchTargetLowering::SimplifyDemandedBitsForTargetNode(
11721 SDValue Op, const APInt &OriginalDemandedBits,
11722 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
11723 unsigned Depth) const {
11724 EVT VT = Op.getValueType();
11725 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
11726 unsigned Opc = Op.getOpcode();
11727 switch (Opc) {
11728 default:
11729 break;
11730 case LoongArchISD::CRC_W_B_W:
11731 case LoongArchISD::CRC_W_H_W:
11732 case LoongArchISD::CRCC_W_B_W:
11733 case LoongArchISD::CRCC_W_H_W: {
11734 KnownBits KnownSrc;
11735 APInt DemandedSrcBits =
11736 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: (Opc == LoongArchISD::CRC_W_B_W ||
11737 Opc == LoongArchISD::CRCC_W_B_W)
11738 ? 8
11739 : 16);
11740 return SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: DemandedSrcBits,
11741 DemandedElts: OriginalDemandedElts, Known&: KnownSrc, TLO, Depth: Depth + 1);
11742 }
11743 case LoongArchISD::VMSKLTZ:
11744 case LoongArchISD::XVMSKLTZ: {
11745 SDValue Src = Op.getOperand(i: 0);
11746 MVT SrcVT = Src.getSimpleValueType();
11747 unsigned SrcBits = SrcVT.getScalarSizeInBits();
11748 unsigned NumElts = SrcVT.getVectorNumElements();
11749
11750 // If we don't need the sign bits at all just return zero.
11751 if (OriginalDemandedBits.countr_zero() >= NumElts)
11752 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
11753
11754 // Only demand the vector elements of the sign bits we need.
11755 APInt KnownUndef, KnownZero;
11756 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(width: NumElts);
11757 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedElts, KnownUndef, KnownZero,
11758 TLO, Depth: Depth + 1))
11759 return true;
11760
11761 Known.Zero = KnownZero.zext(width: BitWidth);
11762 Known.Zero.setHighBits(BitWidth - NumElts);
11763
11764 // [X]VMSKLTZ only uses the MSB from each vector element.
11765 KnownBits KnownSrc;
11766 APInt DemandedSrcBits = APInt::getSignMask(BitWidth: SrcBits);
11767 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, Known&: KnownSrc, TLO,
11768 Depth: Depth + 1))
11769 return true;
11770
11771 if (KnownSrc.One[SrcBits - 1])
11772 Known.One.setLowBits(NumElts);
11773 else if (KnownSrc.Zero[SrcBits - 1])
11774 Known.Zero.setLowBits(NumElts);
11775
11776 // Attempt to avoid multi-use ops if we don't need anything from it.
11777 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
11778 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
11779 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT, Operand: NewSrc));
11780 return false;
11781 }
11782 }
11783
11784 return TargetLowering::SimplifyDemandedBitsForTargetNode(
11785 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
11786}
11787
11788bool LoongArchTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
11789 unsigned Opc = VecOp.getOpcode();
11790
11791 // Assume target opcodes can't be scalarized.
11792 // TODO - do we have any exceptions?
11793 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
11794 return false;
11795
11796 // If the vector op is not supported, try to convert to scalar.
11797 EVT VecVT = VecOp.getValueType();
11798 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
11799 return true;
11800
11801 // If the vector op is supported, but the scalar op is not, the transform may
11802 // not be worthwhile.
11803 EVT ScalarVT = VecVT.getScalarType();
11804 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
11805}
11806
11807TargetLowering::ExtractSubvectorCost
11808LoongArchTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
11809 unsigned Index) const {
11810 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
11811 return ExtractSubvectorCost::Expensive;
11812
11813 // Extract a 128-bit subvector from index 0 of a 256-bit vector is free.
11814 if (Index == 0)
11815 return ExtractSubvectorCost::Free;
11816 return ExtractSubvectorCost::Expensive;
11817}
11818
11819bool LoongArchTargetLowering::isExtractVecEltCheap(EVT VT,
11820 unsigned Index) const {
11821 EVT EltVT = VT.getScalarType();
11822
11823 // Extract a scalar FP value from index 0 of a vector is free.
11824 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
11825}
11826
11827bool LoongArchTargetLowering::hasInlineStackProbe(
11828 const MachineFunction &MF) const {
11829
11830 // If the function specifically requests inline stack probes, emit them.
11831 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
11832 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
11833 "inline-asm";
11834
11835 return false;
11836}
11837
11838unsigned LoongArchTargetLowering::getStackProbeSize(const MachineFunction &MF,
11839 Align StackAlign) const {
11840 // The default stack probe size is 4096 if the function has no
11841 // stack-probe-size attribute.
11842 const Function &Fn = MF.getFunction();
11843 unsigned StackProbeSize =
11844 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
11845 // Round down to the stack alignment.
11846 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
11847 return StackProbeSize ? StackProbeSize : StackAlign.value();
11848}
11849
11850SDValue
11851LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
11852 SelectionDAG &DAG) const {
11853 MachineFunction &MF = DAG.getMachineFunction();
11854 if (!hasInlineStackProbe(MF))
11855 return SDValue();
11856
11857 const MVT GRLenVT = Subtarget.getGRLenVT();
11858 // Get the inputs.
11859 SDValue Chain = Op.getOperand(i: 0);
11860 SDValue Size = Op.getOperand(i: 1);
11861
11862 const MaybeAlign Align =
11863 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
11864 const SDLoc dl(Op);
11865 const EVT VT = Op.getValueType();
11866
11867 // Construct the new SP value in a GPR.
11868 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: LoongArch::R3, VT: GRLenVT);
11869 Chain = SP.getValue(R: 1);
11870 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: GRLenVT, N1: SP, N2: Size);
11871 if (Align)
11872 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
11873 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
11874
11875 // Set the real SP to the new value with a probing loop.
11876 Chain = DAG.getNode(Opcode: LoongArchISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
11877 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
11878}
11879
11880MachineBasicBlock *
11881LoongArchTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
11882 MachineBasicBlock *MBB) const {
11883 MachineFunction &MF = *MBB->getParent();
11884 MachineBasicBlock::iterator MBBI = MI.getIterator();
11885 DebugLoc DL = MBB->findDebugLoc(MBBI);
11886 const Register TargetReg = MI.getOperand(i: 0).getReg();
11887
11888 const LoongArchInstrInfo *TII = Subtarget.getInstrInfo();
11889 const bool IsLA64 = Subtarget.is64Bit();
11890 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
11891 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
11892 const uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
11893
11894 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
11895 MachineBasicBlock *const LoopTestMBB =
11896 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
11897 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11898 MachineBasicBlock *const ExitMBB =
11899 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
11900 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11901 const Register SPReg = LoongArch::R3;
11902 const Register ScratchReg =
11903 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
11904
11905 // ScratchReg = ProbeSize
11906 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
11907
11908 // LoopTest:
11909 // sub.{w/d} $sp, $sp, ScratchReg
11910 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
11911 MCID: TII->get(Opcode: IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W), DestReg: SPReg)
11912 .addReg(RegNo: SPReg)
11913 .addReg(RegNo: ScratchReg);
11914
11915 // st.{w/d} $zero, $sp, 0
11916 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
11917 MCID: TII->get(Opcode: IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
11918 .addReg(RegNo: LoongArch::R0)
11919 .addReg(RegNo: SPReg)
11920 .addImm(Val: 0);
11921
11922 // bltu TargetReg, $sp, LoopTest
11923 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::BLTU))
11924 .addReg(RegNo: TargetReg)
11925 .addReg(RegNo: SPReg)
11926 .addMBB(MBB: LoopTestMBB);
11927
11928 // move $sp, TargetReg
11929 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::OR), DestReg: SPReg)
11930 .addReg(RegNo: TargetReg)
11931 .addReg(RegNo: LoongArch::R0);
11932
11933 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
11934 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
11935
11936 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11937 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
11938 MBB->addSuccessor(Succ: LoopTestMBB);
11939
11940 MI.eraseFromParent();
11941 MF.getInfo<LoongArchMachineFunctionInfo>()->setDynamicAllocation();
11942 return ExitMBB->begin()->getParent();
11943}
11944