1//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that LoongArch uses to lower LLVM code into
10// a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LoongArchISelLowering.h"
15#include "LoongArch.h"
16#include "LoongArchMachineFunctionInfo.h"
17#include "LoongArchRegisterInfo.h"
18#include "LoongArchSelectionDAGInfo.h"
19#include "LoongArchSubtarget.h"
20#include "MCTargetDesc/LoongArchBaseInfo.h"
21#include "MCTargetDesc/LoongArchMCTargetDesc.h"
22#include "MCTargetDesc/LoongArchMatInt.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/IntrinsicsLoongArch.h"
33#include "llvm/Support/CodeGen.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include <llvm/Analysis/VectorUtils.h>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "loongarch-isel-lowering"
43
44STATISTIC(NumTailCalls, "Number of tail calls");
45
46enum MaterializeFPImm {
47 NoMaterializeFPImm = 0,
48 MaterializeFPImm2Ins = 2,
49 MaterializeFPImm3Ins = 3,
50 MaterializeFPImm4Ins = 4,
51 MaterializeFPImm5Ins = 5,
52 MaterializeFPImm6Ins = 6
53};
54
55static cl::opt<MaterializeFPImm> MaterializeFPImmInsNum(
56 "loongarch-materialize-float-imm", cl::Hidden,
57 cl::desc("Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
60 cl::init(Val: MaterializeFPImm3Ins),
61 cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"),
62 clEnumValN(MaterializeFPImm2Ins, "2",
63 "Materialize FP immediate within 2 instructions"),
64 clEnumValN(MaterializeFPImm3Ins, "3",
65 "Materialize FP immediate within 3 instructions"),
66 clEnumValN(MaterializeFPImm4Ins, "4",
67 "Materialize FP immediate within 4 instructions"),
68 clEnumValN(MaterializeFPImm5Ins, "5",
69 "Materialize FP immediate within 5 instructions"),
70 clEnumValN(MaterializeFPImm6Ins, "6",
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
73
74static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
75 cl::desc("Trap on integer division by zero."),
76 cl::init(Val: false));
77
78LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
79 const LoongArchSubtarget &STI)
80 : TargetLowering(TM, STI), Subtarget(STI) {
81
82 MVT GRLenVT = Subtarget.getGRLenVT();
83
84 // Set up the register classes.
85
86 addRegisterClass(VT: GRLenVT, RC: &LoongArch::GPRRegClass);
87 if (Subtarget.hasBasicF())
88 addRegisterClass(VT: MVT::f32, RC: &LoongArch::FPR32RegClass);
89 if (Subtarget.hasBasicD())
90 addRegisterClass(VT: MVT::f64, RC: &LoongArch::FPR64RegClass);
91
92 static const MVT::SimpleValueType LSXVTs[] = {
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
94 static const MVT::SimpleValueType LASXVTs[] = {
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
96
97 if (Subtarget.hasExtLSX())
98 for (MVT VT : LSXVTs)
99 addRegisterClass(VT, RC: &LoongArch::LSX128RegClass);
100
101 if (Subtarget.hasExtLASX())
102 for (MVT VT : LASXVTs)
103 addRegisterClass(VT, RC: &LoongArch::LASX256RegClass);
104
105 // Set operations for LA32 and LA64.
106
107 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: GRLenVT,
108 MemVT: MVT::i1, Action: Promote);
109
110 setOperationAction(Op: ISD::SHL_PARTS, VT: GRLenVT, Action: Custom);
111 setOperationAction(Op: ISD::SRA_PARTS, VT: GRLenVT, Action: Custom);
112 setOperationAction(Op: ISD::SRL_PARTS, VT: GRLenVT, Action: Custom);
113 setOperationAction(Op: ISD::FP_TO_SINT, VT: GRLenVT, Action: Custom);
114 setOperationAction(Op: ISD::ROTL, VT: GRLenVT, Action: Expand);
115 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Expand);
116
117 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
118 ISD::JumpTable, ISD::GlobalTLSAddress},
119 VT: GRLenVT, Action: Custom);
120
121 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: GRLenVT, Action: Custom);
122
123 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: GRLenVT, Action: Custom);
124 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
125 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
126 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
127
128 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
129 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
130
131 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
132 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
133 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
134
135 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
136
137 // BITREV/REVB requires the 32S feature.
138 if (STI.has32S()) {
139 // Expand bitreverse.i16 with native-width bitrev and shift for now, before
140 // we get to know which of sll and revb.2h is faster.
141 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
142 setOperationAction(Op: ISD::BITREVERSE, VT: GRLenVT, Action: Legal);
143
144 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
145 // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
146 // and i32 could still be byte-swapped relatively cheaply.
147 setOperationAction(Op: ISD::BSWAP, VT: MVT::i16, Action: Custom);
148 } else {
149 setOperationAction(Op: ISD::BSWAP, VT: GRLenVT, Action: Expand);
150 setOperationAction(Op: ISD::CTTZ, VT: GRLenVT, Action: Expand);
151 setOperationAction(Op: ISD::CTLZ, VT: GRLenVT, Action: Expand);
152 setOperationAction(Op: ISD::ROTR, VT: GRLenVT, Action: Expand);
153 setOperationAction(Op: ISD::SELECT, VT: GRLenVT, Action: Custom);
154 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
155 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
156 }
157
158 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
159 setOperationAction(Op: ISD::BR_CC, VT: GRLenVT, Action: Expand);
160 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
161 setOperationAction(Op: ISD::SELECT_CC, VT: GRLenVT, Action: Expand);
162 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
163 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: GRLenVT, Action: Expand);
164
165 setOperationAction(Op: ISD::FP_TO_UINT, VT: GRLenVT, Action: Custom);
166 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Expand);
167
168 // Set operations for LA64 only.
169
170 if (Subtarget.is64Bit()) {
171 setOperationAction(Op: ISD::ADD, VT: MVT::i32, Action: Custom);
172 setOperationAction(Op: ISD::SUB, VT: MVT::i32, Action: Custom);
173 setOperationAction(Op: ISD::SHL, VT: MVT::i32, Action: Custom);
174 setOperationAction(Op: ISD::SRA, VT: MVT::i32, Action: Custom);
175 setOperationAction(Op: ISD::SRL, VT: MVT::i32, Action: Custom);
176 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
177 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
178 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Custom);
179 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Custom);
180 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
181 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Custom);
182 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
183 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i32, Action: Custom);
184 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i32, Action: Custom);
185 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
186 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
187 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i32, Action: Custom);
188
189 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Custom);
190 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Custom);
191 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: MVT::i32,
192 Action: Custom);
193 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
194 }
195
196 // Set operations for LA32 only.
197
198 if (!Subtarget.is64Bit()) {
199 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
200 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
201 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i64, Action: Custom);
202 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
203 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i64, Action: Custom);
204 if (Subtarget.hasBasicD())
205 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
206 }
207
208 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
209
210 static const ISD::CondCode FPCCToExpand[] = {
211 ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
212 ISD::SETGE, ISD::SETNE, ISD::SETGT};
213
214 // Set operations for 'F' feature.
215
216 if (Subtarget.hasBasicF()) {
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
218 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
220 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
221 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
222
223 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
224 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
225 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
226 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Legal);
227 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
228 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
229 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
230 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
231 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
232 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
233 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
234 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Legal);
235 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
236 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
237 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
238 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
239 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
240 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32,
241 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
242 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32,
243 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
244 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
245 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
246 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
247 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
248 setOperationAction(Op: ISD::GET_ROUNDING, VT: GRLenVT, Action: Custom);
249
250 if (Subtarget.is64Bit())
251 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
252
253 if (!Subtarget.hasBasicD()) {
254 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
255 if (Subtarget.is64Bit()) {
256 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
257 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
258 }
259 }
260 }
261
262 // Set operations for 'D' feature.
263
264 if (Subtarget.hasBasicD()) {
265 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
266 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
267 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
268 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
269 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
271 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
272
273 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
274 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
275 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
276 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
277 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
278 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Legal);
279 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
280 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
281 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
282 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
283 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
284 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Legal);
285 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
286 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
287 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
288 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
289 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
290 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
291 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64,
292 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
293 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
294 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
295 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
296
297 if (Subtarget.is64Bit())
298 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
299 }
300
301 // Set operations for 'LSX' feature.
302
303 if (Subtarget.hasExtLSX()) {
304 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
305 // Expand all truncating stores and extending loads.
306 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
307 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
308 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
309 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
310 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
311 }
312 // By default everything must be expanded. Then we will selectively turn
313 // on ones that can be effectively codegen'd.
314 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
315 setOperationAction(Op, VT, Action: Expand);
316 }
317
318 for (MVT VT : LSXVTs) {
319 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
320 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
321 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
322
323 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
324 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
325 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
326
327 setOperationAction(Op: ISD::SETCC, VT, Action: Legal);
328 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
329 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
330 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
331 }
332 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
333 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
334 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
335 Action: Legal);
336 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
337 VT, Action: Legal);
338 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
339 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
340 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
341 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
342 setCondCodeAction(
343 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
344 Action: Expand);
345 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
346 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
347 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
348 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
349 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
350 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
351 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
352 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
353 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
354 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
355 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
356 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
357 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
358 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
359 }
360 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
361 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
362 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
363 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
364 for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
365 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Action: Legal);
366 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
367 }
368 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Custom);
369 for (MVT VT : {MVT::v4f32, MVT::v2f64}) {
370 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
371 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
372 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
373 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
374 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
375 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
376 ISD::SETUGE, ISD::SETUGT},
377 VT, Action: Expand);
378 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
379 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
380 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
381 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
382 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
383 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
384 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
385 }
386 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Legal);
387 setOperationAction(Ops: ISD::FCEIL, VTs: {MVT::f32, MVT::f64}, Action: Legal);
388 setOperationAction(Ops: ISD::FFLOOR, VTs: {MVT::f32, MVT::f64}, Action: Legal);
389 setOperationAction(Ops: ISD::FTRUNC, VTs: {MVT::f32, MVT::f64}, Action: Legal);
390 setOperationAction(Ops: ISD::FROUNDEVEN, VTs: {MVT::f32, MVT::f64}, Action: Legal);
391
392 for (MVT VT :
393 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
394 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
395 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
396 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
397 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
398 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
399 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
400 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
401 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
402 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
403 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
404 }
405 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Custom);
406 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
407 // We want to legalize this to an f64 load rather than an i64 load.
408 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Custom);
409 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
410 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
411 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
412 MVT::v16i64})
413 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Custom);
414 }
415
416 // Set operations for 'LASX' feature.
417
418 if (Subtarget.hasExtLASX()) {
419 for (MVT VT : LASXVTs) {
420 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
421 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
422 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
423
424 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
425 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
426 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
427 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
428 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT, Action: Legal);
429
430 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
431 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
432 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
433 }
434 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
435 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
436 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
437 Action: Legal);
438 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
439 VT, Action: Legal);
440 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
441 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
442 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
443 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
444 setCondCodeAction(
445 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
446 Action: Expand);
447 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
448 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
449 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
450 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
451 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
452 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
453 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
454 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
455 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
456 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
457 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
458 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
459 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
460 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
461 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
462 }
463 for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
464 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
465 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
466 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
467 for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
468 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
469 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Legal);
470 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
471 }
472 for (MVT VT : {MVT::v8f32, MVT::v4f64}) {
473 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
474 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
475 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
476 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
477 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
478 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
479 ISD::SETUGE, ISD::SETUGT},
480 VT, Action: Expand);
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
482 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
483 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
484 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
485 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
486 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
487 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
488 }
489 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v4f32, Action: Custom);
490 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f64, Action: Custom);
491 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
492 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Legal);
493 setOperationAction(Op: ISD::ZERO_EXTEND, VT, Action: Legal);
494 setOperationAction(Op: ISD::ANY_EXTEND, VT, Action: Custom);
495 }
496 for (MVT VT :
497 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
498 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Legal);
499 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Legal);
500 }
501 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
502 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Legal);
503 }
504
505 // Set DAG combine for LA32 and LA64.
506 if (Subtarget.hasBasicF()) {
507 setTargetDAGCombine(ISD::SINT_TO_FP);
508 }
509
510 setTargetDAGCombine(ISD::AND);
511 setTargetDAGCombine(ISD::OR);
512 setTargetDAGCombine(ISD::SRL);
513 setTargetDAGCombine(ISD::SETCC);
514
515 // On targets with the 32S feature, `select` is expanded into
516 // maskeqz + masknez + or (3 instructions), which is more expensive than
517 // on most other architectures where a single cmov-like instruction
518 // suffices. Enable a combine that can turn
519 // select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
520 // select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
521 // for binop in {add, or, xor, sub}, replacing the 3-insn select (plus
522 // the original binop) with a single mask instruction plus the binop.
523 if (Subtarget.has32S())
524 setTargetDAGCombine(ISD::SELECT);
525
526 // Set DAG combine for 'LSX' feature.
527
528 if (Subtarget.hasExtLSX()) {
529 setTargetDAGCombine(ISD::ADD);
530 setTargetDAGCombine(ISD::SUB);
531 setTargetDAGCombine(ISD::SHL);
532 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
533 setTargetDAGCombine(ISD::BITCAST);
534 setTargetDAGCombine(ISD::VSELECT);
535 setTargetDAGCombine(ISD::FP_TO_SINT);
536 setTargetDAGCombine(ISD::FP_TO_UINT);
537 setTargetDAGCombine(ISD::UINT_TO_FP);
538 setTargetDAGCombine(ISD::ZERO_EXTEND);
539 setTargetDAGCombine(ISD::SIGN_EXTEND);
540 }
541
542 // Set DAG combine for 'LASX' feature.
543 if (Subtarget.hasExtLASX()) {
544 setTargetDAGCombine(ISD::ANY_EXTEND);
545 setTargetDAGCombine(ISD::CONCAT_VECTORS);
546 }
547
548 // Compute derived properties from the register classes.
549 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
550
551 setStackPointerRegisterToSaveRestore(LoongArch::R3);
552
553 setBooleanContents(ZeroOrOneBooleanContent);
554 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
555
556 setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());
557
558 setMinCmpXchgSizeInBits(32);
559
560 // Function alignments.
561 setMinFunctionAlignment(Align(4));
562 // Set preferred alignments.
563 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
564 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
565 setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
566
567 // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
568 if (Subtarget.hasLAMCAS())
569 setMinCmpXchgSizeInBits(8);
570
571 if (Subtarget.hasSCQ()) {
572 setMaxAtomicSizeInBitsSupported(128);
573 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i128, Action: Custom);
574 }
575
576 // Disable strict node mutation.
577 IsStrictFPEnabled = true;
578}
579
580bool LoongArchTargetLowering::isOffsetFoldingLegal(
581 const GlobalAddressSDNode *GA) const {
582 // In order to maximise the opportunity for common subexpression elimination,
583 // keep a separate ADD node for the global address offset instead of folding
584 // it in the global address node. Later peephole optimisations may choose to
585 // fold it back in when profitable.
586 return false;
587}
588
589SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
590 SelectionDAG &DAG) const {
591 switch (Op.getOpcode()) {
592 case ISD::ATOMIC_FENCE:
593 return lowerATOMIC_FENCE(Op, DAG);
594 case ISD::EH_DWARF_CFA:
595 return lowerEH_DWARF_CFA(Op, DAG);
596 case ISD::GlobalAddress:
597 return lowerGlobalAddress(Op, DAG);
598 case ISD::GlobalTLSAddress:
599 return lowerGlobalTLSAddress(Op, DAG);
600 case ISD::INTRINSIC_WO_CHAIN:
601 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
602 case ISD::INTRINSIC_W_CHAIN:
603 return lowerINTRINSIC_W_CHAIN(Op, DAG);
604 case ISD::INTRINSIC_VOID:
605 return lowerINTRINSIC_VOID(Op, DAG);
606 case ISD::BlockAddress:
607 return lowerBlockAddress(Op, DAG);
608 case ISD::JumpTable:
609 return lowerJumpTable(Op, DAG);
610 case ISD::SHL_PARTS:
611 return lowerShiftLeftParts(Op, DAG);
612 case ISD::SRA_PARTS:
613 return lowerShiftRightParts(Op, DAG, IsSRA: true);
614 case ISD::SRL_PARTS:
615 return lowerShiftRightParts(Op, DAG, IsSRA: false);
616 case ISD::ConstantPool:
617 return lowerConstantPool(Op, DAG);
618 case ISD::FP_TO_SINT:
619 return lowerFP_TO_SINT(Op, DAG);
620 case ISD::FP_TO_UINT:
621 return lowerFP_TO_UINT(Op, DAG);
622 case ISD::BITCAST:
623 return lowerBITCAST(Op, DAG);
624 case ISD::UINT_TO_FP:
625 return lowerUINT_TO_FP(Op, DAG);
626 case ISD::SINT_TO_FP:
627 return lowerSINT_TO_FP(Op, DAG);
628 case ISD::VASTART:
629 return lowerVASTART(Op, DAG);
630 case ISD::FRAMEADDR:
631 return lowerFRAMEADDR(Op, DAG);
632 case ISD::RETURNADDR:
633 return lowerRETURNADDR(Op, DAG);
634 case ISD::SET_ROUNDING:
635 return lowerSET_ROUNDING(Op, DAG);
636 case ISD::GET_ROUNDING:
637 return lowerGET_ROUNDING(Op, DAG);
638 case ISD::WRITE_REGISTER:
639 return lowerWRITE_REGISTER(Op, DAG);
640 case ISD::INSERT_VECTOR_ELT:
641 return lowerINSERT_VECTOR_ELT(Op, DAG);
642 case ISD::EXTRACT_VECTOR_ELT:
643 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
644 case ISD::BUILD_VECTOR:
645 return lowerBUILD_VECTOR(Op, DAG);
646 case ISD::CONCAT_VECTORS:
647 return lowerCONCAT_VECTORS(Op, DAG);
648 case ISD::VECTOR_SHUFFLE:
649 return lowerVECTOR_SHUFFLE(Op, DAG);
650 case ISD::BITREVERSE:
651 return lowerBITREVERSE(Op, DAG);
652 case ISD::SCALAR_TO_VECTOR:
653 return lowerSCALAR_TO_VECTOR(Op, DAG);
654 case ISD::PREFETCH:
655 return lowerPREFETCH(Op, DAG);
656 case ISD::SELECT:
657 return lowerSELECT(Op, DAG);
658 case ISD::BRCOND:
659 return lowerBRCOND(Op, DAG);
660 case ISD::FP_TO_FP16:
661 return lowerFP_TO_FP16(Op, DAG);
662 case ISD::FP16_TO_FP:
663 return lowerFP16_TO_FP(Op, DAG);
664 case ISD::FP_TO_BF16:
665 return lowerFP_TO_BF16(Op, DAG);
666 case ISD::BF16_TO_FP:
667 return lowerBF16_TO_FP(Op, DAG);
668 case ISD::VECREDUCE_ADD:
669 return lowerVECREDUCE_ADD(Op, DAG);
670 case ISD::ROTL:
671 case ISD::ROTR:
672 return lowerRotate(Op, DAG);
673 case ISD::VECREDUCE_AND:
674 case ISD::VECREDUCE_OR:
675 case ISD::VECREDUCE_XOR:
676 case ISD::VECREDUCE_SMAX:
677 case ISD::VECREDUCE_SMIN:
678 case ISD::VECREDUCE_UMAX:
679 case ISD::VECREDUCE_UMIN:
680 return lowerVECREDUCE(Op, DAG);
681 case ISD::ConstantFP:
682 return lowerConstantFP(Op, DAG);
683 case ISD::SETCC:
684 return lowerSETCC(Op, DAG);
685 case ISD::FP_ROUND:
686 return lowerFP_ROUND(Op, DAG);
687 case ISD::FP_EXTEND:
688 return lowerFP_EXTEND(Op, DAG);
689 case ISD::SIGN_EXTEND_VECTOR_INREG:
690 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
691 case ISD::DYNAMIC_STACKALLOC:
692 return lowerDYNAMIC_STACKALLOC(Op, DAG);
693 case ISD::ANY_EXTEND:
694 return lowerANY_EXTEND(Op, DAG);
695 }
696 return SDValue();
697}
698
699// Helper to attempt to return a cheaper, bit-inverted version of \p V.
700static SDValue isNOT(SDValue V, SelectionDAG &DAG) {
701 // TODO: don't always ignore oneuse constraints.
702 V = peekThroughBitcasts(V);
703 EVT VT = V.getValueType();
704
705 // Match not(xor X, -1) -> X.
706 if (V.getOpcode() == ISD::XOR &&
707 (ISD::isBuildVectorAllOnes(N: V.getOperand(i: 1).getNode()) ||
708 isAllOnesConstant(V: V.getOperand(i: 1))))
709 return V.getOperand(i: 0);
710
711 // Match not(extract_subvector(not(X)) -> extract_subvector(X).
712 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
713 (isNullConstant(V: V.getOperand(i: 1)) || V.getOperand(i: 0).hasOneUse())) {
714 if (SDValue Not = isNOT(V: V.getOperand(i: 0), DAG)) {
715 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
716 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Not), VT, N1: Not,
717 N2: V.getOperand(i: 1));
718 }
719 }
720
721 // Match not(SplatVector(not(X)) -> SplatVector(X).
722 if (V.getOpcode() == ISD::BUILD_VECTOR) {
723 if (SDValue SplatValue =
724 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
725 if (!V->isOnlyUserOf(N: SplatValue.getNode()))
726 return SDValue();
727
728 if (SDValue Not = isNOT(V: SplatValue, DAG)) {
729 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
730 return DAG.getSplat(VT, DL: SDLoc(Not), Op: Not);
731 }
732 }
733 }
734
735 // Match not(or(not(X),not(Y))) -> and(X, Y).
736 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
737 V.getOperand(i: 0).hasOneUse() && V.getOperand(i: 1).hasOneUse()) {
738 // TODO: Handle cases with single NOT operand -> VANDN
739 if (SDValue Op1 = isNOT(V: V.getOperand(i: 1), DAG))
740 if (SDValue Op0 = isNOT(V: V.getOperand(i: 0), DAG))
741 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(V), VT, N1: DAG.getBitcast(VT, V: Op0),
742 N2: DAG.getBitcast(VT, V: Op1));
743 }
744
745 // TODO: Add more matching patterns. Such as,
746 // not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
747 // not(slt(C, X)) -> slt(X - 1, C)
748 return SDValue();
749}
750
751// Combine two ISD::FP_ROUND / LoongArchISD::VFCVT nodes with same type to
752// LoongArchISD::VFCVT. For example:
753// x1 = fp_round x, 0
754// y1 = fp_round y, 0
755// z = concat_vectors x1, y1
756// Or
757// x1 = LoongArch::VFCVT undef, x
758// y1 = LoongArch::VFCVT undef, y
759// z = LoongArchISD::VPACKEV y1, x1; or LoongArchISD::VPERMI y1, x1, 68
760// can be combined to:
761// z = LoongArch::VFCVT y, x
762static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG,
763 const LoongArchSubtarget &Subtarget) {
764 assert(((N->getOpcode() == ISD::CONCAT_VECTORS && N->getNumOperands() == 2) ||
765 (N->getOpcode() == LoongArchISD::VPACKEV) ||
766 (N->getOpcode() == LoongArchISD::VPERMI)) &&
767 "Invalid Node");
768
769 SDValue Op0 = peekThroughBitcasts(V: N->getOperand(Num: 0));
770 SDValue Op1 = peekThroughBitcasts(V: N->getOperand(Num: 1));
771 unsigned Opcode0 = Op0.getOpcode();
772 unsigned Opcode1 = Op1.getOpcode();
773 if (Opcode0 != Opcode1)
774 return SDValue();
775
776 if (Opcode0 != ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
777 return SDValue();
778
779 // Check if two nodes have only one use.
780 if (!Op0.hasOneUse() || !Op1.hasOneUse())
781 return SDValue();
782
783 EVT VT = N.getValueType();
784 EVT SVT0 = Op0.getValueType();
785 EVT SVT1 = Op1.getValueType();
786 // Check if two nodes have the same result type.
787 if (SVT0 != SVT1)
788 return SDValue();
789
790 // Check if two nodes have the same operand type.
791 EVT SSVT0 = Op0.getOperand(i: 0).getValueType();
792 EVT SSVT1 = Op1.getOperand(i: 0).getValueType();
793 if (SSVT0 != SSVT1)
794 return SDValue();
795
796 if (N->getOpcode() == ISD::CONCAT_VECTORS && Opcode0 == ISD::FP_ROUND) {
797 if (Subtarget.hasExtLASX() && VT.is256BitVector() && SVT0 == MVT::v4f32 &&
798 SSVT0 == MVT::v4f64) {
799 // A vector_shuffle is required in the final step, as xvfcvt instruction
800 // operates on each 128-bit segament as a lane.
801 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v8f32,
802 N1: Op1.getOperand(i: 0), N2: Op0.getOperand(i: 0));
803 SDValue Undef = DAG.getUNDEF(VT: Res.getValueType());
804 // After VFCVT, the high part of Res comes from the high parts of Op0 and
805 // Op1, and the low part comes from the low parts of Op0 and Op1. However,
806 // the desired order requires Op0 to fully occupy the lower half and Op1
807 // the upper half of Res. The Mask reorders the elements of Res to achieve
808 // this:
809 // - The first four elements (0, 1, 4, 5) come from Op0.
810 // - The next four elements (2, 3, 6, 7) come from Op1.
811 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
812 Res = DAG.getVectorShuffle(VT: Res.getValueType(), dl: DL, N1: Res, N2: Undef, Mask);
813 return DAG.getBitcast(VT, V: Res);
814 }
815 }
816
817 if ((N->getOpcode() == LoongArchISD::VPACKEV ||
818 N->getOpcode() == LoongArchISD::VPERMI) &&
819 Opcode0 == LoongArchISD::VFCVT) {
820 // For VPACKEV or VPERMI, check if the first operation of VFCVT is undef.
821 if (!Op0.getOperand(i: 0).isUndef() || !Op1.getOperand(i: 0).isUndef())
822 return SDValue();
823
824 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
825 return SDValue();
826
827 if (N->getOpcode() == LoongArchISD::VPACKEV &&
828 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
829 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32,
830 N1: Op0.getOperand(i: 1), N2: Op1.getOperand(i: 1));
831 return DAG.getBitcast(VT, V: Res);
832 }
833
834 if (N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
835 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
836 if (Imm != 68)
837 return SDValue();
838 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Op0.getOperand(i: 1),
839 N2: Op1.getOperand(i: 1));
840 }
841 }
842
843 return SDValue();
844}
845
846SDValue LoongArchTargetLowering::lowerFP_ROUND(SDValue Op,
847 SelectionDAG &DAG) const {
848 SDLoc DL(Op);
849 SDValue In = Op.getOperand(i: 0);
850 MVT VT = Op.getSimpleValueType();
851 MVT SVT = In.getSimpleValueType();
852
853 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
854 SDValue Lo, Hi;
855 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: In, DL);
856 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT, N1: Hi, N2: Lo);
857 }
858
859 return SDValue();
860}
861
862SDValue LoongArchTargetLowering::lowerFP_EXTEND(SDValue Op,
863 SelectionDAG &DAG) const {
864
865 SDLoc DL(Op);
866 EVT VT = Op.getValueType();
867 SDValue Src = Op->getOperand(Num: 0);
868 EVT SVT = Src.getValueType();
869
870 bool V2F32ToV2F64 =
871 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
872 bool V4F32ToV4F64 =
873 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
874 if (!V2F32ToV2F64 && !V4F32ToV4F64)
875 return SDValue();
876
877 // Check if Op is the high part of vector.
878 auto CheckVecHighPart = [](SDValue Op) {
879 Op = peekThroughBitcasts(V: Op);
880 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
881 SDValue SOp = Op.getOperand(i: 0);
882 EVT SVT = SOp.getValueType();
883 if (!SVT.isVector() || (SVT.getVectorNumElements() % 2 != 0))
884 return SDValue();
885
886 const uint64_t Imm = Op.getConstantOperandVal(i: 1);
887 if (Imm == SVT.getVectorNumElements() / 2)
888 return SOp;
889 return SDValue();
890 }
891 return SDValue();
892 };
893
894 unsigned Opcode;
895 SDValue VFCVTOp;
896 EVT WideOpVT = SVT.getSimpleVT().getDoubleNumVectorElementsVT();
897 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
898
899 // If the operand of ISD::FP_EXTEND comes from the high part of vector,
900 // generate LoongArchISD::VFCVTH, otherwise LoongArchISD::VFCVTL.
901 if (SDValue V = CheckVecHighPart(Src)) {
902 assert(V.getValueSizeInBits() == WideOpVT.getSizeInBits() &&
903 "Unexpected wide vector");
904 Opcode = LoongArchISD::VFCVTH;
905 VFCVTOp = DAG.getBitcast(VT: WideOpVT, V);
906 } else {
907 Opcode = LoongArchISD::VFCVTL;
908 VFCVTOp = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideOpVT,
909 N1: DAG.getUNDEF(VT: WideOpVT), N2: Src, N3: ZeroIdx);
910 }
911
912 // v2f64 = fp_extend v2f32
913 if (V2F32ToV2F64)
914 return DAG.getNode(Opcode, DL, VT, Operand: VFCVTOp);
915
916 // v4f64 = fp_extend v4f32
917 if (V4F32ToV4F64) {
918 // XVFCVT instruction operates on each 128-bit segment as a lane, so a
919 // vector_shuffle is required firstly.
920 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
921 SDValue Res = DAG.getVectorShuffle(VT: WideOpVT, dl: DL, N1: VFCVTOp,
922 N2: DAG.getUNDEF(VT: WideOpVT), Mask);
923 Res = DAG.getNode(Opcode, DL, VT, Operand: Res);
924 return Res;
925 }
926
927 return SDValue();
928}
929
930SDValue LoongArchTargetLowering::lowerConstantFP(SDValue Op,
931 SelectionDAG &DAG) const {
932 EVT VT = Op.getValueType();
933 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
934 const APFloat &FPVal = CFP->getValueAPF();
935 SDLoc DL(CFP);
936
937 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
938 (VT == MVT::f64 && Subtarget.hasBasicD()));
939
940 // If value is 0.0 or -0.0, just ignore it.
941 if (FPVal.isZero())
942 return SDValue();
943
944 // If lsx enabled, use cheaper 'vldi' instruction if possible.
945 if (isFPImmVLDILegal(Imm: FPVal, VT))
946 return SDValue();
947
948 // Construct as integer, and move to float register.
949 APInt INTVal = FPVal.bitcastToAPInt();
950
951 // If more than MaterializeFPImmInsNum instructions will be used to
952 // generate the INTVal and move it to float register, fallback to
953 // use floating point load from the constant pool.
954 auto Seq = LoongArchMatInt::generateInstSeq(Val: INTVal.getSExtValue());
955 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
956 if (InsNum > MaterializeFPImmInsNum && !FPVal.isOne())
957 return SDValue();
958
959 switch (VT.getSimpleVT().SimpleTy) {
960 default:
961 llvm_unreachable("Unexpected floating point type!");
962 break;
963 case MVT::f32: {
964 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i32);
965 if (Subtarget.is64Bit())
966 NewVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: NewVal);
967 return DAG.getNode(Opcode: Subtarget.is64Bit() ? LoongArchISD::MOVGR2FR_W_LA64
968 : LoongArchISD::MOVGR2FR_W,
969 DL, VT, Operand: NewVal);
970 }
971 case MVT::f64: {
972 if (Subtarget.is64Bit()) {
973 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i64);
974 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D, DL, VT, Operand: NewVal);
975 }
976 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
977 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
978 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D_LO_HI, DL, VT, N1: Lo, N2: Hi);
979 }
980 }
981
982 return SDValue();
983}
984
985// Ensure SETCC result and operand have the same bit width; isel does not
986// support mismatched widths.
987SDValue LoongArchTargetLowering::lowerSETCC(SDValue Op,
988 SelectionDAG &DAG) const {
989 SDLoc DL(Op);
990 EVT ResultVT = Op.getValueType();
991 EVT OperandVT = Op.getOperand(i: 0).getValueType();
992
993 EVT SetCCResultVT =
994 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT);
995
996 if (ResultVT == SetCCResultVT)
997 return Op;
998
999 assert(Op.getOperand(0).getValueType() == Op.getOperand(1).getValueType() &&
1000 "SETCC operands must have the same type!");
1001
1002 SDValue SetCCNode =
1003 DAG.getNode(Opcode: ISD::SETCC, DL, VT: SetCCResultVT, N1: Op.getOperand(i: 0),
1004 N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2));
1005
1006 if (ResultVT.bitsGT(VT: SetCCResultVT))
1007 SetCCNode = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: ResultVT, Operand: SetCCNode);
1008 else if (ResultVT.bitsLT(VT: SetCCResultVT))
1009 SetCCNode = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: SetCCNode);
1010
1011 return SetCCNode;
1012}
1013
1014// Lower sext_invec using vslti instructions.
1015// For example:
1016// %b = sext <4 x i16> %a to <4 x i32>
1017// can be lowered to:
1018// VSLTI_H vr2, vr1, 0
1019// VILVL.H vr1, vr2, vr1
1020SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1021 SDValue Op, SelectionDAG &DAG) const {
1022 SDLoc DL(Op);
1023 SDValue Src = Op.getOperand(i: 0);
1024 MVT SrcVT = Src.getSimpleValueType();
1025 MVT DstVT = Op.getSimpleValueType();
1026
1027 if (!SrcVT.is128BitVector())
1028 return SDValue();
1029
1030 // lower to VSLTI + VILVL if extend could be done in single step.
1031 if (DstVT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits() == 2) {
1032 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1033 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Src, N2: Zero,
1034 N3: DAG.getCondCode(Cond: ISD::SETLT));
1035 SDValue LoInterleaved =
1036 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Src);
1037
1038 return DAG.getBitcast(VT: DstVT, V: LoInterleaved);
1039 }
1040
1041 return SDValue();
1042}
1043
1044// ANY_EXTEND can be replaced by ZERO_EXTEND when LASX is enabled.
1045SDValue LoongArchTargetLowering::lowerANY_EXTEND(SDValue Op,
1046 SelectionDAG &DAG) const {
1047 assert(Subtarget.hasExtLASX());
1048 // We don't have corresponding instrunction for ANY_EXTEND, lowering it to
1049 // ZERO_EXTEND won't break its semantics, while avoid scalar extract/insert.
1050 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: Op.getValueType(),
1051 Operand: Op.getOperand(i: 0));
1052}
1053
1054// Lower vecreduce_add using vhaddw instructions.
1055// For Example:
1056// call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
1057// can be lowered to:
1058// VHADDW_D_W vr0, vr0, vr0
1059// VHADDW_Q_D vr0, vr0, vr0
1060// VPICKVE2GR_D a0, vr0, 0
1061// ADDI_W a0, a0, 0
1062SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
1063 SelectionDAG &DAG) const {
1064
1065 SDLoc DL(Op);
1066 MVT OpVT = Op.getSimpleValueType();
1067 SDValue Val = Op.getOperand(i: 0);
1068
1069 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1070 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1071 unsigned ResBits = OpVT.getScalarSizeInBits();
1072
1073 unsigned LegalVecSize = 128;
1074 bool isLASX256Vector =
1075 Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
1076
1077 // Ensure operand type legal or enable it legal.
1078 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1079 Val = DAG.WidenVector(N: Val, DL);
1080 }
1081
1082 // NumEles is designed for iterations count, v4i32 for LSX
1083 // and v8i32 for LASX should have the same count.
1084 if (isLASX256Vector) {
1085 NumEles /= 2;
1086 LegalVecSize = 256;
1087 }
1088
1089 EleBits *= 2;
1090 for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1091 EleBits = std::min(a: EleBits, b: 64u);
1092 MVT IntTy = MVT::getIntegerVT(BitWidth: EleBits);
1093 MVT VecTy = MVT::getVectorVT(VT: IntTy, NumElements: LegalVecSize / EleBits);
1094 Val = DAG.getNode(Opcode: LoongArchISD::VHADDW, DL, VT: VecTy, N1: Val, N2: Val);
1095 }
1096
1097 if (isLASX256Vector) {
1098 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: Val,
1099 N2: DAG.getConstant(Val: 2, DL, VT: Subtarget.getGRLenVT()));
1100 Val = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::v4i64, N1: Tmp, N2: Val);
1101 }
1102
1103 Val = DAG.getBitcast(VT: MVT::getVectorVT(VT: OpVT, NumElements: LegalVecSize / ResBits), V: Val);
1104 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1105 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
1106}
1107
1108// Lower vecreduce_and/or/xor/[s/u]max/[s/u]min.
1109// For Example:
1110// call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
1111// can be lowered to:
1112// VBSRL_V vr1, vr0, 8
1113// VMAX_W vr0, vr1, vr0
1114// VBSRL_V vr1, vr0, 4
1115// VMAX_W vr0, vr1, vr0
1116// VPICKVE2GR_W a0, vr0, 0
1117// For 256 bit vector, it is illegal and will be spilt into
1118// two 128 bit vector by default then processed by this.
1119SDValue LoongArchTargetLowering::lowerVECREDUCE(SDValue Op,
1120 SelectionDAG &DAG) const {
1121 SDLoc DL(Op);
1122
1123 MVT OpVT = Op.getSimpleValueType();
1124 SDValue Val = Op.getOperand(i: 0);
1125
1126 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1127 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1128
1129 // Ensure operand type legal or enable it legal.
1130 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1131 Val = DAG.WidenVector(N: Val, DL);
1132 }
1133
1134 unsigned Opcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
1135 MVT VecTy = Val.getSimpleValueType();
1136 MVT GRLenVT = Subtarget.getGRLenVT();
1137
1138 for (int i = NumEles; i > 1; i /= 2) {
1139 SDValue ShiftAmt = DAG.getConstant(Val: i * EleBits / 16, DL, VT: GRLenVT);
1140 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: VecTy, N1: Val, N2: ShiftAmt);
1141 Val = DAG.getNode(Opcode, DL, VT: VecTy, N1: Tmp, N2: Val);
1142 }
1143
1144 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1145 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
1146}
1147
1148SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
1149 SelectionDAG &DAG) const {
1150 unsigned IsData = Op.getConstantOperandVal(i: 4);
1151
1152 // We don't support non-data prefetch.
1153 // Just preserve the chain.
1154 if (!IsData)
1155 return Op.getOperand(i: 0);
1156
1157 return Op;
1158}
1159
1160SDValue LoongArchTargetLowering::lowerRotate(SDValue Op,
1161 SelectionDAG &DAG) const {
1162 MVT VT = Op.getSimpleValueType();
1163 assert(VT.isVector() && "Unexpected type");
1164
1165 SDLoc DL(Op);
1166 SDValue R = Op.getOperand(i: 0);
1167 SDValue Amt = Op.getOperand(i: 1);
1168 unsigned Opcode = Op.getOpcode();
1169 unsigned EltSizeInBits = VT.getScalarSizeInBits();
1170
1171 auto checkCstSplat = [](SDValue V, APInt &CstSplatValue) {
1172 if (V.getOpcode() != ISD::BUILD_VECTOR)
1173 return false;
1174 if (SDValue SplatValue =
1175 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
1176 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
1177 CstSplatValue = C->getAPIntValue();
1178 return true;
1179 }
1180 }
1181 return false;
1182 };
1183
1184 // Check for constant splat rotation amount.
1185 APInt CstSplatValue;
1186 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1187 bool isROTL = Opcode == ISD::ROTL;
1188
1189 // Check for splat rotate by zero.
1190 if (IsCstSplat && CstSplatValue.urem(RHS: EltSizeInBits) == 0)
1191 return R;
1192
1193 // LoongArch targets always prefer ISD::ROTR.
1194 if (isROTL) {
1195 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
1196 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: R,
1197 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: Amt));
1198 }
1199
1200 // Rotate by a immediate.
1201 if (IsCstSplat) {
1202 // ISD::ROTR: Attemp to rotate by a positive immediate.
1203 SDValue Bits = DAG.getConstant(Val: EltSizeInBits, DL, VT);
1204 if (SDValue Urem =
1205 DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL, VT, Ops: {Amt, Bits}))
1206 return DAG.getNode(Opcode, DL, VT, N1: R, N2: Urem);
1207 }
1208
1209 return Op;
1210}
1211
1212// Return true if Val is equal to (setcc LHS, RHS, CC).
1213// Return false if Val is the inverse of (setcc LHS, RHS, CC).
1214// Otherwise, return std::nullopt.
1215static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
1216 ISD::CondCode CC, SDValue Val) {
1217 assert(Val->getOpcode() == ISD::SETCC);
1218 SDValue LHS2 = Val.getOperand(i: 0);
1219 SDValue RHS2 = Val.getOperand(i: 1);
1220 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
1221
1222 if (LHS == LHS2 && RHS == RHS2) {
1223 if (CC == CC2)
1224 return true;
1225 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1226 return false;
1227 } else if (LHS == RHS2 && RHS == LHS2) {
1228 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
1229 if (CC == CC2)
1230 return true;
1231 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1232 return false;
1233 }
1234
1235 return std::nullopt;
1236}
1237
1238static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
1239 const LoongArchSubtarget &Subtarget) {
1240 SDValue CondV = N->getOperand(Num: 0);
1241 SDValue TrueV = N->getOperand(Num: 1);
1242 SDValue FalseV = N->getOperand(Num: 2);
1243 MVT VT = N->getSimpleValueType(ResNo: 0);
1244 SDLoc DL(N);
1245
1246 // (select c, -1, y) -> -c | y
1247 if (isAllOnesConstant(V: TrueV)) {
1248 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1249 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1250 }
1251 // (select c, y, -1) -> (c-1) | y
1252 if (isAllOnesConstant(V: FalseV)) {
1253 SDValue Neg =
1254 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1255 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1256 }
1257
1258 // (select c, 0, y) -> (c-1) & y
1259 if (isNullConstant(V: TrueV)) {
1260 SDValue Neg =
1261 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1262 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1263 }
1264 // (select c, y, 0) -> -c & y
1265 if (isNullConstant(V: FalseV)) {
1266 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1267 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1268 }
1269
1270 // select c, ~x, x --> xor -c, x
1271 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
1272 const APInt &TrueVal = TrueV->getAsAPIntVal();
1273 const APInt &FalseVal = FalseV->getAsAPIntVal();
1274 if (~TrueVal == FalseVal) {
1275 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1276 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
1277 }
1278 }
1279
1280 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
1281 // when both truev and falsev are also setcc.
1282 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
1283 FalseV.getOpcode() == ISD::SETCC) {
1284 SDValue LHS = CondV.getOperand(i: 0);
1285 SDValue RHS = CondV.getOperand(i: 1);
1286 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1287
1288 // (select x, x, y) -> x | y
1289 // (select !x, x, y) -> x & y
1290 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
1291 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
1292 N2: DAG.getFreeze(V: FalseV));
1293 }
1294 // (select x, y, x) -> x & y
1295 // (select !x, y, x) -> x | y
1296 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
1297 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
1298 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
1299 }
1300 }
1301
1302 return SDValue();
1303}
1304
1305// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
1306// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
1307// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
1308// being `0` or `-1`. In such cases we can replace `select` with `and`.
1309// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
1310// than `c0`?
1311static SDValue
1312foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
1313 const LoongArchSubtarget &Subtarget) {
1314 unsigned SelOpNo = 0;
1315 SDValue Sel = BO->getOperand(Num: 0);
1316 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1317 SelOpNo = 1;
1318 Sel = BO->getOperand(Num: 1);
1319 }
1320
1321 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1322 return SDValue();
1323
1324 unsigned ConstSelOpNo = 1;
1325 unsigned OtherSelOpNo = 2;
1326 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
1327 ConstSelOpNo = 2;
1328 OtherSelOpNo = 1;
1329 }
1330 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
1331 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
1332 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
1333 return SDValue();
1334
1335 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
1336 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
1337 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
1338 return SDValue();
1339
1340 SDLoc DL(Sel);
1341 EVT VT = BO->getValueType(ResNo: 0);
1342
1343 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1344 if (SelOpNo == 1)
1345 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
1346
1347 SDValue NewConstOp =
1348 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
1349 if (!NewConstOp)
1350 return SDValue();
1351
1352 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
1353 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
1354 return SDValue();
1355
1356 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
1357 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1358 if (SelOpNo == 1)
1359 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
1360 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
1361
1362 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1363 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1364 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
1365}
1366
1367// Changes the condition code and swaps operands if necessary, so the SetCC
1368// operation matches one of the comparisons supported directly by branches
1369// in the LoongArch ISA. May adjust compares to favor compare with 0 over
1370// compare with 1/-1.
1371static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1372 ISD::CondCode &CC, SelectionDAG &DAG) {
1373 // If this is a single bit test that can't be handled by ANDI, shift the
1374 // bit to be tested to the MSB and perform a signed compare with 0.
1375 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
1376 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1377 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
1378 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
1379 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
1380 unsigned ShAmt = 0;
1381 if (isPowerOf2_64(Value: Mask)) {
1382 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
1383 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
1384 } else {
1385 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
1386 }
1387
1388 LHS = LHS.getOperand(i: 0);
1389 if (ShAmt != 0)
1390 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
1391 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
1392 return;
1393 }
1394 }
1395
1396 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
1397 int64_t C = RHSC->getSExtValue();
1398 switch (CC) {
1399 default:
1400 break;
1401 case ISD::SETGT:
1402 // Convert X > -1 to X >= 0.
1403 if (C == -1) {
1404 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1405 CC = ISD::SETGE;
1406 return;
1407 }
1408 break;
1409 case ISD::SETLT:
1410 // Convert X < 1 to 0 >= X.
1411 if (C == 1) {
1412 RHS = LHS;
1413 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1414 CC = ISD::SETGE;
1415 return;
1416 }
1417 break;
1418 }
1419 }
1420
1421 switch (CC) {
1422 default:
1423 break;
1424 case ISD::SETGT:
1425 case ISD::SETLE:
1426 case ISD::SETUGT:
1427 case ISD::SETULE:
1428 CC = ISD::getSetCCSwappedOperands(Operation: CC);
1429 std::swap(a&: LHS, b&: RHS);
1430 break;
1431 }
1432}
1433
1434SDValue LoongArchTargetLowering::lowerSELECT(SDValue Op,
1435 SelectionDAG &DAG) const {
1436 SDValue CondV = Op.getOperand(i: 0);
1437 SDValue TrueV = Op.getOperand(i: 1);
1438 SDValue FalseV = Op.getOperand(i: 2);
1439 SDLoc DL(Op);
1440 MVT VT = Op.getSimpleValueType();
1441 MVT GRLenVT = Subtarget.getGRLenVT();
1442
1443 if (SDValue V = combineSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
1444 return V;
1445
1446 if (Op.hasOneUse()) {
1447 unsigned UseOpc = Op->user_begin()->getOpcode();
1448 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
1449 SDNode *BinOp = *Op->user_begin();
1450 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
1451 DAG, Subtarget)) {
1452 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
1453 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
1454 // may return a constant node and cause crash in lowerSELECT.
1455 if (NewSel.getOpcode() == ISD::SELECT)
1456 return lowerSELECT(Op: NewSel, DAG);
1457 return NewSel;
1458 }
1459 }
1460 }
1461
1462 // If the condition is not an integer SETCC which operates on GRLenVT, we need
1463 // to emit a LoongArchISD::SELECT_CC comparing the condition to zero. i.e.:
1464 // (select condv, truev, falsev)
1465 // -> (loongarchisd::select_cc condv, zero, setne, truev, falsev)
1466 if (CondV.getOpcode() != ISD::SETCC ||
1467 CondV.getOperand(i: 0).getSimpleValueType() != GRLenVT) {
1468 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: GRLenVT);
1469 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
1470
1471 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
1472
1473 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1474 }
1475
1476 // If the CondV is the output of a SETCC node which operates on GRLenVT
1477 // inputs, then merge the SETCC node into the lowered LoongArchISD::SELECT_CC
1478 // to take advantage of the integer compare+branch instructions. i.e.: (select
1479 // (setcc lhs, rhs, cc), truev, falsev)
1480 // -> (loongarchisd::select_cc lhs, rhs, cc, truev, falsev)
1481 SDValue LHS = CondV.getOperand(i: 0);
1482 SDValue RHS = CondV.getOperand(i: 1);
1483 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1484
1485 // Special case for a select of 2 constants that have a difference of 1.
1486 // Normally this is done by DAGCombine, but if the select is introduced by
1487 // type legalization or op legalization, we miss it. Restricting to SETLT
1488 // case for now because that is what signed saturating add/sub need.
1489 // FIXME: We don't need the condition to be SETLT or even a SETCC,
1490 // but we would probably want to swap the true/false values if the condition
1491 // is SETGE/SETLE to avoid an XORI.
1492 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
1493 CCVal == ISD::SETLT) {
1494 const APInt &TrueVal = TrueV->getAsAPIntVal();
1495 const APInt &FalseVal = FalseV->getAsAPIntVal();
1496 if (TrueVal - 1 == FalseVal)
1497 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
1498 if (TrueVal + 1 == FalseVal)
1499 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
1500 }
1501
1502 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1503 // 1 < x ? x : 1 -> 0 < x ? x : 1
1504 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
1505 RHS == TrueV && LHS == FalseV) {
1506 LHS = DAG.getConstant(Val: 0, DL, VT);
1507 // 0 <u x is the same as x != 0.
1508 if (CCVal == ISD::SETULT) {
1509 std::swap(a&: LHS, b&: RHS);
1510 CCVal = ISD::SETNE;
1511 }
1512 }
1513
1514 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
1515 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
1516 RHS == FalseV) {
1517 RHS = DAG.getConstant(Val: 0, DL, VT);
1518 }
1519
1520 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1521
1522 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
1523 // (select (setcc lhs, rhs, CC), constant, falsev)
1524 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
1525 std::swap(a&: TrueV, b&: FalseV);
1526 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
1527 }
1528
1529 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
1530 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1531}
1532
1533SDValue LoongArchTargetLowering::lowerBRCOND(SDValue Op,
1534 SelectionDAG &DAG) const {
1535 SDValue CondV = Op.getOperand(i: 1);
1536 SDLoc DL(Op);
1537 MVT GRLenVT = Subtarget.getGRLenVT();
1538
1539 if (CondV.getOpcode() == ISD::SETCC) {
1540 if (CondV.getOperand(i: 0).getValueType() == GRLenVT) {
1541 SDValue LHS = CondV.getOperand(i: 0);
1542 SDValue RHS = CondV.getOperand(i: 1);
1543 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1544
1545 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1546
1547 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1548 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1549 N1: Op.getOperand(i: 0), N2: LHS, N3: RHS, N4: TargetCC,
1550 N5: Op.getOperand(i: 2));
1551 } else if (CondV.getOperand(i: 0).getValueType().isFloatingPoint()) {
1552 return DAG.getNode(Opcode: LoongArchISD::BRCOND, DL, VT: Op.getValueType(),
1553 N1: Op.getOperand(i: 0), N2: CondV, N3: Op.getOperand(i: 2));
1554 }
1555 }
1556
1557 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1558 N1: Op.getOperand(i: 0), N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT),
1559 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
1560}
1561
1562SDValue
1563LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
1564 SelectionDAG &DAG) const {
1565 SDLoc DL(Op);
1566 MVT OpVT = Op.getSimpleValueType();
1567
1568 SDValue Vector = DAG.getUNDEF(VT: OpVT);
1569 SDValue Val = Op.getOperand(i: 0);
1570 SDValue Idx = DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT());
1571
1572 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: OpVT, N1: Vector, N2: Val, N3: Idx);
1573}
1574
1575SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
1576 SelectionDAG &DAG) const {
1577 EVT ResTy = Op->getValueType(ResNo: 0);
1578 SDValue Src = Op->getOperand(Num: 0);
1579 SDLoc DL(Op);
1580
1581 // LoongArchISD::BITREV_8B is not supported on LA32.
1582 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1583 return SDValue();
1584
1585 EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
1586 unsigned int OrigEltNum = ResTy.getVectorNumElements();
1587 unsigned int NewEltNum = NewVT.getVectorNumElements();
1588
1589 SDValue NewSrc = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: NewVT, Operand: Src);
1590
1591 SmallVector<SDValue, 8> Ops;
1592 for (unsigned int i = 0; i < NewEltNum; i++) {
1593 SDValue Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: NewSrc,
1594 N2: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
1595 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1596 ? (unsigned)LoongArchISD::BITREV_8B
1597 : (unsigned)ISD::BITREVERSE;
1598 Ops.push_back(Elt: DAG.getNode(Opcode: RevOp, DL, VT: MVT::i64, Operand: Op));
1599 }
1600 SDValue Res =
1601 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResTy, Operand: DAG.getBuildVector(VT: NewVT, DL, Ops));
1602
1603 switch (ResTy.getSimpleVT().SimpleTy) {
1604 default:
1605 return SDValue();
1606 case MVT::v16i8:
1607 case MVT::v32i8:
1608 return Res;
1609 case MVT::v8i16:
1610 case MVT::v16i16:
1611 case MVT::v4i32:
1612 case MVT::v8i32: {
1613 SmallVector<int, 32> Mask;
1614 for (unsigned int i = 0; i < NewEltNum; i++)
1615 for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
1616 Mask.push_back(Elt: j + (OrigEltNum / NewEltNum) * i);
1617 return DAG.getVectorShuffle(VT: ResTy, dl: DL, N1: Res, N2: DAG.getUNDEF(VT: ResTy), Mask);
1618 }
1619 }
1620}
1621
1622// Widen element type to get a new mask value (if possible).
1623// For example:
1624// shufflevector <4 x i32> %a, <4 x i32> %b,
1625// <4 x i32> <i32 6, i32 7, i32 2, i32 3>
1626// is equivalent to:
1627// shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 1>
1628// can be lowered to:
1629// VPACKOD_D vr0, vr0, vr1
1630static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1631 SDValue V1, SDValue V2, SelectionDAG &DAG) {
1632 unsigned EltBits = VT.getScalarSizeInBits();
1633
1634 if (EltBits > 32 || EltBits == 1)
1635 return SDValue();
1636
1637 SmallVector<int, 8> NewMask;
1638 if (widenShuffleMaskElts(M: Mask, NewMask)) {
1639 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: EltBits * 2)
1640 : MVT::getIntegerVT(BitWidth: EltBits * 2);
1641 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
1642 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT)) {
1643 SDValue NewV1 = DAG.getBitcast(VT: NewVT, V: V1);
1644 SDValue NewV2 = DAG.getBitcast(VT: NewVT, V: V2);
1645 return DAG.getBitcast(
1646 VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: NewV1, N2: NewV2, Mask: NewMask));
1647 }
1648 }
1649
1650 return SDValue();
1651}
1652
1653/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
1654/// instruction.
1655// The funciton matches elements from one of the input vector shuffled to the
1656// left or right with zeroable elements 'shifted in'. It handles both the
1657// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
1658// lane.
1659// Mostly copied from X86.
1660static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
1661 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
1662 int MaskOffset, const APInt &Zeroable) {
1663 int Size = Mask.size();
1664 unsigned SizeInBits = Size * ScalarSizeInBits;
1665
1666 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
1667 for (int i = 0; i < Size; i += Scale)
1668 for (int j = 0; j < Shift; ++j)
1669 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
1670 return false;
1671
1672 return true;
1673 };
1674
1675 auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
1676 int Step = 1) {
1677 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
1678 if (!(Mask[i] == -1 || Mask[i] == Low))
1679 return false;
1680 return true;
1681 };
1682
1683 auto MatchShift = [&](int Shift, int Scale, bool Left) {
1684 for (int i = 0; i != Size; i += Scale) {
1685 unsigned Pos = Left ? i + Shift : i;
1686 unsigned Low = Left ? i : i + Shift;
1687 unsigned Len = Scale - Shift;
1688 if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
1689 return -1;
1690 }
1691
1692 int ShiftEltBits = ScalarSizeInBits * Scale;
1693 bool ByteShift = ShiftEltBits > 64;
1694 Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1695 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1696 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1697
1698 // Normalize the scale for byte shifts to still produce an i64 element
1699 // type.
1700 Scale = ByteShift ? Scale / 2 : Scale;
1701
1702 // We need to round trip through the appropriate type for the shift.
1703 MVT ShiftSVT = MVT::getIntegerVT(BitWidth: ScalarSizeInBits * Scale);
1704 ShiftVT = ByteShift ? MVT::getVectorVT(VT: MVT::i8, NumElements: SizeInBits / 8)
1705 : MVT::getVectorVT(VT: ShiftSVT, NumElements: Size / Scale);
1706 return (int)ShiftAmt;
1707 };
1708
1709 unsigned MaxWidth = 128;
1710 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1711 for (int Shift = 1; Shift != Scale; ++Shift)
1712 for (bool Left : {true, false})
1713 if (CheckZeros(Shift, Scale, Left)) {
1714 int ShiftAmt = MatchShift(Shift, Scale, Left);
1715 if (0 < ShiftAmt)
1716 return ShiftAmt;
1717 }
1718
1719 // no match
1720 return -1;
1721}
1722
1723/// Lower VECTOR_SHUFFLE as shift (if possible).
1724///
1725/// For example:
1726/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1727/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
1728/// is lowered to:
1729/// (VBSLL_V $v0, $v0, 4)
1730///
1731/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1732/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
1733/// is lowered to:
1734/// (VSLLI_D $v0, $v0, 32)
1735static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef<int> Mask,
1736 MVT VT, SDValue V1, SDValue V2,
1737 SelectionDAG &DAG,
1738 const LoongArchSubtarget &Subtarget,
1739 const APInt &Zeroable) {
1740 int Size = Mask.size();
1741 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
1742
1743 MVT ShiftVT;
1744 SDValue V = V1;
1745 unsigned Opcode;
1746
1747 // Try to match shuffle against V1 shift.
1748 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1749 Mask, MaskOffset: 0, Zeroable);
1750
1751 // If V1 failed, try to match shuffle against V2 shift.
1752 if (ShiftAmt < 0) {
1753 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1754 Mask, MaskOffset: Size, Zeroable);
1755 V = V2;
1756 }
1757
1758 if (ShiftAmt < 0)
1759 return SDValue();
1760
1761 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
1762 "Illegal integer vector type");
1763 V = DAG.getBitcast(VT: ShiftVT, V);
1764 V = DAG.getNode(Opcode, DL, VT: ShiftVT, N1: V,
1765 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: Subtarget.getGRLenVT()));
1766 return DAG.getBitcast(VT, V);
1767}
1768
1769/// Determine whether a range fits a regular pattern of values.
1770/// This function accounts for the possibility of jumping over the End iterator.
1771template <typename ValType>
1772static bool
1773fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,
1774 unsigned CheckStride,
1775 typename SmallVectorImpl<ValType>::const_iterator End,
1776 ValType ExpectedIndex, unsigned ExpectedIndexStride) {
1777 auto &I = Begin;
1778
1779 while (I != End) {
1780 if (*I != -1 && *I != ExpectedIndex)
1781 return false;
1782 ExpectedIndex += ExpectedIndexStride;
1783
1784 // Incrementing past End is undefined behaviour so we must increment one
1785 // step at a time and check for End at each step.
1786 for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
1787 ; // Empty loop body.
1788 }
1789 return true;
1790}
1791
1792/// Compute whether each element of a shuffle is zeroable.
1793///
1794/// A "zeroable" vector shuffle element is one which can be lowered to zero.
1795static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
1796 SDValue V2, APInt &KnownUndef,
1797 APInt &KnownZero) {
1798 int Size = Mask.size();
1799 KnownUndef = KnownZero = APInt::getZero(numBits: Size);
1800
1801 V1 = peekThroughBitcasts(V: V1);
1802 V2 = peekThroughBitcasts(V: V2);
1803
1804 bool V1IsZero = ISD::isBuildVectorAllZeros(N: V1.getNode());
1805 bool V2IsZero = ISD::isBuildVectorAllZeros(N: V2.getNode());
1806
1807 int VectorSizeInBits = V1.getValueSizeInBits();
1808 int ScalarSizeInBits = VectorSizeInBits / Size;
1809 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
1810 (void)ScalarSizeInBits;
1811
1812 for (int i = 0; i < Size; ++i) {
1813 int M = Mask[i];
1814 if (M < 0) {
1815 KnownUndef.setBit(i);
1816 continue;
1817 }
1818 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
1819 KnownZero.setBit(i);
1820 continue;
1821 }
1822 }
1823}
1824
1825/// Test whether a shuffle mask is equivalent within each sub-lane.
1826///
1827/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
1828/// non-trivial to compute in the face of undef lanes. The representation is
1829/// suitable for use with existing 128-bit shuffles as entries from the second
1830/// vector have been remapped to [LaneSize, 2*LaneSize).
1831static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
1832 ArrayRef<int> Mask,
1833 SmallVectorImpl<int> &RepeatedMask) {
1834 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
1835 RepeatedMask.assign(NumElts: LaneSize, Elt: -1);
1836 int Size = Mask.size();
1837 for (int i = 0; i < Size; ++i) {
1838 assert(Mask[i] == -1 || Mask[i] >= 0);
1839 if (Mask[i] < 0)
1840 continue;
1841 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
1842 // This entry crosses lanes, so there is no way to model this shuffle.
1843 return false;
1844
1845 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
1846 // Adjust second vector indices to start at LaneSize instead of Size.
1847 int LocalM =
1848 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1849 if (RepeatedMask[i % LaneSize] < 0)
1850 // This is the first non-undef entry in this slot of a 128-bit lane.
1851 RepeatedMask[i % LaneSize] = LocalM;
1852 else if (RepeatedMask[i % LaneSize] != LocalM)
1853 // Found a mismatch with the repeated mask.
1854 return false;
1855 }
1856 return true;
1857}
1858
1859/// Attempts to match vector shuffle as byte rotation.
1860static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
1861 ArrayRef<int> Mask) {
1862
1863 SDValue Lo, Hi;
1864 SmallVector<int, 16> RepeatedMask;
1865
1866 if (!isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask, RepeatedMask))
1867 return -1;
1868
1869 int NumElts = RepeatedMask.size();
1870 int Rotation = 0;
1871 int Scale = 16 / NumElts;
1872
1873 for (int i = 0; i < NumElts; ++i) {
1874 int M = RepeatedMask[i];
1875 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1876 "Unexpected mask index.");
1877 if (M < 0)
1878 continue;
1879
1880 // Determine where a rotated vector would have started.
1881 int StartIdx = i - (M % NumElts);
1882 if (StartIdx == 0)
1883 return -1;
1884
1885 // If we found the tail of a vector the rotation must be the missing
1886 // front. If we found the head of a vector, it must be how much of the
1887 // head.
1888 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1889
1890 if (Rotation == 0)
1891 Rotation = CandidateRotation;
1892 else if (Rotation != CandidateRotation)
1893 return -1;
1894
1895 // Compute which value this mask is pointing at.
1896 SDValue MaskV = M < NumElts ? V1 : V2;
1897
1898 // Compute which of the two target values this index should be assigned
1899 // to. This reflects whether the high elements are remaining or the low
1900 // elements are remaining.
1901 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
1902
1903 // Either set up this value if we've not encountered it before, or check
1904 // that it remains consistent.
1905 if (!TargetV)
1906 TargetV = MaskV;
1907 else if (TargetV != MaskV)
1908 return -1;
1909 }
1910
1911 // Check that we successfully analyzed the mask, and normalize the results.
1912 assert(Rotation != 0 && "Failed to locate a viable rotation!");
1913 assert((Lo || Hi) && "Failed to find a rotated input vector!");
1914 if (!Lo)
1915 Lo = Hi;
1916 else if (!Hi)
1917 Hi = Lo;
1918
1919 V1 = Lo;
1920 V2 = Hi;
1921
1922 return Rotation * Scale;
1923}
1924
1925/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
1926///
1927/// For example:
1928/// %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
1929/// <2 x i32> <i32 3, i32 0>
1930/// is lowered to:
1931/// (VBSRL_V $v1, $v1, 8)
1932/// (VBSLL_V $v0, $v0, 8)
1933/// (VOR_V $v0, $V0, $v1)
1934static SDValue
1935lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1936 SDValue V1, SDValue V2, SelectionDAG &DAG,
1937 const LoongArchSubtarget &Subtarget) {
1938
1939 SDValue Lo = V1, Hi = V2;
1940 int ByteRotation = matchShuffleAsByteRotate(VT, V1&: Lo, V2&: Hi, Mask);
1941 if (ByteRotation <= 0)
1942 return SDValue();
1943
1944 MVT ByteVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VT.getSizeInBits() / 8);
1945 Lo = DAG.getBitcast(VT: ByteVT, V: Lo);
1946 Hi = DAG.getBitcast(VT: ByteVT, V: Hi);
1947
1948 int LoByteShift = 16 - ByteRotation;
1949 int HiByteShift = ByteRotation;
1950 MVT GRLenVT = Subtarget.getGRLenVT();
1951
1952 SDValue LoShift = DAG.getNode(Opcode: LoongArchISD::VBSLL, DL, VT: ByteVT, N1: Lo,
1953 N2: DAG.getConstant(Val: LoByteShift, DL, VT: GRLenVT));
1954 SDValue HiShift = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: ByteVT, N1: Hi,
1955 N2: DAG.getConstant(Val: HiByteShift, DL, VT: GRLenVT));
1956 return DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::OR, DL, VT: ByteVT, N1: LoShift, N2: HiShift));
1957}
1958
1959/// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
1960///
1961/// For example:
1962/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1963/// <4 x i32> <i32 0, i32 4, i32 1, i32 4>
1964/// %3 = bitcast <4 x i32> %2 to <2 x i64>
1965/// is lowered to:
1966/// (VREPLI $v1, 0)
1967/// (VILVL $v0, $v1, $v0)
1968static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL,
1969 ArrayRef<int> Mask, MVT VT,
1970 SDValue V1, SDValue V2,
1971 SelectionDAG &DAG,
1972 const APInt &Zeroable) {
1973 int Bits = VT.getSizeInBits();
1974 int EltBits = VT.getScalarSizeInBits();
1975 int NumElements = VT.getVectorNumElements();
1976
1977 if (Zeroable.isAllOnes())
1978 return DAG.getConstant(Val: 0, DL, VT);
1979
1980 // Define a helper function to check a particular ext-scale and lower to it if
1981 // valid.
1982 auto Lower = [&](int Scale) -> SDValue {
1983 SDValue InputV;
1984 bool AnyExt = true;
1985 int Offset = 0;
1986 for (int i = 0; i < NumElements; i++) {
1987 int M = Mask[i];
1988 if (M < 0)
1989 continue;
1990 if (i % Scale != 0) {
1991 // Each of the extended elements need to be zeroable.
1992 if (!Zeroable[i])
1993 return SDValue();
1994
1995 AnyExt = false;
1996 continue;
1997 }
1998
1999 // Each of the base elements needs to be consecutive indices into the
2000 // same input vector.
2001 SDValue V = M < NumElements ? V1 : V2;
2002 M = M % NumElements;
2003 if (!InputV) {
2004 InputV = V;
2005 Offset = M - (i / Scale);
2006
2007 // These offset can't be handled
2008 if (Offset % (NumElements / Scale))
2009 return SDValue();
2010 } else if (InputV != V)
2011 return SDValue();
2012
2013 if (M != (Offset + (i / Scale)))
2014 return SDValue(); // Non-consecutive strided elements.
2015 }
2016
2017 // If we fail to find an input, we have a zero-shuffle which should always
2018 // have already been handled.
2019 if (!InputV)
2020 return SDValue();
2021
2022 do {
2023 unsigned VilVLoHi = LoongArchISD::VILVL;
2024 if (Offset >= (NumElements / 2)) {
2025 VilVLoHi = LoongArchISD::VILVH;
2026 Offset -= (NumElements / 2);
2027 }
2028
2029 MVT InputVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits), NumElements);
2030 SDValue Ext =
2031 AnyExt ? DAG.getFreeze(V: InputV) : DAG.getConstant(Val: 0, DL, VT: InputVT);
2032 InputV = DAG.getBitcast(VT: InputVT, V: InputV);
2033 Ext = DAG.getBitcast(VT: InputVT, V: Ext);
2034 InputV = DAG.getNode(Opcode: VilVLoHi, DL, VT: InputVT, N1: Ext, N2: InputV);
2035 Scale /= 2;
2036 EltBits *= 2;
2037 NumElements /= 2;
2038 } while (Scale > 1);
2039 return DAG.getBitcast(VT, V: InputV);
2040 };
2041
2042 // Each iteration, try extending the elements half as much, but into twice as
2043 // many elements.
2044 for (int NumExtElements = Bits / 64; NumExtElements < NumElements;
2045 NumExtElements *= 2) {
2046 if (SDValue V = Lower(NumElements / NumExtElements))
2047 return V;
2048 }
2049 return SDValue();
2050}
2051
2052/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
2053///
2054/// VREPLVEI performs vector broadcast based on an element specified by an
2055/// integer immediate, with its mask being similar to:
2056/// <x, x, x, ...>
2057/// where x is any valid index.
2058///
2059/// When undef's appear in the mask they are treated as if they were whatever
2060/// value is necessary in order to fit the above form.
2061static SDValue
2062lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2063 SDValue V1, SelectionDAG &DAG,
2064 const LoongArchSubtarget &Subtarget) {
2065 int SplatIndex = -1;
2066 for (const auto &M : Mask) {
2067 if (M != -1) {
2068 SplatIndex = M;
2069 break;
2070 }
2071 }
2072
2073 if (SplatIndex == -1)
2074 return DAG.getUNDEF(VT);
2075
2076 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2077 if (fitsRegularPattern<int>(Begin: Mask.begin(), CheckStride: 1, End: Mask.end(), ExpectedIndex: SplatIndex, ExpectedIndexStride: 0)) {
2078 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2079 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2080 }
2081
2082 return SDValue();
2083}
2084
2085/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
2086///
2087/// VSHUF4I splits the vector into blocks of four elements, then shuffles these
2088/// elements according to a <4 x i2> constant (encoded as an integer immediate).
2089///
2090/// It is therefore possible to lower into VSHUF4I when the mask takes the form:
2091/// <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2092/// When undef's appear they are treated as if they were whatever value is
2093/// necessary in order to fit the above forms.
2094///
2095/// For example:
2096/// %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2097/// <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2098/// i32 7, i32 6, i32 5, i32 4>
2099/// is lowered to:
2100/// (VSHUF4I_H $v0, $v1, 27)
2101/// where the 27 comes from:
2102/// 3 + (2 << 2) + (1 << 4) + (0 << 6)
2103static SDValue
2104lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2105 SDValue V1, SDValue V2, SelectionDAG &DAG,
2106 const LoongArchSubtarget &Subtarget) {
2107
2108 unsigned SubVecSize = 4;
2109 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2110 SubVecSize = 2;
2111
2112 int SubMask[4] = {-1, -1, -1, -1};
2113 for (unsigned i = 0; i < SubVecSize; ++i) {
2114 for (unsigned j = i; j < Mask.size(); j += SubVecSize) {
2115 int M = Mask[j];
2116
2117 // Convert from vector index to 4-element subvector index
2118 // If an index refers to an element outside of the subvector then give up
2119 if (M != -1) {
2120 M -= 4 * (j / SubVecSize);
2121 if (M < 0 || M >= 4)
2122 return SDValue();
2123 }
2124
2125 // If the mask has an undef, replace it with the current index.
2126 // Note that it might still be undef if the current index is also undef
2127 if (SubMask[i] == -1)
2128 SubMask[i] = M;
2129 // Check that non-undef values are the same as in the mask. If they
2130 // aren't then give up
2131 else if (M != -1 && M != SubMask[i])
2132 return SDValue();
2133 }
2134 }
2135
2136 // Calculate the immediate. Replace any remaining undefs with zero
2137 int Imm = 0;
2138 for (int i = SubVecSize - 1; i >= 0; --i) {
2139 int M = SubMask[i];
2140
2141 if (M == -1)
2142 M = 0;
2143
2144 Imm <<= 2;
2145 Imm |= M & 0x3;
2146 }
2147
2148 MVT GRLenVT = Subtarget.getGRLenVT();
2149
2150 // Return vshuf4i.d
2151 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2152 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2153 N3: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2154
2155 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT, N1: V1,
2156 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2157}
2158
2159/// Lower VECTOR_SHUFFLE whose result is the reversed source vector.
2160///
2161/// It is possible to do optimization for VECTOR_SHUFFLE performing vector
2162/// reverse whose mask likes:
2163/// <7, 6, 5, 4, 3, 2, 1, 0>
2164///
2165/// When undef's appear in the mask they are treated as if they were whatever
2166/// value is necessary in order to fit the above forms.
2167static SDValue
2168lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2169 SDValue V1, SelectionDAG &DAG,
2170 const LoongArchSubtarget &Subtarget) {
2171 // Only vectors with i8/i16 elements which cannot match other patterns
2172 // directly needs to do this.
2173 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2174 VT != MVT::v16i16)
2175 return SDValue();
2176
2177 if (!ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: Mask.size()))
2178 return SDValue();
2179
2180 int WidenNumElts = VT.getVectorNumElements() / 4;
2181 SmallVector<int, 16> WidenMask(WidenNumElts, -1);
2182 for (int i = 0; i < WidenNumElts; ++i)
2183 WidenMask[i] = WidenNumElts - 1 - i;
2184
2185 MVT WidenVT = MVT::getVectorVT(
2186 VT: VT.getVectorElementType() == MVT::i8 ? MVT::i32 : MVT::i64, NumElements: WidenNumElts);
2187 SDValue NewV1 = DAG.getBitcast(VT: WidenVT, V: V1);
2188 SDValue WidenRev = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: NewV1,
2189 N2: DAG.getUNDEF(VT: WidenVT), Mask: WidenMask);
2190
2191 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT,
2192 N1: DAG.getBitcast(VT, V: WidenRev),
2193 N2: DAG.getConstant(Val: 27, DL, VT: Subtarget.getGRLenVT()));
2194}
2195
2196/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).
2197///
2198/// VPACKEV interleaves the even elements from each vector.
2199///
2200/// It is possible to lower into VPACKEV when the mask consists of two of the
2201/// following forms interleaved:
2202/// <0, 2, 4, ...>
2203/// <n, n+2, n+4, ...>
2204/// where n is the number of elements in the vector.
2205/// For example:
2206/// <0, 0, 2, 2, 4, 4, ...>
2207/// <0, n, 2, n+2, 4, n+4, ...>
2208///
2209/// When undef's appear in the mask they are treated as if they were whatever
2210/// value is necessary in order to fit the above forms.
2211static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2212 MVT VT, SDValue V1, SDValue V2,
2213 SelectionDAG &DAG) {
2214
2215 const auto &Begin = Mask.begin();
2216 const auto &End = Mask.end();
2217 SDValue OriV1 = V1, OriV2 = V2;
2218
2219 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2220 V1 = OriV1;
2221 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2222 V1 = OriV2;
2223 else
2224 return SDValue();
2225
2226 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2227 V2 = OriV1;
2228 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2229 V2 = OriV2;
2230 else
2231 return SDValue();
2232
2233 return DAG.getNode(Opcode: LoongArchISD::VPACKEV, DL, VT, N1: V2, N2: V1);
2234}
2235
2236/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).
2237///
2238/// VPACKOD interleaves the odd elements from each vector.
2239///
2240/// It is possible to lower into VPACKOD when the mask consists of two of the
2241/// following forms interleaved:
2242/// <1, 3, 5, ...>
2243/// <n+1, n+3, n+5, ...>
2244/// where n is the number of elements in the vector.
2245/// For example:
2246/// <1, 1, 3, 3, 5, 5, ...>
2247/// <1, n+1, 3, n+3, 5, n+5, ...>
2248///
2249/// When undef's appear in the mask they are treated as if they were whatever
2250/// value is necessary in order to fit the above forms.
2251static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2252 MVT VT, SDValue V1, SDValue V2,
2253 SelectionDAG &DAG) {
2254
2255 const auto &Begin = Mask.begin();
2256 const auto &End = Mask.end();
2257 SDValue OriV1 = V1, OriV2 = V2;
2258
2259 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2260 V1 = OriV1;
2261 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2262 V1 = OriV2;
2263 else
2264 return SDValue();
2265
2266 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2267 V2 = OriV1;
2268 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2269 V2 = OriV2;
2270 else
2271 return SDValue();
2272
2273 return DAG.getNode(Opcode: LoongArchISD::VPACKOD, DL, VT, N1: V2, N2: V1);
2274}
2275
2276/// Lower VECTOR_SHUFFLE into VILVH (if possible).
2277///
2278/// VILVH interleaves consecutive elements from the left (highest-indexed) half
2279/// of each vector.
2280///
2281/// It is possible to lower into VILVH when the mask consists of two of the
2282/// following forms interleaved:
2283/// <x, x+1, x+2, ...>
2284/// <n+x, n+x+1, n+x+2, ...>
2285/// where n is the number of elements in the vector and x is half n.
2286/// For example:
2287/// <x, x, x+1, x+1, x+2, x+2, ...>
2288/// <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2289///
2290/// When undef's appear in the mask they are treated as if they were whatever
2291/// value is necessary in order to fit the above forms.
2292static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef<int> Mask,
2293 MVT VT, SDValue V1, SDValue V2,
2294 SelectionDAG &DAG) {
2295
2296 const auto &Begin = Mask.begin();
2297 const auto &End = Mask.end();
2298 unsigned HalfSize = Mask.size() / 2;
2299 SDValue OriV1 = V1, OriV2 = V2;
2300
2301 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2302 V1 = OriV1;
2303 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2304 V1 = OriV2;
2305 else
2306 return SDValue();
2307
2308 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2309 V2 = OriV1;
2310 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize,
2311 ExpectedIndexStride: 1))
2312 V2 = OriV2;
2313 else
2314 return SDValue();
2315
2316 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2317}
2318
2319/// Lower VECTOR_SHUFFLE into VILVL (if possible).
2320///
2321/// VILVL interleaves consecutive elements from the right (lowest-indexed) half
2322/// of each vector.
2323///
2324/// It is possible to lower into VILVL when the mask consists of two of the
2325/// following forms interleaved:
2326/// <0, 1, 2, ...>
2327/// <n, n+1, n+2, ...>
2328/// where n is the number of elements in the vector.
2329/// For example:
2330/// <0, 0, 1, 1, 2, 2, ...>
2331/// <0, n, 1, n+1, 2, n+2, ...>
2332///
2333/// When undef's appear in the mask they are treated as if they were whatever
2334/// value is necessary in order to fit the above forms.
2335static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef<int> Mask,
2336 MVT VT, SDValue V1, SDValue V2,
2337 SelectionDAG &DAG) {
2338
2339 const auto &Begin = Mask.begin();
2340 const auto &End = Mask.end();
2341 SDValue OriV1 = V1, OriV2 = V2;
2342
2343 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2344 V1 = OriV1;
2345 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2346 V1 = OriV2;
2347 else
2348 return SDValue();
2349
2350 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2351 V2 = OriV1;
2352 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2353 V2 = OriV2;
2354 else
2355 return SDValue();
2356
2357 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2358}
2359
2360/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).
2361///
2362/// VPICKEV copies the even elements of each vector into the result vector.
2363///
2364/// It is possible to lower into VPICKEV when the mask consists of two of the
2365/// following forms concatenated:
2366/// <0, 2, 4, ...>
2367/// <n, n+2, n+4, ...>
2368/// where n is the number of elements in the vector.
2369/// For example:
2370/// <0, 2, 4, ..., 0, 2, 4, ...>
2371/// <0, 2, 4, ..., n, n+2, n+4, ...>
2372///
2373/// When undef's appear in the mask they are treated as if they were whatever
2374/// value is necessary in order to fit the above forms.
2375static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2376 MVT VT, SDValue V1, SDValue V2,
2377 SelectionDAG &DAG) {
2378
2379 const auto &Begin = Mask.begin();
2380 const auto &Mid = Mask.begin() + Mask.size() / 2;
2381 const auto &End = Mask.end();
2382 SDValue OriV1 = V1, OriV2 = V2;
2383
2384 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2))
2385 V1 = OriV1;
2386 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2387 V1 = OriV2;
2388 else
2389 return SDValue();
2390
2391 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2392 V2 = OriV1;
2393 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2394 V2 = OriV2;
2395
2396 else
2397 return SDValue();
2398
2399 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2400}
2401
2402/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).
2403///
2404/// VPICKOD copies the odd elements of each vector into the result vector.
2405///
2406/// It is possible to lower into VPICKOD when the mask consists of two of the
2407/// following forms concatenated:
2408/// <1, 3, 5, ...>
2409/// <n+1, n+3, n+5, ...>
2410/// where n is the number of elements in the vector.
2411/// For example:
2412/// <1, 3, 5, ..., 1, 3, 5, ...>
2413/// <1, 3, 5, ..., n+1, n+3, n+5, ...>
2414///
2415/// When undef's appear in the mask they are treated as if they were whatever
2416/// value is necessary in order to fit the above forms.
2417static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2418 MVT VT, SDValue V1, SDValue V2,
2419 SelectionDAG &DAG) {
2420
2421 const auto &Begin = Mask.begin();
2422 const auto &Mid = Mask.begin() + Mask.size() / 2;
2423 const auto &End = Mask.end();
2424 SDValue OriV1 = V1, OriV2 = V2;
2425
2426 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2))
2427 V1 = OriV1;
2428 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2429 V1 = OriV2;
2430 else
2431 return SDValue();
2432
2433 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2434 V2 = OriV1;
2435 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2436 V2 = OriV2;
2437 else
2438 return SDValue();
2439
2440 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
2441}
2442
2443/// Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
2444///
2445/// VEXTRINS copies one element of a vector into any place of the result
2446/// vector and makes no change to the rest elements of the result vector.
2447///
2448/// It is possible to lower into VEXTRINS when the mask takes the form:
2449/// <0, 1, 2, ..., n+i, ..., n-1> or <n, n+1, n+2, ..., i, ..., 2n-1> or
2450/// <0, 1, 2, ..., i, ..., n-1> or <n, n+1, n+2, ..., n+i, ..., 2n-1>
2451/// where n is the number of elements in the vector and i is in [0, n).
2452/// For example:
2453/// <0, 1, 2, 3, 4, 5, 6, 8> , <2, 9, 10, 11, 12, 13, 14, 15> ,
2454/// <0, 1, 2, 6, 4, 5, 6, 7> , <8, 9, 10, 11, 12, 9, 14, 15>
2455///
2456/// When undef's appear in the mask they are treated as if they were whatever
2457/// value is necessary in order to fit the above forms.
2458static SDValue
2459lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2460 SDValue V1, SDValue V2, SelectionDAG &DAG,
2461 const LoongArchSubtarget &Subtarget) {
2462 unsigned NumElts = VT.getVectorNumElements();
2463 MVT EltVT = VT.getVectorElementType();
2464 MVT GRLenVT = Subtarget.getGRLenVT();
2465
2466 if (Mask.size() != NumElts)
2467 return SDValue();
2468
2469 auto tryLowerToExtrAndIns = [&](unsigned Base) -> SDValue {
2470 int DiffCount = 0;
2471 int DiffPos = -1;
2472 for (unsigned i = 0; i < NumElts; ++i) {
2473 if (Mask[i] == -1)
2474 continue;
2475 if (Mask[i] != int(Base + i)) {
2476 ++DiffCount;
2477 DiffPos = int(i);
2478 if (DiffCount > 1)
2479 return SDValue();
2480 }
2481 }
2482
2483 // Need exactly one differing element to lower into VEXTRINS.
2484 if (DiffCount != 1)
2485 return SDValue();
2486
2487 // DiffMask must be in [0, 2N).
2488 int DiffMask = Mask[DiffPos];
2489 if (DiffMask < 0 || DiffMask >= int(2 * NumElts))
2490 return SDValue();
2491
2492 // Determine source vector and source index.
2493 SDValue SrcVec;
2494 unsigned SrcIdx;
2495 if (unsigned(DiffMask) < NumElts) {
2496 SrcVec = V1;
2497 SrcIdx = unsigned(DiffMask);
2498 } else {
2499 SrcVec = V2;
2500 SrcIdx = unsigned(DiffMask) - NumElts;
2501 }
2502
2503 // Replace with EXTRACT_VECTOR_ELT + INSERT_VECTOR_ELT, it will match the
2504 // patterns of VEXTRINS in tablegen.
2505 SDValue Extracted = DAG.getNode(
2506 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT.isFloatingPoint() ? EltVT : GRLenVT,
2507 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdx, DL, VT: GRLenVT));
2508 SDValue Result =
2509 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: (Base == 0) ? V1 : V2,
2510 N2: Extracted, N3: DAG.getConstant(Val: DiffPos, DL, VT: GRLenVT));
2511
2512 return Result;
2513 };
2514
2515 // Try [0, n-1) insertion then [n, 2n-1) insertion.
2516 if (SDValue Result = tryLowerToExtrAndIns(0))
2517 return Result;
2518 return tryLowerToExtrAndIns(NumElts);
2519}
2520
2521// Check the Mask and then build SrcVec and MaskImm infos which will
2522// be used to build LoongArchISD nodes for VPERMI_W or XVPERMI_W.
2523// On success, return true. Otherwise, return false.
2524static bool buildVPERMIInfo(ArrayRef<int> Mask, SDValue V1, SDValue V2,
2525 SmallVectorImpl<SDValue> &SrcVec,
2526 unsigned &MaskImm) {
2527 unsigned MaskSize = Mask.size();
2528
2529 auto isValid = [&](int M, int Off) {
2530 return (M == -1) || (M >= Off && M < Off + 4);
2531 };
2532
2533 auto buildImm = [&](int MLo, int MHi, unsigned Off, unsigned I) {
2534 auto immPart = [&](int M, unsigned Off) {
2535 return (M == -1 ? 0 : (M - Off)) & 0x3;
2536 };
2537 MaskImm |= immPart(MLo, Off) << (I * 2);
2538 MaskImm |= immPart(MHi, Off) << ((I + 1) * 2);
2539 };
2540
2541 for (unsigned i = 0; i < 4; i += 2) {
2542 int MLo = Mask[i];
2543 int MHi = Mask[i + 1];
2544
2545 if (MaskSize == 8) { // Only v8i32/v8f32 need this check.
2546 auto isValid2 = [&](int &M, int M2) {
2547 // If high half index is undef, it's always valid.
2548 if (M2 == -1)
2549 return true;
2550 if (M == -1) {
2551 // If low half index is undef, use index from high half,
2552 // remapped to low half.
2553 if ((M2 % MaskSize) < 4)
2554 return false;
2555 M = M2 - 4;
2556 return true;
2557 }
2558 // Index in low half must be same as index in high half.
2559 return M2 == M + 4;
2560 };
2561 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2562 return false;
2563 }
2564
2565 if (isValid(MLo, 0) && isValid(MHi, 0)) {
2566 SrcVec.push_back(Elt: V1);
2567 buildImm(MLo, MHi, 0, i);
2568 } else if (isValid(MLo, MaskSize) && isValid(MHi, MaskSize)) {
2569 SrcVec.push_back(Elt: V2);
2570 buildImm(MLo, MHi, MaskSize, i);
2571 } else {
2572 return false;
2573 }
2574 }
2575
2576 return true;
2577}
2578
2579/// Lower VECTOR_SHUFFLE into VPERMI (if possible).
2580///
2581/// VPERMI selects two elements from each of the two vectors based on the
2582/// mask and places them in the corresponding positions of the result vector
2583/// in order. Only v4i32 and v4f32 types are allowed.
2584///
2585/// It is possible to lower into VPERMI when the mask consists of two of the
2586/// following forms concatenated:
2587/// <i, j, u, v>
2588/// <u, v, i, j>
2589/// where i,j are in [0,4) and u,v are in [4, 8).
2590/// For example:
2591/// <2, 3, 4, 5>
2592/// <5, 7, 0, 2>
2593///
2594/// When undef's appear in the mask they are treated as if they were whatever
2595/// value is necessary in order to fit the above forms.
2596static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef<int> Mask,
2597 MVT VT, SDValue V1, SDValue V2,
2598 SelectionDAG &DAG,
2599 const LoongArchSubtarget &Subtarget) {
2600 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2601 Mask.size() != VT.getVectorNumElements())
2602 return SDValue();
2603
2604 SmallVector<SDValue, 2> SrcVec;
2605 unsigned MaskImm = 0;
2606 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2607 return SDValue();
2608
2609 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2610 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2611}
2612
2613/// Lower VECTOR_SHUFFLE into VSHUF.
2614///
2615/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and
2616/// adding it as an operand to the resulting VSHUF.
2617static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef<int> Mask,
2618 MVT VT, SDValue V1, SDValue V2,
2619 SelectionDAG &DAG,
2620 const LoongArchSubtarget &Subtarget) {
2621
2622 SmallVector<SDValue, 16> Ops;
2623 for (auto M : Mask)
2624 Ops.push_back(Elt: DAG.getSignedConstant(Val: M, DL, VT: Subtarget.getGRLenVT()));
2625
2626 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
2627 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops);
2628
2629 // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.
2630 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>
2631 // VSHF concatenates the vectors in a bitwise fashion:
2632 // <0b00, 0b01> + <0b10, 0b11> ->
2633 // 0b0100 + 0b1110 -> 0b01001110
2634 // <0b10, 0b11, 0b00, 0b01>
2635 // We must therefore swap the operands to get the correct result.
2636 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
2637}
2638
2639/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
2640///
2641/// This routine breaks down the specific type of 128-bit shuffle and
2642/// dispatches to the lowering routines accordingly.
2643static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2644 SDValue V1, SDValue V2, SelectionDAG &DAG,
2645 const LoongArchSubtarget &Subtarget) {
2646 assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||
2647 VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||
2648 VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&
2649 "Vector type is unsupported for lsx!");
2650 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
2651 "Two operands have different types!");
2652 assert(VT.getVectorNumElements() == Mask.size() &&
2653 "Unexpected mask size for shuffle!");
2654 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
2655
2656 APInt KnownUndef, KnownZero;
2657 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
2658 APInt Zeroable = KnownUndef | KnownZero;
2659
2660 SDValue Result;
2661 // TODO: Add more comparison patterns.
2662 if (V2.isUndef()) {
2663 if ((Result =
2664 lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
2665 return Result;
2666 if ((Result =
2667 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2668 return Result;
2669 if ((Result =
2670 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
2671 return Result;
2672
2673 // TODO: This comment may be enabled in the future to better match the
2674 // pattern for instruction selection.
2675 /* V2 = V1; */
2676 }
2677
2678 // It is recommended not to change the pattern comparison order for better
2679 // performance.
2680 if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))
2681 return Result;
2682 if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))
2683 return Result;
2684 if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))
2685 return Result;
2686 if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))
2687 return Result;
2688 if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))
2689 return Result;
2690 if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
2691 return Result;
2692 if ((VT.SimpleTy == MVT::v2i64 || VT.SimpleTy == MVT::v2f64) &&
2693 (Result =
2694 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2695 return Result;
2696 if ((Result =
2697 lowerVECTOR_SHUFFLE_VEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2698 return Result;
2699 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
2700 Zeroable)))
2701 return Result;
2702 if ((Result =
2703 lowerVECTOR_SHUFFLE_VPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2704 return Result;
2705 if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
2706 Zeroable)))
2707 return Result;
2708 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
2709 Subtarget)))
2710 return Result;
2711 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
2712 return NewShuffle;
2713 if ((Result =
2714 lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2715 return Result;
2716 return SDValue();
2717}
2718
2719/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
2720///
2721/// It is a XVREPLVEI when the mask is:
2722/// <x, x, x, ..., x+n, x+n, x+n, ...>
2723/// where the number of x is equal to n and n is half the length of vector.
2724///
2725/// When undef's appear in the mask they are treated as if they were whatever
2726/// value is necessary in order to fit the above form.
2727static SDValue
2728lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2729 SDValue V1, SelectionDAG &DAG,
2730 const LoongArchSubtarget &Subtarget) {
2731 int SplatIndex = -1;
2732 for (const auto &M : Mask) {
2733 if (M != -1) {
2734 SplatIndex = M;
2735 break;
2736 }
2737 }
2738
2739 if (SplatIndex == -1)
2740 return DAG.getUNDEF(VT);
2741
2742 const auto &Begin = Mask.begin();
2743 const auto &End = Mask.end();
2744 int HalfSize = Mask.size() / 2;
2745
2746 if (SplatIndex >= HalfSize)
2747 return SDValue();
2748
2749 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2750 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: End - HalfSize, ExpectedIndex: SplatIndex, ExpectedIndexStride: 0) &&
2751 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 1, End, ExpectedIndex: SplatIndex + HalfSize,
2752 ExpectedIndexStride: 0)) {
2753 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2754 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2755 }
2756
2757 return SDValue();
2758}
2759
2760/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
2761static SDValue
2762lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2763 SDValue V1, SDValue V2, SelectionDAG &DAG,
2764 const LoongArchSubtarget &Subtarget) {
2765 // XVSHUF4I_D must be handled separately because it is different from other
2766 // types of [X]VSHUF4I instructions.
2767 if (Mask.size() == 4) {
2768 unsigned MaskImm = 0;
2769 for (int i = 1; i >= 0; --i) {
2770 int MLo = Mask[i];
2771 int MHi = Mask[i + 2];
2772 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2773 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2774 return SDValue();
2775 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2776 return SDValue();
2777
2778 MaskImm <<= 2;
2779 if (MLo != -1)
2780 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2781 else if (MHi != -1)
2782 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2783 }
2784
2785 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2786 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2787 }
2788
2789 return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget);
2790}
2791
2792/// Lower VECTOR_SHUFFLE into XVREPLVE0 (if possible).
2793///
2794/// It is a XVREPLVE0 when the VECTOR_SHUFFLE is a broadcast of element 0,
2795/// i.e. the mask is:
2796/// <0, 0, 0, ...>
2797///
2798/// When undef's appear in the mask they are treated as if they were whatever
2799/// value is necessary in order to fit the above form.
2800static SDValue lowerVECTOR_SHUFFLE_XVREPLVE0(const SDLoc &DL,
2801 ArrayRef<int> Mask, MVT VT,
2802 SDValue V1, SelectionDAG &DAG) {
2803 if (!ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts: Mask.size()))
2804 return SDValue();
2805
2806 return DAG.getNode(Opcode: LoongArchISD::XVREPLVE0, DL, VT, Operand: V1);
2807}
2808
2809/// Lower VECTOR_SHUFFLE into XVPERMI (if possible).
2810static SDValue
2811lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2812 SDValue V1, SDValue V2, SelectionDAG &DAG,
2813 const LoongArchSubtarget &Subtarget) {
2814 MVT GRLenVT = Subtarget.getGRLenVT();
2815 unsigned MaskSize = Mask.size();
2816 if (MaskSize != VT.getVectorNumElements())
2817 return SDValue();
2818
2819 // Consider XVPERMI_W.
2820 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2821 SmallVector<SDValue, 2> SrcVec;
2822 unsigned MaskImm = 0;
2823 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2824 return SDValue();
2825
2826 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2827 N3: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2828 }
2829
2830 // Consider XVPERMI_D.
2831 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2832 unsigned MaskImm = 0;
2833 for (unsigned i = 0; i < MaskSize; ++i) {
2834 if (Mask[i] == -1)
2835 continue;
2836 if (Mask[i] >= (int)MaskSize)
2837 return SDValue();
2838 MaskImm |= Mask[i] << (i * 2);
2839 }
2840
2841 return DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT, N1: V1,
2842 N2: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2843 }
2844
2845 return SDValue();
2846}
2847
2848/// Lower VECTOR_SHUFFLE into XVPERM (if possible).
2849static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef<int> Mask,
2850 MVT VT, SDValue V1, SelectionDAG &DAG,
2851 const LoongArchSubtarget &Subtarget) {
2852 // LoongArch LASX only have XVPERM_W.
2853 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2854 return SDValue();
2855
2856 unsigned NumElts = VT.getVectorNumElements();
2857 unsigned HalfSize = NumElts / 2;
2858 bool FrontLo = true, FrontHi = true;
2859 bool BackLo = true, BackHi = true;
2860
2861 auto inRange = [](int val, int low, int high) {
2862 return (val == -1) || (val >= low && val < high);
2863 };
2864
2865 for (unsigned i = 0; i < HalfSize; ++i) {
2866 int Fronti = Mask[i];
2867 int Backi = Mask[i + HalfSize];
2868
2869 FrontLo &= inRange(Fronti, 0, HalfSize);
2870 FrontHi &= inRange(Fronti, HalfSize, NumElts);
2871 BackLo &= inRange(Backi, 0, HalfSize);
2872 BackHi &= inRange(Backi, HalfSize, NumElts);
2873 }
2874
2875 // If both the lower and upper 128-bit parts access only one half of the
2876 // vector (either lower or upper), avoid using xvperm.w. The latency of
2877 // xvperm.w(3) is higher than using xvshuf(1) and xvori(1).
2878 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2879 return SDValue();
2880
2881 SmallVector<SDValue, 8> Masks;
2882 MVT GRLenVT = Subtarget.getGRLenVT();
2883 for (unsigned i = 0; i < NumElts; ++i)
2884 Masks.push_back(Elt: Mask[i] == -1 ? DAG.getUNDEF(VT: GRLenVT)
2885 : DAG.getConstant(Val: Mask[i], DL, VT: GRLenVT));
2886 SDValue MaskVec = DAG.getBuildVector(VT: MVT::v8i32, DL, Ops: Masks);
2887
2888 return DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT, N1: V1, N2: MaskVec);
2889}
2890
2891/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
2892static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2893 MVT VT, SDValue V1, SDValue V2,
2894 SelectionDAG &DAG) {
2895 return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);
2896}
2897
2898/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
2899static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2900 MVT VT, SDValue V1, SDValue V2,
2901 SelectionDAG &DAG) {
2902 return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);
2903}
2904
2905/// Lower VECTOR_SHUFFLE into XVILVH (if possible).
2906static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef<int> Mask,
2907 MVT VT, SDValue V1, SDValue V2,
2908 SelectionDAG &DAG) {
2909
2910 const auto &Begin = Mask.begin();
2911 const auto &End = Mask.end();
2912 unsigned HalfSize = Mask.size() / 2;
2913 unsigned LeftSize = HalfSize / 2;
2914 SDValue OriV1 = V1, OriV2 = V2;
2915
2916 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2917 ExpectedIndexStride: 1) &&
2918 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize, ExpectedIndexStride: 1))
2919 V1 = OriV1;
2920 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize,
2921 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2922 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2923 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2924 V1 = OriV2;
2925 else
2926 return SDValue();
2927
2928 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2929 ExpectedIndexStride: 1) &&
2930 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize,
2931 ExpectedIndexStride: 1))
2932 V2 = OriV1;
2933 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize,
2934 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2935 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2936 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2937 V2 = OriV2;
2938 else
2939 return SDValue();
2940
2941 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2942}
2943
2944/// Lower VECTOR_SHUFFLE into XVILVL (if possible).
2945static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef<int> Mask,
2946 MVT VT, SDValue V1, SDValue V2,
2947 SelectionDAG &DAG) {
2948
2949 const auto &Begin = Mask.begin();
2950 const auto &End = Mask.end();
2951 unsigned HalfSize = Mask.size() / 2;
2952 SDValue OriV1 = V1, OriV2 = V2;
2953
2954 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2955 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2956 V1 = OriV1;
2957 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1) &&
2958 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2959 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2960 V1 = OriV2;
2961 else
2962 return SDValue();
2963
2964 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2965 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2966 V2 = OriV1;
2967 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(),
2968 ExpectedIndexStride: 1) &&
2969 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2970 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2971 V2 = OriV2;
2972 else
2973 return SDValue();
2974
2975 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2976}
2977
2978/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
2979static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2980 MVT VT, SDValue V1, SDValue V2,
2981 SelectionDAG &DAG) {
2982
2983 const auto &Begin = Mask.begin();
2984 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2985 const auto &Mid = Mask.begin() + Mask.size() / 2;
2986 const auto &RightMid = Mask.end() - Mask.size() / 4;
2987 const auto &End = Mask.end();
2988 unsigned HalfSize = Mask.size() / 2;
2989 SDValue OriV1 = V1, OriV2 = V2;
2990
2991 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
2992 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
2993 V1 = OriV1;
2994 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
2995 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
2996 V1 = OriV2;
2997 else
2998 return SDValue();
2999
3000 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
3001 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
3002 V2 = OriV1;
3003 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
3004 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
3005 V2 = OriV2;
3006
3007 else
3008 return SDValue();
3009
3010 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
3011}
3012
3013/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
3014static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
3015 MVT VT, SDValue V1, SDValue V2,
3016 SelectionDAG &DAG) {
3017
3018 const auto &Begin = Mask.begin();
3019 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3020 const auto &Mid = Mask.begin() + Mask.size() / 2;
3021 const auto &RightMid = Mask.end() - Mask.size() / 4;
3022 const auto &End = Mask.end();
3023 unsigned HalfSize = Mask.size() / 2;
3024 SDValue OriV1 = V1, OriV2 = V2;
3025
3026 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3027 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3028 V1 = OriV1;
3029 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3030 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize + 1,
3031 ExpectedIndexStride: 2))
3032 V1 = OriV2;
3033 else
3034 return SDValue();
3035
3036 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3037 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3038 V2 = OriV1;
3039 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3040 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize + 1,
3041 ExpectedIndexStride: 2))
3042 V2 = OriV2;
3043 else
3044 return SDValue();
3045
3046 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
3047}
3048
3049/// Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
3050static SDValue
3051lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3052 SDValue V1, SDValue V2, SelectionDAG &DAG,
3053 const LoongArchSubtarget &Subtarget) {
3054 int NumElts = VT.getVectorNumElements();
3055 int HalfSize = NumElts / 2;
3056 MVT EltVT = VT.getVectorElementType();
3057 MVT GRLenVT = Subtarget.getGRLenVT();
3058
3059 if ((int)Mask.size() != NumElts)
3060 return SDValue();
3061
3062 auto tryLowerToExtrAndIns = [&](int Base) -> SDValue {
3063 SmallVector<int> DiffPos;
3064 for (int i = 0; i < NumElts; ++i) {
3065 if (Mask[i] == -1)
3066 continue;
3067 if (Mask[i] != Base + i) {
3068 DiffPos.push_back(Elt: i);
3069 if (DiffPos.size() > 2)
3070 return SDValue();
3071 }
3072 }
3073
3074 // Need exactly two differing element to lower into XVEXTRINS.
3075 // If only one differing element, the element at a distance of
3076 // HalfSize from it must be undef.
3077 if (DiffPos.size() == 1) {
3078 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3079 DiffPos.push_back(Elt: DiffPos[0] + HalfSize);
3080 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3081 DiffPos.insert(I: DiffPos.begin(), Elt: DiffPos[0] - HalfSize);
3082 else
3083 return SDValue();
3084 }
3085 if (DiffPos.size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3086 return SDValue();
3087
3088 // DiffMask must be in its low or high part.
3089 int DiffMaskLo = Mask[DiffPos[0]];
3090 int DiffMaskHi = Mask[DiffPos[1]];
3091 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3092 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3093 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3094 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3095 return SDValue();
3096 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3097 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3098 return SDValue();
3099 if (DiffMaskHi != DiffMaskLo + HalfSize)
3100 return SDValue();
3101
3102 // Determine source vector and source index.
3103 SDValue SrcVec = (DiffMaskLo < HalfSize) ? V1 : V2;
3104 int SrcIdxLo =
3105 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3106 bool IsEltFP = EltVT.isFloatingPoint();
3107
3108 // Replace with 2*EXTRACT_VECTOR_ELT + 2*INSERT_VECTOR_ELT, it will match
3109 // the patterns of XVEXTRINS in tablegen.
3110 SDValue BaseVec = (Base == 0) ? V1 : V2;
3111 SDValue EltLo =
3112 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3113 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo, DL, VT: GRLenVT));
3114 SDValue InsLo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: BaseVec, N2: EltLo,
3115 N3: DAG.getConstant(Val: DiffPos[0], DL, VT: GRLenVT));
3116 SDValue EltHi =
3117 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3118 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo + HalfSize, DL, VT: GRLenVT));
3119 SDValue Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: InsLo, N2: EltHi,
3120 N3: DAG.getConstant(Val: DiffPos[1], DL, VT: GRLenVT));
3121
3122 return Result;
3123 };
3124
3125 // Try [0, n-1) insertion then [n, 2n-1) insertion.
3126 if (SDValue Result = tryLowerToExtrAndIns(0))
3127 return Result;
3128 return tryLowerToExtrAndIns(NumElts);
3129}
3130
3131/// Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
3132static SDValue
3133lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3134 SDValue V1, SDValue V2, SelectionDAG &DAG,
3135 const LoongArchSubtarget &Subtarget) {
3136 // LoongArch LASX only supports xvinsve0.{w/d}.
3137 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3138 VT != MVT::v4f64)
3139 return SDValue();
3140
3141 MVT GRLenVT = Subtarget.getGRLenVT();
3142 int MaskSize = Mask.size();
3143 assert(MaskSize == (int)VT.getVectorNumElements() && "Unexpected mask size");
3144
3145 // Check if exactly one element of the Mask is replaced by 'Replaced', while
3146 // all other elements are either 'Base + i' or undef (-1). On success, return
3147 // the index of the replaced element. Otherwise, just return -1.
3148 auto checkReplaceOne = [&](int Base, int Replaced) -> int {
3149 int Idx = -1;
3150 for (int i = 0; i < MaskSize; ++i) {
3151 if (Mask[i] == Base + i || Mask[i] == -1)
3152 continue;
3153 if (Mask[i] != Replaced)
3154 return -1;
3155 if (Idx == -1)
3156 Idx = i;
3157 else
3158 return -1;
3159 }
3160 return Idx;
3161 };
3162
3163 // Case 1: the lowest element of V2 replaces one element in V1.
3164 int Idx = checkReplaceOne(0, MaskSize);
3165 if (Idx != -1)
3166 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V1, N2: V2,
3167 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3168
3169 // Case 2: the lowest element of V1 replaces one element in V2.
3170 Idx = checkReplaceOne(MaskSize, 0);
3171 if (Idx != -1)
3172 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V2, N2: V1,
3173 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3174
3175 return SDValue();
3176}
3177
3178/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).
3179static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,
3180 MVT VT, SDValue V1, SDValue V2,
3181 SelectionDAG &DAG) {
3182
3183 int MaskSize = Mask.size();
3184 int HalfSize = Mask.size() / 2;
3185 const auto &Begin = Mask.begin();
3186 const auto &Mid = Mask.begin() + HalfSize;
3187 const auto &End = Mask.end();
3188
3189 // VECTOR_SHUFFLE concatenates the vectors:
3190 // <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>
3191 // shuffling ->
3192 // <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>
3193 //
3194 // XVSHUF concatenates the vectors:
3195 // <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>
3196 // shuffling ->
3197 // <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>
3198 SmallVector<SDValue, 8> MaskAlloc;
3199 for (auto it = Begin; it < Mid; it++) {
3200 if (*it < 0) // UNDEF
3201 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3202 else if ((*it >= 0 && *it < HalfSize) ||
3203 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3204 int M = *it < HalfSize ? *it : *it - HalfSize;
3205 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3206 } else
3207 return SDValue();
3208 }
3209 assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");
3210
3211 for (auto it = Mid; it < End; it++) {
3212 if (*it < 0) // UNDEF
3213 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3214 else if ((*it >= HalfSize && *it < MaskSize) ||
3215 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3216 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3217 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3218 } else
3219 return SDValue();
3220 }
3221 assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");
3222
3223 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
3224 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops: MaskAlloc);
3225 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
3226}
3227
3228/// Shuffle vectors by lane to generate more optimized instructions.
3229/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
3230///
3231/// Therefore, except for the following four cases, other cases are regarded
3232/// as cross-lane shuffles, where optimization is relatively limited.
3233///
3234/// - Shuffle high, low lanes of two inputs vector
3235/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
3236/// - Shuffle low, high lanes of two inputs vector
3237/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
3238/// - Shuffle low, low lanes of two inputs vector
3239/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
3240/// - Shuffle high, high lanes of two inputs vector
3241/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
3242///
3243/// The first case is the closest to LoongArch instructions and the other
3244/// cases need to be converted to it for processing.
3245///
3246/// This function will return true for the last three cases above and will
3247/// modify V1, V2 and Mask. Otherwise, return false for the first case and
3248/// cross-lane shuffle cases.
3249static bool canonicalizeShuffleVectorByLane(
3250 const SDLoc &DL, MutableArrayRef<int> Mask, MVT VT, SDValue &V1,
3251 SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
3252
3253 enum HalfMaskType { HighLaneTy, LowLaneTy, None };
3254
3255 int MaskSize = Mask.size();
3256 int HalfSize = Mask.size() / 2;
3257 MVT GRLenVT = Subtarget.getGRLenVT();
3258
3259 HalfMaskType preMask = None, postMask = None;
3260
3261 if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3262 return M < 0 || (M >= 0 && M < HalfSize) ||
3263 (M >= MaskSize && M < MaskSize + HalfSize);
3264 }))
3265 preMask = HighLaneTy;
3266 else if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3267 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3268 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3269 }))
3270 preMask = LowLaneTy;
3271
3272 if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3273 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3274 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3275 }))
3276 postMask = LowLaneTy;
3277 else if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3278 return M < 0 || (M >= 0 && M < HalfSize) ||
3279 (M >= MaskSize && M < MaskSize + HalfSize);
3280 }))
3281 postMask = HighLaneTy;
3282
3283 // The pre-half of mask is high lane type, and the post-half of mask
3284 // is low lane type, which is closest to the LoongArch instructions.
3285 //
3286 // Note: In the LoongArch architecture, the high lane of mask corresponds
3287 // to the lower 128-bit of vector register, and the low lane of mask
3288 // corresponds the higher 128-bit of vector register.
3289 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3290 return false;
3291 }
3292 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3293 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3294 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3295 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3296 V1 = DAG.getBitcast(VT, V: V1);
3297
3298 if (!V2.isUndef()) {
3299 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3300 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3301 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3302 V2 = DAG.getBitcast(VT, V: V2);
3303 }
3304
3305 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3306 *it = *it < 0 ? *it : *it - HalfSize;
3307 }
3308 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3309 *it = *it < 0 ? *it : *it + HalfSize;
3310 }
3311 } else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3312 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3313 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3314 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3315 V1 = DAG.getBitcast(VT, V: V1);
3316
3317 if (!V2.isUndef()) {
3318 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3319 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3320 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3321 V2 = DAG.getBitcast(VT, V: V2);
3322 }
3323
3324 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3325 *it = *it < 0 ? *it : *it - HalfSize;
3326 }
3327 } else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3328 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3329 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3330 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3331 V1 = DAG.getBitcast(VT, V: V1);
3332
3333 if (!V2.isUndef()) {
3334 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3335 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3336 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3337 V2 = DAG.getBitcast(VT, V: V2);
3338 }
3339
3340 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3341 *it = *it < 0 ? *it : *it + HalfSize;
3342 }
3343 } else { // cross-lane
3344 return false;
3345 }
3346
3347 return true;
3348}
3349
3350/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
3351/// Only for 256-bit vector.
3352///
3353/// For example:
3354/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
3355/// <4 x i64> <i32 0, i32 3, i32 2, i32 0>
3356/// is lowerded to:
3357/// (XVPERMI $xr2, $xr0, 78)
3358/// (XVSHUF $xr1, $xr2, $xr0)
3359/// (XVORI $xr0, $xr1, 0)
3360static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL,
3361 ArrayRef<int> Mask,
3362 MVT VT, SDValue V1,
3363 SDValue V2,
3364 SelectionDAG &DAG) {
3365 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
3366 int Size = Mask.size();
3367 int LaneSize = Size / 2;
3368
3369 bool LaneCrossing[2] = {false, false};
3370 for (int i = 0; i < Size; ++i)
3371 if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
3372 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
3373
3374 // Ensure that all lanes ared involved.
3375 if (!LaneCrossing[0] && !LaneCrossing[1])
3376 return SDValue();
3377
3378 SmallVector<int> InLaneMask;
3379 InLaneMask.assign(in_start: Mask.begin(), in_end: Mask.end());
3380 for (int i = 0; i < Size; ++i) {
3381 int &M = InLaneMask[i];
3382 if (M < 0)
3383 continue;
3384 if (((M % Size) / LaneSize) != (i / LaneSize))
3385 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
3386 }
3387
3388 SDValue Flipped = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3389 Flipped = DAG.getVectorShuffle(VT: MVT::v4i64, dl: DL, N1: Flipped,
3390 N2: DAG.getUNDEF(VT: MVT::v4i64), Mask: {2, 3, 0, 1});
3391 Flipped = DAG.getBitcast(VT, V: Flipped);
3392 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: Flipped, Mask: InLaneMask);
3393}
3394
3395/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
3396///
3397/// This routine breaks down the specific type of 256-bit shuffle and
3398/// dispatches to the lowering routines accordingly.
3399static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3400 SDValue V1, SDValue V2, SelectionDAG &DAG,
3401 const LoongArchSubtarget &Subtarget) {
3402 assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||
3403 VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||
3404 VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&
3405 "Vector type is unsupported for lasx!");
3406 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
3407 "Two operands have different types!");
3408 assert(VT.getVectorNumElements() == Mask.size() &&
3409 "Unexpected mask size for shuffle!");
3410 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
3411 assert(Mask.size() >= 4 && "Mask size is less than 4.");
3412
3413 APInt KnownUndef, KnownZero;
3414 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
3415 APInt Zeroable = KnownUndef | KnownZero;
3416
3417 SDValue Result;
3418 // TODO: Add more comparison patterns.
3419 if (V2.isUndef()) {
3420 if ((Result =
3421 lowerVECTOR_SHUFFLE_XVREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
3422 return Result;
3423 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG,
3424 Subtarget)))
3425 return Result;
3426 if ((Result = lowerVECTOR_SHUFFLE_XVREPLVE0(DL, Mask, VT, V1, DAG)))
3427 return Result;
3428 // Try to widen vectors to gain more optimization opportunities.
3429 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
3430 return NewShuffle;
3431 if ((Result =
3432 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3433 return Result;
3434 if ((Result = lowerVECTOR_SHUFFLE_XVPERM(DL, Mask, VT, V1, DAG, Subtarget)))
3435 return Result;
3436 if ((Result =
3437 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
3438 return Result;
3439
3440 // TODO: This comment may be enabled in the future to better match the
3441 // pattern for instruction selection.
3442 /* V2 = V1; */
3443 }
3444
3445 // It is recommended not to change the pattern comparison order for better
3446 // performance.
3447 if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, Mask, VT, V1, V2, DAG)))
3448 return Result;
3449 if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, Mask, VT, V1, V2, DAG)))
3450 return Result;
3451 if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, Mask, VT, V1, V2, DAG)))
3452 return Result;
3453 if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, Mask, VT, V1, V2, DAG)))
3454 return Result;
3455 if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, Mask, VT, V1, V2, DAG)))
3456 return Result;
3457 if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, Mask, VT, V1, V2, DAG)))
3458 return Result;
3459 if ((VT.SimpleTy == MVT::v4i64 || VT.SimpleTy == MVT::v4f64) &&
3460 (Result =
3461 lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3462 return Result;
3463 if ((Result =
3464 lowerVECTOR_SHUFFLE_XVEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3465 return Result;
3466 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
3467 Zeroable)))
3468 return Result;
3469 if ((Result =
3470 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3471 return Result;
3472 if ((Result =
3473 lowerVECTOR_SHUFFLE_XVINSVE0(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3474 return Result;
3475 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
3476 Subtarget)))
3477 return Result;
3478
3479 // canonicalize non cross-lane shuffle vector
3480 SmallVector<int> NewMask(Mask);
3481 if (canonicalizeShuffleVectorByLane(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget))
3482 return lower256BitShuffle(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget);
3483
3484 // FIXME: Handling the remaining cases earlier can degrade performance
3485 // in some situations. Further analysis is required to enable more
3486 // effective optimizations.
3487 if (V2.isUndef()) {
3488 if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, Mask: NewMask, VT,
3489 V1, V2, DAG)))
3490 return Result;
3491 }
3492
3493 if (SDValue NewShuffle = widenShuffleMask(DL, Mask: NewMask, VT, V1, V2, DAG))
3494 return NewShuffle;
3495 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, Mask: NewMask, VT, V1, V2, DAG)))
3496 return Result;
3497
3498 return SDValue();
3499}
3500
3501SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3502 SelectionDAG &DAG) const {
3503 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
3504 ArrayRef<int> OrigMask = SVOp->getMask();
3505 SDValue V1 = Op.getOperand(i: 0);
3506 SDValue V2 = Op.getOperand(i: 1);
3507 MVT VT = Op.getSimpleValueType();
3508 int NumElements = VT.getVectorNumElements();
3509 SDLoc DL(Op);
3510
3511 bool V1IsUndef = V1.isUndef();
3512 bool V2IsUndef = V2.isUndef();
3513 if (V1IsUndef && V2IsUndef)
3514 return DAG.getUNDEF(VT);
3515
3516 // When we create a shuffle node we put the UNDEF node to second operand,
3517 // but in some cases the first operand may be transformed to UNDEF.
3518 // In this case we should just commute the node.
3519 if (V1IsUndef)
3520 return DAG.getCommutedVectorShuffle(SV: *SVOp);
3521
3522 // Check for non-undef masks pointing at an undef vector and make the masks
3523 // undef as well. This makes it easier to match the shuffle based solely on
3524 // the mask.
3525 if (V2IsUndef &&
3526 any_of(Range&: OrigMask, P: [NumElements](int M) { return M >= NumElements; })) {
3527 SmallVector<int, 8> NewMask(OrigMask);
3528 for (int &M : NewMask)
3529 if (M >= NumElements)
3530 M = -1;
3531 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask: NewMask);
3532 }
3533
3534 // Check for illegal shuffle mask element index values.
3535 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
3536 (void)MaskUpperLimit;
3537 assert(llvm::all_of(OrigMask,
3538 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
3539 "Out of bounds shuffle index");
3540
3541 // For each vector width, delegate to a specialized lowering routine.
3542 if (VT.is128BitVector())
3543 return lower128BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3544
3545 if (VT.is256BitVector())
3546 return lower256BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3547
3548 return SDValue();
3549}
3550
3551SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
3552 SelectionDAG &DAG) const {
3553 // Custom lower to ensure the libcall return is passed in an FPR on hard
3554 // float ABIs.
3555 SDLoc DL(Op);
3556 MakeLibCallOptions CallOptions;
3557 SDValue Op0 = Op.getOperand(i: 0);
3558 SDValue Chain = SDValue();
3559 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
3560 SDValue Res;
3561 std::tie(args&: Res, args&: Chain) =
3562 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
3563 if (Subtarget.is64Bit())
3564 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3565 return DAG.getBitcast(VT: MVT::i32, V: Res);
3566}
3567
3568SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
3569 SelectionDAG &DAG) const {
3570 // Custom lower to ensure the libcall argument is passed in an FPR on hard
3571 // float ABIs.
3572 SDLoc DL(Op);
3573 MakeLibCallOptions CallOptions;
3574 SDValue Op0 = Op.getOperand(i: 0);
3575 SDValue Chain = SDValue();
3576 SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3577 DL, VT: MVT::f32, Operand: Op0)
3578 : DAG.getBitcast(VT: MVT::f32, V: Op0);
3579 SDValue Res;
3580 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
3581 CallOptions, dl: DL, Chain);
3582 return Res;
3583}
3584
3585SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
3586 SelectionDAG &DAG) const {
3587 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3588 SDLoc DL(Op);
3589 MakeLibCallOptions CallOptions;
3590 RTLIB::Libcall LC =
3591 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
3592 SDValue Res =
3593 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
3594 if (Subtarget.is64Bit())
3595 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3596 return DAG.getBitcast(VT: MVT::i32, V: Res);
3597}
3598
3599SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
3600 SelectionDAG &DAG) const {
3601 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3602 MVT VT = Op.getSimpleValueType();
3603 SDLoc DL(Op);
3604 Op = DAG.getNode(
3605 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
3606 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
3607 SDValue Res = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3608 DL, VT: MVT::f32, Operand: Op)
3609 : DAG.getBitcast(VT: MVT::f32, V: Op);
3610 if (VT != MVT::f32)
3611 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
3612 return Res;
3613}
3614
3615// Lower BUILD_VECTOR as broadcast load (if possible).
3616// For example:
3617// %a = load i8, ptr %ptr
3618// %b = build_vector %a, %a, %a, %a
3619// is lowered to :
3620// (VLDREPL_B $a0, 0)
3621static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp,
3622 const SDLoc &DL,
3623 SelectionDAG &DAG) {
3624 MVT VT = BVOp->getSimpleValueType(ResNo: 0);
3625 int NumOps = BVOp->getNumOperands();
3626
3627 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3628 "Unsupported vector type for broadcast.");
3629
3630 SDValue IdentitySrc;
3631 bool IsIdeneity = true;
3632
3633 for (int i = 0; i != NumOps; i++) {
3634 SDValue Op = BVOp->getOperand(Num: i);
3635 if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
3636 IsIdeneity = false;
3637 break;
3638 }
3639 IdentitySrc = BVOp->getOperand(Num: 0);
3640 }
3641
3642 // make sure that this load is valid and only has one user.
3643 if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(N: IdentitySrc.getNode()))
3644 return SDValue();
3645
3646 auto *LN = cast<LoadSDNode>(Val&: IdentitySrc);
3647 auto ExtType = LN->getExtensionType();
3648
3649 if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
3650 VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) {
3651 // Indexed loads and stores are not supported on LoongArch.
3652 assert(LN->isUnindexed() && "Unexpected indexed load.");
3653
3654 SDVTList Tys = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3655 // The offset operand of unindexed load is always undefined, so there is
3656 // no need to pass it to VLDREPL.
3657 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3658 SDValue BCast = DAG.getNode(Opcode: LoongArchISD::VLDREPL, DL, VTList: Tys, Ops);
3659 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: BCast.getValue(R: 1));
3660 return BCast;
3661 }
3662 return SDValue();
3663}
3664
3665// Sequentially insert elements from Ops into Vector, from low to high indices.
3666// Note: Ops can have fewer elements than Vector.
3667static void fillVector(ArrayRef<SDValue> Ops, SelectionDAG &DAG, SDLoc DL,
3668 const LoongArchSubtarget &Subtarget, SDValue &Vector,
3669 EVT ResTy) {
3670 assert(Ops.size() <= ResTy.getVectorNumElements());
3671
3672 SDValue Op0 = Ops[0];
3673 if (!Op0.isUndef())
3674 Vector = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: ResTy, Operand: Op0);
3675 for (unsigned i = 1; i < Ops.size(); ++i) {
3676 SDValue Opi = Ops[i];
3677 if (Opi.isUndef())
3678 continue;
3679 Vector = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Vector, N2: Opi,
3680 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3681 }
3682}
3683
3684// Build a ResTy subvector from Node, taking NumElts elements starting at index
3685// 'first'.
3686static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node,
3687 SelectionDAG &DAG, SDLoc DL,
3688 const LoongArchSubtarget &Subtarget,
3689 EVT ResTy, unsigned first) {
3690 unsigned NumElts = ResTy.getVectorNumElements();
3691
3692 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3693
3694 SmallVector<SDValue, 16> Ops(Node->op_begin() + first,
3695 Node->op_begin() + first + NumElts);
3696 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3697 fillVector(Ops, DAG, DL, Subtarget, Vector, ResTy);
3698 return Vector;
3699}
3700
3701SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
3702 SelectionDAG &DAG) const {
3703 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Val&: Op);
3704 MVT VT = Node->getSimpleValueType(ResNo: 0);
3705 EVT ResTy = Op->getValueType(ResNo: 0);
3706 unsigned NumElts = ResTy.getVectorNumElements();
3707 SDLoc DL(Op);
3708 APInt SplatValue, SplatUndef;
3709 unsigned SplatBitSize;
3710 bool HasAnyUndefs;
3711 bool IsConstant = false;
3712 bool UseSameConstant = true;
3713 SDValue ConstantValue;
3714 bool Is128Vec = ResTy.is128BitVector();
3715 bool Is256Vec = ResTy.is256BitVector();
3716
3717 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3718 (!Subtarget.hasExtLASX() || !Is256Vec))
3719 return SDValue();
3720
3721 if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(BVOp: Node, DL, DAG))
3722 return Result;
3723
3724 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3725 /*MinSplatBits=*/8) &&
3726 SplatBitSize <= 64) {
3727 // We can only cope with 8, 16, 32, or 64-bit elements.
3728 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3729 SplatBitSize != 64)
3730 return SDValue();
3731
3732 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3733 // We can only handle 64-bit elements that are within
3734 // the signed 10-bit range or match vldi patterns on 32-bit targets.
3735 // See the BUILD_VECTOR case in LoongArchDAGToDAGISel::Select().
3736 if (!SplatValue.isSignedIntN(N: 10) &&
3737 !isImmVLDILegalForMode1(SplatValue, SplatBitSize).first)
3738 return SDValue();
3739 if ((Is128Vec && ResTy == MVT::v4i32) ||
3740 (Is256Vec && ResTy == MVT::v8i32))
3741 return Op;
3742 }
3743
3744 EVT ViaVecTy;
3745
3746 switch (SplatBitSize) {
3747 default:
3748 return SDValue();
3749 case 8:
3750 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3751 break;
3752 case 16:
3753 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3754 break;
3755 case 32:
3756 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3757 break;
3758 case 64:
3759 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3760 break;
3761 }
3762
3763 // SelectionDAG::getConstant will promote SplatValue appropriately.
3764 SDValue Result = DAG.getConstant(Val: SplatValue, DL, VT: ViaVecTy);
3765
3766 // Bitcast to the type we originally wanted.
3767 if (ViaVecTy != ResTy)
3768 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Node), VT: ResTy, Operand: Result);
3769
3770 return Result;
3771 }
3772
3773 if (DAG.isSplatValue(V: Op, /*AllowUndefs=*/false))
3774 return Op;
3775
3776 for (unsigned i = 0; i < NumElts; ++i) {
3777 SDValue Opi = Node->getOperand(Num: i);
3778 if (isIntOrFPConstant(V: Opi)) {
3779 IsConstant = true;
3780 if (!ConstantValue.getNode())
3781 ConstantValue = Opi;
3782 else if (ConstantValue != Opi)
3783 UseSameConstant = false;
3784 }
3785 }
3786
3787 // If the type of BUILD_VECTOR is v2f64, custom legalizing it has no benefits.
3788 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3789 SDValue Result = DAG.getSplatBuildVector(VT: ResTy, DL, Op: ConstantValue);
3790 for (unsigned i = 0; i < NumElts; ++i) {
3791 SDValue Opi = Node->getOperand(Num: i);
3792 if (!isIntOrFPConstant(V: Opi))
3793 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Result, N2: Opi,
3794 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3795 }
3796 return Result;
3797 }
3798
3799 if (!IsConstant) {
3800 // If the BUILD_VECTOR has a repeated pattern, use INSERT_VECTOR_ELT to fill
3801 // the sub-sequence of the vector and then broadcast the sub-sequence.
3802 //
3803 // TODO: If the BUILD_VECTOR contains undef elements, consider falling
3804 // back to use INSERT_VECTOR_ELT to materialize the vector, because it
3805 // generates worse code in some cases. This could be further optimized
3806 // with more consideration.
3807 SmallVector<SDValue> Sequence;
3808 BitVector UndefElements;
3809 if (Node->getRepeatedSequence(Sequence, UndefElements: &UndefElements) &&
3810 UndefElements.count() == 0) {
3811 // Using LSX instructions to fill the sub-sequence of 256-bits vector,
3812 // because the high part can be simply treated as undef.
3813 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3814 EVT FillTy = Is256Vec
3815 ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext())
3816 : ResTy;
3817 SDValue FillVec =
3818 Is256Vec ? DAG.getExtractSubvector(DL, VT: FillTy, Vec: Vector, Idx: 0) : Vector;
3819
3820 fillVector(Ops: Sequence, DAG, DL, Subtarget, Vector&: FillVec, ResTy: FillTy);
3821
3822 unsigned SeqLen = Sequence.size();
3823 unsigned SplatLen = NumElts / SeqLen;
3824 MVT SplatEltTy = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits() * SeqLen);
3825 MVT SplatTy = MVT::getVectorVT(VT: SplatEltTy, NumElements: SplatLen);
3826
3827 // If size of the sub-sequence is half of a 256-bits vector, bitcast the
3828 // vector to v4i64 type in order to match the pattern of XVREPLVE0Q.
3829 if (SplatEltTy == MVT::i128)
3830 SplatTy = MVT::v4i64;
3831
3832 SDValue SplatVec;
3833 SDValue SrcVec = DAG.getBitcast(
3834 VT: SplatTy,
3835 V: Is256Vec ? DAG.getInsertSubvector(DL, Vec: Vector, SubVec: FillVec, Idx: 0) : FillVec);
3836 if (Is256Vec) {
3837 SplatVec =
3838 DAG.getNode(Opcode: (SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
3839 : LoongArchISD::XVREPLVE0,
3840 DL, VT: SplatTy, Operand: SrcVec);
3841 } else {
3842 SplatVec = DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT: SplatTy, N1: SrcVec,
3843 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
3844 }
3845
3846 return DAG.getBitcast(VT: ResTy, V: SplatVec);
3847 }
3848
3849 // Use INSERT_VECTOR_ELT operations rather than expand to stores, because
3850 // using memory operations is much lower.
3851 //
3852 // For 256-bit vectors, normally split into two halves and concatenate.
3853 // Special case: for v8i32/v8f32/v4i64/v4f64, if the upper half has only
3854 // one non-undef element, skip spliting to avoid a worse result.
3855 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
3856 ResTy == MVT::v4f64) {
3857 unsigned NonUndefCount = 0;
3858 for (unsigned i = NumElts / 2; i < NumElts; ++i) {
3859 if (!Node->getOperand(Num: i).isUndef()) {
3860 ++NonUndefCount;
3861 if (NonUndefCount > 1)
3862 break;
3863 }
3864 }
3865 if (NonUndefCount == 1)
3866 return fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy, first: 0);
3867 }
3868
3869 EVT VecTy =
3870 Is256Vec ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext()) : ResTy;
3871 SDValue Vector =
3872 fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy: VecTy, first: 0);
3873
3874 if (Is128Vec)
3875 return Vector;
3876
3877 SDValue VectorHi = fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget,
3878 ResTy: VecTy, first: NumElts / 2);
3879
3880 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResTy, N1: Vector, N2: VectorHi);
3881 }
3882
3883 return SDValue();
3884}
3885
3886SDValue LoongArchTargetLowering::lowerCONCAT_VECTORS(SDValue Op,
3887 SelectionDAG &DAG) const {
3888 SDLoc DL(Op);
3889 MVT ResVT = Op.getSimpleValueType();
3890 assert(ResVT.is256BitVector() && Op.getNumOperands() == 2);
3891
3892 if (Op.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
3893 Op.getOperand(i: 1).getOpcode() == ISD::TRUNCATE)
3894 return Op;
3895
3896 unsigned NumOperands = Op.getNumOperands();
3897 unsigned NumFreezeUndef = 0;
3898 unsigned NumZero = 0;
3899 unsigned NumNonZero = 0;
3900 unsigned NonZeros = 0;
3901 SmallSet<SDValue, 4> Undefs;
3902 for (unsigned i = 0; i != NumOperands; ++i) {
3903 SDValue SubVec = Op.getOperand(i);
3904 if (SubVec.isUndef())
3905 continue;
3906 if (ISD::isFreezeUndef(N: SubVec.getNode())) {
3907 // If the freeze(undef) has multiple uses then we must fold to zero.
3908 if (SubVec.hasOneUse()) {
3909 ++NumFreezeUndef;
3910 } else {
3911 ++NumZero;
3912 Undefs.insert(V: SubVec);
3913 }
3914 } else if (ISD::isBuildVectorAllZeros(N: SubVec.getNode()))
3915 ++NumZero;
3916 else {
3917 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
3918 NonZeros |= 1 << i;
3919 ++NumNonZero;
3920 }
3921 }
3922
3923 // If we have more than 2 non-zeros, build each half separately.
3924 if (NumNonZero > 2) {
3925 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
3926 ArrayRef<SDUse> Ops = Op->ops();
3927 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3928 Ops: Ops.slice(N: 0, M: NumOperands / 2));
3929 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3930 Ops: Ops.slice(N: NumOperands / 2));
3931 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
3932 }
3933
3934 // Otherwise, build it up through insert_subvectors.
3935 SDValue Vec = NumZero ? DAG.getConstant(Val: 0, DL, VT: ResVT)
3936 : (NumFreezeUndef ? DAG.getFreeze(V: DAG.getUNDEF(VT: ResVT))
3937 : DAG.getUNDEF(VT: ResVT));
3938
3939 // Replace Undef operands with ZeroVector.
3940 for (SDValue U : Undefs)
3941 DAG.ReplaceAllUsesWith(From: U, To: DAG.getConstant(Val: 0, DL, VT: U.getSimpleValueType()));
3942
3943 MVT SubVT = Op.getOperand(i: 0).getSimpleValueType();
3944 unsigned NumSubElems = SubVT.getVectorNumElements();
3945 for (unsigned i = 0; i != NumOperands; ++i) {
3946 if ((NonZeros & (1 << i)) == 0)
3947 continue;
3948
3949 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ResVT, N1: Vec, N2: Op.getOperand(i),
3950 N3: DAG.getVectorIdxConstant(Val: i * NumSubElems, DL));
3951 }
3952
3953 return Vec;
3954}
3955
3956SDValue
3957LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
3958 SelectionDAG &DAG) const {
3959 MVT EltVT = Op.getSimpleValueType();
3960 SDValue Vec = Op->getOperand(Num: 0);
3961 EVT VecTy = Vec->getValueType(ResNo: 0);
3962 SDValue Idx = Op->getOperand(Num: 1);
3963 SDLoc DL(Op);
3964 MVT GRLenVT = Subtarget.getGRLenVT();
3965
3966 assert(VecTy.is256BitVector() && "Unexpected EXTRACT_VECTOR_ELT vector type");
3967
3968 if (isa<ConstantSDNode>(Val: Idx))
3969 return Op;
3970
3971 switch (VecTy.getSimpleVT().SimpleTy) {
3972 default:
3973 llvm_unreachable("Unexpected type");
3974 case MVT::v32i8:
3975 case MVT::v16i16:
3976 case MVT::v4i64:
3977 case MVT::v4f64: {
3978 // Extract the high half subvector and place it to the low half of a new
3979 // vector. It doesn't matter what the high half of the new vector is.
3980 EVT HalfTy = VecTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3981 SDValue VecHi =
3982 DAG.getExtractSubvector(DL, VT: HalfTy, Vec, Idx: HalfTy.getVectorNumElements());
3983 SDValue TmpVec =
3984 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: VecTy, N1: DAG.getUNDEF(VT: VecTy),
3985 N2: VecHi, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3986
3987 // Shuffle the origin Vec and the TmpVec using MaskVec, the lowest element
3988 // of MaskVec is Idx, the rest do not matter. ResVec[0] will hold the
3989 // desired element.
3990 SDValue IdxCp =
3991 Subtarget.is64Bit()
3992 ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Idx)
3993 : DAG.getBitcast(VT: MVT::f32, V: Idx);
3994 SDValue IdxVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v8f32, Operand: IdxCp);
3995 SDValue MaskVec =
3996 DAG.getBitcast(VT: (VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, V: IdxVec);
3997 SDValue ResVec =
3998 DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT: VecTy, N1: MaskVec, N2: TmpVec, N3: Vec);
3999
4000 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: ResVec,
4001 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
4002 }
4003 case MVT::v8i32:
4004 case MVT::v8f32: {
4005 SDValue SplatIdx = DAG.getSplatBuildVector(VT: MVT::v8i32, DL, Op: Idx);
4006 SDValue SplatValue =
4007 DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT: VecTy, N1: Vec, N2: SplatIdx);
4008
4009 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: SplatValue,
4010 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
4011 }
4012 }
4013}
4014
4015SDValue
4016LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4017 SelectionDAG &DAG) const {
4018 MVT VT = Op.getSimpleValueType();
4019 MVT EltVT = VT.getVectorElementType();
4020 unsigned NumElts = VT.getVectorNumElements();
4021 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
4022 SDLoc DL(Op);
4023 SDValue Op0 = Op.getOperand(i: 0);
4024 SDValue Op1 = Op.getOperand(i: 1);
4025 SDValue Op2 = Op.getOperand(i: 2);
4026
4027 if (isa<ConstantSDNode>(Val: Op2))
4028 return Op;
4029
4030 MVT IdxTy = MVT::getIntegerVT(BitWidth: EltSizeInBits);
4031 MVT IdxVTy = MVT::getVectorVT(VT: IdxTy, NumElements: NumElts);
4032
4033 if (!isTypeLegal(VT) || !isTypeLegal(VT: IdxVTy))
4034 return SDValue();
4035
4036 SDValue SplatElt = DAG.getSplatBuildVector(VT, DL, Op: Op1);
4037 SmallVector<SDValue, 32> RawIndices;
4038 SDValue SplatIdx;
4039 SDValue Indices;
4040
4041 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4042 MVT PairVTy = MVT::getVectorVT(VT: MVT::i32, NumElements: NumElts * 2);
4043 for (unsigned i = 0; i < NumElts; ++i) {
4044 RawIndices.push_back(Elt: Op2);
4045 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4046 }
4047 SplatIdx = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4048 SplatIdx = DAG.getBitcast(VT: IdxVTy, V: SplatIdx);
4049
4050 RawIndices.clear();
4051 for (unsigned i = 0; i < NumElts; ++i) {
4052 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: MVT::i32));
4053 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4054 }
4055 Indices = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4056 Indices = DAG.getBitcast(VT: IdxVTy, V: Indices);
4057 } else {
4058 SplatIdx = DAG.getSplatBuildVector(VT: IdxVTy, DL, Op: Op2);
4059
4060 for (unsigned i = 0; i < NumElts; ++i)
4061 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
4062 Indices = DAG.getBuildVector(VT: IdxVTy, DL, Ops: RawIndices);
4063 }
4064
4065 // insert vec, elt, idx
4066 // =>
4067 // select (splatidx == {0,1,2...}) ? splatelt : vec
4068 SDValue SelectCC =
4069 DAG.getSetCC(DL, VT: IdxVTy, LHS: SplatIdx, RHS: Indices, Cond: ISD::CondCode::SETEQ);
4070 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectCC, N2: SplatElt, N3: Op0);
4071}
4072
4073SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4074 SelectionDAG &DAG) const {
4075 SDLoc DL(Op);
4076 SyncScope::ID FenceSSID =
4077 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4078
4079 // singlethread fences only synchronize with signal handlers on the same
4080 // thread and thus only need to preserve instruction order, not actually
4081 // enforce memory ordering.
4082 if (FenceSSID == SyncScope::SingleThread)
4083 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4084 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
4085
4086 return Op;
4087}
4088
4089static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL,
4090 MVT GRLenVT, SDValue RMValue) {
4091 // LLVM rounding mode encoding differs from LoongArch FCSR encoding:
4092 // LLVM: 0=RTZ, 1=RNE, 2=RUP, 3=RDN
4093 // FCSR: 0=RNE, 1=RZ, 2=RP, 3=RN
4094 //
4095 // The conversion swaps encodings 0 and 1 while preserving 2 and 3.
4096 // Since the transformation is self-inverse, it applies in both directions:
4097 // LLVM RM <-> LoongArch FCSR RM
4098 //
4099 // Transformation: RM ^ (~(RM >> 1) & 1)
4100 SDValue ShiftRight1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4101 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4102
4103 SDValue SwapMask = DAG.getNode(Opcode: ISD::AND, DL, VT: GRLenVT,
4104 N1: DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: ShiftRight1,
4105 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT)),
4106 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4107
4108 return DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: RMValue, N2: SwapMask);
4109}
4110
4111SDValue LoongArchTargetLowering::lowerSET_ROUNDING(SDValue Op,
4112 SelectionDAG &DAG) const {
4113 MVT GRLenVT = Subtarget.getGRLenVT();
4114 SDLoc DL(Op);
4115 SDValue Chain = Op.getOperand(i: 0);
4116 SDValue RMValue = Op.getOperand(i: 1);
4117
4118 if (auto *CVal = dyn_cast<ConstantSDNode>(Val&: RMValue)) {
4119 uint64_t RM = CVal->getZExtValue();
4120 if (RM > 3) {
4121 MachineFunction &MF = DAG.getMachineFunction();
4122 LLVMContext &C = MF.getFunction().getContext();
4123 C.diagnose(DI: DiagnosticInfoUnsupported(
4124 MF.getFunction(),
4125 "rounding mode is not supported by LoongArch hardware",
4126 DiagnosticLocation(DL.getDebugLoc()), DS_Error));
4127 return Chain;
4128 }
4129 }
4130
4131 RMValue = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: RMValue);
4132 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4133
4134 // The RM field in FCSR is at bits [9:8]. Shift the rounding mode value
4135 // into position before writing via WRFCSR.
4136 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: RMValue,
4137 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4138
4139 // FCSR3 is an alias of the RM field; writing it avoids clobbering
4140 // unrelated fields in FCSR0.
4141 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4142 MachineSDNode *RN = DAG.getMachineNode(Opcode: LoongArch::WRFCSR, dl: DL, VT: MVT::Other,
4143 Op1: FCSRNo, Op2: RMValue, Op3: Chain);
4144 return SDValue(RN, 0);
4145}
4146
4147SDValue LoongArchTargetLowering::lowerGET_ROUNDING(SDValue Op,
4148 SelectionDAG &DAG) const {
4149 MVT GRLenVT = Subtarget.getGRLenVT();
4150 SDLoc DL(Op);
4151 SDValue Chain = Op->getOperand(Num: 0);
4152
4153 // FCSR3 is an alias of the RM field.
4154 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4155 MachineSDNode *FCSR = DAG.getMachineNode(Opcode: LoongArch::RDFCSR, dl: DL, VT1: GRLenVT,
4156 VT2: MVT::Other, Op1: FCSRNo, Op2: Chain);
4157 SDValue RMValue = SDValue(FCSR, 0);
4158 Chain = SDValue(FCSR, 1);
4159
4160 // The RM field in FCSR is at bits [9:8].
4161 RMValue = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4162 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4163 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4164
4165 SDValue RetVal = DAG.getZExtOrTrunc(Op: RMValue, DL, VT: Op.getValueType());
4166 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl: DL);
4167}
4168
4169SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
4170 SelectionDAG &DAG) const {
4171
4172 if (Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i32) {
4173 DAG.getContext()->emitError(
4174 ErrorStr: "On LA64, only 64-bit registers can be written.");
4175 return Op.getOperand(i: 0);
4176 }
4177
4178 if (!Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i64) {
4179 DAG.getContext()->emitError(
4180 ErrorStr: "On LA32, only 32-bit registers can be written.");
4181 return Op.getOperand(i: 0);
4182 }
4183
4184 return Op;
4185}
4186
4187SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
4188 SelectionDAG &DAG) const {
4189 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 0))) {
4190 DAG.getContext()->emitError(ErrorStr: "argument to '__builtin_frame_address' must "
4191 "be a constant integer");
4192 return SDValue();
4193 }
4194
4195 MachineFunction &MF = DAG.getMachineFunction();
4196 MF.getFrameInfo().setFrameAddressIsTaken(true);
4197 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4198 EVT VT = Op.getValueType();
4199 SDLoc DL(Op);
4200 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
4201 unsigned Depth = Op.getConstantOperandVal(i: 0);
4202 int GRLenInBytes = Subtarget.getGRLen() / 8;
4203
4204 while (Depth--) {
4205 int Offset = -(GRLenInBytes * 2);
4206 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
4207 N2: DAG.getSignedConstant(Val: Offset, DL, VT));
4208 FrameAddr =
4209 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
4210 }
4211 return FrameAddr;
4212}
4213
4214SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
4215 SelectionDAG &DAG) const {
4216 // Currently only support lowering return address for current frame.
4217 if (Op.getConstantOperandVal(i: 0) != 0) {
4218 DAG.getContext()->emitError(
4219 ErrorStr: "return address can only be determined for the current frame");
4220 return SDValue();
4221 }
4222
4223 MachineFunction &MF = DAG.getMachineFunction();
4224 MF.getFrameInfo().setReturnAddressIsTaken(true);
4225 MVT GRLenVT = Subtarget.getGRLenVT();
4226
4227 // Return the value of the return address register, marking it an implicit
4228 // live-in.
4229 Register Reg = MF.addLiveIn(PReg: Subtarget.getRegisterInfo()->getRARegister(),
4230 RC: getRegClassFor(VT: GRLenVT));
4231 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg, VT: GRLenVT);
4232}
4233
4234SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
4235 SelectionDAG &DAG) const {
4236 MachineFunction &MF = DAG.getMachineFunction();
4237 auto Size = Subtarget.getGRLen() / 8;
4238 auto FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: 0, IsImmutable: false);
4239 return DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4240}
4241
4242SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,
4243 SelectionDAG &DAG) const {
4244 MachineFunction &MF = DAG.getMachineFunction();
4245 auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();
4246
4247 SDLoc DL(Op);
4248 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4249 VT: getPointerTy(DL: MF.getDataLayout()));
4250
4251 // vastart just stores the address of the VarArgsFrameIndex slot into the
4252 // memory location argument.
4253 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4254 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
4255 PtrInfo: MachinePointerInfo(SV));
4256}
4257
4258SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
4259 SelectionDAG &DAG) const {
4260 SDLoc DL(Op);
4261 SDValue Op0 = Op.getOperand(i: 0);
4262 EVT VT = Op.getValueType();
4263 EVT Op0VT = Op0.getValueType();
4264
4265 if (VT.isVector()) {
4266 if (VT.getScalarSizeInBits() != Op0VT.getScalarSizeInBits())
4267 return SDValue();
4268 return Op;
4269 }
4270
4271 if ((DAG.SignBitIsZero(Op: Op0) || Op->getFlags().hasNonNeg()) &&
4272 !isOperationLegal(Op: ISD::UINT_TO_FP, VT: Op0VT) &&
4273 isOperationLegal(Op: ISD::SINT_TO_FP, VT: Op0VT))
4274 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: Op0);
4275
4276 // We can't do uint64 -> double -> float because of double-rounding issue.
4277 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4278 Op0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i64, Operand: Op0);
4279 SDValue Conv = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::v2f64, Operand: Op0);
4280 Conv = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f64, N1: Conv,
4281 N2: DAG.getIntPtrConstant(Val: 0, DL));
4282 return Conv;
4283 }
4284
4285 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4286 return SDValue();
4287
4288 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4289 !Subtarget.hasBasicD() && "unexpected target features");
4290
4291 if (Op0->getOpcode() == ISD::AND) {
4292 auto *C = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
4293 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4294 return Op;
4295 }
4296
4297 if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&
4298 Op0.getConstantOperandVal(i: 1) < UINT64_C(0X1F) &&
4299 Op0.getConstantOperandVal(i: 2) == UINT64_C(0))
4300 return Op;
4301
4302 if (Op0.getOpcode() == ISD::AssertZext &&
4303 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLT(VT: MVT::i32))
4304 return Op;
4305
4306 EVT OpVT = Op0.getValueType();
4307 EVT RetVT = Op.getValueType();
4308 RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
4309 MakeLibCallOptions CallOptions;
4310 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4311 SDValue Chain = SDValue();
4312 SDValue Result;
4313 std::tie(args&: Result, args&: Chain) =
4314 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4315 return Result;
4316}
4317
4318SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
4319 SelectionDAG &DAG) const {
4320 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4321 !Subtarget.hasBasicD() && "unexpected target features");
4322
4323 SDLoc DL(Op);
4324 SDValue Op0 = Op.getOperand(i: 0);
4325
4326 if ((Op0.getOpcode() == ISD::AssertSext ||
4327 Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&
4328 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLE(VT: MVT::i32))
4329 return Op;
4330
4331 EVT OpVT = Op0.getValueType();
4332 EVT RetVT = Op.getValueType();
4333 RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
4334 MakeLibCallOptions CallOptions;
4335 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4336 SDValue Chain = SDValue();
4337 SDValue Result;
4338 std::tie(args&: Result, args&: Chain) =
4339 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4340 return Result;
4341}
4342
4343SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
4344 SelectionDAG &DAG) const {
4345
4346 SDLoc DL(Op);
4347 EVT VT = Op.getValueType();
4348 SDValue Op0 = Op.getOperand(i: 0);
4349 EVT Op0VT = Op0.getValueType();
4350
4351 if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4352 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4353 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4354 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: NewOp0);
4355 }
4356 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4357 SDValue Lo, Hi;
4358 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
4359 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
4360 }
4361 return Op;
4362}
4363
4364SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
4365 SelectionDAG &DAG) const {
4366
4367 SDLoc DL(Op);
4368 SDValue Op0 = Op.getOperand(i: 0);
4369
4370 if (Op0.getValueType() == MVT::f16)
4371 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
4372
4373 if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4374 !Subtarget.hasBasicD()) {
4375 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f32, Operand: Op0);
4376 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Dst);
4377 }
4378
4379 EVT FPTy = EVT::getFloatingPointVT(BitWidth: Op.getValueSizeInBits());
4380 SDValue Trunc = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FPTy, Operand: Op0);
4381 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op.getValueType(), Operand: Trunc);
4382}
4383
4384SDValue LoongArchTargetLowering::lowerFP_TO_UINT(SDValue Op,
4385 SelectionDAG &DAG) const {
4386 if (!Subtarget.hasExtLSX())
4387 return SDValue();
4388
4389 SDLoc DL(Op);
4390 SDValue Src = Op.getOperand(i: 0);
4391 EVT VT = Op.getValueType();
4392 EVT SrcVT = Src.getValueType();
4393
4394 if (VT != MVT::i64)
4395 return SDValue();
4396
4397 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4398 return SDValue();
4399
4400 if (SrcVT == MVT::f32)
4401 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Src);
4402 Src = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2f64, Operand: Src);
4403 SDValue Conv = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: MVT::v2i64, Operand: Src);
4404 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Conv,
4405 N2: DAG.getIntPtrConstant(Val: 0, DL));
4406}
4407
4408static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
4409 SelectionDAG &DAG, unsigned Flags) {
4410 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
4411}
4412
4413static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
4414 SelectionDAG &DAG, unsigned Flags) {
4415 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
4416 TargetFlags: Flags);
4417}
4418
4419static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
4420 SelectionDAG &DAG, unsigned Flags) {
4421 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
4422 Offset: N->getOffset(), TargetFlags: Flags);
4423}
4424
4425static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
4426 SelectionDAG &DAG, unsigned Flags) {
4427 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
4428}
4429
4430template <class NodeTy>
4431SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4432 CodeModel::Model M,
4433 bool IsLocal) const {
4434 SDLoc DL(N);
4435 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4436 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
4437 SDValue Load;
4438
4439 switch (M) {
4440 default:
4441 report_fatal_error(reason: "Unsupported code model");
4442
4443 case CodeModel::Large: {
4444 assert(Subtarget.is64Bit() && "Large code model requires LA64");
4445
4446 // This is not actually used, but is necessary for successfully matching
4447 // the PseudoLA_*_LARGE nodes.
4448 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4449 if (IsLocal) {
4450 // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that
4451 // eventually becomes the desired 5-insn code sequence.
4452 Load = SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL_LARGE, dl: DL, VT: Ty,
4453 Op1: Tmp, Op2: Addr),
4454 0);
4455 } else {
4456 // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that
4457 // eventually becomes the desired 5-insn code sequence.
4458 Load = SDValue(
4459 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT_LARGE, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr),
4460 0);
4461 }
4462 break;
4463 }
4464
4465 case CodeModel::Small:
4466 case CodeModel::Medium:
4467 if (IsLocal) {
4468 // This generates the pattern (PseudoLA_PCREL sym), which
4469 //
4470 // for la32r expands to:
4471 // (addi.w (pcaddu12i %pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4472 //
4473 // for la32s and la64 expands to:
4474 // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).
4475 Load = SDValue(
4476 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL, dl: DL, VT: Ty, Op1: Addr), 0);
4477 } else {
4478 // This generates the pattern (PseudoLA_GOT sym), which
4479 //
4480 // for la32r expands to:
4481 // (ld.w (pcaddu12i %got_pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4482 //
4483 // for la32s and la64 expands to:
4484 // (ld.w/d (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).
4485 Load =
4486 SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT, dl: DL, VT: Ty, Op1: Addr), 0);
4487 }
4488 }
4489
4490 if (!IsLocal) {
4491 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4492 MachineFunction &MF = DAG.getMachineFunction();
4493 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4494 PtrInfo: MachinePointerInfo::getGOT(MF),
4495 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4496 MachineMemOperand::MOInvariant,
4497 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4498 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
4499 }
4500
4501 return Load;
4502}
4503
4504SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,
4505 SelectionDAG &DAG) const {
4506 return getAddr(N: cast<BlockAddressSDNode>(Val&: Op), DAG,
4507 M: DAG.getTarget().getCodeModel());
4508}
4509
4510SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,
4511 SelectionDAG &DAG) const {
4512 return getAddr(N: cast<JumpTableSDNode>(Val&: Op), DAG,
4513 M: DAG.getTarget().getCodeModel());
4514}
4515
4516SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,
4517 SelectionDAG &DAG) const {
4518 return getAddr(N: cast<ConstantPoolSDNode>(Val&: Op), DAG,
4519 M: DAG.getTarget().getCodeModel());
4520}
4521
4522SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,
4523 SelectionDAG &DAG) const {
4524 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4525 assert(N->getOffset() == 0 && "unexpected offset in global node");
4526 auto CM = DAG.getTarget().getCodeModel();
4527 const GlobalValue *GV = N->getGlobal();
4528
4529 if (GV->isDSOLocal() && isa<GlobalVariable>(Val: GV)) {
4530 if (auto GCM = dyn_cast<GlobalVariable>(Val: GV)->getCodeModel())
4531 CM = *GCM;
4532 }
4533
4534 return getAddr(N, DAG, M: CM, IsLocal: GV->isDSOLocal());
4535}
4536
4537SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
4538 SelectionDAG &DAG,
4539 unsigned Opc, bool UseGOT,
4540 bool Large) const {
4541 SDLoc DL(N);
4542 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4543 MVT GRLenVT = Subtarget.getGRLenVT();
4544
4545 // This is not actually used, but is necessary for successfully matching the
4546 // PseudoLA_*_LARGE nodes.
4547 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4548 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4549
4550 // Only IE needs an extra argument for large code model.
4551 SDValue Offset = Opc == LoongArch::PseudoLA_TLS_IE_LARGE
4552 ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4553 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4554
4555 // If it is LE for normal/medium code model, the add tp operation will occur
4556 // during the pseudo-instruction expansion.
4557 if (Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4558 return Offset;
4559
4560 if (UseGOT) {
4561 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4562 MachineFunction &MF = DAG.getMachineFunction();
4563 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4564 PtrInfo: MachinePointerInfo::getGOT(MF),
4565 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4566 MachineMemOperand::MOInvariant,
4567 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4568 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Offset.getNode()), NewMemRefs: {MemOp});
4569 }
4570
4571 // Add the thread pointer.
4572 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Offset,
4573 N2: DAG.getRegister(Reg: LoongArch::R2, VT: GRLenVT));
4574}
4575
4576SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
4577 SelectionDAG &DAG,
4578 unsigned Opc,
4579 bool Large) const {
4580 SDLoc DL(N);
4581 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4582 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
4583
4584 // This is not actually used, but is necessary for successfully matching the
4585 // PseudoLA_*_LARGE nodes.
4586 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4587
4588 // Use a PC-relative addressing mode to access the dynamic GOT address.
4589 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4590 SDValue Load = Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4591 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4592
4593 // Prepare argument list to generate call.
4594 ArgListTy Args;
4595 Args.emplace_back(args&: Load, args&: CallTy);
4596
4597 // Setup call to __tls_get_addr.
4598 TargetLowering::CallLoweringInfo CLI(DAG);
4599 CLI.setDebugLoc(DL)
4600 .setChain(DAG.getEntryNode())
4601 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
4602 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
4603 ArgsList: std::move(Args));
4604
4605 return LowerCallTo(CLI).first;
4606}
4607
4608SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
4609 SelectionDAG &DAG, unsigned Opc,
4610 bool Large) const {
4611 SDLoc DL(N);
4612 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4613 const GlobalValue *GV = N->getGlobal();
4614
4615 // This is not actually used, but is necessary for successfully matching the
4616 // PseudoLA_*_LARGE nodes.
4617 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4618
4619 // Use a PC-relative addressing mode to access the global dynamic GOT address.
4620 // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).
4621 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
4622 return Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4623 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4624}
4625
4626SDValue
4627LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,
4628 SelectionDAG &DAG) const {
4629 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4630 CallingConv::GHC)
4631 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4632
4633 bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;
4634 assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");
4635
4636 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4637 assert(N->getOffset() == 0 && "unexpected offset in global node");
4638
4639 if (DAG.getTarget().useEmulatedTLS())
4640 reportFatalUsageError(reason: "the emulated TLS is prohibited");
4641
4642 bool IsDesc = DAG.getTarget().useTLSDESC();
4643
4644 switch (getTargetMachine().getTLSModel(GV: N->getGlobal())) {
4645 case TLSModel::GeneralDynamic:
4646 // In this model, application code calls the dynamic linker function
4647 // __tls_get_addr to locate TLS offsets into the dynamic thread vector at
4648 // runtime.
4649 if (!IsDesc)
4650 return getDynamicTLSAddr(N, DAG,
4651 Opc: Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4652 : LoongArch::PseudoLA_TLS_GD,
4653 Large);
4654 break;
4655 case TLSModel::LocalDynamic:
4656 // Same as GeneralDynamic, except for assembly modifiers and relocation
4657 // records.
4658 if (!IsDesc)
4659 return getDynamicTLSAddr(N, DAG,
4660 Opc: Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4661 : LoongArch::PseudoLA_TLS_LD,
4662 Large);
4663 break;
4664 case TLSModel::InitialExec:
4665 // This model uses the GOT to resolve TLS offsets.
4666 return getStaticTLSAddr(N, DAG,
4667 Opc: Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4668 : LoongArch::PseudoLA_TLS_IE,
4669 /*UseGOT=*/true, Large);
4670 case TLSModel::LocalExec:
4671 // This model is used when static linking as the TLS offsets are resolved
4672 // during program linking.
4673 //
4674 // This node doesn't need an extra argument for the large code model.
4675 return getStaticTLSAddr(N, DAG, Opc: LoongArch::PseudoLA_TLS_LE,
4676 /*UseGOT=*/false, Large);
4677 }
4678
4679 return getTLSDescAddr(N, DAG,
4680 Opc: Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
4681 : LoongArch::PseudoLA_TLS_DESC,
4682 Large);
4683}
4684
4685template <unsigned N>
4686static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp,
4687 SelectionDAG &DAG, bool IsSigned = false) {
4688 auto *CImm = cast<ConstantSDNode>(Val: Op->getOperand(Num: ImmOp));
4689 // Check the ImmArg.
4690 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
4691 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
4692 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) +
4693 ": argument out of range.");
4694 return DAG.getNode(Opcode: ISD::UNDEF, DL: SDLoc(Op), VT: Op.getValueType());
4695 }
4696 return SDValue();
4697}
4698
4699SDValue
4700LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
4701 SelectionDAG &DAG) const {
4702 switch (Op.getConstantOperandVal(i: 0)) {
4703 default:
4704 return SDValue(); // Don't custom lower most intrinsics.
4705 case Intrinsic::thread_pointer: {
4706 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4707 return DAG.getRegister(Reg: LoongArch::R2, VT: PtrVT);
4708 }
4709 case Intrinsic::loongarch_lsx_vpickve2gr_d:
4710 case Intrinsic::loongarch_lsx_vpickve2gr_du:
4711 case Intrinsic::loongarch_lsx_vreplvei_d:
4712 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
4713 return checkIntrinsicImmArg<1>(Op, ImmOp: 2, DAG);
4714 case Intrinsic::loongarch_lsx_vreplvei_w:
4715 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
4716 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
4717 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
4718 case Intrinsic::loongarch_lasx_xvpickve_d:
4719 case Intrinsic::loongarch_lasx_xvpickve_d_f:
4720 return checkIntrinsicImmArg<2>(Op, ImmOp: 2, DAG);
4721 case Intrinsic::loongarch_lasx_xvinsve0_d:
4722 return checkIntrinsicImmArg<2>(Op, ImmOp: 3, DAG);
4723 case Intrinsic::loongarch_lsx_vsat_b:
4724 case Intrinsic::loongarch_lsx_vsat_bu:
4725 case Intrinsic::loongarch_lsx_vrotri_b:
4726 case Intrinsic::loongarch_lsx_vsllwil_h_b:
4727 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
4728 case Intrinsic::loongarch_lsx_vsrlri_b:
4729 case Intrinsic::loongarch_lsx_vsrari_b:
4730 case Intrinsic::loongarch_lsx_vreplvei_h:
4731 case Intrinsic::loongarch_lasx_xvsat_b:
4732 case Intrinsic::loongarch_lasx_xvsat_bu:
4733 case Intrinsic::loongarch_lasx_xvrotri_b:
4734 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
4735 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
4736 case Intrinsic::loongarch_lasx_xvsrlri_b:
4737 case Intrinsic::loongarch_lasx_xvsrari_b:
4738 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
4739 case Intrinsic::loongarch_lasx_xvpickve_w:
4740 case Intrinsic::loongarch_lasx_xvpickve_w_f:
4741 return checkIntrinsicImmArg<3>(Op, ImmOp: 2, DAG);
4742 case Intrinsic::loongarch_lasx_xvinsve0_w:
4743 return checkIntrinsicImmArg<3>(Op, ImmOp: 3, DAG);
4744 case Intrinsic::loongarch_lsx_vsat_h:
4745 case Intrinsic::loongarch_lsx_vsat_hu:
4746 case Intrinsic::loongarch_lsx_vrotri_h:
4747 case Intrinsic::loongarch_lsx_vsllwil_w_h:
4748 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
4749 case Intrinsic::loongarch_lsx_vsrlri_h:
4750 case Intrinsic::loongarch_lsx_vsrari_h:
4751 case Intrinsic::loongarch_lsx_vreplvei_b:
4752 case Intrinsic::loongarch_lasx_xvsat_h:
4753 case Intrinsic::loongarch_lasx_xvsat_hu:
4754 case Intrinsic::loongarch_lasx_xvrotri_h:
4755 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
4756 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
4757 case Intrinsic::loongarch_lasx_xvsrlri_h:
4758 case Intrinsic::loongarch_lasx_xvsrari_h:
4759 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
4760 return checkIntrinsicImmArg<4>(Op, ImmOp: 2, DAG);
4761 case Intrinsic::loongarch_lsx_vsrlni_b_h:
4762 case Intrinsic::loongarch_lsx_vsrani_b_h:
4763 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
4764 case Intrinsic::loongarch_lsx_vsrarni_b_h:
4765 case Intrinsic::loongarch_lsx_vssrlni_b_h:
4766 case Intrinsic::loongarch_lsx_vssrani_b_h:
4767 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
4768 case Intrinsic::loongarch_lsx_vssrani_bu_h:
4769 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
4770 case Intrinsic::loongarch_lsx_vssrarni_b_h:
4771 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
4772 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
4773 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
4774 case Intrinsic::loongarch_lasx_xvsrani_b_h:
4775 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
4776 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
4777 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
4778 case Intrinsic::loongarch_lasx_xvssrani_b_h:
4779 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
4780 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
4781 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
4782 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
4783 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
4784 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
4785 return checkIntrinsicImmArg<4>(Op, ImmOp: 3, DAG);
4786 case Intrinsic::loongarch_lsx_vsat_w:
4787 case Intrinsic::loongarch_lsx_vsat_wu:
4788 case Intrinsic::loongarch_lsx_vrotri_w:
4789 case Intrinsic::loongarch_lsx_vsllwil_d_w:
4790 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
4791 case Intrinsic::loongarch_lsx_vsrlri_w:
4792 case Intrinsic::loongarch_lsx_vsrari_w:
4793 case Intrinsic::loongarch_lsx_vslei_bu:
4794 case Intrinsic::loongarch_lsx_vslei_hu:
4795 case Intrinsic::loongarch_lsx_vslei_wu:
4796 case Intrinsic::loongarch_lsx_vslei_du:
4797 case Intrinsic::loongarch_lsx_vslti_bu:
4798 case Intrinsic::loongarch_lsx_vslti_hu:
4799 case Intrinsic::loongarch_lsx_vslti_wu:
4800 case Intrinsic::loongarch_lsx_vslti_du:
4801 case Intrinsic::loongarch_lsx_vbsll_v:
4802 case Intrinsic::loongarch_lsx_vbsrl_v:
4803 case Intrinsic::loongarch_lasx_xvsat_w:
4804 case Intrinsic::loongarch_lasx_xvsat_wu:
4805 case Intrinsic::loongarch_lasx_xvrotri_w:
4806 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
4807 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
4808 case Intrinsic::loongarch_lasx_xvsrlri_w:
4809 case Intrinsic::loongarch_lasx_xvsrari_w:
4810 case Intrinsic::loongarch_lasx_xvslei_bu:
4811 case Intrinsic::loongarch_lasx_xvslei_hu:
4812 case Intrinsic::loongarch_lasx_xvslei_wu:
4813 case Intrinsic::loongarch_lasx_xvslei_du:
4814 case Intrinsic::loongarch_lasx_xvslti_bu:
4815 case Intrinsic::loongarch_lasx_xvslti_hu:
4816 case Intrinsic::loongarch_lasx_xvslti_wu:
4817 case Intrinsic::loongarch_lasx_xvslti_du:
4818 case Intrinsic::loongarch_lasx_xvbsll_v:
4819 case Intrinsic::loongarch_lasx_xvbsrl_v:
4820 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG);
4821 case Intrinsic::loongarch_lsx_vseqi_b:
4822 case Intrinsic::loongarch_lsx_vseqi_h:
4823 case Intrinsic::loongarch_lsx_vseqi_w:
4824 case Intrinsic::loongarch_lsx_vseqi_d:
4825 case Intrinsic::loongarch_lsx_vslei_b:
4826 case Intrinsic::loongarch_lsx_vslei_h:
4827 case Intrinsic::loongarch_lsx_vslei_w:
4828 case Intrinsic::loongarch_lsx_vslei_d:
4829 case Intrinsic::loongarch_lsx_vslti_b:
4830 case Intrinsic::loongarch_lsx_vslti_h:
4831 case Intrinsic::loongarch_lsx_vslti_w:
4832 case Intrinsic::loongarch_lsx_vslti_d:
4833 case Intrinsic::loongarch_lasx_xvseqi_b:
4834 case Intrinsic::loongarch_lasx_xvseqi_h:
4835 case Intrinsic::loongarch_lasx_xvseqi_w:
4836 case Intrinsic::loongarch_lasx_xvseqi_d:
4837 case Intrinsic::loongarch_lasx_xvslei_b:
4838 case Intrinsic::loongarch_lasx_xvslei_h:
4839 case Intrinsic::loongarch_lasx_xvslei_w:
4840 case Intrinsic::loongarch_lasx_xvslei_d:
4841 case Intrinsic::loongarch_lasx_xvslti_b:
4842 case Intrinsic::loongarch_lasx_xvslti_h:
4843 case Intrinsic::loongarch_lasx_xvslti_w:
4844 case Intrinsic::loongarch_lasx_xvslti_d:
4845 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG, /*IsSigned=*/true);
4846 case Intrinsic::loongarch_lsx_vsrlni_h_w:
4847 case Intrinsic::loongarch_lsx_vsrani_h_w:
4848 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
4849 case Intrinsic::loongarch_lsx_vsrarni_h_w:
4850 case Intrinsic::loongarch_lsx_vssrlni_h_w:
4851 case Intrinsic::loongarch_lsx_vssrani_h_w:
4852 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
4853 case Intrinsic::loongarch_lsx_vssrani_hu_w:
4854 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
4855 case Intrinsic::loongarch_lsx_vssrarni_h_w:
4856 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
4857 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
4858 case Intrinsic::loongarch_lsx_vfrstpi_b:
4859 case Intrinsic::loongarch_lsx_vfrstpi_h:
4860 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
4861 case Intrinsic::loongarch_lasx_xvsrani_h_w:
4862 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
4863 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
4864 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
4865 case Intrinsic::loongarch_lasx_xvssrani_h_w:
4866 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
4867 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
4868 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
4869 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
4870 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
4871 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
4872 case Intrinsic::loongarch_lasx_xvfrstpi_b:
4873 case Intrinsic::loongarch_lasx_xvfrstpi_h:
4874 return checkIntrinsicImmArg<5>(Op, ImmOp: 3, DAG);
4875 case Intrinsic::loongarch_lsx_vsat_d:
4876 case Intrinsic::loongarch_lsx_vsat_du:
4877 case Intrinsic::loongarch_lsx_vrotri_d:
4878 case Intrinsic::loongarch_lsx_vsrlri_d:
4879 case Intrinsic::loongarch_lsx_vsrari_d:
4880 case Intrinsic::loongarch_lasx_xvsat_d:
4881 case Intrinsic::loongarch_lasx_xvsat_du:
4882 case Intrinsic::loongarch_lasx_xvrotri_d:
4883 case Intrinsic::loongarch_lasx_xvsrlri_d:
4884 case Intrinsic::loongarch_lasx_xvsrari_d:
4885 return checkIntrinsicImmArg<6>(Op, ImmOp: 2, DAG);
4886 case Intrinsic::loongarch_lsx_vsrlni_w_d:
4887 case Intrinsic::loongarch_lsx_vsrani_w_d:
4888 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
4889 case Intrinsic::loongarch_lsx_vsrarni_w_d:
4890 case Intrinsic::loongarch_lsx_vssrlni_w_d:
4891 case Intrinsic::loongarch_lsx_vssrani_w_d:
4892 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
4893 case Intrinsic::loongarch_lsx_vssrani_wu_d:
4894 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
4895 case Intrinsic::loongarch_lsx_vssrarni_w_d:
4896 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
4897 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
4898 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
4899 case Intrinsic::loongarch_lasx_xvsrani_w_d:
4900 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
4901 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
4902 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
4903 case Intrinsic::loongarch_lasx_xvssrani_w_d:
4904 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
4905 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
4906 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
4907 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
4908 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
4909 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
4910 return checkIntrinsicImmArg<6>(Op, ImmOp: 3, DAG);
4911 case Intrinsic::loongarch_lsx_vsrlni_d_q:
4912 case Intrinsic::loongarch_lsx_vsrani_d_q:
4913 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
4914 case Intrinsic::loongarch_lsx_vsrarni_d_q:
4915 case Intrinsic::loongarch_lsx_vssrlni_d_q:
4916 case Intrinsic::loongarch_lsx_vssrani_d_q:
4917 case Intrinsic::loongarch_lsx_vssrlni_du_q:
4918 case Intrinsic::loongarch_lsx_vssrani_du_q:
4919 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
4920 case Intrinsic::loongarch_lsx_vssrarni_d_q:
4921 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
4922 case Intrinsic::loongarch_lsx_vssrarni_du_q:
4923 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
4924 case Intrinsic::loongarch_lasx_xvsrani_d_q:
4925 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
4926 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
4927 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
4928 case Intrinsic::loongarch_lasx_xvssrani_d_q:
4929 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
4930 case Intrinsic::loongarch_lasx_xvssrani_du_q:
4931 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
4932 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
4933 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
4934 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
4935 return checkIntrinsicImmArg<7>(Op, ImmOp: 3, DAG);
4936 case Intrinsic::loongarch_lsx_vnori_b:
4937 case Intrinsic::loongarch_lsx_vshuf4i_b:
4938 case Intrinsic::loongarch_lsx_vshuf4i_h:
4939 case Intrinsic::loongarch_lsx_vshuf4i_w:
4940 case Intrinsic::loongarch_lasx_xvnori_b:
4941 case Intrinsic::loongarch_lasx_xvshuf4i_b:
4942 case Intrinsic::loongarch_lasx_xvshuf4i_h:
4943 case Intrinsic::loongarch_lasx_xvshuf4i_w:
4944 case Intrinsic::loongarch_lasx_xvpermi_d:
4945 return checkIntrinsicImmArg<8>(Op, ImmOp: 2, DAG);
4946 case Intrinsic::loongarch_lsx_vshuf4i_d:
4947 case Intrinsic::loongarch_lsx_vpermi_w:
4948 case Intrinsic::loongarch_lsx_vbitseli_b:
4949 case Intrinsic::loongarch_lsx_vextrins_b:
4950 case Intrinsic::loongarch_lsx_vextrins_h:
4951 case Intrinsic::loongarch_lsx_vextrins_w:
4952 case Intrinsic::loongarch_lsx_vextrins_d:
4953 case Intrinsic::loongarch_lasx_xvshuf4i_d:
4954 case Intrinsic::loongarch_lasx_xvpermi_w:
4955 case Intrinsic::loongarch_lasx_xvpermi_q:
4956 case Intrinsic::loongarch_lasx_xvbitseli_b:
4957 case Intrinsic::loongarch_lasx_xvextrins_b:
4958 case Intrinsic::loongarch_lasx_xvextrins_h:
4959 case Intrinsic::loongarch_lasx_xvextrins_w:
4960 case Intrinsic::loongarch_lasx_xvextrins_d:
4961 return checkIntrinsicImmArg<8>(Op, ImmOp: 3, DAG);
4962 case Intrinsic::loongarch_lsx_vrepli_b:
4963 case Intrinsic::loongarch_lsx_vrepli_h:
4964 case Intrinsic::loongarch_lsx_vrepli_w:
4965 case Intrinsic::loongarch_lsx_vrepli_d:
4966 case Intrinsic::loongarch_lasx_xvrepli_b:
4967 case Intrinsic::loongarch_lasx_xvrepli_h:
4968 case Intrinsic::loongarch_lasx_xvrepli_w:
4969 case Intrinsic::loongarch_lasx_xvrepli_d:
4970 return checkIntrinsicImmArg<10>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4971 case Intrinsic::loongarch_lsx_vldi:
4972 case Intrinsic::loongarch_lasx_xvldi:
4973 return checkIntrinsicImmArg<13>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4974 }
4975}
4976
4977// Helper function that emits error message for intrinsics with chain and return
4978// merge values of a UNDEF and the chain.
4979static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op,
4980 StringRef ErrorMsg,
4981 SelectionDAG &DAG) {
4982 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
4983 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: Op.getValueType()), Op.getOperand(i: 0)},
4984 dl: SDLoc(Op));
4985}
4986
4987SDValue
4988LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
4989 SelectionDAG &DAG) const {
4990 SDLoc DL(Op);
4991 MVT GRLenVT = Subtarget.getGRLenVT();
4992 EVT VT = Op.getValueType();
4993 SDValue Chain = Op.getOperand(i: 0);
4994 const StringRef ErrorMsgOOR = "argument out of range";
4995 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
4996 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
4997
4998 switch (Op.getConstantOperandVal(i: 1)) {
4999 default:
5000 return Op;
5001 case Intrinsic::loongarch_crc_w_b_w:
5002 case Intrinsic::loongarch_crc_w_h_w:
5003 case Intrinsic::loongarch_crc_w_w_w:
5004 case Intrinsic::loongarch_crc_w_d_w:
5005 case Intrinsic::loongarch_crcc_w_b_w:
5006 case Intrinsic::loongarch_crcc_w_h_w:
5007 case Intrinsic::loongarch_crcc_w_w_w:
5008 case Intrinsic::loongarch_crcc_w_d_w:
5009 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5010 case Intrinsic::loongarch_csrrd_w:
5011 case Intrinsic::loongarch_csrrd_d: {
5012 unsigned Imm = Op.getConstantOperandVal(i: 2);
5013 return !isUInt<14>(x: Imm)
5014 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5015 : DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
5016 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5017 }
5018 case Intrinsic::loongarch_csrwr_w:
5019 case Intrinsic::loongarch_csrwr_d: {
5020 unsigned Imm = Op.getConstantOperandVal(i: 3);
5021 return !isUInt<14>(x: Imm)
5022 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5023 : DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5024 Ops: {Chain, Op.getOperand(i: 2),
5025 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5026 }
5027 case Intrinsic::loongarch_csrxchg_w:
5028 case Intrinsic::loongarch_csrxchg_d: {
5029 unsigned Imm = Op.getConstantOperandVal(i: 4);
5030 return !isUInt<14>(x: Imm)
5031 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5032 : DAG.getNode(Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5033 Ops: {Chain, Op.getOperand(i: 2), Op.getOperand(i: 3),
5034 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5035 }
5036 case Intrinsic::loongarch_iocsrrd_d: {
5037 return DAG.getNode(
5038 Opcode: LoongArchISD::IOCSRRD_D, DL, ResultTys: {GRLenVT, MVT::Other},
5039 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op.getOperand(i: 2))});
5040 }
5041#define IOCSRRD_CASE(NAME, NODE) \
5042 case Intrinsic::loongarch_##NAME: { \
5043 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5044 {Chain, Op.getOperand(2)}); \
5045 }
5046 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5047 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5048 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5049#undef IOCSRRD_CASE
5050 case Intrinsic::loongarch_cpucfg: {
5051 return DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5052 Ops: {Chain, Op.getOperand(i: 2)});
5053 }
5054 case Intrinsic::loongarch_lddir_d: {
5055 unsigned Imm = Op.getConstantOperandVal(i: 3);
5056 return !isUInt<8>(x: Imm)
5057 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5058 : Op;
5059 }
5060 case Intrinsic::loongarch_movfcsr2gr: {
5061 if (!Subtarget.hasBasicF())
5062 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5063 unsigned Imm = Op.getConstantOperandVal(i: 2);
5064 return !isUInt<2>(x: Imm)
5065 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5066 : DAG.getNode(Opcode: LoongArchISD::MOVFCSR2GR, DL, ResultTys: {VT, MVT::Other},
5067 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5068 }
5069 case Intrinsic::loongarch_lsx_vld:
5070 case Intrinsic::loongarch_lsx_vldrepl_b:
5071 case Intrinsic::loongarch_lasx_xvld:
5072 case Intrinsic::loongarch_lasx_xvldrepl_b:
5073 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5074 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5075 : SDValue();
5076 case Intrinsic::loongarch_lsx_vldrepl_h:
5077 case Intrinsic::loongarch_lasx_xvldrepl_h:
5078 return !isShiftedInt<11, 1>(
5079 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5080 ? emitIntrinsicWithChainErrorMessage(
5081 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5082 : SDValue();
5083 case Intrinsic::loongarch_lsx_vldrepl_w:
5084 case Intrinsic::loongarch_lasx_xvldrepl_w:
5085 return !isShiftedInt<10, 2>(
5086 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5087 ? emitIntrinsicWithChainErrorMessage(
5088 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5089 : SDValue();
5090 case Intrinsic::loongarch_lsx_vldrepl_d:
5091 case Intrinsic::loongarch_lasx_xvldrepl_d:
5092 return !isShiftedInt<9, 3>(
5093 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5094 ? emitIntrinsicWithChainErrorMessage(
5095 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5096 : SDValue();
5097 }
5098}
5099
5100// Helper function that emits error message for intrinsics with void return
5101// value and return the chain.
5102static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg,
5103 SelectionDAG &DAG) {
5104
5105 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5106 return Op.getOperand(i: 0);
5107}
5108
5109SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
5110 SelectionDAG &DAG) const {
5111 SDLoc DL(Op);
5112 MVT GRLenVT = Subtarget.getGRLenVT();
5113 SDValue Chain = Op.getOperand(i: 0);
5114 uint64_t IntrinsicEnum = Op.getConstantOperandVal(i: 1);
5115 SDValue Op2 = Op.getOperand(i: 2);
5116 const StringRef ErrorMsgOOR = "argument out of range";
5117 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5118 const StringRef ErrorMsgReqLA32 = "requires loongarch32";
5119 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5120
5121 switch (IntrinsicEnum) {
5122 default:
5123 // TODO: Add more Intrinsics.
5124 return SDValue();
5125 case Intrinsic::loongarch_cacop_d:
5126 case Intrinsic::loongarch_cacop_w: {
5127 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5128 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5129 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5130 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA32, DAG);
5131 // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)
5132 unsigned Imm1 = Op2->getAsZExtVal();
5133 int Imm2 = cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue();
5134 if (!isUInt<5>(x: Imm1) || !isInt<12>(x: Imm2))
5135 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG);
5136 return Op;
5137 }
5138 case Intrinsic::loongarch_dbar: {
5139 unsigned Imm = Op2->getAsZExtVal();
5140 return !isUInt<15>(x: Imm)
5141 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5142 : DAG.getNode(Opcode: LoongArchISD::DBAR, DL, VT: MVT::Other, N1: Chain,
5143 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5144 }
5145 case Intrinsic::loongarch_ibar: {
5146 unsigned Imm = Op2->getAsZExtVal();
5147 return !isUInt<15>(x: Imm)
5148 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5149 : DAG.getNode(Opcode: LoongArchISD::IBAR, DL, VT: MVT::Other, N1: Chain,
5150 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5151 }
5152 case Intrinsic::loongarch_break: {
5153 unsigned Imm = Op2->getAsZExtVal();
5154 return !isUInt<15>(x: Imm)
5155 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5156 : DAG.getNode(Opcode: LoongArchISD::BREAK, DL, VT: MVT::Other, N1: Chain,
5157 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5158 }
5159 case Intrinsic::loongarch_movgr2fcsr: {
5160 if (!Subtarget.hasBasicF())
5161 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5162 unsigned Imm = Op2->getAsZExtVal();
5163 return !isUInt<2>(x: Imm)
5164 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5165 : DAG.getNode(Opcode: LoongArchISD::MOVGR2FCSR, DL, VT: MVT::Other, N1: Chain,
5166 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT),
5167 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT,
5168 Operand: Op.getOperand(i: 3)));
5169 }
5170 case Intrinsic::loongarch_syscall: {
5171 unsigned Imm = Op2->getAsZExtVal();
5172 return !isUInt<15>(x: Imm)
5173 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5174 : DAG.getNode(Opcode: LoongArchISD::SYSCALL, DL, VT: MVT::Other, N1: Chain,
5175 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5176 }
5177#define IOCSRWR_CASE(NAME, NODE) \
5178 case Intrinsic::loongarch_##NAME: { \
5179 SDValue Op3 = Op.getOperand(3); \
5180 return Subtarget.is64Bit() \
5181 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5182 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5183 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5184 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5185 Op3); \
5186 }
5187 IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);
5188 IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);
5189 IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);
5190#undef IOCSRWR_CASE
5191 case Intrinsic::loongarch_iocsrwr_d: {
5192 return !Subtarget.is64Bit()
5193 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5194 : DAG.getNode(Opcode: LoongArchISD::IOCSRWR_D, DL, VT: MVT::Other, N1: Chain,
5195 N2: Op2,
5196 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64,
5197 Operand: Op.getOperand(i: 3)));
5198 }
5199#define ASRT_LE_GT_CASE(NAME) \
5200 case Intrinsic::loongarch_##NAME: { \
5201 return !Subtarget.is64Bit() \
5202 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5203 : Op; \
5204 }
5205 ASRT_LE_GT_CASE(asrtle_d)
5206 ASRT_LE_GT_CASE(asrtgt_d)
5207#undef ASRT_LE_GT_CASE
5208 case Intrinsic::loongarch_ldpte_d: {
5209 unsigned Imm = Op.getConstantOperandVal(i: 3);
5210 return !Subtarget.is64Bit()
5211 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5212 : !isUInt<8>(x: Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5213 : Op;
5214 }
5215 case Intrinsic::loongarch_lsx_vst:
5216 case Intrinsic::loongarch_lasx_xvst:
5217 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue())
5218 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5219 : SDValue();
5220 case Intrinsic::loongarch_lasx_xvstelm_b:
5221 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5222 !isUInt<5>(x: Op.getConstantOperandVal(i: 5)))
5223 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5224 : SDValue();
5225 case Intrinsic::loongarch_lsx_vstelm_b:
5226 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5227 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5228 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5229 : SDValue();
5230 case Intrinsic::loongarch_lasx_xvstelm_h:
5231 return (!isShiftedInt<8, 1>(
5232 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5233 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5234 ? emitIntrinsicErrorMessage(
5235 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5236 : SDValue();
5237 case Intrinsic::loongarch_lsx_vstelm_h:
5238 return (!isShiftedInt<8, 1>(
5239 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5240 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5241 ? emitIntrinsicErrorMessage(
5242 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5243 : SDValue();
5244 case Intrinsic::loongarch_lasx_xvstelm_w:
5245 return (!isShiftedInt<8, 2>(
5246 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5247 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5248 ? emitIntrinsicErrorMessage(
5249 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5250 : SDValue();
5251 case Intrinsic::loongarch_lsx_vstelm_w:
5252 return (!isShiftedInt<8, 2>(
5253 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5254 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5255 ? emitIntrinsicErrorMessage(
5256 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5257 : SDValue();
5258 case Intrinsic::loongarch_lasx_xvstelm_d:
5259 return (!isShiftedInt<8, 3>(
5260 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5261 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5262 ? emitIntrinsicErrorMessage(
5263 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5264 : SDValue();
5265 case Intrinsic::loongarch_lsx_vstelm_d:
5266 return (!isShiftedInt<8, 3>(
5267 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5268 !isUInt<1>(x: Op.getConstantOperandVal(i: 5)))
5269 ? emitIntrinsicErrorMessage(
5270 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5271 : SDValue();
5272 }
5273}
5274
5275SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,
5276 SelectionDAG &DAG) const {
5277 SDLoc DL(Op);
5278 SDValue Lo = Op.getOperand(i: 0);
5279 SDValue Hi = Op.getOperand(i: 1);
5280 SDValue Shamt = Op.getOperand(i: 2);
5281 EVT VT = Lo.getValueType();
5282
5283 // if Shamt-GRLen < 0: // Shamt < GRLen
5284 // Lo = Lo << Shamt
5285 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))
5286 // else:
5287 // Lo = 0
5288 // Hi = Lo << (Shamt-GRLen)
5289
5290 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5291 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5292 SDValue MinusGRLen =
5293 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5294 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5295 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5296 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5297
5298 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
5299 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
5300 SDValue ShiftRightLo =
5301 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: GRLenMinus1Shamt);
5302 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
5303 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
5304 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusGRLen);
5305
5306 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5307
5308 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
5309 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5310
5311 SDValue Parts[2] = {Lo, Hi};
5312 return DAG.getMergeValues(Ops: Parts, dl: DL);
5313}
5314
5315SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,
5316 SelectionDAG &DAG,
5317 bool IsSRA) const {
5318 SDLoc DL(Op);
5319 SDValue Lo = Op.getOperand(i: 0);
5320 SDValue Hi = Op.getOperand(i: 1);
5321 SDValue Shamt = Op.getOperand(i: 2);
5322 EVT VT = Lo.getValueType();
5323
5324 // SRA expansion:
5325 // if Shamt-GRLen < 0: // Shamt < GRLen
5326 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5327 // Hi = Hi >>s Shamt
5328 // else:
5329 // Lo = Hi >>s (Shamt-GRLen);
5330 // Hi = Hi >>s (GRLen-1)
5331 //
5332 // SRL expansion:
5333 // if Shamt-GRLen < 0: // Shamt < GRLen
5334 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5335 // Hi = Hi >>u Shamt
5336 // else:
5337 // Lo = Hi >>u (Shamt-GRLen);
5338 // Hi = 0;
5339
5340 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
5341
5342 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5343 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5344 SDValue MinusGRLen =
5345 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5346 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5347 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5348 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5349
5350 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
5351 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
5352 SDValue ShiftLeftHi =
5353 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: GRLenMinus1Shamt);
5354 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
5355 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
5356 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusGRLen);
5357 SDValue HiFalse =
5358 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: GRLenMinus1) : Zero;
5359
5360 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5361
5362 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
5363 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5364
5365 SDValue Parts[2] = {Lo, Hi};
5366 return DAG.getMergeValues(Ops: Parts, dl: DL);
5367}
5368
5369// Returns the opcode of the target-specific SDNode that implements the 32-bit
5370// form of the given Opcode.
5371static unsigned getLoongArchWOpcode(unsigned Opcode) {
5372 switch (Opcode) {
5373 default:
5374 llvm_unreachable("Unexpected opcode");
5375 case ISD::SDIV:
5376 return LoongArchISD::DIV_W;
5377 case ISD::UDIV:
5378 return LoongArchISD::DIV_WU;
5379 case ISD::SREM:
5380 return LoongArchISD::MOD_W;
5381 case ISD::UREM:
5382 return LoongArchISD::MOD_WU;
5383 case ISD::SHL:
5384 return LoongArchISD::SLL_W;
5385 case ISD::SRA:
5386 return LoongArchISD::SRA_W;
5387 case ISD::SRL:
5388 return LoongArchISD::SRL_W;
5389 case ISD::ROTL:
5390 case ISD::ROTR:
5391 return LoongArchISD::ROTR_W;
5392 case ISD::CTTZ:
5393 return LoongArchISD::CTZ_W;
5394 case ISD::CTLZ:
5395 return LoongArchISD::CLZ_W;
5396 }
5397}
5398
5399// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5400// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
5401// otherwise be promoted to i64, making it difficult to select the
5402// SLL_W/.../*W later one because the fact the operation was originally of
5403// type i8/i16/i32 is lost.
5404static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
5405 unsigned ExtOpc = ISD::ANY_EXTEND) {
5406 SDLoc DL(N);
5407 unsigned WOpcode = getLoongArchWOpcode(Opcode: N->getOpcode());
5408 SDValue NewOp0, NewRes;
5409
5410 switch (NumOp) {
5411 default:
5412 llvm_unreachable("Unexpected NumOp");
5413 case 1: {
5414 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5415 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, Operand: NewOp0);
5416 break;
5417 }
5418 case 2: {
5419 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5420 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5421 if (N->getOpcode() == ISD::ROTL) {
5422 SDValue TmpOp = DAG.getConstant(Val: 32, DL, VT: MVT::i64);
5423 NewOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: TmpOp, N2: NewOp1);
5424 }
5425 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5426 break;
5427 }
5428 // TODO:Handle more NumOp.
5429 }
5430
5431 // ReplaceNodeResults requires we maintain the same type for the return
5432 // value.
5433 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
5434}
5435
5436// Converts the given 32-bit operation to a i64 operation with signed extension
5437// semantic to reduce the signed extension instructions.
5438static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
5439 SDLoc DL(N);
5440 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5441 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5442 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5443 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
5444 N2: DAG.getValueType(MVT::i32));
5445 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
5446}
5447
5448// Helper function that emits error message for intrinsics with/without chain
5449// and return a UNDEF or and the chain as the results.
5450static void emitErrorAndReplaceIntrinsicResults(
5451 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
5452 StringRef ErrorMsg, bool WithChain = true) {
5453 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5454 Results.push_back(Elt: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)));
5455 if (!WithChain)
5456 return;
5457 Results.push_back(Elt: N->getOperand(Num: 0));
5458}
5459
5460template <unsigned N>
5461static void
5462replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl<SDValue> &Results,
5463 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget,
5464 unsigned ResOp) {
5465 const StringRef ErrorMsgOOR = "argument out of range";
5466 unsigned Imm = Node->getConstantOperandVal(Num: 2);
5467 if (!isUInt<N>(Imm)) {
5468 emitErrorAndReplaceIntrinsicResults(N: Node, Results, DAG, ErrorMsg: ErrorMsgOOR,
5469 /*WithChain=*/false);
5470 return;
5471 }
5472 SDLoc DL(Node);
5473 SDValue Vec = Node->getOperand(Num: 1);
5474
5475 SDValue PickElt =
5476 DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), N1: Vec,
5477 N2: DAG.getConstant(Val: Imm, DL, VT: Subtarget.getGRLenVT()),
5478 N3: DAG.getValueType(Vec.getValueType().getVectorElementType()));
5479 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Node->getValueType(ResNo: 0),
5480 Operand: PickElt.getValue(R: 0)));
5481}
5482
5483static void replaceVecCondBranchResults(SDNode *N,
5484 SmallVectorImpl<SDValue> &Results,
5485 SelectionDAG &DAG,
5486 const LoongArchSubtarget &Subtarget,
5487 unsigned ResOp) {
5488 SDLoc DL(N);
5489 SDValue Vec = N->getOperand(Num: 1);
5490
5491 SDValue CB = DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), Operand: Vec);
5492 Results.push_back(
5493 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: CB.getValue(R: 0)));
5494}
5495
5496static void
5497replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
5498 SelectionDAG &DAG,
5499 const LoongArchSubtarget &Subtarget) {
5500 switch (N->getConstantOperandVal(Num: 0)) {
5501 default:
5502 llvm_unreachable("Unexpected Intrinsic.");
5503 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5504 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5505 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5506 break;
5507 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5508 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5509 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5510 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5511 break;
5512 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5513 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5514 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5515 break;
5516 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5517 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5518 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5519 break;
5520 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5521 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5522 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5523 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5524 break;
5525 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5526 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5527 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5528 break;
5529 case Intrinsic::loongarch_lsx_bz_b:
5530 case Intrinsic::loongarch_lsx_bz_h:
5531 case Intrinsic::loongarch_lsx_bz_w:
5532 case Intrinsic::loongarch_lsx_bz_d:
5533 case Intrinsic::loongarch_lasx_xbz_b:
5534 case Intrinsic::loongarch_lasx_xbz_h:
5535 case Intrinsic::loongarch_lasx_xbz_w:
5536 case Intrinsic::loongarch_lasx_xbz_d:
5537 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5538 ResOp: LoongArchISD::VALL_ZERO);
5539 break;
5540 case Intrinsic::loongarch_lsx_bz_v:
5541 case Intrinsic::loongarch_lasx_xbz_v:
5542 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5543 ResOp: LoongArchISD::VANY_ZERO);
5544 break;
5545 case Intrinsic::loongarch_lsx_bnz_b:
5546 case Intrinsic::loongarch_lsx_bnz_h:
5547 case Intrinsic::loongarch_lsx_bnz_w:
5548 case Intrinsic::loongarch_lsx_bnz_d:
5549 case Intrinsic::loongarch_lasx_xbnz_b:
5550 case Intrinsic::loongarch_lasx_xbnz_h:
5551 case Intrinsic::loongarch_lasx_xbnz_w:
5552 case Intrinsic::loongarch_lasx_xbnz_d:
5553 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5554 ResOp: LoongArchISD::VALL_NONZERO);
5555 break;
5556 case Intrinsic::loongarch_lsx_bnz_v:
5557 case Intrinsic::loongarch_lasx_xbnz_v:
5558 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5559 ResOp: LoongArchISD::VANY_NONZERO);
5560 break;
5561 }
5562}
5563
5564static void replaceCMP_XCHG_128Results(SDNode *N,
5565 SmallVectorImpl<SDValue> &Results,
5566 SelectionDAG &DAG) {
5567 assert(N->getValueType(0) == MVT::i128 &&
5568 "AtomicCmpSwap on types less than 128 should be legal");
5569 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
5570
5571 unsigned Opcode;
5572 switch (MemOp->getMergedOrdering()) {
5573 case AtomicOrdering::Acquire:
5574 case AtomicOrdering::AcquireRelease:
5575 case AtomicOrdering::SequentiallyConsistent:
5576 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5577 break;
5578 case AtomicOrdering::Monotonic:
5579 case AtomicOrdering::Release:
5580 Opcode = LoongArch::PseudoCmpXchg128;
5581 break;
5582 default:
5583 llvm_unreachable("Unexpected ordering!");
5584 }
5585
5586 SDLoc DL(N);
5587 auto CmpVal = DAG.SplitScalar(N: N->getOperand(Num: 2), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5588 auto NewVal = DAG.SplitScalar(N: N->getOperand(Num: 3), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5589 SDValue Ops[] = {N->getOperand(Num: 1), CmpVal.first, CmpVal.second,
5590 NewVal.first, NewVal.second, N->getOperand(Num: 0)};
5591
5592 SDNode *CmpSwap = DAG.getMachineNode(
5593 Opcode, dl: SDLoc(N), VTs: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::i64, VT4: MVT::Other),
5594 Ops);
5595 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
5596 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
5597 N1: SDValue(CmpSwap, 0), N2: SDValue(CmpSwap, 1)));
5598 Results.push_back(Elt: SDValue(CmpSwap, 3));
5599}
5600
5601void LoongArchTargetLowering::ReplaceNodeResults(
5602 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
5603 SDLoc DL(N);
5604 EVT VT = N->getValueType(ResNo: 0);
5605 switch (N->getOpcode()) {
5606 default:
5607 llvm_unreachable("Don't know how to legalize this operation");
5608 case ISD::ADD:
5609 case ISD::SUB:
5610 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5611 "Unexpected custom legalisation");
5612 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
5613 break;
5614 case ISD::SDIV:
5615 case ISD::UDIV:
5616 case ISD::SREM:
5617 case ISD::UREM:
5618 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5619 "Unexpected custom legalisation");
5620 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2,
5621 ExtOpc: Subtarget.hasDiv32() && VT == MVT::i32
5622 ? ISD::ANY_EXTEND
5623 : ISD::SIGN_EXTEND));
5624 break;
5625 case ISD::SHL:
5626 case ISD::SRA:
5627 case ISD::SRL:
5628 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5629 "Unexpected custom legalisation");
5630 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
5631 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5632 break;
5633 }
5634 break;
5635 case ISD::ROTL:
5636 case ISD::ROTR:
5637 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5638 "Unexpected custom legalisation");
5639 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5640 break;
5641 case ISD::LOAD: {
5642 // Use an f64 load and a scalar_to_vector for v2f32 loads. This avoids
5643 // scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
5644 // cast since type legalization will try to use an i64 load.
5645 MVT VT = N->getSimpleValueType(ResNo: 0);
5646 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5647 "Unexpected custom legalisation");
5648 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
5649 "Unexpected type action!");
5650 if (!ISD::isNON_EXTLoad(N))
5651 return;
5652 auto *Ld = cast<LoadSDNode>(Val: N);
5653 SDValue Res = DAG.getLoad(VT: MVT::f64, dl: DL, Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5654 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getBaseAlign(),
5655 MMOFlags: Ld->getMemOperand()->getFlags());
5656 SDValue Chain = Res.getValue(R: 1);
5657 MVT VecVT = MVT::getVectorVT(VT: MVT::f64, NumElements: 2);
5658 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecVT, Operand: Res);
5659 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5660 Res = DAG.getBitcast(VT: WideVT, V: Res);
5661 Results.push_back(Elt: Res);
5662 Results.push_back(Elt: Chain);
5663 break;
5664 }
5665 case ISD::FP_TO_SINT: {
5666 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5667 "Unexpected custom legalisation");
5668 SDValue Src = N->getOperand(Num: 0);
5669 EVT FVT = EVT::getFloatingPointVT(BitWidth: N->getValueSizeInBits(ResNo: 0));
5670 if (getTypeAction(Context&: *DAG.getContext(), VT: Src.getValueType()) !=
5671 TargetLowering::TypeSoftenFloat) {
5672 if (!isTypeLegal(VT: Src.getValueType()))
5673 return;
5674 if (Src.getValueType() == MVT::f16)
5675 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
5676 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FVT, Operand: Src);
5677 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Dst));
5678 return;
5679 }
5680 // If the FP type needs to be softened, emit a library call using the 'si'
5681 // version. If we left it to default legalization we'd end up with 'di'.
5682 RTLIB::Libcall LC;
5683 LC = RTLIB::getFPTOSINT(OpVT: Src.getValueType(), RetVT: VT);
5684 MakeLibCallOptions CallOptions;
5685 EVT OpVT = Src.getValueType();
5686 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: VT);
5687 SDValue Chain = SDValue();
5688 SDValue Result;
5689 std::tie(args&: Result, args&: Chain) =
5690 makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl: DL, Chain);
5691 Results.push_back(Elt: Result);
5692 break;
5693 }
5694 case ISD::BITCAST: {
5695 SDValue Src = N->getOperand(Num: 0);
5696 EVT SrcVT = Src.getValueType();
5697 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
5698 Subtarget.hasBasicF()) {
5699 SDValue Dst =
5700 DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Src);
5701 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Dst));
5702 } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
5703 SDValue NewReg = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
5704 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Src);
5705 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64,
5706 N1: NewReg.getValue(R: 0), N2: NewReg.getValue(R: 1));
5707 Results.push_back(Elt: RetReg);
5708 }
5709 break;
5710 }
5711 case ISD::FP_TO_UINT: {
5712 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5713 "Unexpected custom legalisation");
5714 auto &TLI = DAG.getTargetLoweringInfo();
5715 SDValue Tmp1, Tmp2;
5716 TLI.expandFP_TO_UINT(N, Result&: Tmp1, Chain&: Tmp2, DAG);
5717 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Tmp1));
5718 break;
5719 }
5720 case ISD::FP_ROUND: {
5721 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5722 "Unexpected custom legalisation");
5723 // On LSX platforms, rounding from v2f64 to v4f32 (after legalization from
5724 // v2f32) is scalarized. Add a customized v2f32 widening to convert it into
5725 // a target-specific LoongArchISD::VFCVT to optimize it.
5726 SDValue Op0 = N->getOperand(Num: 0);
5727 EVT OpVT = Op0.getValueType();
5728 if (OpVT == MVT::v2f64) {
5729 SDValue Undef = DAG.getUNDEF(VT: OpVT);
5730 SDValue Dst =
5731 DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Undef, N2: Op0);
5732 Results.push_back(Elt: Dst);
5733 }
5734 break;
5735 }
5736 case ISD::BSWAP: {
5737 SDValue Src = N->getOperand(Num: 0);
5738 assert((VT == MVT::i16 || VT == MVT::i32) &&
5739 "Unexpected custom legalization");
5740 MVT GRLenVT = Subtarget.getGRLenVT();
5741 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5742 SDValue Tmp;
5743 switch (VT.getSizeInBits()) {
5744 default:
5745 llvm_unreachable("Unexpected operand width");
5746 case 16:
5747 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2H, DL, VT: GRLenVT, Operand: NewSrc);
5748 break;
5749 case 32:
5750 // Only LA64 will get to here due to the size mismatch between VT and
5751 // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
5752 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2W, DL, VT: GRLenVT, Operand: NewSrc);
5753 break;
5754 }
5755 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5756 break;
5757 }
5758 case ISD::BITREVERSE: {
5759 SDValue Src = N->getOperand(Num: 0);
5760 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
5761 "Unexpected custom legalization");
5762 MVT GRLenVT = Subtarget.getGRLenVT();
5763 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5764 SDValue Tmp;
5765 switch (VT.getSizeInBits()) {
5766 default:
5767 llvm_unreachable("Unexpected operand width");
5768 case 8:
5769 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL, VT: GRLenVT, Operand: NewSrc);
5770 break;
5771 case 32:
5772 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_W, DL, VT: GRLenVT, Operand: NewSrc);
5773 break;
5774 }
5775 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5776 break;
5777 }
5778 case ISD::CTLZ:
5779 case ISD::CTTZ: {
5780 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5781 "Unexpected custom legalisation");
5782 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 1));
5783 break;
5784 }
5785 case ISD::INTRINSIC_W_CHAIN: {
5786 SDValue Chain = N->getOperand(Num: 0);
5787 SDValue Op2 = N->getOperand(Num: 2);
5788 MVT GRLenVT = Subtarget.getGRLenVT();
5789 const StringRef ErrorMsgOOR = "argument out of range";
5790 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5791 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5792
5793 switch (N->getConstantOperandVal(Num: 1)) {
5794 default:
5795 llvm_unreachable("Unexpected Intrinsic.");
5796 case Intrinsic::loongarch_movfcsr2gr: {
5797 if (!Subtarget.hasBasicF()) {
5798 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqF);
5799 return;
5800 }
5801 unsigned Imm = Op2->getAsZExtVal();
5802 if (!isUInt<2>(x: Imm)) {
5803 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5804 return;
5805 }
5806 SDValue MOVFCSR2GRResults = DAG.getNode(
5807 Opcode: LoongArchISD::MOVFCSR2GR, DL: SDLoc(N), ResultTys: {MVT::i64, MVT::Other},
5808 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5809 Results.push_back(
5810 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: MOVFCSR2GRResults.getValue(R: 0)));
5811 Results.push_back(Elt: MOVFCSR2GRResults.getValue(R: 1));
5812 break;
5813 }
5814#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
5815 case Intrinsic::loongarch_##NAME: { \
5816 SDValue NODE = DAG.getNode( \
5817 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5818 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5819 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5820 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5821 Results.push_back(NODE.getValue(1)); \
5822 break; \
5823 }
5824 CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
5825 CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)
5826 CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)
5827 CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)
5828 CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)
5829 CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)
5830#undef CRC_CASE_EXT_BINARYOP
5831
5832#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
5833 case Intrinsic::loongarch_##NAME: { \
5834 SDValue NODE = DAG.getNode( \
5835 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5836 {Chain, Op2, \
5837 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5838 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5839 Results.push_back(NODE.getValue(1)); \
5840 break; \
5841 }
5842 CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
5843 CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)
5844#undef CRC_CASE_EXT_UNARYOP
5845#define CSR_CASE(ID) \
5846 case Intrinsic::loongarch_##ID: { \
5847 if (!Subtarget.is64Bit()) \
5848 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
5849 break; \
5850 }
5851 CSR_CASE(csrrd_d);
5852 CSR_CASE(csrwr_d);
5853 CSR_CASE(csrxchg_d);
5854 CSR_CASE(iocsrrd_d);
5855#undef CSR_CASE
5856 case Intrinsic::loongarch_csrrd_w: {
5857 unsigned Imm = Op2->getAsZExtVal();
5858 if (!isUInt<14>(x: Imm)) {
5859 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5860 return;
5861 }
5862 SDValue CSRRDResults =
5863 DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
5864 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5865 Results.push_back(
5866 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRRDResults.getValue(R: 0)));
5867 Results.push_back(Elt: CSRRDResults.getValue(R: 1));
5868 break;
5869 }
5870 case Intrinsic::loongarch_csrwr_w: {
5871 unsigned Imm = N->getConstantOperandVal(Num: 3);
5872 if (!isUInt<14>(x: Imm)) {
5873 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5874 return;
5875 }
5876 SDValue CSRWRResults =
5877 DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5878 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5879 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5880 Results.push_back(
5881 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRWRResults.getValue(R: 0)));
5882 Results.push_back(Elt: CSRWRResults.getValue(R: 1));
5883 break;
5884 }
5885 case Intrinsic::loongarch_csrxchg_w: {
5886 unsigned Imm = N->getConstantOperandVal(Num: 4);
5887 if (!isUInt<14>(x: Imm)) {
5888 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5889 return;
5890 }
5891 SDValue CSRXCHGResults = DAG.getNode(
5892 Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5893 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5894 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 3)),
5895 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5896 Results.push_back(
5897 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRXCHGResults.getValue(R: 0)));
5898 Results.push_back(Elt: CSRXCHGResults.getValue(R: 1));
5899 break;
5900 }
5901#define IOCSRRD_CASE(NAME, NODE) \
5902 case Intrinsic::loongarch_##NAME: { \
5903 SDValue IOCSRRDResults = \
5904 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5905 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
5906 Results.push_back( \
5907 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
5908 Results.push_back(IOCSRRDResults.getValue(1)); \
5909 break; \
5910 }
5911 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5912 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5913 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5914#undef IOCSRRD_CASE
5915 case Intrinsic::loongarch_cpucfg: {
5916 SDValue CPUCFGResults =
5917 DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5918 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2)});
5919 Results.push_back(
5920 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CPUCFGResults.getValue(R: 0)));
5921 Results.push_back(Elt: CPUCFGResults.getValue(R: 1));
5922 break;
5923 }
5924 case Intrinsic::loongarch_lddir_d: {
5925 if (!Subtarget.is64Bit()) {
5926 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqLA64);
5927 return;
5928 }
5929 break;
5930 }
5931 }
5932 break;
5933 }
5934 case ISD::READ_REGISTER: {
5935 if (Subtarget.is64Bit())
5936 DAG.getContext()->emitError(
5937 ErrorStr: "On LA64, only 64-bit registers can be read.");
5938 else
5939 DAG.getContext()->emitError(
5940 ErrorStr: "On LA32, only 32-bit registers can be read.");
5941 Results.push_back(Elt: DAG.getUNDEF(VT));
5942 Results.push_back(Elt: N->getOperand(Num: 0));
5943 break;
5944 }
5945 case ISD::INTRINSIC_WO_CHAIN: {
5946 replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);
5947 break;
5948 }
5949 case ISD::LROUND: {
5950 SDValue Op0 = N->getOperand(Num: 0);
5951 EVT OpVT = Op0.getValueType();
5952 RTLIB::Libcall LC =
5953 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
5954 MakeLibCallOptions CallOptions;
5955 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
5956 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
5957 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
5958 Results.push_back(Elt: Result);
5959 break;
5960 }
5961 case ISD::ATOMIC_CMP_SWAP: {
5962 replaceCMP_XCHG_128Results(N, Results, DAG);
5963 break;
5964 }
5965 case ISD::TRUNCATE: {
5966 MVT VT = N->getSimpleValueType(ResNo: 0);
5967 if (getTypeAction(Context&: *DAG.getContext(), VT) != TypeWidenVector)
5968 return;
5969
5970 MVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT).getSimpleVT();
5971 SDValue In = N->getOperand(Num: 0);
5972 EVT InVT = In.getValueType();
5973 EVT InEltVT = InVT.getVectorElementType();
5974 EVT EltVT = VT.getVectorElementType();
5975 unsigned MinElts = VT.getVectorNumElements();
5976 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5977 unsigned InBits = InVT.getSizeInBits();
5978
5979 // v8i64 -> (v8i32) -> v8i8
5980 if (InVT == MVT::v8i64 && WidenVT.is128BitVector()) {
5981 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 256 / MinElts), NumElements: MinElts);
5982 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5983 InBits = 256;
5984 }
5985
5986 // v8i32 -> v8i8 / v4i64 -> v4i16 / v4i64 -> v4i8
5987 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
5988 WidenVT.is128BitVector()) {
5989 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / MinElts), NumElements: MinElts);
5990 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5991 InBits = 128;
5992 InEltVT = InVT.getVectorElementType();
5993 }
5994
5995 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
5996 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
5997 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
5998 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
5999 for (unsigned I = 0; I < MinElts; ++I)
6000 TruncMask[I] = Scale * I;
6001
6002 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
6003 MVT SVT = In.getSimpleValueType().getScalarType();
6004 MVT VT = MVT::getVectorVT(VT: SVT, NumElements: WidenNumElts);
6005 SDValue WidenIn =
6006 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT), N2: In,
6007 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6008 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
6009 "Illegal vector type in truncation");
6010 WidenIn = DAG.getBitcast(VT: WidenVT, V: WidenIn);
6011 Results.push_back(
6012 Elt: DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: WidenIn, N2: WidenIn, Mask: TruncMask));
6013 return;
6014 }
6015 }
6016
6017 break;
6018 }
6019 case ISD::SIGN_EXTEND: {
6020 // LASX has native VEXT2XV_* for sign extension.
6021 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
6022 return;
6023
6024 EVT DstVT = N->getValueType(ResNo: 0);
6025 SDValue Src = N->getOperand(Num: 0);
6026 MVT SrcVT = Src.getSimpleValueType();
6027
6028 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6029 unsigned DstEltBits = DstVT.getScalarSizeInBits();
6030 unsigned NumElts = DstVT.getVectorNumElements();
6031
6032 if (SrcVT.getSizeInBits() > 128)
6033 return;
6034
6035 if (!DstVT.isVector() || DstVT.getSizeInBits() <= 128)
6036 return;
6037
6038 // Legalize and extend the src to 128-bit first.
6039 if (SrcVT.getSizeInBits() < 128) {
6040 unsigned WidenSrcElts = 128 / SrcEltBits;
6041 MVT WidenSrcVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: WidenSrcElts);
6042 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WidenSrcVT,
6043 N1: DAG.getUNDEF(VT: WidenSrcVT), N2: Src,
6044 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6045 SrcVT = WidenSrcVT;
6046
6047 unsigned FirstStageEltBits = 128 / NumElts;
6048 MVT FirstStageEltVT = MVT::getIntegerVT(BitWidth: FirstStageEltBits);
6049 MVT FirstStageVT = MVT::getVectorVT(VT: FirstStageEltVT, NumElements: NumElts);
6050 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: FirstStageVT, Operand: Src);
6051 SrcVT = FirstStageVT;
6052 SrcEltBits = FirstStageEltBits;
6053 }
6054
6055 SmallVector<SDValue, 8> Blocks;
6056 Blocks.push_back(Elt: Src);
6057
6058 // Sign-extend the src by using SLTI + VILVL + VILVH recursively.
6059 while (SrcEltBits < DstEltBits) {
6060 unsigned NextEltBits = SrcEltBits * 2;
6061 MVT NextEltVT = MVT::getIntegerVT(BitWidth: NextEltBits);
6062 unsigned CurEltsPerBlock = SrcVT.getVectorNumElements();
6063 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6064 MVT NextBlockVT = MVT::getVectorVT(VT: NextEltVT, NumElements: NextEltsPerBlock);
6065
6066 SmallVector<SDValue, 8> NextBlocks;
6067 NextBlocks.reserve(N: Blocks.size() * 2);
6068 for (SDValue Block : Blocks) {
6069 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
6070 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Block, N2: Zero,
6071 N3: DAG.getCondCode(Cond: ISD::SETLT));
6072 SDValue LoInterleaved =
6073 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Block);
6074 SDValue HiInterleaved =
6075 DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT: SrcVT, N1: Mask, N2: Block);
6076
6077 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: LoInterleaved));
6078 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: HiInterleaved));
6079 }
6080
6081 Blocks = std::move(NextBlocks);
6082 SrcVT = NextBlockVT;
6083 SrcEltBits = NextEltBits;
6084 }
6085
6086 Results.push_back(Elt: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Blocks));
6087 break;
6088 }
6089 case ISD::FP_EXTEND:
6090 // FP_EXTEND may reach here due to the Custom action for v2f32 results, but
6091 // no target-specific lowering is required. Leave it unchanged and rely on
6092 // the default type legalization.
6093 break;
6094 }
6095}
6096
6097/// Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
6098static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL,
6099 SelectionDAG &DAG) {
6100 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDN");
6101
6102 MVT VT = N->getSimpleValueType(ResNo: 0);
6103 if (!VT.is128BitVector() && !VT.is256BitVector())
6104 return SDValue();
6105
6106 SDValue X, Y;
6107 SDValue N0 = N->getOperand(Num: 0);
6108 SDValue N1 = N->getOperand(Num: 1);
6109
6110 if (SDValue Not = isNOT(V: N0, DAG)) {
6111 X = Not;
6112 Y = N1;
6113 } else if (SDValue Not = isNOT(V: N1, DAG)) {
6114 X = Not;
6115 Y = N0;
6116 } else
6117 return SDValue();
6118
6119 X = DAG.getBitcast(VT, V: X);
6120 Y = DAG.getBitcast(VT, V: Y);
6121 return DAG.getNode(Opcode: LoongArchISD::VANDN, DL, VT, N1: X, N2: Y);
6122}
6123
6124static bool isConstantSplatVector(SDValue N, APInt &SplatValue,
6125 unsigned MinSizeInBits) {
6126 N = peekThroughBitcasts(V: N);
6127 BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(Val&: N);
6128
6129 if (!Node)
6130 return false;
6131
6132 APInt SplatUndef;
6133 unsigned SplatBitSize;
6134 bool HasAnyUndefs;
6135
6136 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6137 HasAnyUndefs, MinSplatBits: MinSizeInBits,
6138 /*IsBigEndian=*/isBigEndian: false);
6139}
6140
6141static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex) {
6142 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6143 if (!BV)
6144 return SDValue();
6145
6146 SDValue Src;
6147 int Start = -1;
6148
6149 for (unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6150 SDValue Op = BV->getOperand(Num: i);
6151 if (Op.isUndef())
6152 continue;
6153 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6154 return SDValue();
6155
6156 auto *IdxC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6157 if (!IdxC)
6158 return SDValue();
6159
6160 unsigned EltIdx = IdxC->getZExtValue();
6161 if (Start < 0)
6162 Start = (int)EltIdx - (int)(i * 2);
6163 if (Start < 0 || Start > 1 || EltIdx != (unsigned)(Start + (int)(i * 2)))
6164 return SDValue();
6165
6166 SDValue CurSrc = Op.getOperand(i: 0);
6167 if (!Src)
6168 Src = CurSrc;
6169 else if (Src != CurSrc)
6170 return SDValue();
6171 }
6172
6173 if (!Src || Start < 0)
6174 return SDValue();
6175
6176 StartIndex = (unsigned)Start;
6177 return Src;
6178}
6179
6180static SDValue
6181performHorizWideningCombine(SDNode *N, SelectionDAG &DAG,
6182 const LoongArchSubtarget &Subtarget) {
6183 if (!Subtarget.hasExtLSX())
6184 return SDValue();
6185
6186 unsigned Opc = N->getOpcode();
6187 assert((Opc == ISD::ADD || Opc == ISD::SUB) && "Unexpected opcode");
6188
6189 EVT VT = N->getValueType(ResNo: 0);
6190 SDLoc DL(N);
6191
6192 SDValue LHS = N->getOperand(Num: 0);
6193 SDValue RHS = N->getOperand(Num: 1);
6194
6195 bool isSigned;
6196 unsigned ExtOpc = LHS.getOpcode();
6197 if (ExtOpc == ISD::SIGN_EXTEND)
6198 isSigned = true;
6199 else if (ExtOpc == ISD::ZERO_EXTEND)
6200 isSigned = false;
6201 else
6202 return SDValue();
6203
6204 if (ExtOpc != RHS.getOpcode())
6205 return SDValue();
6206
6207 if (!LHS.hasOneUse() || !RHS.hasOneUse())
6208 return SDValue();
6209
6210 unsigned OddIdx, EvenIdx;
6211 SDValue LHSVec = matchDeinterleaveBuildVector(N: LHS.getOperand(i: 0), StartIndex&: OddIdx);
6212 SDValue RHSVec = matchDeinterleaveBuildVector(N: RHS.getOperand(i: 0), StartIndex&: EvenIdx);
6213
6214 if (!LHSVec || !RHSVec)
6215 return SDValue();
6216 if (OddIdx != 1 || EvenIdx != 0)
6217 return SDValue();
6218 if (LHSVec.getValueType() != RHSVec.getValueType())
6219 return SDValue();
6220
6221 EVT SrcVT = LHSVec.getValueType();
6222 EVT SrcEltVT = SrcVT.getVectorElementType();
6223 EVT DstEltVT = VT.getVectorElementType();
6224 auto &TLI = DAG.getTargetLoweringInfo();
6225
6226 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
6227 return SDValue();
6228 if (!SrcVT.isVector() || !VT.isVector())
6229 return SDValue();
6230 if (SrcVT.getSizeInBits() != VT.getSizeInBits())
6231 return SDValue();
6232 if (DstEltVT.getSizeInBits() != SrcEltVT.getSizeInBits() * 2)
6233 return SDValue();
6234 if (!SrcEltVT.isInteger() || SrcEltVT.getSizeInBits() > 32)
6235 return SDValue();
6236
6237 unsigned TargetOpc;
6238 if (Opc == ISD::ADD)
6239 TargetOpc = isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6240 else
6241 TargetOpc = isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6242
6243 return DAG.getNode(Opcode: TargetOpc, DL, VT, N1: LHSVec, N2: RHSVec);
6244}
6245
6246static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
6247 TargetLowering::DAGCombinerInfo &DCI,
6248 const LoongArchSubtarget &Subtarget) {
6249 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6250 return V;
6251
6252 if (DCI.isBeforeLegalizeOps())
6253 return SDValue();
6254
6255 EVT VT = N->getValueType(ResNo: 0);
6256 if (!VT.isVector())
6257 return SDValue();
6258
6259 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6260 return SDValue();
6261
6262 EVT EltVT = VT.getVectorElementType();
6263 if (!EltVT.isInteger())
6264 return SDValue();
6265
6266 // match:
6267 //
6268 // add
6269 // (and
6270 // (srl X, shift-1) / X
6271 // 1)
6272 // (srl/sra X, shift)
6273
6274 SDValue Add0 = N->getOperand(Num: 0);
6275 SDValue Add1 = N->getOperand(Num: 1);
6276 SDValue And;
6277 SDValue Shr;
6278
6279 if (Add0.getOpcode() == ISD::AND) {
6280 And = Add0;
6281 Shr = Add1;
6282 } else if (Add1.getOpcode() == ISD::AND) {
6283 And = Add1;
6284 Shr = Add0;
6285 } else {
6286 return SDValue();
6287 }
6288
6289 // match:
6290 //
6291 // srl/sra X, shift
6292
6293 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
6294 return SDValue();
6295
6296 SDValue X = Shr.getOperand(i: 0);
6297 SDValue Shift = Shr.getOperand(i: 1);
6298 APInt ShiftVal;
6299
6300 if (!isConstantSplatVector(N: Shift, SplatValue&: ShiftVal, MinSizeInBits: EltVT.getSizeInBits()))
6301 return SDValue();
6302
6303 if (ShiftVal == 0)
6304 return SDValue();
6305
6306 // match:
6307 //
6308 // and
6309 // (srl X, shift-1) / X
6310 // 1
6311
6312 SDValue One = And.getOperand(i: 1);
6313 APInt SplatVal;
6314
6315 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6316 return SDValue();
6317
6318 if (SplatVal != 1)
6319 return SDValue();
6320
6321 if (And.getOperand(i: 0) == X) {
6322 // match:
6323 //
6324 // shift == 1
6325
6326 if (ShiftVal != 1)
6327 return SDValue();
6328 } else {
6329 // match:
6330 //
6331 // srl X, shift-1
6332
6333 SDValue Srl = And.getOperand(i: 0);
6334
6335 if (Srl.getOpcode() != ISD::SRL)
6336 return SDValue();
6337
6338 if (Srl.getOperand(i: 0) != X)
6339 return SDValue();
6340
6341 // match:
6342 //
6343 // shift-1
6344
6345 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
6346
6347 if (!isConstantSplatVector(N: ShiftMinus1, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6348 return SDValue();
6349
6350 if (ShiftVal != (SplatVal + 1))
6351 return SDValue();
6352 }
6353
6354 // We matched a rounded right shift pattern and can lower it
6355 // to a single vector rounded shift instruction.
6356
6357 SDLoc DL(N);
6358 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
6359 : LoongArchISD::VSRAR,
6360 DL, VT, N1: X, N2: Shift);
6361}
6362
6363static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
6364 TargetLowering::DAGCombinerInfo &DCI,
6365 const LoongArchSubtarget &Subtarget) {
6366 if (DCI.isBeforeLegalizeOps())
6367 return SDValue();
6368
6369 SDValue FirstOperand = N->getOperand(Num: 0);
6370 SDValue SecondOperand = N->getOperand(Num: 1);
6371 unsigned FirstOperandOpc = FirstOperand.getOpcode();
6372 EVT ValTy = N->getValueType(ResNo: 0);
6373 SDLoc DL(N);
6374 uint64_t lsb, msb;
6375 unsigned SMIdx, SMLen;
6376 ConstantSDNode *CN;
6377 SDValue NewOperand;
6378 MVT GRLenVT = Subtarget.getGRLenVT();
6379
6380 if (SDValue R = combineAndNotIntoVANDN(N, DL, DAG))
6381 return R;
6382
6383 // BSTRPICK requires the 32S feature.
6384 if (!Subtarget.has32S())
6385 return SDValue();
6386
6387 // Op's second operand must be a shifted mask.
6388 if (!(CN = dyn_cast<ConstantSDNode>(Val&: SecondOperand)) ||
6389 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx&: SMIdx, MaskLen&: SMLen))
6390 return SDValue();
6391
6392 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
6393 // Pattern match BSTRPICK.
6394 // $dst = and ((sra or srl) $src , lsb), (2**len - 1)
6395 // => BSTRPICK $dst, $src, msb, lsb
6396 // where msb = lsb + len - 1
6397
6398 // The second operand of the shift must be an immediate.
6399 if (!(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))))
6400 return SDValue();
6401
6402 lsb = CN->getZExtValue();
6403
6404 // Return if the shifted mask does not start at bit 0 or the sum of its
6405 // length and lsb exceeds the word's size.
6406 if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())
6407 return SDValue();
6408
6409 NewOperand = FirstOperand.getOperand(i: 0);
6410 } else {
6411 // Pattern match BSTRPICK.
6412 // $dst = and $src, (2**len- 1) , if len > 12
6413 // => BSTRPICK $dst, $src, msb, lsb
6414 // where lsb = 0 and msb = len - 1
6415
6416 // If the mask is <= 0xfff, andi can be used instead.
6417 if (CN->getZExtValue() <= 0xfff)
6418 return SDValue();
6419
6420 // Return if the MSB exceeds.
6421 if (SMIdx + SMLen > ValTy.getSizeInBits())
6422 return SDValue();
6423
6424 if (SMIdx > 0) {
6425 // Omit if the constant has more than 2 uses. This a conservative
6426 // decision. Whether it is a win depends on the HW microarchitecture.
6427 // However it should always be better for 1 and 2 uses.
6428 if (CN->use_size() > 2)
6429 return SDValue();
6430 // Return if the constant can be composed by a single LU12I.W.
6431 if ((CN->getZExtValue() & 0xfff) == 0)
6432 return SDValue();
6433 // Return if the constand can be composed by a single ADDI with
6434 // the zero register.
6435 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)
6436 return SDValue();
6437 }
6438
6439 lsb = SMIdx;
6440 NewOperand = FirstOperand;
6441 }
6442
6443 msb = lsb + SMLen - 1;
6444 SDValue NR0 = DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy, N1: NewOperand,
6445 N2: DAG.getConstant(Val: msb, DL, VT: GRLenVT),
6446 N3: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6447 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)
6448 return NR0;
6449 // Try to optimize to
6450 // bstrpick $Rd, $Rs, msb, lsb
6451 // slli $Rd, $Rd, lsb
6452 return DAG.getNode(Opcode: ISD::SHL, DL, VT: ValTy, N1: NR0,
6453 N2: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6454}
6455
6456// Return the original source vector if N consists of the half
6457// of each 128-bit lane.
6458static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow) {
6459 N = peekThroughBitcasts(V: N);
6460
6461 EVT DstVT = N.getValueType();
6462 if (!DstVT.isVector())
6463 return SDValue();
6464
6465 unsigned NumElts = DstVT.getVectorNumElements();
6466
6467 // LSX canonical form:
6468 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6469 SDValue Src = N.getOperand(i: 0);
6470 EVT SrcVT = Src.getValueType();
6471
6472 if (!SrcVT.isVector() || !SrcVT.is128BitVector())
6473 return SDValue();
6474 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6475 return SDValue();
6476 if (SrcVT.getVectorNumElements() != NumElts * 2)
6477 return SDValue();
6478 if (N.getConstantOperandVal(i: 1) != (isLow ? 0 : NumElts))
6479 return SDValue();
6480
6481 return Src;
6482 }
6483
6484 // LASX canonical form:
6485 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6486 if (!BV)
6487 return SDValue();
6488
6489 if (NumElts % 2 != 0)
6490 return SDValue();
6491
6492 SDValue Src;
6493 EVT SrcVT;
6494
6495 for (unsigned I = 0; I != NumElts; ++I) {
6496 SDValue Elt = BV->getOperand(Num: I);
6497 if (Elt.isUndef())
6498 continue;
6499 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6500 return SDValue();
6501
6502 SDValue ThisSrc = Elt.getOperand(i: 0);
6503 SDValue Idx = Elt.getOperand(i: 1);
6504 auto *CI = dyn_cast<ConstantSDNode>(Val&: Idx);
6505 if (!CI)
6506 return SDValue();
6507
6508 if (!Src) {
6509 Src = ThisSrc;
6510 SrcVT = Src.getValueType();
6511 if (!SrcVT.isVector())
6512 return SDValue();
6513
6514 if (!SrcVT.is256BitVector())
6515 return SDValue();
6516 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6517 return SDValue();
6518 if (SrcVT.getVectorNumElements() != NumElts * 2)
6519 return SDValue();
6520 } else if (ThisSrc != Src) {
6521 return SDValue();
6522 }
6523
6524 unsigned Half = NumElts / 2;
6525 unsigned ExpectedIdx = (I < Half) ? I : (I + Half);
6526 ExpectedIdx += isLow ? 0 : Half;
6527
6528 if (CI->getZExtValue() != ExpectedIdx)
6529 return SDValue();
6530 }
6531
6532 return Src;
6533}
6534
6535static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
6536 TargetLowering::DAGCombinerInfo &DCI,
6537 const LoongArchSubtarget &Subtarget) {
6538 assert(N->getOpcode() == ISD::SHL && "Unexpected opcode");
6539
6540 EVT VT = N->getValueType(ResNo: 0);
6541 SDLoc DL(N);
6542
6543 SDValue LHS = N->getOperand(Num: 0);
6544 SDValue RHS = N->getOperand(Num: 1);
6545
6546 bool isSigned;
6547 unsigned ExtOpc = LHS.getOpcode();
6548 if (ExtOpc == ISD::SIGN_EXTEND)
6549 isSigned = true;
6550 else if (ExtOpc == ISD::ZERO_EXTEND)
6551 isSigned = false;
6552 else
6553 return SDValue();
6554
6555 if (!LHS.hasOneUse())
6556 return SDValue();
6557
6558 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
6559 N->getValueSizeInBits(ResNo: 0) != LHS->getOperand(Num: 0).getValueSizeInBits() * 2)
6560 return SDValue();
6561
6562 SDValue Vec = matchHalfOf128BitLanes(N: LHS.getOperand(i: 0), /*isLow=*/true);
6563 if (!Vec)
6564 return SDValue();
6565
6566 EVT SrcVT = Vec.getValueType();
6567 EVT SrcEltVT = SrcVT.getVectorElementType();
6568 EVT DstEltVT = VT.getVectorElementType();
6569 APInt Imm;
6570 if (!isConstantSplatVector(N: RHS, SplatValue&: Imm, MinSizeInBits: DstEltVT.getSizeInBits()))
6571 return SDValue();
6572 if (!Imm.ult(RHS: SrcEltVT.getSizeInBits()))
6573 return SDValue();
6574
6575 unsigned Opc = isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
6576 SDValue Sht = DAG.getConstant(Val: Imm.getZExtValue(), DL, VT: Subtarget.getGRLenVT());
6577 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Sht);
6578}
6579
6580static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
6581 TargetLowering::DAGCombinerInfo &DCI,
6582 const LoongArchSubtarget &Subtarget) {
6583 // BSTRPICK requires the 32S feature.
6584 if (!Subtarget.has32S())
6585 return SDValue();
6586
6587 if (DCI.isBeforeLegalizeOps())
6588 return SDValue();
6589
6590 // $dst = srl (and $src, Mask), Shamt
6591 // =>
6592 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt
6593 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1
6594 //
6595
6596 SDValue FirstOperand = N->getOperand(Num: 0);
6597 ConstantSDNode *CN;
6598 EVT ValTy = N->getValueType(ResNo: 0);
6599 SDLoc DL(N);
6600 MVT GRLenVT = Subtarget.getGRLenVT();
6601 unsigned MaskIdx, MaskLen;
6602 uint64_t Shamt;
6603
6604 // The first operand must be an AND and the second operand of the AND must be
6605 // a shifted mask.
6606 if (FirstOperand.getOpcode() != ISD::AND ||
6607 !(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))) ||
6608 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx, MaskLen))
6609 return SDValue();
6610
6611 // The second operand (shift amount) must be an immediate.
6612 if (!(CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))))
6613 return SDValue();
6614
6615 Shamt = CN->getZExtValue();
6616 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
6617 return DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy,
6618 N1: FirstOperand->getOperand(Num: 0),
6619 N2: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
6620 N3: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6621
6622 return SDValue();
6623}
6624
6625static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
6626 TargetLowering::DAGCombinerInfo &DCI,
6627 const LoongArchSubtarget &Subtarget) {
6628 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6629 return V;
6630
6631 return SDValue();
6632}
6633
6634// Helper to peek through bitops/trunc/setcc to determine size of source vector.
6635// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
6636static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
6637 unsigned Depth) {
6638 // Limit recursion.
6639 if (Depth >= SelectionDAG::MaxRecursionDepth)
6640 return false;
6641 switch (Src.getOpcode()) {
6642 case ISD::SETCC:
6643 case ISD::TRUNCATE:
6644 return Src.getOperand(i: 0).getValueSizeInBits() == Size;
6645 case ISD::FREEZE:
6646 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1);
6647 case ISD::AND:
6648 case ISD::XOR:
6649 case ISD::OR:
6650 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1) &&
6651 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1);
6652 case ISD::SELECT:
6653 case ISD::VSELECT:
6654 return Src.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
6655 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1) &&
6656 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 2), Size, Depth: Depth + 1);
6657 case ISD::BUILD_VECTOR:
6658 return ISD::isBuildVectorAllZeros(N: Src.getNode()) ||
6659 ISD::isBuildVectorAllOnes(N: Src.getNode());
6660 }
6661 return false;
6662}
6663
6664// Helper to push sign extension of vXi1 SETCC result through bitops.
6665static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
6666 SDValue Src, const SDLoc &DL) {
6667 switch (Src.getOpcode()) {
6668 case ISD::SETCC:
6669 case ISD::FREEZE:
6670 case ISD::TRUNCATE:
6671 case ISD::BUILD_VECTOR:
6672 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6673 case ISD::AND:
6674 case ISD::XOR:
6675 case ISD::OR:
6676 return DAG.getNode(
6677 Opcode: Src.getOpcode(), DL, VT: SExtVT,
6678 N1: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 0), DL),
6679 N2: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL));
6680 case ISD::SELECT:
6681 case ISD::VSELECT:
6682 return DAG.getSelect(
6683 DL, VT: SExtVT, Cond: Src.getOperand(i: 0),
6684 LHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL),
6685 RHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 2), DL));
6686 }
6687 llvm_unreachable("Unexpected node type for vXi1 sign extension");
6688}
6689
6690static SDValue
6691performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
6692 TargetLowering::DAGCombinerInfo &DCI,
6693 const LoongArchSubtarget &Subtarget) {
6694 SDLoc DL(N);
6695 EVT VT = N->getValueType(ResNo: 0);
6696 SDValue Src = N->getOperand(Num: 0);
6697 EVT SrcVT = Src.getValueType();
6698
6699 if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse())
6700 return SDValue();
6701
6702 bool UseLASX;
6703 unsigned Opc = ISD::DELETED_NODE;
6704 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6705 EVT EltVT = CmpVT.getVectorElementType();
6706
6707 if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128)
6708 UseLASX = false;
6709 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
6710 CmpVT.getSizeInBits() == 256)
6711 UseLASX = true;
6712 else
6713 return SDValue();
6714
6715 SDValue SrcN1 = Src.getOperand(i: 1);
6716 switch (cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get()) {
6717 default:
6718 break;
6719 case ISD::SETEQ:
6720 // x == 0 => not (vmsknez.b x)
6721 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6722 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
6723 break;
6724 case ISD::SETGT:
6725 // x > -1 => vmskgez.b x
6726 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) && EltVT == MVT::i8)
6727 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6728 break;
6729 case ISD::SETGE:
6730 // x >= 0 => vmskgez.b x
6731 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6732 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6733 break;
6734 case ISD::SETLT:
6735 // x < 0 => vmskltz.{b,h,w,d} x
6736 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) &&
6737 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6738 EltVT == MVT::i64))
6739 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6740 break;
6741 case ISD::SETLE:
6742 // x <= -1 => vmskltz.{b,h,w,d} x
6743 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) &&
6744 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6745 EltVT == MVT::i64))
6746 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6747 break;
6748 case ISD::SETNE:
6749 // x != 0 => vmsknez.b x
6750 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6751 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
6752 break;
6753 }
6754
6755 if (Opc == ISD::DELETED_NODE)
6756 return SDValue();
6757
6758 SDValue V = DAG.getNode(Opcode: Opc, DL, VT: Subtarget.getGRLenVT(), Operand: Src.getOperand(i: 0));
6759 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6760 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6761 return DAG.getBitcast(VT, V);
6762}
6763
6764static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG,
6765 TargetLowering::DAGCombinerInfo &DCI,
6766 const LoongArchSubtarget &Subtarget) {
6767 SDLoc DL(N);
6768 EVT VT = N->getValueType(ResNo: 0);
6769 SDValue Src = N->getOperand(Num: 0);
6770 EVT SrcVT = Src.getValueType();
6771 MVT GRLenVT = Subtarget.getGRLenVT();
6772
6773 if (!DCI.isBeforeLegalizeOps())
6774 return SDValue();
6775
6776 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
6777 return SDValue();
6778
6779 // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
6780 SDValue Res = performSETCC_BITCASTCombine(N, DAG, DCI, Subtarget);
6781 if (Res)
6782 return Res;
6783
6784 // Generate vXi1 using [X]VMSKLTZ
6785 MVT SExtVT;
6786 unsigned Opc;
6787 bool UseLASX = false;
6788 bool PropagateSExt = false;
6789
6790 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
6791 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6792 if (CmpVT.getSizeInBits() > 256)
6793 return SDValue();
6794 }
6795
6796 switch (SrcVT.getSimpleVT().SimpleTy) {
6797 default:
6798 return SDValue();
6799 case MVT::v2i1:
6800 SExtVT = MVT::v2i64;
6801 break;
6802 case MVT::v4i1:
6803 SExtVT = MVT::v4i32;
6804 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6805 SExtVT = MVT::v4i64;
6806 UseLASX = true;
6807 PropagateSExt = true;
6808 }
6809 break;
6810 case MVT::v8i1:
6811 SExtVT = MVT::v8i16;
6812 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6813 SExtVT = MVT::v8i32;
6814 UseLASX = true;
6815 PropagateSExt = true;
6816 }
6817 break;
6818 case MVT::v16i1:
6819 SExtVT = MVT::v16i8;
6820 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6821 SExtVT = MVT::v16i16;
6822 UseLASX = true;
6823 PropagateSExt = true;
6824 }
6825 break;
6826 case MVT::v32i1:
6827 SExtVT = MVT::v32i8;
6828 UseLASX = true;
6829 break;
6830 };
6831 Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
6832 : DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6833
6834 SDValue V;
6835 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
6836 if (Src.getSimpleValueType() == MVT::v32i8) {
6837 SDValue Lo, Hi;
6838 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Src, DL);
6839 Lo = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Lo);
6840 Hi = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Hi);
6841 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: Hi,
6842 N2: DAG.getShiftAmountConstant(Val: 16, VT: GRLenVT, DL));
6843 V = DAG.getNode(Opcode: ISD::OR, DL, VT: GRLenVT, N1: Lo, N2: Hi);
6844 } else if (UseLASX) {
6845 return SDValue();
6846 }
6847 }
6848
6849 if (!V) {
6850 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6851 V = DAG.getNode(Opcode: Opc, DL, VT: GRLenVT, Operand: Src);
6852 }
6853
6854 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6855 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6856 return DAG.getBitcast(VT, V);
6857}
6858
6859static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
6860 TargetLowering::DAGCombinerInfo &DCI,
6861 const LoongArchSubtarget &Subtarget) {
6862 MVT GRLenVT = Subtarget.getGRLenVT();
6863 EVT ValTy = N->getValueType(ResNo: 0);
6864 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
6865 ConstantSDNode *CN0, *CN1;
6866 SDLoc DL(N);
6867 unsigned ValBits = ValTy.getSizeInBits();
6868 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
6869 unsigned Shamt;
6870 bool SwapAndRetried = false;
6871
6872 // BSTRPICK requires the 32S feature.
6873 if (!Subtarget.has32S())
6874 return SDValue();
6875
6876 if (DCI.isBeforeLegalizeOps())
6877 return SDValue();
6878
6879 if (ValBits != 32 && ValBits != 64)
6880 return SDValue();
6881
6882Retry:
6883 // 1st pattern to match BSTRINS:
6884 // R = or (and X, mask0), (and (shl Y, lsb), mask1)
6885 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1
6886 // =>
6887 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6888 if (N0.getOpcode() == ISD::AND &&
6889 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6890 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6891 N1.getOpcode() == ISD::AND && N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
6892 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6893 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6894 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
6895 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6896 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6897 (MaskIdx0 + MaskLen0 <= ValBits)) {
6898 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");
6899 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6900 N2: N1.getOperand(i: 0).getOperand(i: 0),
6901 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6902 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6903 }
6904
6905 // 2nd pattern to match BSTRINS:
6906 // R = or (and X, mask0), (shl (and Y, mask1), lsb)
6907 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)
6908 // =>
6909 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6910 if (N0.getOpcode() == ISD::AND &&
6911 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6912 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6913 N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
6914 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6915 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6916 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6917 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6918 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
6919 (MaskIdx0 + MaskLen0 <= ValBits)) {
6920 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");
6921 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6922 N2: N1.getOperand(i: 0).getOperand(i: 0),
6923 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6924 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6925 }
6926
6927 // 3rd pattern to match BSTRINS:
6928 // R = or (and X, mask0), (and Y, mask1)
6929 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0
6930 // =>
6931 // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb
6932 // where msb = lsb + size - 1
6933 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
6934 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6935 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6936 (MaskIdx0 + MaskLen0 <= 64) &&
6937 (CN1 = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) &&
6938 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6939 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");
6940 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6941 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0), N1,
6942 N2: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT)),
6943 N3: DAG.getConstant(Val: ValBits == 32
6944 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6945 : (MaskIdx0 + MaskLen0 - 1),
6946 DL, VT: GRLenVT),
6947 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6948 }
6949
6950 // 4th pattern to match BSTRINS:
6951 // R = or (and X, mask), (shl Y, shamt)
6952 // where mask = (2**shamt - 1)
6953 // =>
6954 // R = BSTRINS X, Y, ValBits - 1, shamt
6955 // where ValBits = 32 or 64
6956 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&
6957 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6958 isShiftedMask_64(Value: CN0->getZExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6959 MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6960 (Shamt = CN1->getZExtValue()) == MaskLen0 &&
6961 (MaskIdx0 + MaskLen0 <= ValBits)) {
6962 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");
6963 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6964 N2: N1.getOperand(i: 0),
6965 N3: DAG.getConstant(Val: (ValBits - 1), DL, VT: GRLenVT),
6966 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6967 }
6968
6969 // 5th pattern to match BSTRINS:
6970 // R = or (and X, mask), const
6971 // where ~mask = (2**size - 1) << lsb, mask & const = 0
6972 // =>
6973 // R = BSTRINS X, (const >> lsb), msb, lsb
6974 // where msb = lsb + size - 1
6975 if (N0.getOpcode() == ISD::AND &&
6976 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6977 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6978 (CN1 = dyn_cast<ConstantSDNode>(Val&: N1)) &&
6979 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6980 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");
6981 return DAG.getNode(
6982 Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6983 N2: DAG.getSignedConstant(Val: CN1->getSExtValue() >> MaskIdx0, DL, VT: ValTy),
6984 N3: DAG.getConstant(Val: ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6985 : (MaskIdx0 + MaskLen0 - 1),
6986 DL, VT: GRLenVT),
6987 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6988 }
6989
6990 // 6th pattern.
6991 // a = b | ((c & mask) << shamt), where all positions in b to be overwritten
6992 // by the incoming bits are known to be zero.
6993 // =>
6994 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt
6995 //
6996 // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th
6997 // pattern is more common than the 1st. So we put the 1st before the 6th in
6998 // order to match as many nodes as possible.
6999 ConstantSDNode *CNMask, *CNShamt;
7000 unsigned MaskIdx, MaskLen;
7001 if (N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
7002 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7003 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7004 MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7005 CNShamt->getZExtValue() + MaskLen <= ValBits) {
7006 Shamt = CNShamt->getZExtValue();
7007 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);
7008 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7009 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");
7010 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7011 N2: N1.getOperand(i: 0).getOperand(i: 0),
7012 N3: DAG.getConstant(Val: Shamt + MaskLen - 1, DL, VT: GRLenVT),
7013 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
7014 }
7015 }
7016
7017 // 7th pattern.
7018 // a = b | ((c << shamt) & shifted_mask), where all positions in b to be
7019 // overwritten by the incoming bits are known to be zero.
7020 // =>
7021 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx
7022 //
7023 // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd
7024 // before the 7th in order to match as many nodes as possible.
7025 if (N1.getOpcode() == ISD::AND &&
7026 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7027 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7028 N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
7029 (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7030 CNShamt->getZExtValue() == MaskIdx) {
7031 APInt ShMask(ValBits, CNMask->getZExtValue());
7032 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7033 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");
7034 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7035 N2: N1.getOperand(i: 0).getOperand(i: 0),
7036 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7037 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7038 }
7039 }
7040
7041 // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.
7042 if (!SwapAndRetried) {
7043 std::swap(a&: N0, b&: N1);
7044 SwapAndRetried = true;
7045 goto Retry;
7046 }
7047
7048 SwapAndRetried = false;
7049Retry2:
7050 // 8th pattern.
7051 // a = b | (c & shifted_mask), where all positions in b to be overwritten by
7052 // the incoming bits are known to be zero.
7053 // =>
7054 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx
7055 //
7056 // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So
7057 // we put it here in order to match as many nodes as possible or generate less
7058 // instructions.
7059 if (N1.getOpcode() == ISD::AND &&
7060 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7061 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen)) {
7062 APInt ShMask(ValBits, CNMask->getZExtValue());
7063 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7064 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");
7065 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7066 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0),
7067 N1: N1->getOperand(Num: 0),
7068 N2: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT)),
7069 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7070 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7071 }
7072 }
7073 // Swap N0/N1 and retry.
7074 if (!SwapAndRetried) {
7075 std::swap(a&: N0, b&: N1);
7076 SwapAndRetried = true;
7077 goto Retry2;
7078 }
7079
7080 return SDValue();
7081}
7082
7083static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
7084 ExtType = ISD::NON_EXTLOAD;
7085
7086 switch (V.getNode()->getOpcode()) {
7087 case ISD::LOAD: {
7088 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: V.getNode());
7089 if ((LoadNode->getMemoryVT() == MVT::i8) ||
7090 (LoadNode->getMemoryVT() == MVT::i16)) {
7091 ExtType = LoadNode->getExtensionType();
7092 return true;
7093 }
7094 return false;
7095 }
7096 case ISD::AssertSext: {
7097 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7098 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7099 ExtType = ISD::SEXTLOAD;
7100 return true;
7101 }
7102 return false;
7103 }
7104 case ISD::AssertZext: {
7105 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7106 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7107 ExtType = ISD::ZEXTLOAD;
7108 return true;
7109 }
7110 return false;
7111 }
7112 default:
7113 return false;
7114 }
7115
7116 return false;
7117}
7118
7119// Fold a lane-mask extraction that is only compared against zero into one of
7120// all-lane check instructions, which write the results into fcc register, so
7121// BCEQZ could use it directly.
7122//
7123// (VMSKLTZ X) != 0 -> VSETNEZ.V X
7124// (VMSKLTZ X) == 0 -> VSETEQZ.V X
7125//
7126// This is valid when X is the result of a vector compare instruction,
7127// so each lane of X has to be all-ones or all-zeros.
7128static SDValue foldVMskZeroTest(SDValue LHS, SDValue RHS, ISD::CondCode CC,
7129 const SDLoc &DL, SelectionDAG &DAG,
7130 const LoongArchSubtarget &Subtarget) {
7131 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7132 return SDValue();
7133 if (!isNullConstant(V: RHS))
7134 return SDValue();
7135
7136 unsigned MskOpc = LHS.getOpcode();
7137 if (MskOpc != LoongArchISD::VMSKLTZ && MskOpc != LoongArchISD::XVMSKLTZ)
7138 return SDValue();
7139 // Keeping the mask alive for another user would defeat the purpose.
7140 if (!LHS.hasOneUse())
7141 return SDValue();
7142
7143 SDValue Src = LHS.getOperand(i: 0);
7144 EVT SrcVT = Src.getValueType();
7145 // Make sure every lane is all-ones or all-zeros.
7146 if (!SrcVT.isVector() ||
7147 DAG.ComputeNumSignBits(Op: Src) != SrcVT.getScalarSizeInBits())
7148 return SDValue();
7149
7150 return DAG.getNode(Opcode: CC == ISD::SETNE ? LoongArchISD::VANYNONZERO
7151 : LoongArchISD::VALLZERO,
7152 DL, VT: Subtarget.getGRLenVT(), Operand: Src);
7153}
7154
7155// Eliminate redundant truncation and zero-extension nodes.
7156// * Case 1:
7157// +------------+ +------------+ +------------+
7158// | Input1 | | Input2 | | CC |
7159// +------------+ +------------+ +------------+
7160// | | |
7161// V V +----+
7162// +------------+ +------------+ |
7163// | TRUNCATE | | TRUNCATE | |
7164// +------------+ +------------+ |
7165// | | |
7166// V V |
7167// +------------+ +------------+ |
7168// | ZERO_EXT | | ZERO_EXT | |
7169// +------------+ +------------+ |
7170// | | |
7171// | +-------------+ |
7172// V V | |
7173// +----------------+ | |
7174// | AND | | |
7175// +----------------+ | |
7176// | | |
7177// +---------------+ | |
7178// | | |
7179// V V V
7180// +-------------+
7181// | CMP |
7182// +-------------+
7183// * Case 2:
7184// +------------+ +------------+ +-------------+ +------------+ +------------+
7185// | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
7186// +------------+ +------------+ +-------------+ +------------+ +------------+
7187// | | | | |
7188// V | | | |
7189// +------------+ | | | |
7190// | XOR |<---------------------+ | |
7191// +------------+ | | |
7192// | | | |
7193// V V +---------------+ |
7194// +------------+ +------------+ | |
7195// | TRUNCATE | | TRUNCATE | | +-------------------------+
7196// +------------+ +------------+ | |
7197// | | | |
7198// V V | |
7199// +------------+ +------------+ | |
7200// | ZERO_EXT | | ZERO_EXT | | |
7201// +------------+ +------------+ | |
7202// | | | |
7203// V V | |
7204// +----------------+ | |
7205// | AND | | |
7206// +----------------+ | |
7207// | | |
7208// +---------------+ | |
7209// | | |
7210// V V V
7211// +-------------+
7212// | CMP |
7213// +-------------+
7214static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
7215 TargetLowering::DAGCombinerInfo &DCI,
7216 const LoongArchSubtarget &Subtarget) {
7217 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
7218
7219 if (N->getValueType(ResNo: 0) == Subtarget.getGRLenVT())
7220 if (SDValue V = foldVMskZeroTest(LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), CC,
7221 DL: SDLoc(N), DAG, Subtarget))
7222 return V;
7223
7224 SDNode *AndNode = N->getOperand(Num: 0).getNode();
7225 if (AndNode->getOpcode() != ISD::AND)
7226 return SDValue();
7227
7228 SDValue AndInputValue2 = AndNode->getOperand(Num: 1);
7229 if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
7230 return SDValue();
7231
7232 SDValue CmpInputValue = N->getOperand(Num: 1);
7233 SDValue AndInputValue1 = AndNode->getOperand(Num: 0);
7234 if (AndInputValue1.getOpcode() == ISD::XOR) {
7235 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7236 return SDValue();
7237 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: AndInputValue1.getOperand(i: 1));
7238 if (!CN || !CN->isAllOnes())
7239 return SDValue();
7240 CN = dyn_cast<ConstantSDNode>(Val&: CmpInputValue);
7241 if (!CN || !CN->isZero())
7242 return SDValue();
7243 AndInputValue1 = AndInputValue1.getOperand(i: 0);
7244 if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
7245 return SDValue();
7246 } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
7247 if (AndInputValue2 != CmpInputValue)
7248 return SDValue();
7249 } else {
7250 return SDValue();
7251 }
7252
7253 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(Num: 0);
7254 if (TruncValue1.getOpcode() != ISD::TRUNCATE)
7255 return SDValue();
7256
7257 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(Num: 0);
7258 if (TruncValue2.getOpcode() != ISD::TRUNCATE)
7259 return SDValue();
7260
7261 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(Num: 0);
7262 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(Num: 0);
7263 ISD::LoadExtType ExtType1;
7264 ISD::LoadExtType ExtType2;
7265
7266 if (!checkValueWidth(V: TruncInputValue1, ExtType&: ExtType1) ||
7267 !checkValueWidth(V: TruncInputValue2, ExtType&: ExtType2))
7268 return SDValue();
7269
7270 if (TruncInputValue1->getValueType(ResNo: 0) != TruncInputValue2->getValueType(ResNo: 0) ||
7271 AndNode->getValueType(ResNo: 0) != TruncInputValue1->getValueType(ResNo: 0))
7272 return SDValue();
7273
7274 if ((ExtType2 != ISD::ZEXTLOAD) &&
7275 ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
7276 return SDValue();
7277
7278 // These truncation and zero-extension nodes are not necessary, remove them.
7279 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: AndNode->getValueType(ResNo: 0),
7280 N1: TruncInputValue1, N2: TruncInputValue2);
7281 SDValue NewSetCC =
7282 DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: NewAnd, RHS: TruncInputValue2, Cond: CC);
7283 DAG.ReplaceAllUsesWith(From: N, To: NewSetCC.getNode());
7284 return SDValue(N, 0);
7285}
7286
7287// Strip a single outer ISD::SIGN_EXTEND_INREG from \p V, if present, and
7288// return the inner value together with the narrow VT it was extending
7289// from. If no such node is present, returns \p V unchanged and an invalid
7290// EVT.
7291//
7292// i32 (and other sub-GRLen) arithmetic is legalized to operate on the full
7293// GRLen-width register, with a `sign_extend_inreg` re-normalizing the
7294// result back into the narrow type's range afterwards (see e.g. the
7295// `add i32` -> `add` + `sign_extend_inreg ..., i32` legalization). Any
7296// combine that reassociates such a binop must track and reapply this
7297// extension, otherwise the transformed code can produce a value whose
7298// high bits no longer match the narrow-type semantics.
7299static std::pair<SDValue, EVT> stripSignExtendInReg(SDValue V) {
7300 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
7301 return {V.getOperand(i: 0), cast<VTSDNode>(Val: V.getOperand(i: 1))->getVT()};
7302 return {V, EVT()};
7303}
7304
7305// Try to match \p BinV (after optionally stripping an outer
7306// sign_extend_inreg) as a supported binary operation that has \p X as one
7307// of its operands, returning the matched opcode, the other operand (the
7308// "delta"), and the narrow VT of the sign_extend_inreg that was stripped
7309// (invalid EVT if none was present).
7310//
7311// For commutative ops (add/or/xor), \p X may be either operand, since
7312// `binop(X, Y) == binop(Y, X)` and the identity element (0) works on
7313// either side.
7314//
7315// For `sub`, the operation is NOT commutative: `sub(X, Y) != sub(Y, X)`.
7316// Only `sub(X, Y)` (i.e. \p X is the *minuend*, the first operand) can be
7317// rewritten using the identity `X - 0 == X`. If \p X were the *subtrahend*
7318// (second operand, i.e. the pattern is actually `sub(Y, X)`), there is no
7319// way to express `cond ? (Y - X) : X` (or the symmetric case) as
7320// `X op (select ...)` without introducing an extra negation, so that case
7321// must be rejected instead of "optimized" into worse code.
7322static std::tuple<unsigned, SDValue, EVT>
7323matchBinOpWithSharedOperand(SDValue BinV, SDValue X) {
7324 auto [Inner, ExtVT] = stripSignExtendInReg(V: BinV);
7325
7326 unsigned Opc = Inner.getOpcode();
7327 switch (Opc) {
7328 case ISD::ADD:
7329 case ISD::OR:
7330 case ISD::XOR:
7331 if (Inner.getOperand(i: 0) == X)
7332 return {Opc, Inner.getOperand(i: 1), ExtVT};
7333 if (Inner.getOperand(i: 1) == X)
7334 return {Opc, Inner.getOperand(i: 0), ExtVT};
7335 return {0, SDValue(), EVT()};
7336 case ISD::SUB:
7337 // Only accept X as the minuend (first operand); see comment above.
7338 if (Inner.getOperand(i: 0) == X)
7339 return {Opc, Inner.getOperand(i: 1), ExtVT};
7340 return {0, SDValue(), EVT()};
7341 default:
7342 return {0, SDValue(), EVT()};
7343 }
7344}
7345
7346// Try to combine:
7347// select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
7348// select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
7349// for binop in {add, or, xor, sub}, where 0 is the identity element of the
7350// respective operation, additionally handling the common legalized form
7351// where the binop result is wrapped in a `sign_extend_inreg` (as happens
7352// for sub-GRLen types such as i32 on a 64-bit GRLen target). See
7353// matchBinOpWithSharedOperand() for the restrictions applied to
7354// non-commutative operations (currently only `sub`).
7355static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
7356 TargetLowering::DAGCombinerInfo &DCI,
7357 const LoongArchSubtarget &Subtarget) {
7358 if (DCI.isBeforeLegalizeOps())
7359 return SDValue();
7360
7361 EVT VT = N->getValueType(ResNo: 0);
7362 // Restrict to the scalar GRLen integer type that maskeqz/masknez operate
7363 // on; this also naturally excludes float and vector selects.
7364 if (VT != Subtarget.getGRLenVT())
7365 return SDValue();
7366
7367 SDValue Cond = N->getOperand(Num: 0);
7368 SDValue TrueV = N->getOperand(Num: 1);
7369 SDValue FalseV = N->getOperand(Num: 2);
7370 SDLoc DL(N);
7371
7372 auto TryFold = [&](SDValue BinV, SDValue SharedV,
7373 bool BinIsTrueArm) -> SDValue {
7374 auto [Opc, Delta, ExtVT] = matchBinOpWithSharedOperand(BinV, X: SharedV);
7375 if (!Opc)
7376 return SDValue();
7377
7378 // Avoid infinite combine loops: bail out if Delta is trivially the
7379 // same node we would otherwise be selecting on (shouldn't normally
7380 // happen, but guards against degenerate/self-referential IR).
7381 if (Delta.getNode() == N)
7382 return SDValue();
7383
7384 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
7385 SDValue NewSel = BinIsTrueArm ? DAG.getSelect(DL, VT, Cond, LHS: Delta, RHS: Zero)
7386 : DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: Delta);
7387 SDValue NewBin = DAG.getNode(Opcode: Opc, DL, VT, N1: SharedV, N2: NewSel);
7388
7389 // If the original binop result was normalized back into a narrower
7390 // type via sign_extend_inreg (e.g. i32 arithmetic on a 64-bit GRLen
7391 // target), the new binop must be re-normalized the same way: SharedV
7392 // is already known-sign-extended for that narrow type, but NewSel
7393 // (Delta or 0, selected) combined with SharedV via Opc can still
7394 // produce a 64-bit result whose high bits don't match the narrow
7395 // type's sign-extended representation.
7396 if (ExtVT != EVT())
7397 NewBin = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: NewBin,
7398 N2: DAG.getValueType(ExtVT));
7399
7400 return NewBin;
7401 };
7402
7403 if (SDValue R = TryFold(TrueV, FalseV, /*BinIsTrueArm=*/true))
7404 return R;
7405 if (SDValue R = TryFold(FalseV, TrueV, /*BinIsTrueArm=*/false))
7406 return R;
7407
7408 return SDValue();
7409}
7410
7411// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
7412static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
7413 TargetLowering::DAGCombinerInfo &DCI,
7414 const LoongArchSubtarget &Subtarget) {
7415 if (DCI.isBeforeLegalizeOps())
7416 return SDValue();
7417
7418 SDValue Src = N->getOperand(Num: 0);
7419 if (Src.getOpcode() != LoongArchISD::REVB_2W)
7420 return SDValue();
7421
7422 return DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7423 Operand: Src.getOperand(i: 0));
7424}
7425
7426// Perform common combines for BR_CC and SELECT_CC conditions.
7427static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
7428 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
7429 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7430
7431 // As far as arithmetic right shift always saves the sign,
7432 // shift can be omitted.
7433 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
7434 // setge (sra X, N), 0 -> setge X, 0
7435 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
7436 LHS.getOpcode() == ISD::SRA) {
7437 LHS = LHS.getOperand(i: 0);
7438 return true;
7439 }
7440
7441 if (!ISD::isIntEqualitySetCC(Code: CCVal))
7442 return false;
7443
7444 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
7445 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
7446 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
7447 LHS.getOperand(i: 0).getValueType() == Subtarget.getGRLenVT()) {
7448 // If we're looking for eq 0 instead of ne 0, we need to invert the
7449 // condition.
7450 bool Invert = CCVal == ISD::SETEQ;
7451 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
7452 if (Invert)
7453 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7454
7455 RHS = LHS.getOperand(i: 1);
7456 LHS = LHS.getOperand(i: 0);
7457 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
7458
7459 CC = DAG.getCondCode(Cond: CCVal);
7460 return true;
7461 }
7462
7463 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, GRLen-1-C), 0, ge/lt)
7464 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
7465 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
7466 SDValue LHS0 = LHS.getOperand(i: 0);
7467 if (LHS0.getOpcode() == ISD::AND &&
7468 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7469 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
7470 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
7471 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
7472 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
7473 CC = DAG.getCondCode(Cond: CCVal);
7474
7475 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
7476 LHS = LHS0.getOperand(i: 0);
7477 if (ShAmt != 0)
7478 LHS =
7479 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
7480 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
7481 return true;
7482 }
7483 }
7484 }
7485
7486 // (X, 1, setne) -> (X, 0, seteq) if we can prove X is 0/1.
7487 // This can occur when legalizing some floating point comparisons.
7488 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
7489 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
7490 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7491 CC = DAG.getCondCode(Cond: CCVal);
7492 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
7493 return true;
7494 }
7495
7496 // Fold ((shl (extract_vector_elt X, I), GRLen - EleBits)), 0, eq/ne) ->
7497 // ((extract_vector_elt X, I), 0, eq/ne)
7498 if (isNullConstant(V: RHS) && (CCVal == ISD::SETEQ || CCVal == ISD::SETNE) &&
7499 LHS.getOpcode() == ISD::SHL && LHS.hasOneUse() &&
7500 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
7501 SDValue Ext = LHS.getOperand(i: 0);
7502 unsigned Sht = LHS.getConstantOperandVal(i: 1);
7503 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
7504 SDValue Vec = Ext.getOperand(i: 0);
7505 unsigned EleBits = Vec.getScalarValueSizeInBits();
7506 if ((EleBits + Sht) == Subtarget.getGRLen()) {
7507 LHS = Ext;
7508 return true;
7509 }
7510 }
7511 }
7512
7513 // Fold (C1, C2, cond) -> (0, 0, seteq/setne)
7514 if (isa<ConstantSDNode>(Val: LHS) && isa<ConstantSDNode>(Val: RHS)) {
7515 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
7516 EVT VT = LHS.getValueType();
7517 EVT SetCCResVT =
7518 TLI->getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7519 if (SDValue Folded = DAG.FoldSetCC(VT: SetCCResVT, N1: LHS, N2: RHS, Cond: CCVal, dl: DL)) {
7520 LHS = DAG.getConstant(Val: 0, DL, VT);
7521 RHS = DAG.getConstant(Val: 0, DL, VT);
7522 CC = DAG.getCondCode(Cond: !isNullConstant(V: Folded) ? ISD::SETEQ : ISD::SETNE);
7523 return true;
7524 }
7525 }
7526
7527 return false;
7528}
7529
7530static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG,
7531 TargetLowering::DAGCombinerInfo &DCI,
7532 const LoongArchSubtarget &Subtarget) {
7533 SDValue LHS = N->getOperand(Num: 1);
7534 SDValue RHS = N->getOperand(Num: 2);
7535 SDValue CC = N->getOperand(Num: 3);
7536 SDLoc DL(N);
7537
7538 // CC was folded into V, so always return ISD::SETNE is fine.
7539 if (SDValue V = foldVMskZeroTest(LHS, RHS, CC: cast<CondCodeSDNode>(Val&: CC)->get(),
7540 DL, DAG, Subtarget))
7541 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
7542 N1: N->getOperand(Num: 0), N2: V,
7543 N3: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()),
7544 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: N->getOperand(Num: 4));
7545
7546 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7547 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
7548 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
7549
7550 return SDValue();
7551}
7552
7553static SDValue performSELECT_CCCombine(SDNode *N, SelectionDAG &DAG,
7554 TargetLowering::DAGCombinerInfo &DCI,
7555 const LoongArchSubtarget &Subtarget) {
7556 // Transform
7557 SDValue LHS = N->getOperand(Num: 0);
7558 SDValue RHS = N->getOperand(Num: 1);
7559 SDValue CC = N->getOperand(Num: 2);
7560 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7561 SDValue TrueV = N->getOperand(Num: 3);
7562 SDValue FalseV = N->getOperand(Num: 4);
7563 SDLoc DL(N);
7564 EVT VT = N->getValueType(ResNo: 0);
7565
7566 // If the True and False values are the same, we don't need a select_cc.
7567 if (TrueV == FalseV)
7568 return TrueV;
7569
7570 // (select (x < 0), y, z) -> x >> (GRLEN - 1) & (y - z) + z
7571 // (select (x >= 0), y, z) -> x >> (GRLEN - 1) & (z - y) + y
7572 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
7573 isNullConstant(V: RHS) &&
7574 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
7575 if (CCVal == ISD::CondCode::SETGE)
7576 std::swap(a&: TrueV, b&: FalseV);
7577
7578 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
7579 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
7580 // Only handle simm12, if it is not in this range, it can be considered as
7581 // register.
7582 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
7583 isInt<12>(x: TrueSImm - FalseSImm)) {
7584 SDValue SRA =
7585 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
7586 N2: DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT));
7587 SDValue AND =
7588 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
7589 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
7590 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
7591 }
7592
7593 if (CCVal == ISD::CondCode::SETGE)
7594 std::swap(a&: TrueV, b&: FalseV);
7595 }
7596
7597 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7598 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
7599 Ops: {LHS, RHS, CC, TrueV, FalseV});
7600
7601 return SDValue();
7602}
7603
7604template <unsigned N>
7605static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp,
7606 SelectionDAG &DAG,
7607 const LoongArchSubtarget &Subtarget,
7608 bool IsSigned = false) {
7609 SDLoc DL(Node);
7610 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7611 // Check the ImmArg.
7612 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7613 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7614 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7615 ": argument out of range.");
7616 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: Subtarget.getGRLenVT());
7617 }
7618 return DAG.getConstant(Val: CImm->getZExtValue(), DL, VT: Subtarget.getGRLenVT());
7619}
7620
7621template <unsigned N>
7622static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,
7623 SelectionDAG &DAG, bool IsSigned = false) {
7624 SDLoc DL(Node);
7625 EVT ResTy = Node->getValueType(ResNo: 0);
7626 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7627
7628 // Check the ImmArg.
7629 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7630 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7631 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7632 ": argument out of range.");
7633 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7634 }
7635 return DAG.getConstant(
7636 Val: APInt(ResTy.getScalarType().getSizeInBits(),
7637 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
7638 DL, VT: ResTy);
7639}
7640
7641static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG) {
7642 SDLoc DL(Node);
7643 EVT ResTy = Node->getValueType(ResNo: 0);
7644 SDValue Vec = Node->getOperand(Num: 2);
7645 SDValue Mask = DAG.getConstant(Val: Vec.getScalarValueSizeInBits() - 1, DL, VT: ResTy);
7646 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Vec, N2: Mask);
7647}
7648
7649static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG) {
7650 SDLoc DL(Node);
7651 EVT ResTy = Node->getValueType(ResNo: 0);
7652 SDValue One = DAG.getConstant(Val: 1, DL, VT: ResTy);
7653 SDValue Bit =
7654 DAG.getNode(Opcode: ISD::SHL, DL, VT: ResTy, N1: One, N2: truncateVecElts(Node, DAG));
7655
7656 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1),
7657 N2: DAG.getNOT(DL, Val: Bit, VT: ResTy));
7658}
7659
7660template <unsigned N>
7661static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG) {
7662 SDLoc DL(Node);
7663 EVT ResTy = Node->getValueType(ResNo: 0);
7664 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7665 // Check the unsigned ImmArg.
7666 if (!isUInt<N>(CImm->getZExtValue())) {
7667 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7668 ": argument out of range.");
7669 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7670 }
7671
7672 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7673 SDValue Mask = DAG.getConstant(Val: ~BitImm, DL, VT: ResTy);
7674
7675 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: Mask);
7676}
7677
7678template <unsigned N>
7679static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG) {
7680 SDLoc DL(Node);
7681 EVT ResTy = Node->getValueType(ResNo: 0);
7682 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7683 // Check the unsigned ImmArg.
7684 if (!isUInt<N>(CImm->getZExtValue())) {
7685 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7686 ": argument out of range.");
7687 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7688 }
7689
7690 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7691 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7692 return DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7693}
7694
7695template <unsigned N>
7696static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG) {
7697 SDLoc DL(Node);
7698 EVT ResTy = Node->getValueType(ResNo: 0);
7699 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7700 // Check the unsigned ImmArg.
7701 if (!isUInt<N>(CImm->getZExtValue())) {
7702 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7703 ": argument out of range.");
7704 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7705 }
7706
7707 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7708 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7709 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7710}
7711
7712template <unsigned W>
7713static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG,
7714 unsigned ResOp) {
7715 unsigned Imm = N->getConstantOperandVal(Num: 2);
7716 if (!isUInt<W>(Imm)) {
7717 const StringRef ErrorMsg = "argument out of range";
7718 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
7719 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
7720 }
7721 SDLoc DL(N);
7722 SDValue Vec = N->getOperand(Num: 1);
7723 SDValue Idx = DAG.getConstant(Val: Imm, DL, VT: MVT::i32);
7724 SDValue EltVT = DAG.getValueType(Vec.getValueType().getVectorElementType());
7725 return DAG.getNode(Opcode: ResOp, DL, VT: N->getValueType(ResNo: 0), N1: Vec, N2: Idx, N3: EltVT);
7726}
7727
7728static SDValue
7729performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
7730 TargetLowering::DAGCombinerInfo &DCI,
7731 const LoongArchSubtarget &Subtarget) {
7732 SDLoc DL(N);
7733 switch (N->getConstantOperandVal(Num: 0)) {
7734 default:
7735 break;
7736 case Intrinsic::loongarch_lsx_vadd_b:
7737 case Intrinsic::loongarch_lsx_vadd_h:
7738 case Intrinsic::loongarch_lsx_vadd_w:
7739 case Intrinsic::loongarch_lsx_vadd_d:
7740 case Intrinsic::loongarch_lasx_xvadd_b:
7741 case Intrinsic::loongarch_lasx_xvadd_h:
7742 case Intrinsic::loongarch_lasx_xvadd_w:
7743 case Intrinsic::loongarch_lasx_xvadd_d:
7744 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7745 N2: N->getOperand(Num: 2));
7746 case Intrinsic::loongarch_lsx_vaddi_bu:
7747 case Intrinsic::loongarch_lsx_vaddi_hu:
7748 case Intrinsic::loongarch_lsx_vaddi_wu:
7749 case Intrinsic::loongarch_lsx_vaddi_du:
7750 case Intrinsic::loongarch_lasx_xvaddi_bu:
7751 case Intrinsic::loongarch_lasx_xvaddi_hu:
7752 case Intrinsic::loongarch_lasx_xvaddi_wu:
7753 case Intrinsic::loongarch_lasx_xvaddi_du:
7754 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7755 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7756 case Intrinsic::loongarch_lsx_vsub_b:
7757 case Intrinsic::loongarch_lsx_vsub_h:
7758 case Intrinsic::loongarch_lsx_vsub_w:
7759 case Intrinsic::loongarch_lsx_vsub_d:
7760 case Intrinsic::loongarch_lasx_xvsub_b:
7761 case Intrinsic::loongarch_lasx_xvsub_h:
7762 case Intrinsic::loongarch_lasx_xvsub_w:
7763 case Intrinsic::loongarch_lasx_xvsub_d:
7764 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7765 N2: N->getOperand(Num: 2));
7766 case Intrinsic::loongarch_lsx_vsubi_bu:
7767 case Intrinsic::loongarch_lsx_vsubi_hu:
7768 case Intrinsic::loongarch_lsx_vsubi_wu:
7769 case Intrinsic::loongarch_lsx_vsubi_du:
7770 case Intrinsic::loongarch_lasx_xvsubi_bu:
7771 case Intrinsic::loongarch_lasx_xvsubi_hu:
7772 case Intrinsic::loongarch_lasx_xvsubi_wu:
7773 case Intrinsic::loongarch_lasx_xvsubi_du:
7774 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7775 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7776 case Intrinsic::loongarch_lsx_vneg_b:
7777 case Intrinsic::loongarch_lsx_vneg_h:
7778 case Intrinsic::loongarch_lsx_vneg_w:
7779 case Intrinsic::loongarch_lsx_vneg_d:
7780 case Intrinsic::loongarch_lasx_xvneg_b:
7781 case Intrinsic::loongarch_lasx_xvneg_h:
7782 case Intrinsic::loongarch_lasx_xvneg_w:
7783 case Intrinsic::loongarch_lasx_xvneg_d:
7784 return DAG.getNode(
7785 Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0),
7786 N1: DAG.getConstant(
7787 Val: APInt(N->getValueType(ResNo: 0).getScalarType().getSizeInBits(), 0,
7788 /*isSigned=*/true),
7789 DL: SDLoc(N), VT: N->getValueType(ResNo: 0)),
7790 N2: N->getOperand(Num: 1));
7791 case Intrinsic::loongarch_lsx_vmax_b:
7792 case Intrinsic::loongarch_lsx_vmax_h:
7793 case Intrinsic::loongarch_lsx_vmax_w:
7794 case Intrinsic::loongarch_lsx_vmax_d:
7795 case Intrinsic::loongarch_lasx_xvmax_b:
7796 case Intrinsic::loongarch_lasx_xvmax_h:
7797 case Intrinsic::loongarch_lasx_xvmax_w:
7798 case Intrinsic::loongarch_lasx_xvmax_d:
7799 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7800 N2: N->getOperand(Num: 2));
7801 case Intrinsic::loongarch_lsx_vmax_bu:
7802 case Intrinsic::loongarch_lsx_vmax_hu:
7803 case Intrinsic::loongarch_lsx_vmax_wu:
7804 case Intrinsic::loongarch_lsx_vmax_du:
7805 case Intrinsic::loongarch_lasx_xvmax_bu:
7806 case Intrinsic::loongarch_lasx_xvmax_hu:
7807 case Intrinsic::loongarch_lasx_xvmax_wu:
7808 case Intrinsic::loongarch_lasx_xvmax_du:
7809 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7810 N2: N->getOperand(Num: 2));
7811 case Intrinsic::loongarch_lsx_vmaxi_b:
7812 case Intrinsic::loongarch_lsx_vmaxi_h:
7813 case Intrinsic::loongarch_lsx_vmaxi_w:
7814 case Intrinsic::loongarch_lsx_vmaxi_d:
7815 case Intrinsic::loongarch_lasx_xvmaxi_b:
7816 case Intrinsic::loongarch_lasx_xvmaxi_h:
7817 case Intrinsic::loongarch_lasx_xvmaxi_w:
7818 case Intrinsic::loongarch_lasx_xvmaxi_d:
7819 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7820 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7821 case Intrinsic::loongarch_lsx_vmaxi_bu:
7822 case Intrinsic::loongarch_lsx_vmaxi_hu:
7823 case Intrinsic::loongarch_lsx_vmaxi_wu:
7824 case Intrinsic::loongarch_lsx_vmaxi_du:
7825 case Intrinsic::loongarch_lasx_xvmaxi_bu:
7826 case Intrinsic::loongarch_lasx_xvmaxi_hu:
7827 case Intrinsic::loongarch_lasx_xvmaxi_wu:
7828 case Intrinsic::loongarch_lasx_xvmaxi_du:
7829 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7830 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7831 case Intrinsic::loongarch_lsx_vmin_b:
7832 case Intrinsic::loongarch_lsx_vmin_h:
7833 case Intrinsic::loongarch_lsx_vmin_w:
7834 case Intrinsic::loongarch_lsx_vmin_d:
7835 case Intrinsic::loongarch_lasx_xvmin_b:
7836 case Intrinsic::loongarch_lasx_xvmin_h:
7837 case Intrinsic::loongarch_lasx_xvmin_w:
7838 case Intrinsic::loongarch_lasx_xvmin_d:
7839 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7840 N2: N->getOperand(Num: 2));
7841 case Intrinsic::loongarch_lsx_vmin_bu:
7842 case Intrinsic::loongarch_lsx_vmin_hu:
7843 case Intrinsic::loongarch_lsx_vmin_wu:
7844 case Intrinsic::loongarch_lsx_vmin_du:
7845 case Intrinsic::loongarch_lasx_xvmin_bu:
7846 case Intrinsic::loongarch_lasx_xvmin_hu:
7847 case Intrinsic::loongarch_lasx_xvmin_wu:
7848 case Intrinsic::loongarch_lasx_xvmin_du:
7849 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7850 N2: N->getOperand(Num: 2));
7851 case Intrinsic::loongarch_lsx_vmini_b:
7852 case Intrinsic::loongarch_lsx_vmini_h:
7853 case Intrinsic::loongarch_lsx_vmini_w:
7854 case Intrinsic::loongarch_lsx_vmini_d:
7855 case Intrinsic::loongarch_lasx_xvmini_b:
7856 case Intrinsic::loongarch_lasx_xvmini_h:
7857 case Intrinsic::loongarch_lasx_xvmini_w:
7858 case Intrinsic::loongarch_lasx_xvmini_d:
7859 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7860 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7861 case Intrinsic::loongarch_lsx_vmini_bu:
7862 case Intrinsic::loongarch_lsx_vmini_hu:
7863 case Intrinsic::loongarch_lsx_vmini_wu:
7864 case Intrinsic::loongarch_lsx_vmini_du:
7865 case Intrinsic::loongarch_lasx_xvmini_bu:
7866 case Intrinsic::loongarch_lasx_xvmini_hu:
7867 case Intrinsic::loongarch_lasx_xvmini_wu:
7868 case Intrinsic::loongarch_lasx_xvmini_du:
7869 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7870 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7871 case Intrinsic::loongarch_lsx_vmul_b:
7872 case Intrinsic::loongarch_lsx_vmul_h:
7873 case Intrinsic::loongarch_lsx_vmul_w:
7874 case Intrinsic::loongarch_lsx_vmul_d:
7875 case Intrinsic::loongarch_lasx_xvmul_b:
7876 case Intrinsic::loongarch_lasx_xvmul_h:
7877 case Intrinsic::loongarch_lasx_xvmul_w:
7878 case Intrinsic::loongarch_lasx_xvmul_d:
7879 return DAG.getNode(Opcode: ISD::MUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7880 N2: N->getOperand(Num: 2));
7881 case Intrinsic::loongarch_lsx_vmadd_b:
7882 case Intrinsic::loongarch_lsx_vmadd_h:
7883 case Intrinsic::loongarch_lsx_vmadd_w:
7884 case Intrinsic::loongarch_lsx_vmadd_d:
7885 case Intrinsic::loongarch_lasx_xvmadd_b:
7886 case Intrinsic::loongarch_lasx_xvmadd_h:
7887 case Intrinsic::loongarch_lasx_xvmadd_w:
7888 case Intrinsic::loongarch_lasx_xvmadd_d: {
7889 EVT ResTy = N->getValueType(ResNo: 0);
7890 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7891 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7892 N2: N->getOperand(Num: 3)));
7893 }
7894 case Intrinsic::loongarch_lsx_vmsub_b:
7895 case Intrinsic::loongarch_lsx_vmsub_h:
7896 case Intrinsic::loongarch_lsx_vmsub_w:
7897 case Intrinsic::loongarch_lsx_vmsub_d:
7898 case Intrinsic::loongarch_lasx_xvmsub_b:
7899 case Intrinsic::loongarch_lasx_xvmsub_h:
7900 case Intrinsic::loongarch_lasx_xvmsub_w:
7901 case Intrinsic::loongarch_lasx_xvmsub_d: {
7902 EVT ResTy = N->getValueType(ResNo: 0);
7903 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7904 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7905 N2: N->getOperand(Num: 3)));
7906 }
7907 case Intrinsic::loongarch_lsx_vdiv_b:
7908 case Intrinsic::loongarch_lsx_vdiv_h:
7909 case Intrinsic::loongarch_lsx_vdiv_w:
7910 case Intrinsic::loongarch_lsx_vdiv_d:
7911 case Intrinsic::loongarch_lasx_xvdiv_b:
7912 case Intrinsic::loongarch_lasx_xvdiv_h:
7913 case Intrinsic::loongarch_lasx_xvdiv_w:
7914 case Intrinsic::loongarch_lasx_xvdiv_d:
7915 return DAG.getNode(Opcode: ISD::SDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7916 N2: N->getOperand(Num: 2));
7917 case Intrinsic::loongarch_lsx_vdiv_bu:
7918 case Intrinsic::loongarch_lsx_vdiv_hu:
7919 case Intrinsic::loongarch_lsx_vdiv_wu:
7920 case Intrinsic::loongarch_lsx_vdiv_du:
7921 case Intrinsic::loongarch_lasx_xvdiv_bu:
7922 case Intrinsic::loongarch_lasx_xvdiv_hu:
7923 case Intrinsic::loongarch_lasx_xvdiv_wu:
7924 case Intrinsic::loongarch_lasx_xvdiv_du:
7925 return DAG.getNode(Opcode: ISD::UDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7926 N2: N->getOperand(Num: 2));
7927 case Intrinsic::loongarch_lsx_vmod_b:
7928 case Intrinsic::loongarch_lsx_vmod_h:
7929 case Intrinsic::loongarch_lsx_vmod_w:
7930 case Intrinsic::loongarch_lsx_vmod_d:
7931 case Intrinsic::loongarch_lasx_xvmod_b:
7932 case Intrinsic::loongarch_lasx_xvmod_h:
7933 case Intrinsic::loongarch_lasx_xvmod_w:
7934 case Intrinsic::loongarch_lasx_xvmod_d:
7935 return DAG.getNode(Opcode: ISD::SREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7936 N2: N->getOperand(Num: 2));
7937 case Intrinsic::loongarch_lsx_vmod_bu:
7938 case Intrinsic::loongarch_lsx_vmod_hu:
7939 case Intrinsic::loongarch_lsx_vmod_wu:
7940 case Intrinsic::loongarch_lsx_vmod_du:
7941 case Intrinsic::loongarch_lasx_xvmod_bu:
7942 case Intrinsic::loongarch_lasx_xvmod_hu:
7943 case Intrinsic::loongarch_lasx_xvmod_wu:
7944 case Intrinsic::loongarch_lasx_xvmod_du:
7945 return DAG.getNode(Opcode: ISD::UREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7946 N2: N->getOperand(Num: 2));
7947 case Intrinsic::loongarch_lsx_vand_v:
7948 case Intrinsic::loongarch_lasx_xvand_v:
7949 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7950 N2: N->getOperand(Num: 2));
7951 case Intrinsic::loongarch_lsx_vor_v:
7952 case Intrinsic::loongarch_lasx_xvor_v:
7953 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7954 N2: N->getOperand(Num: 2));
7955 case Intrinsic::loongarch_lsx_vxor_v:
7956 case Intrinsic::loongarch_lasx_xvxor_v:
7957 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7958 N2: N->getOperand(Num: 2));
7959 case Intrinsic::loongarch_lsx_vnor_v:
7960 case Intrinsic::loongarch_lasx_xvnor_v: {
7961 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7962 N2: N->getOperand(Num: 2));
7963 return DAG.getNOT(DL, Val: Res, VT: Res->getValueType(ResNo: 0));
7964 }
7965 case Intrinsic::loongarch_lsx_vandi_b:
7966 case Intrinsic::loongarch_lasx_xvandi_b:
7967 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7968 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7969 case Intrinsic::loongarch_lsx_vori_b:
7970 case Intrinsic::loongarch_lasx_xvori_b:
7971 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7972 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7973 case Intrinsic::loongarch_lsx_vxori_b:
7974 case Intrinsic::loongarch_lasx_xvxori_b:
7975 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7976 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7977 case Intrinsic::loongarch_lsx_vsll_b:
7978 case Intrinsic::loongarch_lsx_vsll_h:
7979 case Intrinsic::loongarch_lsx_vsll_w:
7980 case Intrinsic::loongarch_lsx_vsll_d:
7981 case Intrinsic::loongarch_lasx_xvsll_b:
7982 case Intrinsic::loongarch_lasx_xvsll_h:
7983 case Intrinsic::loongarch_lasx_xvsll_w:
7984 case Intrinsic::loongarch_lasx_xvsll_d:
7985 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7986 N2: truncateVecElts(Node: N, DAG));
7987 case Intrinsic::loongarch_lsx_vslli_b:
7988 case Intrinsic::loongarch_lasx_xvslli_b:
7989 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7990 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7991 case Intrinsic::loongarch_lsx_vslli_h:
7992 case Intrinsic::loongarch_lasx_xvslli_h:
7993 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7994 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7995 case Intrinsic::loongarch_lsx_vslli_w:
7996 case Intrinsic::loongarch_lasx_xvslli_w:
7997 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7998 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7999 case Intrinsic::loongarch_lsx_vslli_d:
8000 case Intrinsic::loongarch_lasx_xvslli_d:
8001 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8002 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8003 case Intrinsic::loongarch_lsx_vsrl_b:
8004 case Intrinsic::loongarch_lsx_vsrl_h:
8005 case Intrinsic::loongarch_lsx_vsrl_w:
8006 case Intrinsic::loongarch_lsx_vsrl_d:
8007 case Intrinsic::loongarch_lasx_xvsrl_b:
8008 case Intrinsic::loongarch_lasx_xvsrl_h:
8009 case Intrinsic::loongarch_lasx_xvsrl_w:
8010 case Intrinsic::loongarch_lasx_xvsrl_d:
8011 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8012 N2: truncateVecElts(Node: N, DAG));
8013 case Intrinsic::loongarch_lsx_vsrli_b:
8014 case Intrinsic::loongarch_lasx_xvsrli_b:
8015 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8016 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
8017 case Intrinsic::loongarch_lsx_vsrli_h:
8018 case Intrinsic::loongarch_lasx_xvsrli_h:
8019 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8020 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
8021 case Intrinsic::loongarch_lsx_vsrli_w:
8022 case Intrinsic::loongarch_lasx_xvsrli_w:
8023 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8024 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8025 case Intrinsic::loongarch_lsx_vsrli_d:
8026 case Intrinsic::loongarch_lasx_xvsrli_d:
8027 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8028 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8029 case Intrinsic::loongarch_lsx_vsra_b:
8030 case Intrinsic::loongarch_lsx_vsra_h:
8031 case Intrinsic::loongarch_lsx_vsra_w:
8032 case Intrinsic::loongarch_lsx_vsra_d:
8033 case Intrinsic::loongarch_lasx_xvsra_b:
8034 case Intrinsic::loongarch_lasx_xvsra_h:
8035 case Intrinsic::loongarch_lasx_xvsra_w:
8036 case Intrinsic::loongarch_lasx_xvsra_d:
8037 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8038 N2: truncateVecElts(Node: N, DAG));
8039 case Intrinsic::loongarch_lsx_vsrai_b:
8040 case Intrinsic::loongarch_lasx_xvsrai_b:
8041 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8042 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
8043 case Intrinsic::loongarch_lsx_vsrai_h:
8044 case Intrinsic::loongarch_lasx_xvsrai_h:
8045 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8046 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
8047 case Intrinsic::loongarch_lsx_vsrai_w:
8048 case Intrinsic::loongarch_lasx_xvsrai_w:
8049 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8050 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8051 case Intrinsic::loongarch_lsx_vsrai_d:
8052 case Intrinsic::loongarch_lasx_xvsrai_d:
8053 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8054 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8055 case Intrinsic::loongarch_lsx_vclz_b:
8056 case Intrinsic::loongarch_lsx_vclz_h:
8057 case Intrinsic::loongarch_lsx_vclz_w:
8058 case Intrinsic::loongarch_lsx_vclz_d:
8059 case Intrinsic::loongarch_lasx_xvclz_b:
8060 case Intrinsic::loongarch_lasx_xvclz_h:
8061 case Intrinsic::loongarch_lasx_xvclz_w:
8062 case Intrinsic::loongarch_lasx_xvclz_d:
8063 return DAG.getNode(Opcode: ISD::CTLZ, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
8064 case Intrinsic::loongarch_lsx_vpcnt_b:
8065 case Intrinsic::loongarch_lsx_vpcnt_h:
8066 case Intrinsic::loongarch_lsx_vpcnt_w:
8067 case Intrinsic::loongarch_lsx_vpcnt_d:
8068 case Intrinsic::loongarch_lasx_xvpcnt_b:
8069 case Intrinsic::loongarch_lasx_xvpcnt_h:
8070 case Intrinsic::loongarch_lasx_xvpcnt_w:
8071 case Intrinsic::loongarch_lasx_xvpcnt_d:
8072 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
8073 case Intrinsic::loongarch_lsx_vbitclr_b:
8074 case Intrinsic::loongarch_lsx_vbitclr_h:
8075 case Intrinsic::loongarch_lsx_vbitclr_w:
8076 case Intrinsic::loongarch_lsx_vbitclr_d:
8077 case Intrinsic::loongarch_lasx_xvbitclr_b:
8078 case Intrinsic::loongarch_lasx_xvbitclr_h:
8079 case Intrinsic::loongarch_lasx_xvbitclr_w:
8080 case Intrinsic::loongarch_lasx_xvbitclr_d:
8081 return lowerVectorBitClear(Node: N, DAG);
8082 case Intrinsic::loongarch_lsx_vbitclri_b:
8083 case Intrinsic::loongarch_lasx_xvbitclri_b:
8084 return lowerVectorBitClearImm<3>(Node: N, DAG);
8085 case Intrinsic::loongarch_lsx_vbitclri_h:
8086 case Intrinsic::loongarch_lasx_xvbitclri_h:
8087 return lowerVectorBitClearImm<4>(Node: N, DAG);
8088 case Intrinsic::loongarch_lsx_vbitclri_w:
8089 case Intrinsic::loongarch_lasx_xvbitclri_w:
8090 return lowerVectorBitClearImm<5>(Node: N, DAG);
8091 case Intrinsic::loongarch_lsx_vbitclri_d:
8092 case Intrinsic::loongarch_lasx_xvbitclri_d:
8093 return lowerVectorBitClearImm<6>(Node: N, DAG);
8094 case Intrinsic::loongarch_lsx_vbitset_b:
8095 case Intrinsic::loongarch_lsx_vbitset_h:
8096 case Intrinsic::loongarch_lsx_vbitset_w:
8097 case Intrinsic::loongarch_lsx_vbitset_d:
8098 case Intrinsic::loongarch_lasx_xvbitset_b:
8099 case Intrinsic::loongarch_lasx_xvbitset_h:
8100 case Intrinsic::loongarch_lasx_xvbitset_w:
8101 case Intrinsic::loongarch_lasx_xvbitset_d: {
8102 EVT VecTy = N->getValueType(ResNo: 0);
8103 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8104 return DAG.getNode(
8105 Opcode: ISD::OR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8106 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8107 }
8108 case Intrinsic::loongarch_lsx_vbitseti_b:
8109 case Intrinsic::loongarch_lasx_xvbitseti_b:
8110 return lowerVectorBitSetImm<3>(Node: N, DAG);
8111 case Intrinsic::loongarch_lsx_vbitseti_h:
8112 case Intrinsic::loongarch_lasx_xvbitseti_h:
8113 return lowerVectorBitSetImm<4>(Node: N, DAG);
8114 case Intrinsic::loongarch_lsx_vbitseti_w:
8115 case Intrinsic::loongarch_lasx_xvbitseti_w:
8116 return lowerVectorBitSetImm<5>(Node: N, DAG);
8117 case Intrinsic::loongarch_lsx_vbitseti_d:
8118 case Intrinsic::loongarch_lasx_xvbitseti_d:
8119 return lowerVectorBitSetImm<6>(Node: N, DAG);
8120 case Intrinsic::loongarch_lsx_vbitrev_b:
8121 case Intrinsic::loongarch_lsx_vbitrev_h:
8122 case Intrinsic::loongarch_lsx_vbitrev_w:
8123 case Intrinsic::loongarch_lsx_vbitrev_d:
8124 case Intrinsic::loongarch_lasx_xvbitrev_b:
8125 case Intrinsic::loongarch_lasx_xvbitrev_h:
8126 case Intrinsic::loongarch_lasx_xvbitrev_w:
8127 case Intrinsic::loongarch_lasx_xvbitrev_d: {
8128 EVT VecTy = N->getValueType(ResNo: 0);
8129 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8130 return DAG.getNode(
8131 Opcode: ISD::XOR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8132 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8133 }
8134 case Intrinsic::loongarch_lsx_vbitrevi_b:
8135 case Intrinsic::loongarch_lasx_xvbitrevi_b:
8136 return lowerVectorBitRevImm<3>(Node: N, DAG);
8137 case Intrinsic::loongarch_lsx_vbitrevi_h:
8138 case Intrinsic::loongarch_lasx_xvbitrevi_h:
8139 return lowerVectorBitRevImm<4>(Node: N, DAG);
8140 case Intrinsic::loongarch_lsx_vbitrevi_w:
8141 case Intrinsic::loongarch_lasx_xvbitrevi_w:
8142 return lowerVectorBitRevImm<5>(Node: N, DAG);
8143 case Intrinsic::loongarch_lsx_vbitrevi_d:
8144 case Intrinsic::loongarch_lasx_xvbitrevi_d:
8145 return lowerVectorBitRevImm<6>(Node: N, DAG);
8146 case Intrinsic::loongarch_lsx_vfadd_s:
8147 case Intrinsic::loongarch_lsx_vfadd_d:
8148 case Intrinsic::loongarch_lasx_xvfadd_s:
8149 case Intrinsic::loongarch_lasx_xvfadd_d:
8150 return DAG.getNode(Opcode: ISD::FADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8151 N2: N->getOperand(Num: 2));
8152 case Intrinsic::loongarch_lsx_vfsub_s:
8153 case Intrinsic::loongarch_lsx_vfsub_d:
8154 case Intrinsic::loongarch_lasx_xvfsub_s:
8155 case Intrinsic::loongarch_lasx_xvfsub_d:
8156 return DAG.getNode(Opcode: ISD::FSUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8157 N2: N->getOperand(Num: 2));
8158 case Intrinsic::loongarch_lsx_vfmul_s:
8159 case Intrinsic::loongarch_lsx_vfmul_d:
8160 case Intrinsic::loongarch_lasx_xvfmul_s:
8161 case Intrinsic::loongarch_lasx_xvfmul_d:
8162 return DAG.getNode(Opcode: ISD::FMUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8163 N2: N->getOperand(Num: 2));
8164 case Intrinsic::loongarch_lsx_vfdiv_s:
8165 case Intrinsic::loongarch_lsx_vfdiv_d:
8166 case Intrinsic::loongarch_lasx_xvfdiv_s:
8167 case Intrinsic::loongarch_lasx_xvfdiv_d:
8168 return DAG.getNode(Opcode: ISD::FDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8169 N2: N->getOperand(Num: 2));
8170 case Intrinsic::loongarch_lsx_vfmadd_s:
8171 case Intrinsic::loongarch_lsx_vfmadd_d:
8172 case Intrinsic::loongarch_lasx_xvfmadd_s:
8173 case Intrinsic::loongarch_lasx_xvfmadd_d:
8174 return DAG.getNode(Opcode: ISD::FMA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8175 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
8176 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
8177 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8178 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8179 N3: legalizeIntrinsicImmArg<4>(Node: N, ImmOp: 3, DAG, Subtarget));
8180 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
8181 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
8182 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8183 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8184 N3: legalizeIntrinsicImmArg<3>(Node: N, ImmOp: 3, DAG, Subtarget));
8185 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
8186 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
8187 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8188 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8189 N3: legalizeIntrinsicImmArg<2>(Node: N, ImmOp: 3, DAG, Subtarget));
8190 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
8191 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8192 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8193 N3: legalizeIntrinsicImmArg<1>(Node: N, ImmOp: 3, DAG, Subtarget));
8194 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
8195 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
8196 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
8197 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
8198 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
8199 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
8200 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
8201 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
8202 return DAG.getNode(Opcode: LoongArchISD::VREPLGR2VR, DL, VT: N->getValueType(ResNo: 0),
8203 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8204 Operand: N->getOperand(Num: 1)));
8205 case Intrinsic::loongarch_lsx_vreplve_b:
8206 case Intrinsic::loongarch_lsx_vreplve_h:
8207 case Intrinsic::loongarch_lsx_vreplve_w:
8208 case Intrinsic::loongarch_lsx_vreplve_d:
8209 case Intrinsic::loongarch_lasx_xvreplve_b:
8210 case Intrinsic::loongarch_lasx_xvreplve_h:
8211 case Intrinsic::loongarch_lasx_xvreplve_w:
8212 case Intrinsic::loongarch_lasx_xvreplve_d:
8213 return DAG.getNode(Opcode: LoongArchISD::VREPLVE, DL, VT: N->getValueType(ResNo: 0),
8214 N1: N->getOperand(Num: 1),
8215 N2: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8216 Operand: N->getOperand(Num: 2)));
8217 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8218 if (!Subtarget.is64Bit())
8219 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8220 break;
8221 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8222 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8223 if (!Subtarget.is64Bit())
8224 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8225 break;
8226 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8227 if (!Subtarget.is64Bit())
8228 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8229 break;
8230 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8231 if (!Subtarget.is64Bit())
8232 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8233 break;
8234 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8235 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8236 if (!Subtarget.is64Bit())
8237 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8238 break;
8239 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8240 if (!Subtarget.is64Bit())
8241 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8242 break;
8243 case Intrinsic::loongarch_lsx_bz_b:
8244 case Intrinsic::loongarch_lsx_bz_h:
8245 case Intrinsic::loongarch_lsx_bz_w:
8246 case Intrinsic::loongarch_lsx_bz_d:
8247 case Intrinsic::loongarch_lasx_xbz_b:
8248 case Intrinsic::loongarch_lasx_xbz_h:
8249 case Intrinsic::loongarch_lasx_xbz_w:
8250 case Intrinsic::loongarch_lasx_xbz_d:
8251 if (!Subtarget.is64Bit())
8252 return DAG.getNode(Opcode: LoongArchISD::VALL_ZERO, DL, VT: N->getValueType(ResNo: 0),
8253 Operand: N->getOperand(Num: 1));
8254 break;
8255 case Intrinsic::loongarch_lsx_bz_v:
8256 case Intrinsic::loongarch_lasx_xbz_v:
8257 if (!Subtarget.is64Bit())
8258 return DAG.getNode(Opcode: LoongArchISD::VANY_ZERO, DL, VT: N->getValueType(ResNo: 0),
8259 Operand: N->getOperand(Num: 1));
8260 break;
8261 case Intrinsic::loongarch_lsx_bnz_b:
8262 case Intrinsic::loongarch_lsx_bnz_h:
8263 case Intrinsic::loongarch_lsx_bnz_w:
8264 case Intrinsic::loongarch_lsx_bnz_d:
8265 case Intrinsic::loongarch_lasx_xbnz_b:
8266 case Intrinsic::loongarch_lasx_xbnz_h:
8267 case Intrinsic::loongarch_lasx_xbnz_w:
8268 case Intrinsic::loongarch_lasx_xbnz_d:
8269 if (!Subtarget.is64Bit())
8270 return DAG.getNode(Opcode: LoongArchISD::VALL_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8271 Operand: N->getOperand(Num: 1));
8272 break;
8273 case Intrinsic::loongarch_lsx_bnz_v:
8274 case Intrinsic::loongarch_lasx_xbnz_v:
8275 if (!Subtarget.is64Bit())
8276 return DAG.getNode(Opcode: LoongArchISD::VANY_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8277 Operand: N->getOperand(Num: 1));
8278 break;
8279 case Intrinsic::loongarch_lasx_concat_128_s:
8280 case Intrinsic::loongarch_lasx_concat_128_d:
8281 case Intrinsic::loongarch_lasx_concat_128:
8282 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
8283 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2));
8284 }
8285 return SDValue();
8286}
8287
8288static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG,
8289 TargetLowering::DAGCombinerInfo &DCI,
8290 const LoongArchSubtarget &Subtarget) {
8291 // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
8292 // conversion is unnecessary and can be replaced with the
8293 // MOVFR2GR_S_LA64 operand.
8294 SDValue Op0 = N->getOperand(Num: 0);
8295 if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8296 return Op0.getOperand(i: 0);
8297 return SDValue();
8298}
8299
8300static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG,
8301 TargetLowering::DAGCombinerInfo &DCI,
8302 const LoongArchSubtarget &Subtarget) {
8303 // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
8304 // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
8305 // operand.
8306 SDValue Op0 = N->getOperand(Num: 0);
8307 if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8308 assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
8309 "Unexpected value type!");
8310 return Op0.getOperand(i: 0);
8311 }
8312 return SDValue();
8313}
8314
8315static SDValue
8316performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG,
8317 TargetLowering::DAGCombinerInfo &DCI) {
8318 MVT VT = N->getSimpleValueType(ResNo: 0);
8319 unsigned NumBits = VT.getScalarSizeInBits();
8320
8321 // Simplify the inputs.
8322 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8323 APInt DemandedMask(APInt::getAllOnes(numBits: NumBits));
8324 if (TLI.SimplifyDemandedBits(Op: SDValue(N, 0), DemandedBits: DemandedMask, DCI))
8325 return SDValue(N, 0);
8326
8327 return SDValue();
8328}
8329
8330static SDValue
8331performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG,
8332 TargetLowering::DAGCombinerInfo &DCI,
8333 const LoongArchSubtarget &Subtarget) {
8334 SDValue Op0 = N->getOperand(Num: 0);
8335 SDLoc DL(N);
8336
8337 // If the input to SplitPairF64 is just BuildPairF64 then the operation is
8338 // redundant. Instead, use BuildPairF64's operands directly.
8339 if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8340 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
8341
8342 if (Op0->isUndef()) {
8343 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
8344 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
8345 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8346 }
8347
8348 // It's cheaper to materialise two 32-bit integers than to load a double
8349 // from the constant pool and transfer it to integer registers through the
8350 // stack.
8351 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
8352 APInt V = C->getValueAPF().bitcastToAPInt();
8353 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
8354 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
8355 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8356 }
8357
8358 return SDValue();
8359}
8360
8361/// Do target-specific dag combines on LoongArchISD::VANDN nodes.
8362static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG,
8363 TargetLowering::DAGCombinerInfo &DCI,
8364 const LoongArchSubtarget &Subtarget) {
8365 SDValue N0 = N->getOperand(Num: 0);
8366 SDValue N1 = N->getOperand(Num: 1);
8367 MVT VT = N->getSimpleValueType(ResNo: 0);
8368 SDLoc DL(N);
8369
8370 // VANDN(undef, x) -> 0
8371 // VANDN(x, undef) -> 0
8372 if (N0.isUndef() || N1.isUndef())
8373 return DAG.getConstant(Val: 0, DL, VT);
8374
8375 // VANDN(0, x) -> x
8376 if (ISD::isBuildVectorAllZeros(N: N0.getNode()))
8377 return N1;
8378
8379 // VANDN(x, 0) -> 0
8380 if (ISD::isBuildVectorAllZeros(N: N1.getNode()))
8381 return DAG.getConstant(Val: 0, DL, VT);
8382
8383 // VANDN(x, -1) -> NOT(x) -> XOR(x, -1)
8384 if (ISD::isBuildVectorAllOnes(N: N1.getNode()))
8385 return DAG.getNOT(DL, Val: N0, VT);
8386
8387 // Turn VANDN back to AND if input is inverted.
8388 if (SDValue Not = isNOT(V: N0, DAG))
8389 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: DAG.getBitcast(VT, V: Not), N2: N1);
8390
8391 // Folds for better commutativity:
8392 if (N1->hasOneUse()) {
8393 // VANDN(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
8394 if (SDValue Not = isNOT(V: N1, DAG))
8395 return DAG.getNOT(
8396 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)), VT);
8397
8398 // VANDN(x, SplatVector(Imm)) -> AND(NOT(x), NOT(SplatVector(~Imm)))
8399 // -> NOT(OR(x, SplatVector(-Imm))
8400 // Combination is performed only when VT is v16i8/v32i8, using `vnori.b` to
8401 // gain benefits.
8402 if (!DCI.isBeforeLegalizeOps() && (VT == MVT::v16i8 || VT == MVT::v32i8) &&
8403 N1.getOpcode() == ISD::BUILD_VECTOR) {
8404 if (SDValue SplatValue =
8405 cast<BuildVectorSDNode>(Val: N1.getNode())->getSplatValue()) {
8406 if (!N1->isOnlyUserOf(N: SplatValue.getNode()))
8407 return SDValue();
8408
8409 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
8410 uint8_t NCVal = static_cast<uint8_t>(~(C->getSExtValue()));
8411 SDValue Not =
8412 DAG.getSplat(VT, DL, Op: DAG.getTargetConstant(Val: NCVal, DL, VT: MVT::i8));
8413 return DAG.getNOT(
8414 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)),
8415 VT);
8416 }
8417 }
8418 }
8419 }
8420
8421 return SDValue();
8422}
8423
8424static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp) {
8425 SDLoc DL(N);
8426 EVT VT = N->getValueType(ResNo: 0);
8427 SDValue Src = N->getOperand(Num: 0);
8428 EVT SrcVT = Src.getValueType();
8429
8430 unsigned DstElts = VT.getVectorNumElements();
8431 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8432 unsigned DstEltBits = VT.getScalarSizeInBits();
8433
8434 if (SrcEltBits >= DstEltBits)
8435 return SDValue();
8436
8437 MVT WidenEltVT = MVT::getIntegerVT(BitWidth: DstEltBits);
8438 EVT WidenSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: DstElts);
8439
8440 SDValue Extend = DAG.getNode(Opcode: ExtendOp, DL, VT: WidenSrcVT, Operand: Src);
8441 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Extend);
8442}
8443
8444// Merge two 64 to 32 convert instructions into one,
8445// e.g.
8446// vffint.s.l $vr0, $vr1, $vr2
8447// will convert 4 si64 into 4 float at once.
8448// or
8449// vftintrz.w.d $vr0, $vr1, $vr2
8450// which will convert 4 double into 4 si32 at once.
8451// also deal with their 256-bits LASX version.
8452static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode,
8453 unsigned BlockBits) {
8454 SDLoc DL(N);
8455 MVT DstVT = N->getSimpleValueType(ResNo: 0);
8456 SDValue Src = N->getOperand(Num: 0);
8457 MVT SrcVT = Src.getSimpleValueType();
8458 unsigned SrcBits = SrcVT.getSizeInBits();
8459
8460 SmallVector<SDValue, 4> Blocks;
8461 unsigned BlockNumElts = BlockBits / SrcVT.getScalarSizeInBits();
8462 MVT BlockVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: BlockNumElts);
8463 if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
8464 Src.getOperand(i: 0).getValueType() == BlockVT) {
8465 for (unsigned i = 0; i < Src.getNumOperands(); ++i)
8466 Blocks.push_back(Elt: Src.getOperand(i));
8467 } else if (SrcBits > BlockBits) {
8468 // Wider than one register: extract each BlockBits-wide sub-vector.
8469 for (unsigned i = 0; i < SrcBits / BlockBits; ++i)
8470 Blocks.push_back(
8471 Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: BlockVT, N1: Src,
8472 N2: DAG.getVectorIdxConstant(Val: i * BlockNumElts, DL)));
8473 } else {
8474 BlockBits = SrcBits;
8475 Blocks.push_back(Elt: Src);
8476 }
8477
8478 MVT NativeVecVT = MVT::getVectorVT(VT: DstVT.getScalarType(),
8479 NumElements: BlockBits / DstVT.getScalarSizeInBits());
8480 SmallVector<SDValue, 4> Parts;
8481 for (unsigned i = 0; i < Blocks.size(); i += 2) {
8482 SDValue Lo = Blocks[i];
8483 SDValue Hi = Blocks.size() > 1 ? Blocks[i + 1] : Lo;
8484 SDValue Res = DAG.getNode(Opcode, DL, VT: NativeVecVT, N1: Hi, N2: Lo);
8485
8486 if (BlockBits == 256) {
8487 SDValue Undef = DAG.getUNDEF(VT: NativeVecVT);
8488 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
8489 Res = DAG.getVectorShuffle(VT: NativeVecVT, dl: DL, N1: Res, N2: Undef, Mask);
8490 Res = DAG.getBitcast(VT: NativeVecVT, V: Res);
8491 }
8492
8493 Parts.push_back(Elt: Res);
8494 }
8495
8496 if (Blocks.size() == 1)
8497 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: DstVT, N1: Parts[0],
8498 N2: DAG.getVectorIdxConstant(Val: 0, DL));
8499 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Parts);
8500}
8501
8502static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8503 TargetLowering::DAGCombinerInfo &DCI,
8504 const LoongArchSubtarget &Subtarget) {
8505 SDLoc DL(N);
8506 EVT VT = N->getValueType(ResNo: 0);
8507 SDValue Src = N->getOperand(Num: 0);
8508 EVT SrcVT = Src.getValueType();
8509
8510 if (VT.isVector()) {
8511 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8512 unsigned DstEltBits = VT.getScalarSizeInBits();
8513 unsigned NumElts = VT.getVectorNumElements();
8514 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8515
8516 // Sign-extend src to avoid scalarization.
8517 if (SrcEltBits <= DstEltBits)
8518 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::SIGN_EXTEND);
8519
8520 if (SrcEltBits != 64 || DstEltBits != 32 || !isPowerOf2_32(Value: NumElts))
8521 return SDValue();
8522
8523 if (!SrcVT.isSimple() || !VT.isSimple())
8524 return SDValue();
8525
8526 // Combine [x]vffint.s.l for vector si64 to float conversion.
8527 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFFINT, BlockBits);
8528 }
8529
8530 if (VT != MVT::f32 && VT != MVT::f64)
8531 return SDValue();
8532 if (VT == MVT::f32 && !Subtarget.hasBasicF())
8533 return SDValue();
8534 if (VT == MVT::f64 && !Subtarget.hasBasicD())
8535 return SDValue();
8536
8537 // Only optimize when the source and destination types have the same width.
8538 if (VT.getSizeInBits() != N->getOperand(Num: 0).getValueSizeInBits())
8539 return SDValue();
8540
8541 // If the result of an integer load is only used by an integer-to-float
8542 // conversion, use a fp load instead. This eliminates an integer-to-float-move
8543 // (movgr2fr) instruction.
8544 if (ISD::isNormalLoad(N: Src.getNode()) && Src.hasOneUse() &&
8545 // Do not change the width of a volatile load. This condition check is
8546 // inspired by AArch64.
8547 !cast<LoadSDNode>(Val&: Src)->isVolatile()) {
8548 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: Src);
8549 SDValue Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
8550 PtrInfo: LN0->getPointerInfo(), Alignment: LN0->getAlign(),
8551 MMOFlags: LN0->getMemOperand()->getFlags());
8552
8553 // Make sure successors of the original load stay after it by updating them
8554 // to use the new Chain.
8555 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: Load.getValue(R: 1));
8556 return DAG.getNode(Opcode: LoongArchISD::SITOF, DL: SDLoc(N), VT, Operand: Load);
8557 }
8558
8559 return SDValue();
8560}
8561
8562static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8563 TargetLowering::DAGCombinerInfo &DCI,
8564 const LoongArchSubtarget &Subtarget) {
8565 SDLoc DL(N);
8566 EVT VT = N->getValueType(ResNo: 0);
8567
8568 // Zero-extend src to avoid scalarization.
8569 if (VT.isVector())
8570 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::ZERO_EXTEND);
8571
8572 return SDValue();
8573}
8574
8575// Using [X]VFTINTRZ_W_D for double to signed 32-bit integer conversion.
8576// For example:
8577// v4i32 = fp_to_sint (concat_vectors v2f64, v2f64)
8578// Can be combined into:
8579// v4i32 = VFTINTRZ_W_D v2f64. v2f64
8580static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG,
8581 TargetLowering::DAGCombinerInfo &DCI,
8582 const LoongArchSubtarget &Subtarget) {
8583 if (!Subtarget.hasExtLSX())
8584 return SDValue();
8585
8586 SDLoc DL(N);
8587 EVT DstVT = N->getValueType(ResNo: 0);
8588 SDValue Src = N->getOperand(Num: 0);
8589 EVT SrcVT = Src.getValueType();
8590 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
8591
8592 if (!DstVT.isVector() || !DstVT.isSimple() || !SrcVT.isSimple())
8593 return SDValue();
8594
8595 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8596 unsigned SrcBits = SrcVT.getSizeInBits();
8597 unsigned DstEltBits = DstVT.getScalarSizeInBits();
8598 unsigned NumElts = DstVT.getVectorNumElements();
8599 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8600
8601 if (!isPowerOf2_32(Value: NumElts) || !isPowerOf2_32(Value: DstEltBits))
8602 return SDValue();
8603
8604 if (SrcBits % BlockBits != 0 && SrcBits != 128)
8605 return SDValue();
8606
8607 if (DstEltBits < 32) {
8608 MVT PromoteVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 32), NumElements: NumElts);
8609 SDValue Conv = DAG.getNode(Opcode: N->getOpcode(), DL, VT: PromoteVT, Operand: Src);
8610 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Conv);
8611 }
8612
8613 if (SrcEltBits != 64 || DstEltBits != 32)
8614 return SDValue();
8615
8616 if (!IsSigned) {
8617 // LASX already has pattern for double convert to uint32.
8618 if (Subtarget.hasExtLASX())
8619 return SDValue();
8620 MVT TmpVT = MVT::getVectorVT(VT: MVT::i64, NumElements: NumElts);
8621 SDValue Tmp = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: TmpVT, Operand: Src);
8622 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Tmp);
8623 }
8624
8625 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFTINTRZ, BlockBits);
8626}
8627
8628// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
8629// logical operations, like in the example below.
8630// or (and (truncate x, truncate y)),
8631// (xor (truncate z, build_vector (constants)))
8632// Given a target type \p VT, we generate
8633// or (and x, y), (xor z, zext(build_vector (constants)))
8634// given x, y and z are of type \p VT. We can do so, if operands are either
8635// truncates from VT types, the second operand is a vector of constants, can
8636// be recursively promoted or is an existing extension we can extend further.
8637static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
8638 SelectionDAG &DAG,
8639 const LoongArchSubtarget &Subtarget,
8640 unsigned Depth) {
8641 // Limit recursion to avoid excessive compile times.
8642 if (Depth >= SelectionDAG::MaxRecursionDepth)
8643 return SDValue();
8644
8645 if (!ISD::isBitwiseLogicOp(Opcode: N.getOpcode()))
8646 return SDValue();
8647
8648 SDValue N0 = N.getOperand(i: 0);
8649 SDValue N1 = N.getOperand(i: 1);
8650
8651 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8652 if (!TLI.isOperationLegalOrPromote(Op: N.getOpcode(), VT))
8653 return SDValue();
8654
8655 if (SDValue NN0 =
8656 PromoteMaskArithmetic(N: N0, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8657 N0 = NN0;
8658 else {
8659 // The left side has to be a 'trunc'.
8660 bool LHSTrunc = N0.getOpcode() == ISD::TRUNCATE &&
8661 N0.getOperand(i: 0).getValueType() == VT;
8662 if (LHSTrunc)
8663 N0 = N0.getOperand(i: 0);
8664 else
8665 return SDValue();
8666 }
8667
8668 if (SDValue NN1 =
8669 PromoteMaskArithmetic(N: N1, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8670 N1 = NN1;
8671 else {
8672 // The right side has to be a 'trunc', a (foldable) constant or an
8673 // existing extension we can extend further.
8674 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
8675 N1.getOperand(i: 0).getValueType() == VT;
8676 if (RHSTrunc)
8677 N1 = N1.getOperand(i: 0);
8678 else if (ISD::isExtVecInRegOpcode(Opcode: N1.getOpcode()) && VT.is256BitVector() &&
8679 Subtarget.hasExtLASX() && N1.hasOneUse())
8680 N1 = DAG.getNode(Opcode: N1.getOpcode(), DL, VT, Operand: N1.getOperand(i: 0));
8681 // On 32-bit platform, i64 is an illegal integer scalar type, and
8682 // FoldConstantArithmetic will fail for v4i64. This may be optimized in the
8683 // future.
8684 else if (SDValue Cst =
8685 DAG.FoldConstantArithmetic(Opcode: ISD::ZERO_EXTEND, DL, VT, Ops: {N1}))
8686 N1 = Cst;
8687 else
8688 return SDValue();
8689 }
8690
8691 return DAG.getNode(Opcode: N.getOpcode(), DL, VT, N1: N0, N2: N1);
8692}
8693
8694// On LASX the type v4i1/v8i1/v16i1 may be legalized to v4i32/v8i16/v16i8, which
8695// is LSX-sized register. In most cases we actually compare or select LASX-sized
8696// registers and mixing the two types creates horrible code. This method
8697// optimizes some of the transition sequences.
8698static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
8699 SelectionDAG &DAG,
8700 const LoongArchSubtarget &Subtarget) {
8701 EVT VT = N.getValueType();
8702 assert(VT.isVector() && "Expected vector type");
8703 assert((N.getOpcode() == ISD::ANY_EXTEND ||
8704 N.getOpcode() == ISD::ZERO_EXTEND ||
8705 N.getOpcode() == ISD::SIGN_EXTEND) &&
8706 "Invalid Node");
8707
8708 if (!Subtarget.hasExtLASX() || !VT.is256BitVector())
8709 return SDValue();
8710
8711 SDValue Narrow = N.getOperand(i: 0);
8712 EVT NarrowVT = Narrow.getValueType();
8713
8714 // Generate the wide operation.
8715 SDValue Op = PromoteMaskArithmetic(N: Narrow, DL, VT, DAG, Subtarget, Depth: 0);
8716 if (!Op)
8717 return SDValue();
8718 switch (N.getOpcode()) {
8719 default:
8720 llvm_unreachable("Unexpected opcode");
8721 case ISD::ANY_EXTEND:
8722 return Op;
8723 case ISD::ZERO_EXTEND:
8724 return DAG.getZeroExtendInReg(Op, DL, VT: NarrowVT);
8725 case ISD::SIGN_EXTEND:
8726 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
8727 N2: DAG.getValueType(NarrowVT));
8728 }
8729}
8730
8731static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG,
8732 TargetLowering::DAGCombinerInfo &DCI,
8733 const LoongArchSubtarget &Subtarget) {
8734 EVT VT = N->getValueType(ResNo: 0);
8735 SDLoc DL(N);
8736
8737 if (VT.isVector()) {
8738 if (SDValue R = PromoteMaskArithmetic(N: SDValue(N, 0), DL, DAG, Subtarget))
8739 return R;
8740
8741 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
8742 N->getValueSizeInBits(ResNo: 0) != N->getOperand(Num: 0).getValueSizeInBits() * 2)
8743 return SDValue();
8744
8745 if (SDValue R = matchHalfOf128BitLanes(N: N->getOperand(Num: 0), /*isLow=*/false)) {
8746 if (N->getOpcode() == ISD::SIGN_EXTEND)
8747 return DAG.getNode(Opcode: LoongArchISD::VEXTH, DL, VT, Operand: R);
8748 if (N->getOpcode() == ISD::ZERO_EXTEND)
8749 return DAG.getNode(Opcode: LoongArchISD::VEXTH_U, DL, VT, Operand: R);
8750 }
8751 }
8752
8753 return SDValue();
8754}
8755
8756static SDValue
8757performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
8758 TargetLowering::DAGCombinerInfo &DCI,
8759 const LoongArchSubtarget &Subtarget) {
8760 SDLoc DL(N);
8761 EVT VT = N->getValueType(ResNo: 0);
8762
8763 if (VT.isVector() && N->getNumOperands() == 2)
8764 if (SDValue R = combineFP_ROUND(N: SDValue(N, 0), DL, DAG, Subtarget))
8765 return R;
8766
8767 return SDValue();
8768}
8769
8770static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
8771 TargetLowering::DAGCombinerInfo &DCI,
8772 const LoongArchSubtarget &Subtarget) {
8773 if (DCI.isBeforeLegalizeOps())
8774 return SDValue();
8775
8776 EVT VT = N->getValueType(ResNo: 0);
8777 if (!VT.isVector())
8778 return SDValue();
8779
8780 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
8781 return SDValue();
8782
8783 EVT EltVT = VT.getVectorElementType();
8784 if (!EltVT.isInteger())
8785 return SDValue();
8786
8787 SDValue Cond = N->getOperand(Num: 0);
8788 SDValue TrueVal = N->getOperand(Num: 1);
8789 SDValue FalseVal = N->getOperand(Num: 2);
8790
8791 // match:
8792 //
8793 // vselect (setcc shift, 0, seteq),
8794 // x,
8795 // rounded_shift
8796
8797 if (Cond.getOpcode() != ISD::SETCC)
8798 return SDValue();
8799
8800 if (!ISD::isConstantSplatVectorAllZeros(N: Cond.getOperand(i: 1).getNode()))
8801 return SDValue();
8802
8803 auto *CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2));
8804 if (CC->get() != ISD::SETEQ)
8805 return SDValue();
8806
8807 SDValue Shift = Cond.getOperand(i: 0);
8808
8809 // True branch must be original value:
8810 //
8811 // vselect cond, x, ...
8812
8813 SDValue X = TrueVal;
8814
8815 // Now match rounded shift pattern:
8816 //
8817 // add
8818 // (and
8819 // (srl X, shift-1)
8820 // 1)
8821 // (srl/sra X, shift)
8822
8823 if (FalseVal.getOpcode() != ISD::ADD)
8824 return SDValue();
8825
8826 SDValue Add0 = FalseVal.getOperand(i: 0);
8827 SDValue Add1 = FalseVal.getOperand(i: 1);
8828 SDValue And;
8829 SDValue Shr;
8830
8831 if (Add0.getOpcode() == ISD::AND) {
8832 And = Add0;
8833 Shr = Add1;
8834 } else if (Add1.getOpcode() == ISD::AND) {
8835 And = Add1;
8836 Shr = Add0;
8837 } else {
8838 return SDValue();
8839 }
8840
8841 // match:
8842 //
8843 // srl/sra X, shift
8844
8845 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
8846 return SDValue();
8847
8848 if (Shr.getOperand(i: 0) != X)
8849 return SDValue();
8850
8851 if (Shr.getOperand(i: 1) != Shift)
8852 return SDValue();
8853
8854 // match:
8855 //
8856 // and
8857 // (srl X, shift-1)
8858 // 1
8859
8860 SDValue Srl = And.getOperand(i: 0);
8861 SDValue One = And.getOperand(i: 1);
8862 APInt SplatVal;
8863
8864 if (Srl.getOpcode() != ISD::SRL)
8865 return SDValue();
8866
8867 One = peekThroughBitcasts(V: One);
8868 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
8869 return SDValue();
8870
8871 if (SplatVal != 1)
8872 return SDValue();
8873
8874 if (Srl.getOperand(i: 0) != X)
8875 return SDValue();
8876
8877 // match:
8878 //
8879 // shift-1
8880
8881 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
8882
8883 if (ShiftMinus1.getOpcode() != ISD::ADD)
8884 return SDValue();
8885
8886 if (ShiftMinus1.getOperand(i: 0) != Shift)
8887 return SDValue();
8888
8889 if (!ISD::isConstantSplatVectorAllOnes(N: ShiftMinus1.getOperand(i: 1).getNode()))
8890 return SDValue();
8891
8892 // We matched a rounded right shift pattern and can lower it
8893 // to a single vector rounded shift instruction.
8894
8895 SDLoc DL(N);
8896 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
8897 : LoongArchISD::VSRAR,
8898 DL, VT, N1: X, N2: Shift);
8899}
8900
8901SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
8902 DAGCombinerInfo &DCI) const {
8903 SelectionDAG &DAG = DCI.DAG;
8904 switch (N->getOpcode()) {
8905 default:
8906 break;
8907 case ISD::ADD:
8908 return performADDCombine(N, DAG, DCI, Subtarget);
8909 case ISD::AND:
8910 return performANDCombine(N, DAG, DCI, Subtarget);
8911 case ISD::OR:
8912 return performORCombine(N, DAG, DCI, Subtarget);
8913 case ISD::SETCC:
8914 return performSETCCCombine(N, DAG, DCI, Subtarget);
8915 case ISD::SELECT:
8916 return performSELECTCombine(N, DAG, DCI, Subtarget);
8917 case ISD::SHL:
8918 return performSHLCombine(N, DAG, DCI, Subtarget);
8919 case ISD::SRL:
8920 return performSRLCombine(N, DAG, DCI, Subtarget);
8921 case ISD::SUB:
8922 return performSUBCombine(N, DAG, DCI, Subtarget);
8923 case ISD::BITCAST:
8924 return performBITCASTCombine(N, DAG, DCI, Subtarget);
8925 case ISD::ANY_EXTEND:
8926 case ISD::ZERO_EXTEND:
8927 case ISD::SIGN_EXTEND:
8928 return performEXTENDCombine(N, DAG, DCI, Subtarget);
8929 case ISD::SINT_TO_FP:
8930 return performSINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8931 case ISD::UINT_TO_FP:
8932 return performUINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8933 case ISD::FP_TO_SINT:
8934 case ISD::FP_TO_UINT:
8935 return performFP_TO_INTCombine(N, DAG, DCI, Subtarget);
8936 case LoongArchISD::BITREV_W:
8937 return performBITREV_WCombine(N, DAG, DCI, Subtarget);
8938 case LoongArchISD::BR_CC:
8939 return performBR_CCCombine(N, DAG, DCI, Subtarget);
8940 case LoongArchISD::SELECT_CC:
8941 return performSELECT_CCCombine(N, DAG, DCI, Subtarget);
8942 case ISD::INTRINSIC_WO_CHAIN:
8943 return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
8944 case LoongArchISD::MOVGR2FR_W_LA64:
8945 return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
8946 case LoongArchISD::MOVFR2GR_S_LA64:
8947 return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
8948 case LoongArchISD::CRC_W_B_W:
8949 case LoongArchISD::CRC_W_H_W:
8950 case LoongArchISD::CRCC_W_B_W:
8951 case LoongArchISD::CRCC_W_H_W:
8952 case LoongArchISD::VMSKLTZ:
8953 case LoongArchISD::XVMSKLTZ:
8954 return performDemandedBitsCombine(N, DAG, DCI);
8955 case LoongArchISD::SPLIT_PAIR_F64:
8956 return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
8957 case LoongArchISD::VANDN:
8958 return performVANDNCombine(N, DAG, DCI, Subtarget);
8959 case ISD::CONCAT_VECTORS:
8960 return performCONCAT_VECTORSCombine(N, DAG, DCI, Subtarget);
8961 case ISD::VSELECT:
8962 return performVSELECTCombine(N, DAG, DCI, Subtarget);
8963 case LoongArchISD::VPACKEV:
8964 case LoongArchISD::VPERMI:
8965 if (SDValue Result =
8966 combineFP_ROUND(N: SDValue(N, 0), DL: SDLoc(N), DAG, Subtarget))
8967 return Result;
8968 }
8969 return SDValue();
8970}
8971
8972static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI,
8973 MachineBasicBlock *MBB) {
8974 if (!ZeroDivCheck)
8975 return MBB;
8976
8977 // Build instructions:
8978 // MBB:
8979 // div(or mod) $dst, $dividend, $divisor
8980 // bne $divisor, $zero, SinkMBB
8981 // BreakMBB:
8982 // break 7 // BRK_DIVZERO
8983 // SinkMBB:
8984 // fallthrough
8985 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8986 MachineFunction::iterator It = ++MBB->getIterator();
8987 MachineFunction *MF = MBB->getParent();
8988 auto BreakMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8989 auto SinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8990 MF->insert(MBBI: It, MBB: BreakMBB);
8991 MF->insert(MBBI: It, MBB: SinkMBB);
8992
8993 // Transfer the remainder of MBB and its successor edges to SinkMBB.
8994 SinkMBB->splice(Where: SinkMBB->end(), Other: MBB, From: std::next(x: MI.getIterator()), To: MBB->end());
8995 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
8996
8997 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
8998 DebugLoc DL = MI.getDebugLoc();
8999 MachineOperand &Divisor = MI.getOperand(i: 2);
9000 Register DivisorReg = Divisor.getReg();
9001
9002 // MBB:
9003 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BNE))
9004 .addReg(RegNo: DivisorReg, Flags: getKillRegState(B: Divisor.isKill()))
9005 .addReg(RegNo: LoongArch::R0)
9006 .addMBB(MBB: SinkMBB);
9007 MBB->addSuccessor(Succ: BreakMBB);
9008 MBB->addSuccessor(Succ: SinkMBB);
9009
9010 // BreakMBB:
9011 // See linux header file arch/loongarch/include/uapi/asm/break.h for the
9012 // definition of BRK_DIVZERO.
9013 BuildMI(BB: BreakMBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BREAK)).addImm(Val: 7 /*BRK_DIVZERO*/);
9014 BreakMBB->addSuccessor(Succ: SinkMBB);
9015
9016 // Clear Divisor's kill flag.
9017 Divisor.setIsKill(false);
9018
9019 return SinkMBB;
9020}
9021
9022static MachineBasicBlock *
9023emitVecCondBranchPseudo(MachineInstr &MI, MachineBasicBlock *BB,
9024 const LoongArchSubtarget &Subtarget) {
9025 unsigned CondOpc;
9026 switch (MI.getOpcode()) {
9027 default:
9028 llvm_unreachable("Unexpected opcode");
9029 case LoongArch::PseudoVBZ:
9030 CondOpc = LoongArch::VSETEQZ_V;
9031 break;
9032 case LoongArch::PseudoVBZ_B:
9033 CondOpc = LoongArch::VSETANYEQZ_B;
9034 break;
9035 case LoongArch::PseudoVBZ_H:
9036 CondOpc = LoongArch::VSETANYEQZ_H;
9037 break;
9038 case LoongArch::PseudoVBZ_W:
9039 CondOpc = LoongArch::VSETANYEQZ_W;
9040 break;
9041 case LoongArch::PseudoVBZ_D:
9042 CondOpc = LoongArch::VSETANYEQZ_D;
9043 break;
9044 case LoongArch::PseudoVBNZ:
9045 CondOpc = LoongArch::VSETNEZ_V;
9046 break;
9047 case LoongArch::PseudoVBNZ_B:
9048 CondOpc = LoongArch::VSETALLNEZ_B;
9049 break;
9050 case LoongArch::PseudoVBNZ_H:
9051 CondOpc = LoongArch::VSETALLNEZ_H;
9052 break;
9053 case LoongArch::PseudoVBNZ_W:
9054 CondOpc = LoongArch::VSETALLNEZ_W;
9055 break;
9056 case LoongArch::PseudoVBNZ_D:
9057 CondOpc = LoongArch::VSETALLNEZ_D;
9058 break;
9059 case LoongArch::PseudoXVBZ:
9060 CondOpc = LoongArch::XVSETEQZ_V;
9061 break;
9062 case LoongArch::PseudoXVBZ_B:
9063 CondOpc = LoongArch::XVSETANYEQZ_B;
9064 break;
9065 case LoongArch::PseudoXVBZ_H:
9066 CondOpc = LoongArch::XVSETANYEQZ_H;
9067 break;
9068 case LoongArch::PseudoXVBZ_W:
9069 CondOpc = LoongArch::XVSETANYEQZ_W;
9070 break;
9071 case LoongArch::PseudoXVBZ_D:
9072 CondOpc = LoongArch::XVSETANYEQZ_D;
9073 break;
9074 case LoongArch::PseudoXVBNZ:
9075 CondOpc = LoongArch::XVSETNEZ_V;
9076 break;
9077 case LoongArch::PseudoXVBNZ_B:
9078 CondOpc = LoongArch::XVSETALLNEZ_B;
9079 break;
9080 case LoongArch::PseudoXVBNZ_H:
9081 CondOpc = LoongArch::XVSETALLNEZ_H;
9082 break;
9083 case LoongArch::PseudoXVBNZ_W:
9084 CondOpc = LoongArch::XVSETALLNEZ_W;
9085 break;
9086 case LoongArch::PseudoXVBNZ_D:
9087 CondOpc = LoongArch::XVSETALLNEZ_D;
9088 break;
9089 }
9090
9091 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9092 const BasicBlock *LLVM_BB = BB->getBasicBlock();
9093 DebugLoc DL = MI.getDebugLoc();
9094 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9095 MachineFunction::iterator It = ++BB->getIterator();
9096
9097 MachineFunction *F = BB->getParent();
9098 MachineBasicBlock *FalseBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9099 MachineBasicBlock *TrueBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9100 MachineBasicBlock *SinkBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9101
9102 F->insert(MBBI: It, MBB: FalseBB);
9103 F->insert(MBBI: It, MBB: TrueBB);
9104 F->insert(MBBI: It, MBB: SinkBB);
9105
9106 // Transfer the remainder of MBB and its successor edges to Sink.
9107 SinkBB->splice(Where: SinkBB->end(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
9108 SinkBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
9109
9110 // Insert the real instruction to BB.
9111 Register FCC = MRI.createVirtualRegister(RegClass: &LoongArch::CFRRegClass);
9112 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: CondOpc), DestReg: FCC).addReg(RegNo: MI.getOperand(i: 1).getReg());
9113
9114 // Insert branch.
9115 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::BCNEZ)).addReg(RegNo: FCC).addMBB(MBB: TrueBB);
9116 BB->addSuccessor(Succ: FalseBB);
9117 BB->addSuccessor(Succ: TrueBB);
9118
9119 // FalseBB.
9120 Register RD1 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9121 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD1)
9122 .addReg(RegNo: LoongArch::R0)
9123 .addImm(Val: 0);
9124 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::PseudoBR)).addMBB(MBB: SinkBB);
9125 FalseBB->addSuccessor(Succ: SinkBB);
9126
9127 // TrueBB.
9128 Register RD2 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9129 BuildMI(BB: TrueBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD2)
9130 .addReg(RegNo: LoongArch::R0)
9131 .addImm(Val: 1);
9132 TrueBB->addSuccessor(Succ: SinkBB);
9133
9134 // SinkBB: merge the results.
9135 BuildMI(BB&: *SinkBB, I: SinkBB->begin(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::PHI),
9136 DestReg: MI.getOperand(i: 0).getReg())
9137 .addReg(RegNo: RD1)
9138 .addMBB(MBB: FalseBB)
9139 .addReg(RegNo: RD2)
9140 .addMBB(MBB: TrueBB);
9141
9142 // The pseudo instruction is gone now.
9143 MI.eraseFromParent();
9144 return SinkBB;
9145}
9146
9147static MachineBasicBlock *
9148emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
9149 const LoongArchSubtarget &Subtarget) {
9150 unsigned InsOp;
9151 unsigned BroadcastOp;
9152 unsigned HalfSize;
9153 switch (MI.getOpcode()) {
9154 default:
9155 llvm_unreachable("Unexpected opcode");
9156 case LoongArch::PseudoXVINSGR2VR_B:
9157 HalfSize = 16;
9158 BroadcastOp = LoongArch::XVREPLGR2VR_B;
9159 InsOp = LoongArch::XVEXTRINS_B;
9160 break;
9161 case LoongArch::PseudoXVINSGR2VR_H:
9162 HalfSize = 8;
9163 BroadcastOp = LoongArch::XVREPLGR2VR_H;
9164 InsOp = LoongArch::XVEXTRINS_H;
9165 break;
9166 }
9167 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9168 const TargetRegisterClass *RC = &LoongArch::LASX256RegClass;
9169 const TargetRegisterClass *SubRC = &LoongArch::LSX128RegClass;
9170 DebugLoc DL = MI.getDebugLoc();
9171 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9172 // XDst = vector_insert XSrc, Elt, Idx
9173 Register XDst = MI.getOperand(i: 0).getReg();
9174 Register XSrc = MI.getOperand(i: 1).getReg();
9175 Register Elt = MI.getOperand(i: 2).getReg();
9176 unsigned Idx = MI.getOperand(i: 3).getImm();
9177
9178 if (XSrc.isVirtual() && MRI.getVRegDef(Reg: XSrc)->isImplicitDef() &&
9179 Idx < HalfSize) {
9180 Register ScratchSubReg1 = MRI.createVirtualRegister(RegClass: SubRC);
9181 Register ScratchSubReg2 = MRI.createVirtualRegister(RegClass: SubRC);
9182
9183 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::COPY), DestReg: ScratchSubReg1)
9184 .addReg(RegNo: XSrc, Flags: {}, SubReg: LoongArch::sub_128);
9185 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9186 MCID: TII->get(Opcode: HalfSize == 8 ? LoongArch::VINSGR2VR_H
9187 : LoongArch::VINSGR2VR_B),
9188 DestReg: ScratchSubReg2)
9189 .addReg(RegNo: ScratchSubReg1)
9190 .addReg(RegNo: Elt)
9191 .addImm(Val: Idx);
9192
9193 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::SUBREG_TO_REG), DestReg: XDst)
9194 .addReg(RegNo: ScratchSubReg2)
9195 .addImm(Val: LoongArch::sub_128);
9196 } else {
9197 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9198 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9199
9200 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Elt);
9201
9202 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPERMI_Q), DestReg: ScratchReg2)
9203 .addReg(RegNo: ScratchReg1)
9204 .addReg(RegNo: XSrc)
9205 .addImm(Val: Idx >= HalfSize ? 48 : 18);
9206
9207 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InsOp), DestReg: XDst)
9208 .addReg(RegNo: XSrc)
9209 .addReg(RegNo: ScratchReg2)
9210 .addImm(Val: (Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
9211 }
9212
9213 MI.eraseFromParent();
9214 return BB;
9215}
9216
9217static MachineBasicBlock *emitPseudoCTPOP(MachineInstr &MI,
9218 MachineBasicBlock *BB,
9219 const LoongArchSubtarget &Subtarget) {
9220 assert(Subtarget.hasExtLSX());
9221 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9222 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9223 DebugLoc DL = MI.getDebugLoc();
9224 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9225 Register Dst = MI.getOperand(i: 0).getReg();
9226 Register Src = MI.getOperand(i: 1).getReg();
9227
9228 unsigned BroadcastOp, CTOp, PickOp;
9229 switch (MI.getOpcode()) {
9230 default:
9231 llvm_unreachable("Unexpected opcode");
9232 case LoongArch::PseudoCTPOP_B:
9233 BroadcastOp = LoongArch::VREPLGR2VR_B;
9234 CTOp = LoongArch::VPCNT_B;
9235 PickOp = LoongArch::VPICKVE2GR_B;
9236 break;
9237 case LoongArch::PseudoCTPOP_H:
9238 case LoongArch::PseudoCTPOP_H_LA32:
9239 BroadcastOp = LoongArch::VREPLGR2VR_H;
9240 CTOp = LoongArch::VPCNT_H;
9241 PickOp = LoongArch::VPICKVE2GR_H;
9242 break;
9243 case LoongArch::PseudoCTPOP_W:
9244 case LoongArch::PseudoCTPOP_W_LA32:
9245 BroadcastOp = LoongArch::VREPLGR2VR_W;
9246 CTOp = LoongArch::VPCNT_W;
9247 PickOp = LoongArch::VPICKVE2GR_W;
9248 break;
9249 case LoongArch::PseudoCTPOP_D:
9250 BroadcastOp = LoongArch::VREPLGR2VR_D;
9251 CTOp = LoongArch::VPCNT_D;
9252 PickOp = LoongArch::VPICKVE2GR_D;
9253 break;
9254 }
9255
9256 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9257 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9258 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Src);
9259 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: CTOp), DestReg: ScratchReg2).addReg(RegNo: ScratchReg1);
9260 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PickOp), DestReg: Dst).addReg(RegNo: ScratchReg2).addImm(Val: 0);
9261
9262 MI.eraseFromParent();
9263 return BB;
9264}
9265
9266static MachineBasicBlock *
9267emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
9268 const LoongArchSubtarget &Subtarget) {
9269 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9270 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9271 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
9272 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9273 Register Dst = MI.getOperand(i: 0).getReg();
9274 Register Src = MI.getOperand(i: 1).getReg();
9275 DebugLoc DL = MI.getDebugLoc();
9276 unsigned EleBits = 8;
9277 unsigned NotOpc = 0;
9278 unsigned MskOpc;
9279
9280 switch (MI.getOpcode()) {
9281 default:
9282 llvm_unreachable("Unexpected opcode");
9283 case LoongArch::PseudoVMSKLTZ_B:
9284 MskOpc = LoongArch::VMSKLTZ_B;
9285 break;
9286 case LoongArch::PseudoVMSKLTZ_H:
9287 MskOpc = LoongArch::VMSKLTZ_H;
9288 EleBits = 16;
9289 break;
9290 case LoongArch::PseudoVMSKLTZ_W:
9291 MskOpc = LoongArch::VMSKLTZ_W;
9292 EleBits = 32;
9293 break;
9294 case LoongArch::PseudoVMSKLTZ_D:
9295 MskOpc = LoongArch::VMSKLTZ_D;
9296 EleBits = 64;
9297 break;
9298 case LoongArch::PseudoVMSKGEZ_B:
9299 MskOpc = LoongArch::VMSKGEZ_B;
9300 break;
9301 case LoongArch::PseudoVMSKEQZ_B:
9302 MskOpc = LoongArch::VMSKNZ_B;
9303 NotOpc = LoongArch::VNOR_V;
9304 break;
9305 case LoongArch::PseudoVMSKNEZ_B:
9306 MskOpc = LoongArch::VMSKNZ_B;
9307 break;
9308 case LoongArch::PseudoXVMSKLTZ_B:
9309 MskOpc = LoongArch::XVMSKLTZ_B;
9310 RC = &LoongArch::LASX256RegClass;
9311 break;
9312 case LoongArch::PseudoXVMSKLTZ_H:
9313 MskOpc = LoongArch::XVMSKLTZ_H;
9314 RC = &LoongArch::LASX256RegClass;
9315 EleBits = 16;
9316 break;
9317 case LoongArch::PseudoXVMSKLTZ_W:
9318 MskOpc = LoongArch::XVMSKLTZ_W;
9319 RC = &LoongArch::LASX256RegClass;
9320 EleBits = 32;
9321 break;
9322 case LoongArch::PseudoXVMSKLTZ_D:
9323 MskOpc = LoongArch::XVMSKLTZ_D;
9324 RC = &LoongArch::LASX256RegClass;
9325 EleBits = 64;
9326 break;
9327 case LoongArch::PseudoXVMSKGEZ_B:
9328 MskOpc = LoongArch::XVMSKGEZ_B;
9329 RC = &LoongArch::LASX256RegClass;
9330 break;
9331 case LoongArch::PseudoXVMSKEQZ_B:
9332 MskOpc = LoongArch::XVMSKNZ_B;
9333 NotOpc = LoongArch::XVNOR_V;
9334 RC = &LoongArch::LASX256RegClass;
9335 break;
9336 case LoongArch::PseudoXVMSKNEZ_B:
9337 MskOpc = LoongArch::XVMSKNZ_B;
9338 RC = &LoongArch::LASX256RegClass;
9339 break;
9340 }
9341
9342 Register Msk = MRI.createVirtualRegister(RegClass: RC);
9343 if (NotOpc) {
9344 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
9345 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Tmp).addReg(RegNo: Src);
9346 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: NotOpc), DestReg: Msk)
9347 .addReg(RegNo: Tmp)
9348 .addReg(RegNo: Tmp);
9349 } else {
9350 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Msk).addReg(RegNo: Src);
9351 }
9352
9353 if (TRI->getRegSizeInBits(RC: *RC) > 128) {
9354 Register Lo = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9355 Register Hi = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9356 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Lo)
9357 .addReg(RegNo: Msk)
9358 .addImm(Val: 0);
9359 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Hi)
9360 .addReg(RegNo: Msk)
9361 .addImm(Val: 4);
9362 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9363 MCID: TII->get(Opcode: Subtarget.is64Bit() ? LoongArch::BSTRINS_D
9364 : LoongArch::BSTRINS_W),
9365 DestReg: Dst)
9366 .addReg(RegNo: Lo)
9367 .addReg(RegNo: Hi)
9368 .addImm(Val: 256 / EleBits - 1)
9369 .addImm(Val: 128 / EleBits);
9370 } else {
9371 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::VPICKVE2GR_HU), DestReg: Dst)
9372 .addReg(RegNo: Msk)
9373 .addImm(Val: 0);
9374 }
9375
9376 MI.eraseFromParent();
9377 return BB;
9378}
9379
9380static MachineBasicBlock *
9381emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9382 const LoongArchSubtarget &Subtarget) {
9383 assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
9384 "Unexpected instruction");
9385
9386 MachineFunction &MF = *BB->getParent();
9387 DebugLoc DL = MI.getDebugLoc();
9388 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9389 Register LoReg = MI.getOperand(i: 0).getReg();
9390 Register HiReg = MI.getOperand(i: 1).getReg();
9391 Register SrcReg = MI.getOperand(i: 2).getReg();
9392
9393 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFR2GR_S_64), DestReg: LoReg).addReg(RegNo: SrcReg);
9394 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFRH2GR_S), DestReg: HiReg)
9395 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9396 MI.eraseFromParent(); // The pseudo instruction is gone now.
9397 return BB;
9398}
9399
9400static MachineBasicBlock *
9401emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9402 const LoongArchSubtarget &Subtarget) {
9403 assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
9404 "Unexpected instruction");
9405
9406 MachineFunction &MF = *BB->getParent();
9407 DebugLoc DL = MI.getDebugLoc();
9408 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9409 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9410 Register TmpReg = MRI.createVirtualRegister(RegClass: &LoongArch::FPR64RegClass);
9411 Register DstReg = MI.getOperand(i: 0).getReg();
9412 Register LoReg = MI.getOperand(i: 1).getReg();
9413 Register HiReg = MI.getOperand(i: 2).getReg();
9414
9415 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FR_W_64), DestReg: TmpReg)
9416 .addReg(RegNo: LoReg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()));
9417 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FRH_W), DestReg: DstReg)
9418 .addReg(RegNo: TmpReg)
9419 .addReg(RegNo: HiReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9420 MI.eraseFromParent(); // The pseudo instruction is gone now.
9421 return BB;
9422}
9423
9424static bool isSelectPseudo(MachineInstr &MI) {
9425 switch (MI.getOpcode()) {
9426 default:
9427 return false;
9428 case LoongArch::Select_GPR_Using_CC_GPR:
9429 return true;
9430 }
9431}
9432
9433static MachineBasicBlock *
9434emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB,
9435 const LoongArchSubtarget &Subtarget) {
9436 // To "insert" Select_* instructions, we actually have to insert the triangle
9437 // control-flow pattern. The incoming instructions know the destination vreg
9438 // to set, the condition code register to branch on, the true/false values to
9439 // select between, and the condcode to use to select the appropriate branch.
9440 //
9441 // We produce the following control flow:
9442 // HeadMBB
9443 // | \
9444 // | IfFalseMBB
9445 // | /
9446 // TailMBB
9447 //
9448 // When we find a sequence of selects we attempt to optimize their emission
9449 // by sharing the control flow. Currently we only handle cases where we have
9450 // multiple selects with the exact same condition (same LHS, RHS and CC).
9451 // The selects may be interleaved with other instructions if the other
9452 // instructions meet some requirements we deem safe:
9453 // - They are not pseudo instructions.
9454 // - They are debug instructions. Otherwise,
9455 // - They do not have side-effects, do not access memory and their inputs do
9456 // not depend on the results of the select pseudo-instructions.
9457 // The TrueV/FalseV operands of the selects cannot depend on the result of
9458 // previous selects in the sequence.
9459 // These conditions could be further relaxed. See the X86 target for a
9460 // related approach and more information.
9461
9462 Register LHS = MI.getOperand(i: 1).getReg();
9463 Register RHS;
9464 if (MI.getOperand(i: 2).isReg())
9465 RHS = MI.getOperand(i: 2).getReg();
9466 auto CC = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
9467
9468 SmallVector<MachineInstr *, 4> SelectDebugValues;
9469 SmallSet<Register, 4> SelectDests;
9470 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
9471
9472 MachineInstr *LastSelectPseudo = &MI;
9473 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
9474 SequenceMBBI != E; ++SequenceMBBI) {
9475 if (SequenceMBBI->isDebugInstr())
9476 continue;
9477 if (isSelectPseudo(MI&: *SequenceMBBI)) {
9478 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
9479 !SequenceMBBI->getOperand(i: 2).isReg() ||
9480 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
9481 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
9482 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
9483 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
9484 break;
9485 LastSelectPseudo = &*SequenceMBBI;
9486 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
9487 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
9488 continue;
9489 }
9490 if (SequenceMBBI->hasUnmodeledSideEffects() ||
9491 SequenceMBBI->mayLoadOrStore() ||
9492 SequenceMBBI->usesCustomInsertionHook())
9493 break;
9494 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
9495 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
9496 }))
9497 break;
9498 }
9499
9500 const LoongArchInstrInfo &TII = *Subtarget.getInstrInfo();
9501 const BasicBlock *LLVM_BB = BB->getBasicBlock();
9502 DebugLoc DL = MI.getDebugLoc();
9503 MachineFunction::iterator I = ++BB->getIterator();
9504
9505 MachineBasicBlock *HeadMBB = BB;
9506 MachineFunction *F = BB->getParent();
9507 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9508 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9509
9510 F->insert(MBBI: I, MBB: IfFalseMBB);
9511 F->insert(MBBI: I, MBB: TailMBB);
9512
9513 // Set the call frame size on entry to the new basic blocks.
9514 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
9515 IfFalseMBB->setCallFrameSize(CallFrameSize);
9516 TailMBB->setCallFrameSize(CallFrameSize);
9517
9518 // Transfer debug instructions associated with the selects to TailMBB.
9519 for (MachineInstr *DebugInstr : SelectDebugValues) {
9520 TailMBB->push_back(MI: DebugInstr->removeFromParent());
9521 }
9522
9523 // Move all instructions after the sequence to TailMBB.
9524 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
9525 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
9526 // Update machine-CFG edges by transferring all successors of the current
9527 // block to the new block which will contain the Phi nodes for the selects.
9528 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
9529 // Set the successors for HeadMBB.
9530 HeadMBB->addSuccessor(Succ: IfFalseMBB);
9531 HeadMBB->addSuccessor(Succ: TailMBB);
9532
9533 // Insert appropriate branch.
9534 if (MI.getOperand(i: 2).isImm())
9535 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC))
9536 .addReg(RegNo: LHS)
9537 .addImm(Val: MI.getOperand(i: 2).getImm())
9538 .addMBB(MBB: TailMBB);
9539 else
9540 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: TailMBB);
9541
9542 // IfFalseMBB just falls through to TailMBB.
9543 IfFalseMBB->addSuccessor(Succ: TailMBB);
9544
9545 // Create PHIs for all of the select pseudo-instructions.
9546 auto SelectMBBI = MI.getIterator();
9547 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
9548 auto InsertionPoint = TailMBB->begin();
9549 while (SelectMBBI != SelectEnd) {
9550 auto Next = std::next(x: SelectMBBI);
9551 if (isSelectPseudo(MI&: *SelectMBBI)) {
9552 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
9553 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
9554 MCID: TII.get(Opcode: LoongArch::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
9555 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
9556 .addMBB(MBB: HeadMBB)
9557 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
9558 .addMBB(MBB: IfFalseMBB);
9559 SelectMBBI->eraseFromParent();
9560 }
9561 SelectMBBI = Next;
9562 }
9563
9564 F->getProperties().resetNoPHIs();
9565 return TailMBB;
9566}
9567
9568MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
9569 MachineInstr &MI, MachineBasicBlock *BB) const {
9570 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9571 DebugLoc DL = MI.getDebugLoc();
9572
9573 switch (MI.getOpcode()) {
9574 default:
9575 llvm_unreachable("Unexpected instr type to insert");
9576 case LoongArch::DIV_W:
9577 case LoongArch::DIV_WU:
9578 case LoongArch::MOD_W:
9579 case LoongArch::MOD_WU:
9580 case LoongArch::DIV_D:
9581 case LoongArch::DIV_DU:
9582 case LoongArch::MOD_D:
9583 case LoongArch::MOD_DU:
9584 return insertDivByZeroTrap(MI, MBB: BB);
9585 break;
9586 case LoongArch::WRFCSR: {
9587 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVGR2FCSR),
9588 DestReg: LoongArch::FCSR0 + MI.getOperand(i: 0).getImm())
9589 .addReg(RegNo: MI.getOperand(i: 1).getReg());
9590 MI.eraseFromParent();
9591 return BB;
9592 }
9593 case LoongArch::RDFCSR: {
9594 MachineInstr *ReadFCSR =
9595 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVFCSR2GR),
9596 DestReg: MI.getOperand(i: 0).getReg())
9597 .addReg(RegNo: LoongArch::FCSR0 + MI.getOperand(i: 1).getImm());
9598 ReadFCSR->getOperand(i: 1).setIsUndef();
9599 MI.eraseFromParent();
9600 return BB;
9601 }
9602 case LoongArch::Select_GPR_Using_CC_GPR:
9603 return emitSelectPseudo(MI, BB, Subtarget);
9604 case LoongArch::BuildPairF64Pseudo:
9605 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
9606 case LoongArch::SplitPairF64Pseudo:
9607 return emitSplitPairF64Pseudo(MI, BB, Subtarget);
9608 case LoongArch::PseudoVBZ:
9609 case LoongArch::PseudoVBZ_B:
9610 case LoongArch::PseudoVBZ_H:
9611 case LoongArch::PseudoVBZ_W:
9612 case LoongArch::PseudoVBZ_D:
9613 case LoongArch::PseudoVBNZ:
9614 case LoongArch::PseudoVBNZ_B:
9615 case LoongArch::PseudoVBNZ_H:
9616 case LoongArch::PseudoVBNZ_W:
9617 case LoongArch::PseudoVBNZ_D:
9618 case LoongArch::PseudoXVBZ:
9619 case LoongArch::PseudoXVBZ_B:
9620 case LoongArch::PseudoXVBZ_H:
9621 case LoongArch::PseudoXVBZ_W:
9622 case LoongArch::PseudoXVBZ_D:
9623 case LoongArch::PseudoXVBNZ:
9624 case LoongArch::PseudoXVBNZ_B:
9625 case LoongArch::PseudoXVBNZ_H:
9626 case LoongArch::PseudoXVBNZ_W:
9627 case LoongArch::PseudoXVBNZ_D:
9628 return emitVecCondBranchPseudo(MI, BB, Subtarget);
9629 case LoongArch::PseudoXVINSGR2VR_B:
9630 case LoongArch::PseudoXVINSGR2VR_H:
9631 return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
9632 case LoongArch::PseudoCTPOP_B:
9633 case LoongArch::PseudoCTPOP_H:
9634 case LoongArch::PseudoCTPOP_W:
9635 case LoongArch::PseudoCTPOP_D:
9636 case LoongArch::PseudoCTPOP_H_LA32:
9637 case LoongArch::PseudoCTPOP_W_LA32:
9638 return emitPseudoCTPOP(MI, BB, Subtarget);
9639 case LoongArch::PseudoVMSKLTZ_B:
9640 case LoongArch::PseudoVMSKLTZ_H:
9641 case LoongArch::PseudoVMSKLTZ_W:
9642 case LoongArch::PseudoVMSKLTZ_D:
9643 case LoongArch::PseudoVMSKGEZ_B:
9644 case LoongArch::PseudoVMSKEQZ_B:
9645 case LoongArch::PseudoVMSKNEZ_B:
9646 case LoongArch::PseudoXVMSKLTZ_B:
9647 case LoongArch::PseudoXVMSKLTZ_H:
9648 case LoongArch::PseudoXVMSKLTZ_W:
9649 case LoongArch::PseudoXVMSKLTZ_D:
9650 case LoongArch::PseudoXVMSKGEZ_B:
9651 case LoongArch::PseudoXVMSKEQZ_B:
9652 case LoongArch::PseudoXVMSKNEZ_B:
9653 return emitPseudoVMSKCOND(MI, BB, Subtarget);
9654 case TargetOpcode::STATEPOINT:
9655 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
9656 // while bl call instruction (where statepoint will be lowered at the
9657 // end) has implicit def. This def is early-clobber as it will be set at
9658 // the moment of the call and earlier than any use is read.
9659 // Add this implicit dead def here as a workaround.
9660 MI.addOperand(MF&: *MI.getMF(),
9661 Op: MachineOperand::CreateReg(
9662 Reg: LoongArch::R1, /*isDef*/ true,
9663 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
9664 /*isUndef*/ false, /*isEarlyClobber*/ true));
9665 if (!Subtarget.is64Bit())
9666 report_fatal_error(reason: "STATEPOINT is only supported on 64-bit targets");
9667 return emitPatchPoint(MI, MBB: BB);
9668 case LoongArch::PROBED_STACKALLOC_DYN:
9669 return emitDynamicProbedAlloc(MI, MBB: BB);
9670 }
9671}
9672
9673bool LoongArchTargetLowering::allowsMisalignedMemoryAccesses(
9674 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
9675 unsigned *Fast) const {
9676 if (!Subtarget.hasUAL())
9677 return false;
9678
9679 // TODO: set reasonable speed number.
9680 if (Fast)
9681 *Fast = 1;
9682 return true;
9683}
9684
9685//===----------------------------------------------------------------------===//
9686// Calling Convention Implementation
9687//===----------------------------------------------------------------------===//
9688
9689// Eight general-purpose registers a0-a7 used for passing integer arguments,
9690// with a0-a1 reused to return values. Generally, the GPRs are used to pass
9691// fixed-point arguments, and floating-point arguments when no FPR is available
9692// or with soft float ABI.
9693const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6,
9694 LoongArch::R7, LoongArch::R8, LoongArch::R9,
9695 LoongArch::R10, LoongArch::R11};
9696
9697// PreserveNone calling convention:
9698// Arguments may be passed in any general-purpose registers except:
9699// - R1 : return address register
9700// - R22 : frame pointer
9701// - R31 : base pointer
9702//
9703// All general-purpose registers are treated as caller-saved,
9704// except R1 (RA) and R22 (FP).
9705//
9706// Non-volatile registers are allocated first so that a function
9707// can call normal functions without having to spill and reload
9708// argument registers.
9709const MCPhysReg PreserveNoneArgGPRs[] = {
9710 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
9711 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
9712 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
9713 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
9714 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
9715 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
9716 LoongArch::R20};
9717
9718// Eight floating-point registers fa0-fa7 used for passing floating-point
9719// arguments, and fa0-fa1 are also used to return values.
9720const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2,
9721 LoongArch::F3, LoongArch::F4, LoongArch::F5,
9722 LoongArch::F6, LoongArch::F7};
9723// FPR32 and FPR64 alias each other.
9724const MCPhysReg ArgFPR64s[] = {
9725 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
9726 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
9727
9728const MCPhysReg ArgVRs[] = {LoongArch::VR0, LoongArch::VR1, LoongArch::VR2,
9729 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
9730 LoongArch::VR6, LoongArch::VR7};
9731
9732const MCPhysReg ArgXRs[] = {LoongArch::XR0, LoongArch::XR1, LoongArch::XR2,
9733 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
9734 LoongArch::XR6, LoongArch::XR7};
9735
9736static Register allocateArgGPR(CCState &State) {
9737 switch (State.getCallingConv()) {
9738 case CallingConv::PreserveNone:
9739 if (!State.isVarArg())
9740 return State.AllocateReg(Regs: PreserveNoneArgGPRs);
9741 [[fallthrough]];
9742 default:
9743 return State.AllocateReg(Regs: ArgGPRs);
9744 }
9745}
9746
9747// Pass a 2*GRLen argument that has been split into two GRLen values through
9748// registers or the stack as necessary.
9749static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
9750 CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1,
9751 unsigned ValNo2, MVT ValVT2, MVT LocVT2,
9752 ISD::ArgFlagsTy ArgFlags2) {
9753 unsigned GRLenInBytes = GRLen / 8;
9754 if (Register Reg = allocateArgGPR(State)) {
9755 // At least one half can be passed via register.
9756 State.addLoc(V: CCValAssign::getReg(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(), Reg,
9757 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9758 } else {
9759 // Both halves must be passed on the stack, with proper alignment.
9760 Align StackAlign =
9761 std::max(a: Align(GRLenInBytes), b: ArgFlags1.getNonZeroOrigAlign());
9762 State.addLoc(
9763 V: CCValAssign::getMem(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(),
9764 Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: StackAlign),
9765 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9766 State.addLoc(V: CCValAssign::getMem(
9767 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9768 LocVT: LocVT2, HTP: CCValAssign::Full));
9769 return false;
9770 }
9771 if (Register Reg = allocateArgGPR(State)) {
9772 // The second half can also be passed via register.
9773 State.addLoc(
9774 V: CCValAssign::getReg(ValNo: ValNo2, ValVT: ValVT2, Reg, LocVT: LocVT2, HTP: CCValAssign::Full));
9775 } else {
9776 // The second half is passed via the stack, without additional alignment.
9777 State.addLoc(V: CCValAssign::getMem(
9778 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9779 LocVT: LocVT2, HTP: CCValAssign::Full));
9780 }
9781 return false;
9782}
9783
9784// Implements the LoongArch calling convention. Returns true upon failure.
9785static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
9786 unsigned ValNo, MVT ValVT,
9787 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
9788 CCState &State, bool IsRet, Type *OrigTy) {
9789 unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
9790 assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
9791 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
9792 MVT LocVT = ValVT;
9793
9794 // Any return value split into more than two values can't be returned
9795 // directly.
9796 if (IsRet && ValNo > 1)
9797 return true;
9798
9799 // If passing a variadic argument, or if no FPR is available.
9800 bool UseGPRForFloat = true;
9801
9802 switch (ABI) {
9803 default:
9804 llvm_unreachable("Unexpected ABI");
9805 break;
9806 case LoongArchABI::ABI_ILP32F:
9807 case LoongArchABI::ABI_LP64F:
9808 case LoongArchABI::ABI_ILP32D:
9809 case LoongArchABI::ABI_LP64D:
9810 UseGPRForFloat = ArgFlags.isVarArg();
9811 break;
9812 case LoongArchABI::ABI_ILP32S:
9813 case LoongArchABI::ABI_LP64S:
9814 break;
9815 }
9816
9817 // If this is a variadic argument, the LoongArch calling convention requires
9818 // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
9819 // byte alignment. An aligned register should be used regardless of whether
9820 // the original argument was split during legalisation or not. The argument
9821 // will not be passed by registers if the original type is larger than
9822 // 2*GRLen, so the register alignment rule does not apply.
9823 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
9824 if (ArgFlags.isVarArg() &&
9825 ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
9826 DL.getTypeAllocSize(Ty: OrigTy) == TwoGRLenInBytes) {
9827 unsigned RegIdx = State.getFirstUnallocated(Regs: ArgGPRs);
9828 // Skip 'odd' register if necessary.
9829 if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1)
9830 State.AllocateReg(Regs: ArgGPRs);
9831 }
9832
9833 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
9834 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
9835 State.getPendingArgFlags();
9836
9837 assert(PendingLocs.size() == PendingArgFlags.size() &&
9838 "PendingLocs and PendingArgFlags out of sync");
9839
9840 // FPR32 and FPR64 alias each other.
9841 if (State.getFirstUnallocated(Regs: ArgFPR32s) == std::size(ArgFPR32s))
9842 UseGPRForFloat = true;
9843
9844 if (UseGPRForFloat && ValVT == MVT::f32) {
9845 LocVT = GRLenVT;
9846 LocInfo = CCValAssign::BCvt;
9847 } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
9848 LocVT = MVT::i64;
9849 LocInfo = CCValAssign::BCvt;
9850 } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
9851 // Handle passing f64 on LA32D with a soft float ABI or when floating point
9852 // registers are exhausted.
9853 assert(PendingLocs.empty() && "Can't lower f64 if it is split");
9854 // Depending on available argument GPRS, f64 may be passed in a pair of
9855 // GPRs, split between a GPR and the stack, or passed completely on the
9856 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
9857 // cases.
9858 MCRegister Reg = allocateArgGPR(State);
9859 if (!Reg) {
9860 int64_t StackOffset = State.AllocateStack(Size: 8, Alignment: Align(8));
9861 State.addLoc(
9862 V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9863 return false;
9864 }
9865 LocVT = MVT::i32;
9866 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9867 MCRegister HiReg = allocateArgGPR(State);
9868 if (HiReg) {
9869 State.addLoc(
9870 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: HiReg, LocVT, HTP: LocInfo));
9871 } else {
9872 int64_t StackOffset = State.AllocateStack(Size: 4, Alignment: Align(4));
9873 State.addLoc(
9874 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9875 }
9876 return false;
9877 }
9878
9879 // Split arguments might be passed indirectly, so keep track of the pending
9880 // values.
9881 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
9882 LocVT = GRLenVT;
9883 LocInfo = CCValAssign::Indirect;
9884 PendingLocs.push_back(
9885 Elt: CCValAssign::getPending(ValNo, ValVT, LocVT, HTP: LocInfo));
9886 PendingArgFlags.push_back(Elt: ArgFlags);
9887 if (!ArgFlags.isSplitEnd()) {
9888 return false;
9889 }
9890 }
9891
9892 // If the split argument only had two elements, it should be passed directly
9893 // in registers or on the stack.
9894 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
9895 PendingLocs.size() <= 2) {
9896 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
9897 // Apply the normal calling convention rules to the first half of the
9898 // split argument.
9899 CCValAssign VA = PendingLocs[0];
9900 ISD::ArgFlagsTy AF = PendingArgFlags[0];
9901 PendingLocs.clear();
9902 PendingArgFlags.clear();
9903 return CC_LoongArchAssign2GRLen(GRLen, State, VA1: VA, ArgFlags1: AF, ValNo2: ValNo, ValVT2: ValVT, LocVT2: LocVT,
9904 ArgFlags2: ArgFlags);
9905 }
9906
9907 // Allocate to a register if possible, or else a stack slot.
9908 Register Reg;
9909 unsigned StoreSizeBytes = GRLen / 8;
9910 Align StackAlign = Align(GRLen / 8);
9911
9912 if (ValVT == MVT::f32 && !UseGPRForFloat) {
9913 Reg = State.AllocateReg(Regs: ArgFPR32s);
9914 } else if (ValVT == MVT::f64 && !UseGPRForFloat) {
9915 Reg = State.AllocateReg(Regs: ArgFPR64s);
9916 } else if (ValVT.is128BitVector()) {
9917 Reg = State.AllocateReg(Regs: ArgVRs);
9918 UseGPRForFloat = false;
9919 StoreSizeBytes = 16;
9920 StackAlign = Align(16);
9921 } else if (ValVT.is256BitVector()) {
9922 Reg = State.AllocateReg(Regs: ArgXRs);
9923 UseGPRForFloat = false;
9924 StoreSizeBytes = 32;
9925 StackAlign = Align(32);
9926 } else {
9927 Reg = allocateArgGPR(State);
9928 }
9929
9930 unsigned StackOffset =
9931 Reg ? 0 : State.AllocateStack(Size: StoreSizeBytes, Alignment: StackAlign);
9932
9933 // If we reach this point and PendingLocs is non-empty, we must be at the
9934 // end of a split argument that must be passed indirectly.
9935 if (!PendingLocs.empty()) {
9936 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
9937 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
9938 for (auto &It : PendingLocs) {
9939 if (Reg)
9940 It.convertToReg(Reg);
9941 else
9942 It.convertToMem(Offset: StackOffset);
9943 State.addLoc(V: It);
9944 }
9945 PendingLocs.clear();
9946 PendingArgFlags.clear();
9947 return false;
9948 }
9949 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
9950 "Expected an GRLenVT at this stage");
9951
9952 if (Reg) {
9953 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9954 return false;
9955 }
9956
9957 // When a floating-point value is passed on the stack, no bit-cast is needed.
9958 if (ValVT.isFloatingPoint()) {
9959 LocVT = ValVT;
9960 LocInfo = CCValAssign::Full;
9961 }
9962
9963 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9964 return false;
9965}
9966
9967void LoongArchTargetLowering::analyzeInputArgs(
9968 MachineFunction &MF, CCState &CCInfo,
9969 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9970 LoongArchCCAssignFn Fn) const {
9971 FunctionType *FType = MF.getFunction().getFunctionType();
9972 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
9973 MVT ArgVT = Ins[i].VT;
9974 Type *ArgTy = nullptr;
9975 if (IsRet)
9976 ArgTy = FType->getReturnType();
9977 else if (Ins[i].isOrigArg())
9978 ArgTy = FType->getParamType(i: Ins[i].getOrigArgIndex());
9979 LoongArchABI::ABI ABI =
9980 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9981 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
9982 CCInfo, IsRet, ArgTy)) {
9983 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
9984 << '\n');
9985 llvm_unreachable("");
9986 }
9987 }
9988}
9989
9990void LoongArchTargetLowering::analyzeOutputArgs(
9991 MachineFunction &MF, CCState &CCInfo,
9992 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9993 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const {
9994 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9995 MVT ArgVT = Outs[i].VT;
9996 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9997 LoongArchABI::ABI ABI =
9998 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9999 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
10000 CCInfo, IsRet, OrigTy)) {
10001 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
10002 << "\n");
10003 llvm_unreachable("");
10004 }
10005 }
10006}
10007
10008// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
10009// values.
10010static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
10011 const CCValAssign &VA, const SDLoc &DL) {
10012 switch (VA.getLocInfo()) {
10013 default:
10014 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10015 case CCValAssign::Full:
10016 case CCValAssign::Indirect:
10017 break;
10018 case CCValAssign::BCvt:
10019 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10020 Val = DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Val);
10021 else
10022 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
10023 break;
10024 }
10025 return Val;
10026}
10027
10028static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
10029 const CCValAssign &VA, const SDLoc &DL,
10030 const ISD::InputArg &In,
10031 const LoongArchTargetLowering &TLI) {
10032 MachineFunction &MF = DAG.getMachineFunction();
10033 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10034 EVT LocVT = VA.getLocVT();
10035 SDValue Val;
10036 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
10037 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
10038 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
10039 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
10040
10041 // If input is sign extended from 32 bits, note it for the OptW pass.
10042 if (In.isOrigArg()) {
10043 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
10044 if (OrigArg->getType()->isIntegerTy()) {
10045 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
10046 // An input zero extended from i31 can also be considered sign extended.
10047 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
10048 (BitWidth < 32 && In.Flags.isZExt())) {
10049 LoongArchMachineFunctionInfo *LAFI =
10050 MF.getInfo<LoongArchMachineFunctionInfo>();
10051 LAFI->addSExt32Register(Reg: VReg);
10052 }
10053 }
10054 }
10055
10056 return convertLocVTToValVT(DAG, Val, VA, DL);
10057}
10058
10059// The caller is responsible for loading the full value if the argument is
10060// passed with CCValAssign::Indirect.
10061static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
10062 const CCValAssign &VA, const SDLoc &DL) {
10063 MachineFunction &MF = DAG.getMachineFunction();
10064 MachineFrameInfo &MFI = MF.getFrameInfo();
10065 EVT ValVT = VA.getValVT();
10066 int FI = MFI.CreateFixedObject(Size: ValVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
10067 /*IsImmutable=*/true);
10068 SDValue FIN = DAG.getFrameIndex(
10069 FI, VT: MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0)));
10070
10071 ISD::LoadExtType ExtType;
10072 switch (VA.getLocInfo()) {
10073 default:
10074 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10075 case CCValAssign::Full:
10076 case CCValAssign::Indirect:
10077 case CCValAssign::BCvt:
10078 ExtType = ISD::NON_EXTLOAD;
10079 break;
10080 }
10081 return DAG.getExtLoad(
10082 ExtType, dl: DL, VT: VA.getLocVT(), Chain, Ptr: FIN,
10083 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT: ValVT);
10084}
10085
10086static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain,
10087 const CCValAssign &VA,
10088 const CCValAssign &HiVA,
10089 const SDLoc &DL) {
10090 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
10091 "Unexpected VA");
10092 MachineFunction &MF = DAG.getMachineFunction();
10093 MachineFrameInfo &MFI = MF.getFrameInfo();
10094 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10095
10096 assert(VA.isRegLoc() && "Expected register VA assignment");
10097
10098 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10099 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
10100 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
10101 SDValue Hi;
10102 if (HiVA.isMemLoc()) {
10103 // Second half of f64 is passed on the stack.
10104 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
10105 /*IsImmutable=*/true);
10106 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
10107 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
10108 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10109 } else {
10110 // Second half of f64 is passed in another GPR.
10111 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10112 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
10113 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
10114 }
10115 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
10116}
10117
10118static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
10119 const CCValAssign &VA, const SDLoc &DL) {
10120 EVT LocVT = VA.getLocVT();
10121
10122 switch (VA.getLocInfo()) {
10123 default:
10124 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10125 case CCValAssign::Full:
10126 break;
10127 case CCValAssign::BCvt:
10128 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10129 Val = DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Val);
10130 else
10131 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
10132 break;
10133 }
10134 return Val;
10135}
10136
10137static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
10138 CCValAssign::LocInfo LocInfo,
10139 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
10140 CCState &State) {
10141 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
10142 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
10143 // s0 s1 s2 s3 s4 s5 s6 s7 s8
10144 static const MCPhysReg GPRList[] = {
10145 LoongArch::R23, LoongArch::R24, LoongArch::R25,
10146 LoongArch::R26, LoongArch::R27, LoongArch::R28,
10147 LoongArch::R29, LoongArch::R30, LoongArch::R31};
10148 if (MCRegister Reg = State.AllocateReg(Regs: GPRList)) {
10149 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10150 return false;
10151 }
10152 }
10153
10154 if (LocVT == MVT::f32) {
10155 // Pass in STG registers: F1, F2, F3, F4
10156 // fs0,fs1,fs2,fs3
10157 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
10158 LoongArch::F26, LoongArch::F27};
10159 if (MCRegister Reg = State.AllocateReg(Regs: FPR32List)) {
10160 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10161 return false;
10162 }
10163 }
10164
10165 if (LocVT == MVT::f64) {
10166 // Pass in STG registers: D1, D2, D3, D4
10167 // fs4,fs5,fs6,fs7
10168 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
10169 LoongArch::F30_64, LoongArch::F31_64};
10170 if (MCRegister Reg = State.AllocateReg(Regs: FPR64List)) {
10171 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10172 return false;
10173 }
10174 }
10175
10176 report_fatal_error(reason: "No registers left in GHC calling convention");
10177 return true;
10178}
10179
10180// Transform physical registers into virtual registers.
10181SDValue LoongArchTargetLowering::LowerFormalArguments(
10182 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10183 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
10184 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
10185
10186 MachineFunction &MF = DAG.getMachineFunction();
10187
10188 switch (CallConv) {
10189 default:
10190 llvm_unreachable("Unsupported calling convention");
10191 case CallingConv::C:
10192 case CallingConv::Fast:
10193 case CallingConv::PreserveNone:
10194 case CallingConv::PreserveMost:
10195 break;
10196 case CallingConv::GHC:
10197 if (!MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicF) ||
10198 !MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicD))
10199 report_fatal_error(
10200 reason: "GHC calling convention requires the F and D extensions");
10201 }
10202
10203 const Function &Func = MF.getFunction();
10204 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10205 MVT GRLenVT = Subtarget.getGRLenVT();
10206 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
10207
10208 // Check if this function has any musttail calls. If so, incoming indirect
10209 // arg pointers must be saved in virtual registers so they survive across
10210 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
10211 // when needed to avoid adding register pressure to non-musttail functions.
10212 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
10213 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
10214 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
10215 return CI->isMustTailCall();
10216 return false;
10217 });
10218 });
10219 // Used with varargs to acumulate store chains.
10220 std::vector<SDValue> OutChains;
10221
10222 // Assign locations to all of the incoming arguments.
10223 SmallVector<CCValAssign> ArgLocs;
10224 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10225
10226 if (CallConv == CallingConv::GHC)
10227 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_LoongArch_GHC);
10228 else
10229 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, Fn: CC_LoongArch);
10230
10231 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10232 CCValAssign &VA = ArgLocs[i];
10233 SDValue ArgValue;
10234 // Passing f64 on LA32D with a soft float ABI must be handled as a special
10235 // case.
10236 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10237 assert(VA.needsCustom());
10238 ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
10239 } else if (VA.isRegLoc())
10240 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
10241 else
10242 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
10243 if (VA.getLocInfo() == CCValAssign::Indirect) {
10244 // If the original argument was split and passed by reference, we need to
10245 // load all parts of it here (using the same address).
10246 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
10247 PtrInfo: MachinePointerInfo()));
10248 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
10249 if (HasMusttail) {
10250 LoongArchMachineFunctionInfo *LAFI =
10251 MF.getInfo<LoongArchMachineFunctionInfo>();
10252 Register VReg =
10253 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10254 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
10255 LAFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
10256 }
10257 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
10258 assert(ArgPartOffset == 0);
10259 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
10260 CCValAssign &PartVA = ArgLocs[i + 1];
10261 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
10262 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10263 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
10264 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
10265 PtrInfo: MachinePointerInfo()));
10266 ++i;
10267 ++InsIdx;
10268 }
10269 continue;
10270 }
10271 InVals.push_back(Elt: ArgValue);
10272 }
10273
10274 if (IsVarArg) {
10275 ArrayRef<MCPhysReg> ArgRegs = ArrayRef(ArgGPRs);
10276 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
10277 const TargetRegisterClass *RC = &LoongArch::GPRRegClass;
10278 MachineFrameInfo &MFI = MF.getFrameInfo();
10279 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10280 auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>();
10281
10282 // Offset of the first variable argument from stack pointer, and size of
10283 // the vararg save area. For now, the varargs save area is either zero or
10284 // large enough to hold a0-a7.
10285 int VaArgOffset, VarArgsSaveSize;
10286
10287 // If all registers are allocated, then all varargs must be passed on the
10288 // stack and we don't need to save any argregs.
10289 if (ArgRegs.size() == Idx) {
10290 VaArgOffset = CCInfo.getStackSize();
10291 VarArgsSaveSize = 0;
10292 } else {
10293 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx);
10294 VaArgOffset = -VarArgsSaveSize;
10295 }
10296
10297 // Record the frame index of the first variable argument
10298 // which is a value necessary to VASTART.
10299 int FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10300 LoongArchFI->setVarArgsFrameIndex(FI);
10301
10302 // If saving an odd number of registers then create an extra stack slot to
10303 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures
10304 // offsets to even-numbered registered remain 2*GRLen-aligned.
10305 if (Idx % 2) {
10306 MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset - (int)GRLenInBytes,
10307 IsImmutable: true);
10308 VarArgsSaveSize += GRLenInBytes;
10309 }
10310
10311 // Copy the integer registers that may have been used for passing varargs
10312 // to the vararg save area.
10313 for (unsigned I = Idx; I < ArgRegs.size();
10314 ++I, VaArgOffset += GRLenInBytes) {
10315 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
10316 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
10317 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: GRLenVT);
10318 FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10319 SDValue PtrOff = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10320 SDValue Store = DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: PtrOff,
10321 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10322 cast<StoreSDNode>(Val: Store.getNode())
10323 ->getMemOperand()
10324 ->setValue((Value *)nullptr);
10325 OutChains.push_back(x: Store);
10326 }
10327 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
10328 }
10329
10330 // All stores are grouped in one node to allow the matching between
10331 // the size of Ins and InVals. This only happens for vararg functions.
10332 if (!OutChains.empty()) {
10333 OutChains.push_back(x: Chain);
10334 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
10335 }
10336
10337 return Chain;
10338}
10339
10340bool LoongArchTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10341 return CI->isTailCall();
10342}
10343
10344// Check if the return value is used as only a return value, as otherwise
10345// we can't perform a tail-call.
10346bool LoongArchTargetLowering::isUsedByReturnOnly(SDNode *N,
10347 SDValue &Chain) const {
10348 if (N->getNumValues() != 1)
10349 return false;
10350 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
10351 return false;
10352
10353 SDNode *Copy = *N->user_begin();
10354 if (Copy->getOpcode() != ISD::CopyToReg)
10355 return false;
10356
10357 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
10358 // isn't safe to perform a tail call.
10359 if (Copy->getGluedNode())
10360 return false;
10361
10362 // The copy must be used by a LoongArchISD::RET, and nothing else.
10363 bool HasRet = false;
10364 for (SDNode *Node : Copy->users()) {
10365 if (Node->getOpcode() != LoongArchISD::RET)
10366 return false;
10367 HasRet = true;
10368 }
10369
10370 if (!HasRet)
10371 return false;
10372
10373 Chain = Copy->getOperand(Num: 0);
10374 return true;
10375}
10376
10377// Check whether the call is eligible for tail call optimization.
10378bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
10379 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
10380 const SmallVectorImpl<CCValAssign> &ArgLocs) const {
10381
10382 auto CalleeCC = CLI.CallConv;
10383 auto &Outs = CLI.Outs;
10384 auto &Caller = MF.getFunction();
10385 auto CallerCC = Caller.getCallingConv();
10386
10387 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
10388
10389 // Byval parameters hand the function a pointer directly into the stack area
10390 // we want to reuse during a tail call. Working around this *is* possible
10391 // but less efficient and uglier in LowerCall. For musttail, there is no
10392 // workaround today: a byval arg requires a local copy that becomes invalid
10393 // after the tail call deallocates the caller's frame, so rejecting here
10394 // (and triggering reportFatalInternalError in LowerCall) is safer than
10395 // miscompiling.
10396 for (auto &Arg : Outs)
10397 if (Arg.Flags.isByVal())
10398 return false;
10399
10400 // musttail bypasses the remaining checks: the checks either reject cases
10401 // we handle specially (indirect args are forwarded via incoming pointers,
10402 // stack-passed args reuse the matching incoming layout, sret is forwarded
10403 // like any other pointer arg) or are optimizations not applicable to
10404 // mandatory tail calls.
10405 if (IsMustTail)
10406 return true;
10407
10408 // Do not tail call opt if the stack is used to pass parameters.
10409 if (CCInfo.getStackSize() != 0)
10410 return false;
10411
10412 // Do not tail call opt if any parameters need to be passed indirectly.
10413 for (auto &VA : ArgLocs)
10414 if (VA.getLocInfo() == CCValAssign::Indirect)
10415 return false;
10416
10417 // Do not tail call opt if either caller or callee uses struct return
10418 // semantics.
10419 auto IsCallerStructRet = Caller.hasStructRetAttr();
10420 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
10421 if (IsCallerStructRet || IsCalleeStructRet)
10422 return false;
10423
10424 // The callee has to preserve all registers the caller needs to preserve.
10425 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
10426 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
10427 if (CalleeCC != CallerCC) {
10428 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
10429 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
10430 return false;
10431 }
10432 return true;
10433}
10434
10435static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
10436 return DAG.getDataLayout().getPrefTypeAlign(
10437 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
10438}
10439
10440// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
10441// and output parameter nodes.
10442SDValue
10443LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
10444 SmallVectorImpl<SDValue> &InVals) const {
10445 SelectionDAG &DAG = CLI.DAG;
10446 SDLoc &DL = CLI.DL;
10447 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
10448 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
10449 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
10450 SDValue Chain = CLI.Chain;
10451 SDValue Callee = CLI.Callee;
10452 CallingConv::ID CallConv = CLI.CallConv;
10453 bool IsVarArg = CLI.IsVarArg;
10454 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10455 MVT GRLenVT = Subtarget.getGRLenVT();
10456 bool &IsTailCall = CLI.IsTailCall;
10457
10458 MachineFunction &MF = DAG.getMachineFunction();
10459
10460 // Analyze the operands of the call, assigning locations to each operand.
10461 SmallVector<CCValAssign> ArgLocs;
10462 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10463
10464 if (CallConv == CallingConv::GHC)
10465 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_LoongArch_GHC);
10466 else
10467 analyzeOutputArgs(MF, CCInfo&: ArgCCInfo, Outs, /*IsRet=*/false, CLI: &CLI, Fn: CC_LoongArch);
10468
10469 // Check if it's really possible to do a tail call.
10470 if (IsTailCall)
10471 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
10472
10473 if (IsTailCall)
10474 ++NumTailCalls;
10475 else if (CLI.CB && CLI.CB->isMustTailCall())
10476 report_fatal_error(reason: "failed to perform tail call elimination on a call "
10477 "site marked musttail");
10478
10479 // Get a count of how many bytes are to be pushed on the stack.
10480 unsigned NumBytes = ArgCCInfo.getStackSize();
10481
10482 // Create local copies for byval args.
10483 SmallVector<SDValue> ByValArgs;
10484 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10485 ISD::ArgFlagsTy Flags = Outs[i].Flags;
10486 if (!Flags.isByVal())
10487 continue;
10488
10489 SDValue Arg = OutVals[i];
10490 unsigned Size = Flags.getByValSize();
10491 Align Alignment = Flags.getNonZeroByValAlign();
10492
10493 int FI =
10494 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
10495 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10496 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: GRLenVT);
10497
10498 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
10499 /*IsVolatile=*/isVol: false,
10500 /*AlwaysInline=*/false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
10501 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
10502 ByValArgs.push_back(Elt: FIPtr);
10503 }
10504
10505 if (!IsTailCall)
10506 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
10507
10508 // Copy argument values to their designated locations.
10509 SmallVector<std::pair<Register, SDValue>> RegsToPass;
10510 SmallVector<SDValue> MemOpChains;
10511 SDValue StackPtr;
10512 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
10513 ++i, ++OutIdx) {
10514 CCValAssign &VA = ArgLocs[i];
10515 SDValue ArgValue = OutVals[OutIdx];
10516 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
10517
10518 // Handle passing f64 on LA32D with a soft float ABI as a special case.
10519 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10520 assert(VA.isRegLoc() && "Expected register VA assignment");
10521 assert(VA.needsCustom());
10522 SDValue SplitF64 =
10523 DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10524 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
10525 SDValue Lo = SplitF64.getValue(R: 0);
10526 SDValue Hi = SplitF64.getValue(R: 1);
10527
10528 Register RegLo = VA.getLocReg();
10529 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
10530
10531 // Get the CCValAssign for the Hi part.
10532 CCValAssign &HiVA = ArgLocs[++i];
10533
10534 if (HiVA.isMemLoc()) {
10535 // Second half of f64 is passed on the stack.
10536 if (!StackPtr.getNode())
10537 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10538 SDValue Address =
10539 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10540 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
10541 // Emit the store.
10542 MemOpChains.push_back(Elt: DAG.getStore(
10543 Chain, dl: DL, Val: Hi, Ptr: Address,
10544 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
10545 } else {
10546 // Second half of f64 is passed in another GPR.
10547 Register RegHigh = HiVA.getLocReg();
10548 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
10549 }
10550 continue;
10551 }
10552
10553 // Promote the value if needed.
10554 // For now, only handle fully promoted and indirect arguments.
10555 if (VA.getLocInfo() == CCValAssign::Indirect) {
10556 // For musttail calls, reuse incoming indirect pointers instead of
10557 // creating new stack temporaries. The incoming pointers point to the
10558 // caller's caller's frame, which remains valid after a tail call.
10559 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
10560 LoongArchMachineFunctionInfo *LAFI =
10561 MF.getInfo<LoongArchMachineFunctionInfo>();
10562 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
10563
10564 // Resolve which formal parameter is being passed at this call
10565 // position.
10566 //
10567 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
10568 // but Outs[].OrigArgIndex is an index into a filtered arg list
10569 // (empty types removed, via CallLoweringInfo in the target-
10570 // independent layer). IncomingIndirectArgs is keyed by the
10571 // caller's unfiltered Argument::getArgNo(), so we have to walk
10572 // the caller's formals (same filter) to translate the index.
10573 // This target-independent asymmetry should be normalized so
10574 // backends do not need to re-derive the mapping.
10575 //
10576 // Steps:
10577 // 1. Find the call operand at filtered position CallArgIdx.
10578 // 2. If it is an Argument, use getArgNo() directly (same filter
10579 // for caller formals and call operands).
10580 // 3. Otherwise (computed value), walk the caller's formals and
10581 // skip empty types to map the filtered index to getArgNo().
10582 const Argument *FormalArg = nullptr;
10583 unsigned FilteredIdx = 0;
10584 for (const auto &CallArg : CLI.CB->args()) {
10585 if (CallArg->getType()->isEmptyTy())
10586 continue;
10587 if (FilteredIdx == CallArgIdx) {
10588 FormalArg = dyn_cast<Argument>(Val: CallArg);
10589 break;
10590 }
10591 ++FilteredIdx;
10592 }
10593
10594 // For forwarded args, getArgNo() gives the unfiltered index directly.
10595 // For computed args, walk the caller's formals to resolve it.
10596 unsigned FormalArgIdx = CallArgIdx;
10597 if (FormalArg) {
10598 FormalArgIdx = FormalArg->getArgNo();
10599 } else {
10600 FilteredIdx = 0;
10601 for (const auto &Arg : MF.getFunction().args()) {
10602 if (Arg.getType()->isEmptyTy())
10603 continue;
10604 if (FilteredIdx == CallArgIdx) {
10605 FormalArgIdx = Arg.getArgNo();
10606 break;
10607 }
10608 ++FilteredIdx;
10609 }
10610 }
10611
10612 Register VReg = LAFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
10613 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
10614 // Thread the CopyFromReg output chain through MemOpChains so the
10615 // TokenFactor below sequences the copy with any stores we emit
10616 // for this argument.
10617 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
10618 SDValue IncomingPtr = CopyOp;
10619
10620 if (!FormalArg) {
10621 // Computed value: store into the incoming indirect pointer for the
10622 // same-position formal parameter (musttail guarantees matching
10623 // prototypes, so types match). The pointer survives the tail call
10624 // since it points to the caller's caller's frame.
10625 //
10626 // The data-flow edge through IncomingPtr already prevents the
10627 // store from being scheduled before the CopyFromReg. Threading
10628 // CopyOp.getValue(1) (the copy's output chain) into the store
10629 // makes that ordering explicit on the chain edge as well, which
10630 // is the convention for memory ops chaining off their producers.
10631 MemOpChains.push_back(
10632 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
10633 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10634 // Store any split parts at their respective offsets.
10635 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10636 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10637 SDValue PartValue = OutVals[OutIdx + 1];
10638 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10639 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10640 SDValue Addr =
10641 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
10642 MemOpChains.push_back(
10643 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
10644 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10645 ++i;
10646 ++OutIdx;
10647 }
10648 }
10649 ArgValue = IncomingPtr;
10650
10651 // Skip any remaining split parts (for forwarded args, they are
10652 // covered by the forwarded pointer).
10653 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10654 ++i;
10655 ++OutIdx;
10656 }
10657 } else {
10658 // Store the argument in a stack slot and pass its address.
10659 Align StackAlign =
10660 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
10661 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
10662 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
10663 // If the original argument was split and passed by reference, we need
10664 // to store the required parts of it here (and pass just one address).
10665 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
10666 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10667 assert(ArgPartOffset == 0);
10668 // Calculate the total size to store. We don't have access to what we're
10669 // actually storing other than performing the loop and collecting the
10670 // info.
10671 SmallVector<std::pair<SDValue, SDValue>> Parts;
10672 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
10673 SDValue PartValue = OutVals[OutIdx + 1];
10674 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10675 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10676 EVT PartVT = PartValue.getValueType();
10677 StoredSize += PartVT.getStoreSize();
10678 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
10679 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
10680 ++i;
10681 ++OutIdx;
10682 }
10683 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
10684 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
10685 MemOpChains.push_back(
10686 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
10687 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10688 for (const auto &Part : Parts) {
10689 SDValue PartValue = Part.first;
10690 SDValue PartOffset = Part.second;
10691 SDValue Address =
10692 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
10693 MemOpChains.push_back(
10694 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
10695 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10696 }
10697 ArgValue = SpillSlot;
10698 }
10699 } else {
10700 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL);
10701 }
10702
10703 // Use local copy if it is a byval arg.
10704 if (Flags.isByVal())
10705 ArgValue = ByValArgs[j++];
10706
10707 if (VA.isRegLoc()) {
10708 // Queue up the argument copies and emit them at the end.
10709 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
10710 } else {
10711 assert(VA.isMemLoc() && "Argument not register or memory");
10712 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
10713 "Tail call not allowed if stack is used for passing parameters");
10714
10715 // Work out the address of the stack slot.
10716 if (!StackPtr.getNode())
10717 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10718 SDValue Address =
10719 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10720 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
10721
10722 // Emit the store.
10723 MemOpChains.push_back(
10724 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
10725 }
10726 }
10727
10728 // Join the stores, which are independent of one another.
10729 if (!MemOpChains.empty())
10730 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
10731
10732 SDValue Glue;
10733
10734 // Build a sequence of copy-to-reg nodes, chained and glued together.
10735 for (auto &Reg : RegsToPass) {
10736 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
10737 Glue = Chain.getValue(R: 1);
10738 }
10739
10740 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
10741 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
10742 // split it and then direct call can be matched by PseudoCALL_SMALL.
10743 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
10744 const GlobalValue *GV = S->getGlobal();
10745 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV)
10746 ? LoongArchII::MO_CALL
10747 : LoongArchII::MO_CALL_PLT;
10748 Callee = DAG.getTargetGlobalAddress(GV: S->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: OpFlags);
10749 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
10750 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV: nullptr)
10751 ? LoongArchII::MO_CALL
10752 : LoongArchII::MO_CALL_PLT;
10753 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: OpFlags);
10754 }
10755
10756 // The first call operand is the chain and the second is the target address.
10757 SmallVector<SDValue> Ops;
10758 Ops.push_back(Elt: Chain);
10759 Ops.push_back(Elt: Callee);
10760
10761 // Add argument registers to the end of the list so that they are
10762 // known live into the call.
10763 for (auto &Reg : RegsToPass)
10764 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
10765
10766 if (!IsTailCall) {
10767 // Add a register mask operand representing the call-preserved registers.
10768 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
10769 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
10770 assert(Mask && "Missing call preserved mask for calling convention");
10771 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
10772 }
10773
10774 // Glue the call to the argument copies, if any.
10775 if (Glue.getNode())
10776 Ops.push_back(Elt: Glue);
10777
10778 // Emit the call.
10779 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
10780 unsigned Op;
10781 switch (DAG.getTarget().getCodeModel()) {
10782 default:
10783 report_fatal_error(reason: "Unsupported code model");
10784 case CodeModel::Small:
10785 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
10786 break;
10787 case CodeModel::Medium:
10788 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
10789 break;
10790 case CodeModel::Large:
10791 assert(Subtarget.is64Bit() && "Large code model requires LA64");
10792 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
10793 break;
10794 }
10795
10796 if (IsTailCall) {
10797 MF.getFrameInfo().setHasTailCall();
10798 SDValue Ret = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10799 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
10800 return Ret;
10801 }
10802
10803 Chain = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10804 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
10805 Glue = Chain.getValue(R: 1);
10806
10807 // Mark the end of the call, which is glued to the call itself.
10808 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
10809 Glue = Chain.getValue(R: 1);
10810
10811 // Assign locations to each value returned by this call.
10812 SmallVector<CCValAssign> RVLocs;
10813 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
10814 analyzeInputArgs(MF, CCInfo&: RetCCInfo, Ins, /*IsRet=*/true, Fn: CC_LoongArch);
10815
10816 // Copy all of the result registers out of their specified physreg.
10817 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
10818 auto &VA = RVLocs[i];
10819 // Copy the value out.
10820 SDValue RetValue =
10821 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
10822 // Glue the RetValue to the end of the call sequence.
10823 Chain = RetValue.getValue(R: 1);
10824 Glue = RetValue.getValue(R: 2);
10825
10826 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10827 assert(VA.needsCustom());
10828 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
10829 VT: MVT::i32, Glue);
10830 Chain = RetValue2.getValue(R: 1);
10831 Glue = RetValue2.getValue(R: 2);
10832 RetValue = DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64,
10833 N1: RetValue, N2: RetValue2);
10834 } else
10835 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL);
10836
10837 InVals.push_back(Elt: RetValue);
10838 }
10839
10840 return Chain;
10841}
10842
10843bool LoongArchTargetLowering::CanLowerReturn(
10844 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
10845 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
10846 const Type *RetTy) const {
10847 SmallVector<CCValAssign> RVLocs;
10848 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
10849
10850 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10851 LoongArchABI::ABI ABI =
10852 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10853 if (CC_LoongArch(DL: MF.getDataLayout(), ABI, ValNo: i, ValVT: Outs[i].VT, LocInfo: CCValAssign::Full,
10854 ArgFlags: Outs[i].Flags, State&: CCInfo, /*IsRet=*/true, OrigTy: nullptr))
10855 return false;
10856 }
10857 return true;
10858}
10859
10860SDValue LoongArchTargetLowering::LowerReturn(
10861 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10862 const SmallVectorImpl<ISD::OutputArg> &Outs,
10863 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
10864 SelectionDAG &DAG) const {
10865 // Stores the assignment of the return value to a location.
10866 SmallVector<CCValAssign> RVLocs;
10867
10868 // Info about the registers and stack slot.
10869 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
10870 *DAG.getContext());
10871
10872 analyzeOutputArgs(MF&: DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
10873 CLI: nullptr, Fn: CC_LoongArch);
10874 if (CallConv == CallingConv::GHC && !RVLocs.empty())
10875 report_fatal_error(reason: "GHC functions return void only");
10876 SDValue Glue;
10877 SmallVector<SDValue, 4> RetOps(1, Chain);
10878
10879 // Copy the result values into the output registers.
10880 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
10881 SDValue Val = OutVals[OutIdx];
10882 CCValAssign &VA = RVLocs[i];
10883 assert(VA.isRegLoc() && "Can only return in registers!");
10884
10885 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10886 // Handle returning f64 on LA32D with a soft float ABI.
10887 assert(VA.isRegLoc() && "Expected return via registers");
10888 assert(VA.needsCustom());
10889 SDValue SplitF64 = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10890 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
10891 SDValue Lo = SplitF64.getValue(R: 0);
10892 SDValue Hi = SplitF64.getValue(R: 1);
10893 Register RegLo = VA.getLocReg();
10894 Register RegHi = RVLocs[++i].getLocReg();
10895
10896 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
10897 Glue = Chain.getValue(R: 1);
10898 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
10899 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
10900 Glue = Chain.getValue(R: 1);
10901 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
10902 } else {
10903 // Handle a 'normal' return.
10904 Val = convertValVTToLocVT(DAG, Val, VA, DL);
10905 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
10906
10907 // Guarantee that all emitted copies are stuck together.
10908 Glue = Chain.getValue(R: 1);
10909 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
10910 }
10911 }
10912
10913 RetOps[0] = Chain; // Update chain.
10914
10915 // Add the glue node if we have it.
10916 if (Glue.getNode())
10917 RetOps.push_back(Elt: Glue);
10918
10919 return DAG.getNode(Opcode: LoongArchISD::RET, DL, VT: MVT::Other, Ops: RetOps);
10920}
10921
10922// Check if a constant splat can be generated using [x]vldi, where imm[12] == 1.
10923// Note: The following prefixes are excluded:
10924// imm[11:8] == 4'b0000, 4'b0100, 4'b1000
10925// as they can be represented using [x]vrepli.[whb]
10926std::pair<bool, uint64_t> LoongArchTargetLowering::isImmVLDILegalForMode1(
10927 const APInt &SplatValue, const unsigned SplatBitSize) const {
10928 uint64_t RequiredImm = 0;
10929 uint64_t V = SplatValue.getZExtValue();
10930 if (SplatBitSize == 16 && !(V & 0x00FF)) {
10931 // 4'b0101
10932 RequiredImm = (0b10101 << 8) | (V >> 8);
10933 return {true, RequiredImm};
10934 } else if (SplatBitSize == 32) {
10935 // 4'b0001
10936 if (!(V & 0xFFFF00FF)) {
10937 RequiredImm = (0b10001 << 8) | (V >> 8);
10938 return {true, RequiredImm};
10939 }
10940 // 4'b0010
10941 if (!(V & 0xFF00FFFF)) {
10942 RequiredImm = (0b10010 << 8) | (V >> 16);
10943 return {true, RequiredImm};
10944 }
10945 // 4'b0011
10946 if (!(V & 0x00FFFFFF)) {
10947 RequiredImm = (0b10011 << 8) | (V >> 24);
10948 return {true, RequiredImm};
10949 }
10950 // 4'b0110
10951 if ((V & 0xFFFF00FF) == 0xFF) {
10952 RequiredImm = (0b10110 << 8) | (V >> 8);
10953 return {true, RequiredImm};
10954 }
10955 // 4'b0111
10956 if ((V & 0xFF00FFFF) == 0xFFFF) {
10957 RequiredImm = (0b10111 << 8) | (V >> 16);
10958 return {true, RequiredImm};
10959 }
10960 // 4'b1010
10961 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
10962 RequiredImm =
10963 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10964 return {true, RequiredImm};
10965 }
10966 } else if (SplatBitSize == 64) {
10967 // 4'b1011
10968 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
10969 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
10970 RequiredImm =
10971 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10972 return {true, RequiredImm};
10973 }
10974 // 4'b1100
10975 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
10976 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
10977 RequiredImm =
10978 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
10979 return {true, RequiredImm};
10980 }
10981 // 4'b1001
10982 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
10983 uint8_t res = 0;
10984 for (int i = 0; i < 8; ++i) {
10985 uint8_t byte = x & 0xFF;
10986 if (byte == 0 || byte == 0xFF)
10987 res |= ((byte & 1) << i);
10988 else
10989 return {false, 0};
10990 x >>= 8;
10991 }
10992 return {true, res};
10993 };
10994 auto [IsSame, Suffix] = sameBitsPreByte(V);
10995 if (IsSame) {
10996 RequiredImm = (0b11001 << 8) | Suffix;
10997 return {true, RequiredImm};
10998 }
10999 }
11000 return {false, RequiredImm};
11001}
11002
11003bool LoongArchTargetLowering::isFPImmVLDILegal(const APFloat &Imm,
11004 EVT VT) const {
11005 if (!Subtarget.hasExtLSX())
11006 return false;
11007
11008 if (VT == MVT::f32) {
11009 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
11010 return (masked == 0x3e000000 || masked == 0x40000000);
11011 }
11012
11013 if (VT == MVT::f64) {
11014 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
11015 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
11016 }
11017
11018 return false;
11019}
11020
11021bool LoongArchTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
11022 bool ForCodeSize) const {
11023 // TODO: Maybe need more checks here after vector extension is supported.
11024 if (VT == MVT::f32 && !Subtarget.hasBasicF())
11025 return false;
11026 if (VT == MVT::f64 && !Subtarget.hasBasicD())
11027 return false;
11028 return (Imm.isZero() || Imm.isOne() || isFPImmVLDILegal(Imm, VT));
11029}
11030
11031bool LoongArchTargetLowering::isCheapToSpeculateCttz(Type *) const {
11032 return true;
11033}
11034
11035bool LoongArchTargetLowering::isCheapToSpeculateCtlz(Type *) const {
11036 return true;
11037}
11038
11039bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
11040 const Instruction *I) const {
11041 if (!Subtarget.is64Bit())
11042 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
11043
11044 if (isa<LoadInst>(Val: I))
11045 return true;
11046
11047 // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
11048 // require fences beacuse we can use amswap_db.[w/d].
11049 Type *Ty = I->getOperand(i: 0)->getType();
11050 if (isa<StoreInst>(Val: I) && Ty->isIntegerTy()) {
11051 unsigned Size = Ty->getIntegerBitWidth();
11052 return (Size == 8 || Size == 16);
11053 }
11054
11055 return false;
11056}
11057
11058EVT LoongArchTargetLowering::getSetCCResultType(const DataLayout &DL,
11059 LLVMContext &Context,
11060 EVT VT) const {
11061 if (!VT.isVector())
11062 return getPointerTy(DL);
11063 return VT.changeVectorElementTypeToInteger();
11064}
11065
11066bool LoongArchTargetLowering::canMergeStoresTo(
11067 unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const {
11068 // Do not merge to float value size (128 or 256 bits) if no implicit
11069 // float attribute is set.
11070 bool NoFloat = MF.getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat);
11071 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
11072 if (NoFloat)
11073 return MemVT.getSizeInBits() <= MaxIntSize;
11074
11075 // Make sure we don't merge greater than our maximum supported vector width.
11076 if (Subtarget.hasExtLASX())
11077 MaxIntSize = 256;
11078 else if (Subtarget.hasExtLSX())
11079 MaxIntSize = 128;
11080
11081 return MemVT.getSizeInBits() <= MaxIntSize;
11082}
11083
11084bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
11085 EVT VT = Y.getValueType();
11086
11087 if (VT.isVector())
11088 return Subtarget.hasExtLSX() && VT.isInteger();
11089
11090 return VT.isScalarInteger() && !isa<ConstantSDNode>(Val: Y);
11091}
11092
11093void LoongArchTargetLowering::getTgtMemIntrinsic(
11094 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
11095 MachineFunction &MF, unsigned Intrinsic) const {
11096 switch (Intrinsic) {
11097 default:
11098 return;
11099 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
11100 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
11101 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
11102 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
11103 IntrinsicInfo Info;
11104 Info.opc = ISD::INTRINSIC_W_CHAIN;
11105 Info.memVT = MVT::i32;
11106 Info.ptrVal = I.getArgOperand(i: 0);
11107 Info.offset = 0;
11108 Info.align = Align(4);
11109 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
11110 MachineMemOperand::MOVolatile;
11111 Infos.push_back(Elt: Info);
11112 return;
11113 // TODO: Add more Intrinsics later.
11114 }
11115 }
11116}
11117
11118// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
11119// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
11120// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
11121// regression, we need to implement it manually.
11122void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
11123 AtomicRMWInst::BinOp Op = AI->getOperation();
11124
11125 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
11126 Op == AtomicRMWInst::And) &&
11127 "Unable to expand");
11128 unsigned MinWordSize = 4;
11129
11130 IRBuilder<> Builder(AI);
11131 LLVMContext &Ctx = Builder.getContext();
11132 const DataLayout &DL = AI->getDataLayout();
11133 Type *ValueType = AI->getType();
11134 Type *WordType = Type::getIntNTy(C&: Ctx, N: MinWordSize * 8);
11135
11136 Value *Addr = AI->getPointerOperand();
11137 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
11138 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
11139
11140 Value *AlignedAddr = Builder.CreateIntrinsic(
11141 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
11142 Args: {Addr, ConstantInt::get(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))}, FMFSource: nullptr,
11143 Name: "AlignedAddr");
11144
11145 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
11146 Value *PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
11147 Value *ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
11148 ShiftAmt = Builder.CreateTrunc(V: ShiftAmt, DestTy: WordType, Name: "ShiftAmt");
11149 Value *Mask = Builder.CreateShl(
11150 LHS: ConstantInt::get(Ty: WordType,
11151 V: (1 << (DL.getTypeStoreSize(Ty: ValueType) * 8)) - 1),
11152 RHS: ShiftAmt, Name: "Mask");
11153 Value *Inv_Mask = Builder.CreateNot(V: Mask, Name: "Inv_Mask");
11154 Value *ValOperand_Shifted =
11155 Builder.CreateShl(LHS: Builder.CreateZExt(V: AI->getValOperand(), DestTy: WordType),
11156 RHS: ShiftAmt, Name: "ValOperand_Shifted");
11157 Value *NewOperand;
11158 if (Op == AtomicRMWInst::And)
11159 NewOperand = Builder.CreateOr(LHS: ValOperand_Shifted, RHS: Inv_Mask, Name: "AndOperand");
11160 else
11161 NewOperand = ValOperand_Shifted;
11162
11163 AtomicRMWInst *NewAI =
11164 Builder.CreateAtomicRMW(Op, Ptr: AlignedAddr, Val: NewOperand, Align: Align(MinWordSize),
11165 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
11166
11167 Value *Shift = Builder.CreateLShr(LHS: NewAI, RHS: ShiftAmt, Name: "shifted");
11168 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: ValueType, Name: "extracted");
11169 Value *FinalOldResult = Builder.CreateBitCast(V: Trunc, DestTy: ValueType);
11170 AI->replaceAllUsesWith(V: FinalOldResult);
11171 AI->eraseFromParent();
11172}
11173
11174TargetLowering::AtomicExpansionKind
11175LoongArchTargetLowering::shouldExpandAtomicRMWInIR(
11176 const AtomicRMWInst *AI) const {
11177 // TODO: Add more AtomicRMWInst that needs to be extended.
11178
11179 // Since floating-point operation requires a non-trivial set of data
11180 // operations, use CmpXChg to expand.
11181 if (AI->isFloatingPointOperation() ||
11182 AI->getOperation() == AtomicRMWInst::UIncWrap ||
11183 AI->getOperation() == AtomicRMWInst::UDecWrap ||
11184 AI->getOperation() == AtomicRMWInst::USubCond ||
11185 AI->getOperation() == AtomicRMWInst::USubSat)
11186 return AtomicExpansionKind::CmpXChg;
11187
11188 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
11189 (AI->getOperation() == AtomicRMWInst::Xchg ||
11190 AI->getOperation() == AtomicRMWInst::Add ||
11191 AI->getOperation() == AtomicRMWInst::Sub)) {
11192 return AtomicExpansionKind::None;
11193 }
11194
11195 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11196 if (Subtarget.hasLAMCAS()) {
11197 if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
11198 AI->getOperation() == AtomicRMWInst::Or ||
11199 AI->getOperation() == AtomicRMWInst::Xor))
11200 return AtomicExpansionKind::CustomExpand;
11201 if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
11202 return AtomicExpansionKind::CmpXChg;
11203 }
11204
11205 if (Size == 8 || Size == 16)
11206 return AtomicExpansionKind::MaskedIntrinsic;
11207 return AtomicExpansionKind::None;
11208}
11209
11210static Intrinsic::ID
11211getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
11212 AtomicRMWInst::BinOp BinOp) {
11213 if (GRLen == 64) {
11214 switch (BinOp) {
11215 default:
11216 llvm_unreachable("Unexpected AtomicRMW BinOp");
11217 case AtomicRMWInst::Xchg:
11218 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
11219 case AtomicRMWInst::Add:
11220 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
11221 case AtomicRMWInst::Sub:
11222 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
11223 case AtomicRMWInst::Nand:
11224 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
11225 case AtomicRMWInst::UMax:
11226 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
11227 case AtomicRMWInst::UMin:
11228 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
11229 case AtomicRMWInst::Max:
11230 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
11231 case AtomicRMWInst::Min:
11232 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
11233 // TODO: support other AtomicRMWInst.
11234 }
11235 }
11236
11237 if (GRLen == 32) {
11238 switch (BinOp) {
11239 default:
11240 llvm_unreachable("Unexpected AtomicRMW BinOp");
11241 case AtomicRMWInst::Xchg:
11242 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
11243 case AtomicRMWInst::Add:
11244 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
11245 case AtomicRMWInst::Sub:
11246 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
11247 case AtomicRMWInst::Nand:
11248 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
11249 case AtomicRMWInst::UMax:
11250 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
11251 case AtomicRMWInst::UMin:
11252 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
11253 case AtomicRMWInst::Max:
11254 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
11255 case AtomicRMWInst::Min:
11256 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
11257 // TODO: support other AtomicRMWInst.
11258 }
11259 }
11260
11261 llvm_unreachable("Unexpected GRLen\n");
11262}
11263
11264TargetLowering::AtomicExpansionKind
11265LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
11266 const AtomicCmpXchgInst *CI) const {
11267
11268 if (Subtarget.hasLAMCAS())
11269 return AtomicExpansionKind::None;
11270
11271 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
11272 if (Size == 8 || Size == 16)
11273 return AtomicExpansionKind::MaskedIntrinsic;
11274 return AtomicExpansionKind::None;
11275}
11276
11277Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
11278 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
11279 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
11280 unsigned GRLen = Subtarget.getGRLen();
11281 AtomicOrdering FailOrd = CI->getFailureOrdering();
11282 Value *FailureOrdering =
11283 Builder.getIntN(N: Subtarget.getGRLen(), C: static_cast<uint64_t>(FailOrd));
11284 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
11285 if (GRLen == 64) {
11286 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
11287 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
11288 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
11289 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11290 }
11291 Type *Tys[] = {AlignedAddr->getType()};
11292 Value *Result = Builder.CreateIntrinsic(
11293 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
11294 if (GRLen == 64)
11295 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11296 return Result;
11297}
11298
11299Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic(
11300 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
11301 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
11302 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
11303 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
11304 // mask, as this produces better code than the LL/SC loop emitted by
11305 // int_loongarch_masked_atomicrmw_xchg.
11306 if (AI->getOperation() == AtomicRMWInst::Xchg &&
11307 isa<ConstantInt>(Val: AI->getValOperand())) {
11308 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
11309 if (CVal->isZero())
11310 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
11311 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
11312 Align: AI->getAlign(), Ordering: Ord);
11313 if (CVal->isMinusOne())
11314 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
11315 Align: AI->getAlign(), Ordering: Ord);
11316 }
11317
11318 unsigned GRLen = Subtarget.getGRLen();
11319 Value *Ordering =
11320 Builder.getIntN(N: GRLen, C: static_cast<uint64_t>(AI->getOrdering()));
11321 Type *Tys[] = {AlignedAddr->getType()};
11322 Function *LlwOpScwLoop = Intrinsic::getOrInsertDeclaration(
11323 M: AI->getModule(),
11324 id: getIntrinsicForMaskedAtomicRMWBinOp(GRLen, BinOp: AI->getOperation()), OverloadTys: Tys);
11325
11326 if (GRLen == 64) {
11327 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
11328 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11329 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
11330 }
11331
11332 Value *Result;
11333
11334 // Must pass the shift amount needed to sign extend the loaded value prior
11335 // to performing a signed comparison for min/max. ShiftAmt is the number of
11336 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which
11337 // is the number of bits to left+right shift the value in order to
11338 // sign-extend.
11339 if (AI->getOperation() == AtomicRMWInst::Min ||
11340 AI->getOperation() == AtomicRMWInst::Max) {
11341 const DataLayout &DL = AI->getDataLayout();
11342 unsigned ValWidth =
11343 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
11344 Value *SextShamt =
11345 Builder.CreateSub(LHS: Builder.getIntN(N: GRLen, C: GRLen - ValWidth), RHS: ShiftAmt);
11346 Result = Builder.CreateCall(Callee: LlwOpScwLoop,
11347 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
11348 } else {
11349 Result =
11350 Builder.CreateCall(Callee: LlwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
11351 }
11352
11353 if (GRLen == 64)
11354 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11355 return Result;
11356}
11357
11358bool LoongArchTargetLowering::isFMAFasterThanFMulAndFAdd(
11359 const MachineFunction &MF, EVT VT) const {
11360 VT = VT.getScalarType();
11361
11362 if (!VT.isSimple())
11363 return false;
11364
11365 switch (VT.getSimpleVT().SimpleTy) {
11366 case MVT::f32:
11367 case MVT::f64:
11368 return true;
11369 default:
11370 break;
11371 }
11372
11373 return false;
11374}
11375
11376Register LoongArchTargetLowering::getExceptionPointerRegister(
11377 ExceptionHandling EH, const Constant *PersonalityFn) const {
11378 return LoongArch::R4;
11379}
11380
11381Register LoongArchTargetLowering::getExceptionSelectorRegister(
11382 ExceptionHandling EH, const Constant *PersonalityFn) const {
11383 return LoongArch::R5;
11384}
11385
11386//===----------------------------------------------------------------------===//
11387// Target Optimization Hooks
11388//===----------------------------------------------------------------------===//
11389
11390static int getEstimateRefinementSteps(EVT VT,
11391 const LoongArchSubtarget &Subtarget) {
11392 // Feature FRECIPE instrucions relative accuracy is 2^-14.
11393 // IEEE float has 23 digits and double has 52 digits.
11394 int RefinementSteps = VT.getScalarType() == MVT::f64 ? 2 : 1;
11395 return RefinementSteps;
11396}
11397
11398static bool
11399isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget) {
11400 assert(Subtarget.hasFrecipe() &&
11401 "Reciprocal estimate queried on unsupported target");
11402
11403 if (!VT.isSimple())
11404 return false;
11405
11406 switch (VT.getSimpleVT().SimpleTy) {
11407 case MVT::f32:
11408 // f32 is the base type for reciprocal estimate instructions.
11409 return true;
11410
11411 case MVT::f64:
11412 return Subtarget.hasBasicD();
11413
11414 case MVT::v4f32:
11415 case MVT::v2f64:
11416 return Subtarget.hasExtLSX();
11417
11418 case MVT::v8f32:
11419 case MVT::v4f64:
11420 return Subtarget.hasExtLASX();
11421
11422 default:
11423 return false;
11424 }
11425}
11426
11427SDValue LoongArchTargetLowering::getSqrtEstimate(SDValue Operand,
11428 SelectionDAG &DAG, int Enabled,
11429 int &RefinementSteps,
11430 bool &UseOneConstNR,
11431 bool Reciprocal) const {
11432 assert(Enabled != ReciprocalEstimate::Disabled &&
11433 "Enabled should never be Disabled here");
11434
11435 if (!Subtarget.hasFrecipe())
11436 return SDValue();
11437
11438 SDLoc DL(Operand);
11439 EVT VT = Operand.getValueType();
11440
11441 // Check supported types.
11442 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11443 return SDValue();
11444
11445 // Handle refinement steps.
11446 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11447 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11448
11449 // LoongArch only has FRSQRTE which is 1.0 / sqrt(x).
11450 UseOneConstNR = false;
11451 SDValue Rsqrt = DAG.getNode(Opcode: LoongArchISD::FRSQRTE, DL, VT, Operand);
11452
11453 // If the caller wants 1.0 / sqrt(x), or if further refinement steps
11454 // are needed (which rely on the reciprocal form), return the raw reciprocal
11455 // estimate.
11456 if (Reciprocal || RefinementSteps > 0)
11457 return Rsqrt;
11458
11459 // Otherwise, return sqrt(x) by multiplying with the operand.
11460 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Operand, N2: Rsqrt);
11461}
11462
11463SDValue LoongArchTargetLowering::getRecipEstimate(SDValue Operand,
11464 SelectionDAG &DAG,
11465 int Enabled,
11466 int &RefinementSteps) const {
11467 assert(Enabled != ReciprocalEstimate::Disabled &&
11468 "Enabled should never be Disabled here");
11469
11470 if (!Subtarget.hasFrecipe())
11471 return SDValue();
11472
11473 SDLoc DL(Operand);
11474 EVT VT = Operand.getValueType();
11475
11476 // Check supported types.
11477 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11478 return SDValue();
11479
11480 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11481 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11482
11483 // FRECIPE computes 1.0 / x.
11484 return DAG.getNode(Opcode: LoongArchISD::FRECIPE, DL, VT, Operand);
11485}
11486
11487//===----------------------------------------------------------------------===//
11488// LoongArch Inline Assembly Support
11489//===----------------------------------------------------------------------===//
11490
11491LoongArchTargetLowering::ConstraintType
11492LoongArchTargetLowering::getConstraintType(StringRef Constraint) const {
11493 // LoongArch specific constraints in GCC: config/loongarch/constraints.md
11494 //
11495 // 'f': A floating-point register (if available).
11496 // 'k': A memory operand whose address is formed by a base register and
11497 // (optionally scaled) index register.
11498 // 'l': A signed 16-bit constant.
11499 // 'm': A memory operand whose address is formed by a base register and
11500 // offset that is suitable for use in instructions with the same
11501 // addressing mode as st.w and ld.w.
11502 // 'q': A general-purpose register except for $r0 and $r1 (for the csrxchg
11503 // instruction)
11504 // 'I': A signed 12-bit constant (for arithmetic instructions).
11505 // 'J': Integer zero.
11506 // 'K': An unsigned 12-bit constant (for logic instructions).
11507 // "ZB": An address that is held in a general-purpose register. The offset is
11508 // zero.
11509 // "ZC": A memory operand whose address is formed by a base register and
11510 // offset that is suitable for use in instructions with the same
11511 // addressing mode as ll.w and sc.w.
11512 if (Constraint.size() == 1) {
11513 switch (Constraint[0]) {
11514 default:
11515 break;
11516 case 'f':
11517 case 'q':
11518 return C_RegisterClass;
11519 case 'l':
11520 case 'I':
11521 case 'J':
11522 case 'K':
11523 return C_Immediate;
11524 case 'k':
11525 return C_Memory;
11526 }
11527 }
11528
11529 if (Constraint == "ZC" || Constraint == "ZB")
11530 return C_Memory;
11531
11532 // 'm' is handled here.
11533 return TargetLowering::getConstraintType(Constraint);
11534}
11535
11536InlineAsm::ConstraintCode LoongArchTargetLowering::getInlineAsmMemConstraint(
11537 StringRef ConstraintCode) const {
11538 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
11539 .Case(S: "k", Value: InlineAsm::ConstraintCode::k)
11540 .Case(S: "ZB", Value: InlineAsm::ConstraintCode::ZB)
11541 .Case(S: "ZC", Value: InlineAsm::ConstraintCode::ZC)
11542 .Default(Value: TargetLowering::getInlineAsmMemConstraint(ConstraintCode));
11543}
11544
11545std::pair<unsigned, const TargetRegisterClass *>
11546LoongArchTargetLowering::getRegForInlineAsmConstraint(
11547 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
11548 // First, see if this is a constraint that directly corresponds to a LoongArch
11549 // register class.
11550 if (Constraint.size() == 1) {
11551 switch (Constraint[0]) {
11552 case 'r':
11553 // TODO: Support fixed vectors up to GRLen?
11554 if (VT.isVector())
11555 break;
11556 return std::make_pair(x: 0U, y: &LoongArch::GPRRegClass);
11557 case 'q':
11558 return std::make_pair(x: 0U, y: &LoongArch::GPRNoR0R1RegClass);
11559 case 'f':
11560 if (Subtarget.hasBasicF() && VT == MVT::f32)
11561 return std::make_pair(x: 0U, y: &LoongArch::FPR32RegClass);
11562 if (Subtarget.hasBasicD() && VT == MVT::f64)
11563 return std::make_pair(x: 0U, y: &LoongArch::FPR64RegClass);
11564 if (Subtarget.hasExtLSX() &&
11565 TRI->isTypeLegalForClass(RC: LoongArch::LSX128RegClass, T: VT))
11566 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11567 if (Subtarget.hasExtLSX() && VT == MVT::i128)
11568 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11569 if (Subtarget.hasExtLASX() &&
11570 TRI->isTypeLegalForClass(RC: LoongArch::LASX256RegClass, T: VT))
11571 return std::make_pair(x: 0U, y: &LoongArch::LASX256RegClass);
11572 break;
11573 default:
11574 break;
11575 }
11576 }
11577
11578 // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen
11579 // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm
11580 // constraints while the official register name is prefixed with a '$'. So we
11581 // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.)
11582 // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is
11583 // case insensitive, so no need to convert the constraint to upper case here.
11584 //
11585 // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly
11586 // decode the usage of register name aliases into their official names. And
11587 // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use
11588 // official register names.
11589 if (Constraint.starts_with(Prefix: "{$r") || Constraint.starts_with(Prefix: "{$f") ||
11590 Constraint.starts_with(Prefix: "{$vr") || Constraint.starts_with(Prefix: "{$xr")) {
11591 bool IsFP = Constraint[2] == 'f';
11592 std::pair<StringRef, StringRef> Temp = Constraint.split(Separator: '$');
11593 std::pair<unsigned, const TargetRegisterClass *> R;
11594 R = TargetLowering::getRegForInlineAsmConstraint(
11595 TRI, Constraint: join_items(Separator: "", Items&: Temp.first, Items&: Temp.second), VT);
11596 // Match those names to the widest floating point register type available.
11597 if (IsFP) {
11598 unsigned RegNo = R.first;
11599 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
11600 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
11601 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
11602 return std::make_pair(x&: DReg, y: &LoongArch::FPR64RegClass);
11603 }
11604 }
11605 }
11606 return R;
11607 }
11608
11609 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
11610}
11611
11612void LoongArchTargetLowering::LowerAsmOperandForConstraint(
11613 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
11614 SelectionDAG &DAG) const {
11615 // Currently only support length 1 constraints.
11616 if (Constraint.size() == 1) {
11617 switch (Constraint[0]) {
11618 case 'l':
11619 // Validate & create a 16-bit signed immediate operand.
11620 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11621 uint64_t CVal = C->getSExtValue();
11622 if (isInt<16>(x: CVal))
11623 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11624 VT: Subtarget.getGRLenVT()));
11625 }
11626 return;
11627 case 'I':
11628 // Validate & create a 12-bit signed immediate operand.
11629 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11630 uint64_t CVal = C->getSExtValue();
11631 if (isInt<12>(x: CVal))
11632 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11633 VT: Subtarget.getGRLenVT()));
11634 }
11635 return;
11636 case 'J':
11637 // Validate & create an integer zero operand.
11638 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
11639 if (C->getZExtValue() == 0)
11640 Ops.push_back(
11641 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11642 return;
11643 case 'K':
11644 // Validate & create a 12-bit unsigned immediate operand.
11645 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11646 uint64_t CVal = C->getZExtValue();
11647 if (isUInt<12>(x: CVal))
11648 Ops.push_back(
11649 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11650 }
11651 return;
11652 default:
11653 break;
11654 }
11655 }
11656 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
11657}
11658
11659#define GET_REGISTER_MATCHER
11660#include "LoongArchGenAsmMatcher.inc"
11661
11662Register
11663LoongArchTargetLowering::getRegisterByName(const char *RegName, LLT VT,
11664 const MachineFunction &MF) const {
11665 std::pair<StringRef, StringRef> Name = StringRef(RegName).split(Separator: '$');
11666 std::string NewRegName = Name.second.str();
11667 Register Reg = MatchRegisterAltName(Name: NewRegName);
11668 if (!Reg)
11669 Reg = MatchRegisterName(Name: NewRegName);
11670 if (!Reg)
11671 return Reg;
11672 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11673 if (!ReservedRegs.test(Idx: Reg))
11674 report_fatal_error(reason: Twine("Trying to obtain non-reserved register \"" +
11675 StringRef(RegName) + "\"."));
11676 return Reg;
11677}
11678
11679bool LoongArchTargetLowering::decomposeMulByConstant(LLVMContext &Context,
11680 EVT VT, SDValue C) const {
11681 // TODO: Support vectors.
11682 if (!VT.isScalarInteger())
11683 return false;
11684
11685 // Omit the optimization if the data size exceeds GRLen.
11686 if (VT.getSizeInBits() > Subtarget.getGRLen())
11687 return false;
11688
11689 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
11690 const APInt &Imm = ConstNode->getAPIntValue();
11691 // Break MUL into (SLLI + ADD/SUB) or ALSL.
11692 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
11693 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
11694 return true;
11695 // Break MUL into (ALSL x, (SLLI x, imm0), imm1).
11696 if (ConstNode->hasOneUse() &&
11697 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
11698 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
11699 return true;
11700 // Break (MUL x, imm) into (ADD (SLLI x, s0), (SLLI x, s1)),
11701 // in which the immediate has two set bits. Or Break (MUL x, imm)
11702 // into (SUB (SLLI x, s0), (SLLI x, s1)), in which the immediate
11703 // equals to (1 << s0) - (1 << s1).
11704 if (ConstNode->hasOneUse() && !(Imm.sge(RHS: -2048) && Imm.sle(RHS: 4095))) {
11705 unsigned Shifts = Imm.countr_zero();
11706 // Reject immediates which can be composed via a single LUI.
11707 if (Shifts >= 12)
11708 return false;
11709 // Reject multiplications can be optimized to
11710 // (SLLI (ALSL x, x, 1/2/3/4), s).
11711 APInt ImmPop = Imm.ashr(ShiftAmt: Shifts);
11712 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
11713 return false;
11714 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`,
11715 // since it needs one more instruction than other 3 cases.
11716 APInt ImmSmall = APInt(Imm.getBitWidth(), 1ULL << Shifts, true);
11717 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
11718 (ImmSmall - Imm).isPowerOf2())
11719 return true;
11720 }
11721 }
11722
11723 return false;
11724}
11725
11726bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
11727 const AddrMode &AM,
11728 Type *Ty, unsigned AS,
11729 Instruction *I) const {
11730 // LoongArch has four basic addressing modes:
11731 // 1. reg
11732 // 2. reg + 12-bit signed offset
11733 // 3. reg + 14-bit signed offset left-shifted by 2
11734 // 4. reg1 + reg2
11735 // TODO: Add more checks after support vector extension.
11736
11737 // No global is ever allowed as a base.
11738 if (AM.BaseGV)
11739 return false;
11740
11741 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
11742 // with `UAL` feature.
11743 if (!isInt<12>(x: AM.BaseOffs) &&
11744 !(isShiftedInt<14, 2>(x: AM.BaseOffs) && Subtarget.hasUAL()))
11745 return false;
11746
11747 switch (AM.Scale) {
11748 case 0:
11749 // "r+i" or just "i", depending on HasBaseReg.
11750 break;
11751 case 1:
11752 // "r+r+i" is not allowed.
11753 if (AM.HasBaseReg && AM.BaseOffs)
11754 return false;
11755 // Otherwise we have "r+r" or "r+i".
11756 break;
11757 case 2:
11758 // "2*r+r" or "2*r+i" is not allowed.
11759 if (AM.HasBaseReg || AM.BaseOffs)
11760 return false;
11761 // Allow "2*r" as "r+r".
11762 break;
11763 default:
11764 return false;
11765 }
11766
11767 return true;
11768}
11769
11770bool LoongArchTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
11771 return isInt<12>(x: Imm);
11772}
11773
11774bool LoongArchTargetLowering::isLegalAddImmediate(int64_t Imm) const {
11775 return isInt<12>(x: Imm);
11776}
11777
11778bool LoongArchTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11779 // Zexts are free if they can be combined with a load.
11780 // Don't advertise i32->i64 zextload as being free for LA64. It interacts
11781 // poorly with type legalization of compares preferring sext.
11782 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
11783 EVT MemVT = LD->getMemoryVT();
11784 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11785 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
11786 LD->getExtensionType() == ISD::ZEXTLOAD))
11787 return true;
11788 }
11789
11790 return TargetLowering::isZExtFree(Val, VT2);
11791}
11792
11793bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT,
11794 EVT DstVT) const {
11795 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
11796}
11797
11798bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const {
11799 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
11800}
11801
11802bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
11803 // TODO: Support vectors.
11804 if (Y.getValueType().isVector())
11805 return false;
11806
11807 return !isa<ConstantSDNode>(Val: Y);
11808}
11809
11810ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
11811 // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
11812 return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
11813}
11814
11815bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
11816 Type *Ty, bool IsSigned) const {
11817 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
11818 return true;
11819
11820 return IsSigned;
11821}
11822
11823bool LoongArchTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
11824 // Return false to suppress the unnecessary extensions if the LibCall
11825 // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
11826 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
11827 Type.getSizeInBits() < Subtarget.getGRLen()))
11828 return false;
11829 return true;
11830}
11831
11832// memcpy, and other memory intrinsics, typically tries to use wider load/store
11833// if the source/dest is aligned and the copy size is large enough. We therefore
11834// want to align such objects passed to memory intrinsics.
11835bool LoongArchTargetLowering::shouldAlignPointerArgs(CallInst *CI,
11836 unsigned &MinSize,
11837 Align &PrefAlign) const {
11838 if (!isa<MemIntrinsic>(Val: CI))
11839 return false;
11840
11841 if (Subtarget.is64Bit()) {
11842 MinSize = 8;
11843 PrefAlign = Align(8);
11844 } else {
11845 MinSize = 4;
11846 PrefAlign = Align(4);
11847 }
11848
11849 return true;
11850}
11851
11852TargetLoweringBase::LegalizeTypeAction
11853LoongArchTargetLowering::getPreferredVectorAction(MVT VT) const {
11854 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
11855 VT.getVectorElementType() != MVT::i1)
11856 return TypeWidenVector;
11857
11858 return TargetLoweringBase::getPreferredVectorAction(VT);
11859}
11860
11861bool LoongArchTargetLowering::splitValueIntoRegisterParts(
11862 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
11863 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
11864 bool IsABIRegCopy = CC.has_value();
11865 EVT ValueVT = Val.getValueType();
11866
11867 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11868 PartVT == MVT::f32) {
11869 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
11870 // nan, and cast to f32.
11871 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
11872 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
11873 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
11874 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
11875 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Val);
11876 Parts[0] = Val;
11877 return true;
11878 }
11879
11880 return false;
11881}
11882
11883SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
11884 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
11885 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
11886 bool IsABIRegCopy = CC.has_value();
11887
11888 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11889 PartVT == MVT::f32) {
11890 SDValue Val = Parts[0];
11891
11892 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
11893 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
11894 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
11895 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
11896 return Val;
11897 }
11898
11899 return SDValue();
11900}
11901
11902MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
11903 CallingConv::ID CC,
11904 EVT VT) const {
11905 // Use f32 to pass f16.
11906 if (VT == MVT::f16 && Subtarget.hasBasicF())
11907 return MVT::f32;
11908
11909 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
11910}
11911
11912unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
11913 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
11914 // Use f32 to pass f16.
11915 if (VT == MVT::f16 && Subtarget.hasBasicF())
11916 return 1;
11917
11918 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
11919}
11920
11921void LoongArchTargetLowering::computeKnownBitsForTargetNode(
11922 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
11923 const SelectionDAG &DAG, unsigned Depth) const {
11924 unsigned Opc = Op.getOpcode();
11925 Known.resetAll();
11926 switch (Opc) {
11927 default:
11928 break;
11929 case LoongArchISD::VANYNONZERO:
11930 case LoongArchISD::VALLZERO: {
11931 // MOVCF2GR zero-extend the i1 cond to GPR.
11932 Known.Zero.setBitsFrom(1);
11933 break;
11934 }
11935 case LoongArchISD::VPICK_ZEXT_ELT: {
11936 assert(isa<VTSDNode>(Op->getOperand(2)) && "Unexpected operand!");
11937 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 2))->getVT();
11938 unsigned VTBits = VT.getScalarSizeInBits();
11939 assert(Known.getBitWidth() >= VTBits && "Unexpected width!");
11940 Known.Zero.setBitsFrom(VTBits);
11941 break;
11942 }
11943 }
11944}
11945
11946bool LoongArchTargetLowering::SimplifyDemandedBitsForTargetNode(
11947 SDValue Op, const APInt &OriginalDemandedBits,
11948 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
11949 unsigned Depth) const {
11950 EVT VT = Op.getValueType();
11951 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
11952 unsigned Opc = Op.getOpcode();
11953 switch (Opc) {
11954 default:
11955 break;
11956 case LoongArchISD::CRC_W_B_W:
11957 case LoongArchISD::CRC_W_H_W:
11958 case LoongArchISD::CRCC_W_B_W:
11959 case LoongArchISD::CRCC_W_H_W: {
11960 KnownBits KnownSrc;
11961 APInt DemandedSrcBits =
11962 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: (Opc == LoongArchISD::CRC_W_B_W ||
11963 Opc == LoongArchISD::CRCC_W_B_W)
11964 ? 8
11965 : 16);
11966 return SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: DemandedSrcBits,
11967 DemandedElts: OriginalDemandedElts, Known&: KnownSrc, TLO, Depth: Depth + 1);
11968 }
11969 case LoongArchISD::VMSKLTZ:
11970 case LoongArchISD::XVMSKLTZ: {
11971 SDValue Src = Op.getOperand(i: 0);
11972 MVT SrcVT = Src.getSimpleValueType();
11973 unsigned SrcBits = SrcVT.getScalarSizeInBits();
11974 unsigned NumElts = SrcVT.getVectorNumElements();
11975
11976 // If we don't need the sign bits at all just return zero.
11977 if (OriginalDemandedBits.countr_zero() >= NumElts)
11978 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
11979
11980 // Only demand the vector elements of the sign bits we need.
11981 APInt KnownUndef, KnownZero;
11982 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(width: NumElts);
11983 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedElts, KnownUndef, KnownZero,
11984 TLO, Depth: Depth + 1))
11985 return true;
11986
11987 Known.Zero = KnownZero.zext(width: BitWidth);
11988 Known.Zero.setHighBits(BitWidth - NumElts);
11989
11990 // [X]VMSKLTZ only uses the MSB from each vector element.
11991 KnownBits KnownSrc;
11992 APInt DemandedSrcBits = APInt::getSignMask(BitWidth: SrcBits);
11993 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, Known&: KnownSrc, TLO,
11994 Depth: Depth + 1))
11995 return true;
11996
11997 if (KnownSrc.One[SrcBits - 1])
11998 Known.One.setLowBits(NumElts);
11999 else if (KnownSrc.Zero[SrcBits - 1])
12000 Known.Zero.setLowBits(NumElts);
12001
12002 // Attempt to avoid multi-use ops if we don't need anything from it.
12003 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
12004 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
12005 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT, Operand: NewSrc));
12006 return false;
12007 }
12008 }
12009
12010 return TargetLowering::SimplifyDemandedBitsForTargetNode(
12011 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
12012}
12013
12014bool LoongArchTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
12015 unsigned Opc = VecOp.getOpcode();
12016
12017 // Assume target opcodes can't be scalarized.
12018 // TODO - do we have any exceptions?
12019 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
12020 return false;
12021
12022 // If the vector op is not supported, try to convert to scalar.
12023 EVT VecVT = VecOp.getValueType();
12024 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
12025 return true;
12026
12027 // If the vector op is supported, but the scalar op is not, the transform may
12028 // not be worthwhile.
12029 EVT ScalarVT = VecVT.getScalarType();
12030 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
12031}
12032
12033TargetLowering::ExtractSubvectorCost
12034LoongArchTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
12035 unsigned Index) const {
12036 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
12037 return ExtractSubvectorCost::Expensive;
12038
12039 // Extract a 128-bit subvector from index 0 of a 256-bit vector is free.
12040 if (Index == 0)
12041 return ExtractSubvectorCost::Free;
12042 return ExtractSubvectorCost::Expensive;
12043}
12044
12045bool LoongArchTargetLowering::isExtractVecEltCheap(EVT VT,
12046 unsigned Index) const {
12047 EVT EltVT = VT.getScalarType();
12048
12049 // Extract a scalar FP value from index 0 of a vector is free.
12050 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
12051}
12052
12053bool LoongArchTargetLowering::hasInlineStackProbe(
12054 const MachineFunction &MF) const {
12055
12056 // If the function specifically requests inline stack probes, emit them.
12057 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
12058 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
12059 "inline-asm";
12060
12061 return false;
12062}
12063
12064unsigned LoongArchTargetLowering::getStackProbeSize(const MachineFunction &MF,
12065 Align StackAlign) const {
12066 // The default stack probe size is 4096 if the function has no
12067 // stack-probe-size attribute.
12068 const Function &Fn = MF.getFunction();
12069 unsigned StackProbeSize =
12070 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
12071 // Round down to the stack alignment.
12072 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
12073 return StackProbeSize ? StackProbeSize : StackAlign.value();
12074}
12075
12076SDValue
12077LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
12078 SelectionDAG &DAG) const {
12079 MachineFunction &MF = DAG.getMachineFunction();
12080 if (!hasInlineStackProbe(MF))
12081 return SDValue();
12082
12083 const MVT GRLenVT = Subtarget.getGRLenVT();
12084 // Get the inputs.
12085 SDValue Chain = Op.getOperand(i: 0);
12086 SDValue Size = Op.getOperand(i: 1);
12087
12088 const MaybeAlign Align =
12089 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
12090 const SDLoc dl(Op);
12091 const EVT VT = Op.getValueType();
12092
12093 // Construct the new SP value in a GPR.
12094 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: LoongArch::R3, VT: GRLenVT);
12095 Chain = SP.getValue(R: 1);
12096 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: GRLenVT, N1: SP, N2: Size);
12097 if (Align)
12098 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
12099 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
12100
12101 // Set the real SP to the new value with a probing loop.
12102 Chain = DAG.getNode(Opcode: LoongArchISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
12103 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
12104}
12105
12106MachineBasicBlock *
12107LoongArchTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
12108 MachineBasicBlock *MBB) const {
12109 MachineFunction &MF = *MBB->getParent();
12110 MachineBasicBlock::iterator MBBI = MI.getIterator();
12111 DebugLoc DL = MBB->findDebugLoc(MBBI);
12112 const Register TargetReg = MI.getOperand(i: 0).getReg();
12113
12114 const LoongArchInstrInfo *TII = Subtarget.getInstrInfo();
12115 const bool IsLA64 = Subtarget.is64Bit();
12116 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
12117 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
12118 const uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
12119
12120 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
12121 MachineBasicBlock *const LoopTestMBB =
12122 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12123 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
12124 MachineBasicBlock *const ExitMBB =
12125 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12126 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
12127 const Register SPReg = LoongArch::R3;
12128 const Register ScratchReg =
12129 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
12130
12131 // ScratchReg = ProbeSize
12132 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
12133
12134 // LoopTest:
12135 // sub.{w/d} $sp, $sp, ScratchReg
12136 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12137 MCID: TII->get(Opcode: IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W), DestReg: SPReg)
12138 .addReg(RegNo: SPReg)
12139 .addReg(RegNo: ScratchReg);
12140
12141 // st.{w/d} $zero, $sp, 0
12142 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12143 MCID: TII->get(Opcode: IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
12144 .addReg(RegNo: LoongArch::R0)
12145 .addReg(RegNo: SPReg)
12146 .addImm(Val: 0);
12147
12148 // bltu TargetReg, $sp, LoopTest
12149 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::BLTU))
12150 .addReg(RegNo: TargetReg)
12151 .addReg(RegNo: SPReg)
12152 .addMBB(MBB: LoopTestMBB);
12153
12154 // move $sp, TargetReg
12155 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::OR), DestReg: SPReg)
12156 .addReg(RegNo: TargetReg)
12157 .addReg(RegNo: LoongArch::R0);
12158
12159 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
12160 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
12161
12162 LoopTestMBB->addSuccessor(Succ: ExitMBB);
12163 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
12164 MBB->addSuccessor(Succ: LoopTestMBB);
12165
12166 MI.eraseFromParent();
12167 MF.getInfo<LoongArchMachineFunctionInfo>()->setDynamicAllocation();
12168 return ExitMBB->begin()->getParent();
12169}
12170