1//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that LoongArch uses to lower LLVM code into
10// a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LoongArchISelLowering.h"
15#include "LoongArch.h"
16#include "LoongArchMachineFunctionInfo.h"
17#include "LoongArchRegisterInfo.h"
18#include "LoongArchSelectionDAGInfo.h"
19#include "LoongArchSubtarget.h"
20#include "MCTargetDesc/LoongArchBaseInfo.h"
21#include "MCTargetDesc/LoongArchMCTargetDesc.h"
22#include "MCTargetDesc/LoongArchMatInt.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/IntrinsicsLoongArch.h"
33#include "llvm/Support/CodeGen.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include <llvm/Analysis/VectorUtils.h>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "loongarch-isel-lowering"
43
44STATISTIC(NumTailCalls, "Number of tail calls");
45
46enum MaterializeFPImm {
47 NoMaterializeFPImm = 0,
48 MaterializeFPImm2Ins = 2,
49 MaterializeFPImm3Ins = 3,
50 MaterializeFPImm4Ins = 4,
51 MaterializeFPImm5Ins = 5,
52 MaterializeFPImm6Ins = 6
53};
54
55static cl::opt<MaterializeFPImm> MaterializeFPImmInsNum(
56 "loongarch-materialize-float-imm", cl::Hidden,
57 cl::desc("Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
60 cl::init(Val: MaterializeFPImm3Ins),
61 cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"),
62 clEnumValN(MaterializeFPImm2Ins, "2",
63 "Materialize FP immediate within 2 instructions"),
64 clEnumValN(MaterializeFPImm3Ins, "3",
65 "Materialize FP immediate within 3 instructions"),
66 clEnumValN(MaterializeFPImm4Ins, "4",
67 "Materialize FP immediate within 4 instructions"),
68 clEnumValN(MaterializeFPImm5Ins, "5",
69 "Materialize FP immediate within 5 instructions"),
70 clEnumValN(MaterializeFPImm6Ins, "6",
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
73
74static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
75 cl::desc("Trap on integer division by zero."),
76 cl::init(Val: false));
77
78LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
79 const LoongArchSubtarget &STI)
80 : TargetLowering(TM, STI), Subtarget(STI) {
81
82 MVT GRLenVT = Subtarget.getGRLenVT();
83
84 // Set up the register classes.
85
86 addRegisterClass(VT: GRLenVT, RC: &LoongArch::GPRRegClass);
87 if (Subtarget.hasBasicF())
88 addRegisterClass(VT: MVT::f32, RC: &LoongArch::FPR32RegClass);
89 if (Subtarget.hasBasicD())
90 addRegisterClass(VT: MVT::f64, RC: &LoongArch::FPR64RegClass);
91
92 static const MVT::SimpleValueType LSXVTs[] = {
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
94 static const MVT::SimpleValueType LASXVTs[] = {
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
96
97 if (Subtarget.hasExtLSX())
98 for (MVT VT : LSXVTs)
99 addRegisterClass(VT, RC: &LoongArch::LSX128RegClass);
100
101 if (Subtarget.hasExtLASX())
102 for (MVT VT : LASXVTs)
103 addRegisterClass(VT, RC: &LoongArch::LASX256RegClass);
104
105 // Set operations for LA32 and LA64.
106
107 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: GRLenVT,
108 MemVT: MVT::i1, Action: Promote);
109
110 setOperationAction(Op: ISD::SHL_PARTS, VT: GRLenVT, Action: Custom);
111 setOperationAction(Op: ISD::SRA_PARTS, VT: GRLenVT, Action: Custom);
112 setOperationAction(Op: ISD::SRL_PARTS, VT: GRLenVT, Action: Custom);
113 setOperationAction(Op: ISD::FP_TO_SINT, VT: GRLenVT, Action: Custom);
114 setOperationAction(Op: ISD::ROTL, VT: GRLenVT, Action: Expand);
115 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Expand);
116
117 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
118 ISD::JumpTable, ISD::GlobalTLSAddress},
119 VT: GRLenVT, Action: Custom);
120
121 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: GRLenVT, Action: Custom);
122
123 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: GRLenVT, Action: Custom);
124 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
125 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
126 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
127
128 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
129 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
130
131 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
132 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
133 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
134
135 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
136
137 // BITREV/REVB requires the 32S feature.
138 if (STI.has32S()) {
139 // Expand bitreverse.i16 with native-width bitrev and shift for now, before
140 // we get to know which of sll and revb.2h is faster.
141 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
142 setOperationAction(Op: ISD::BITREVERSE, VT: GRLenVT, Action: Legal);
143
144 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
145 // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
146 // and i32 could still be byte-swapped relatively cheaply.
147 setOperationAction(Op: ISD::BSWAP, VT: MVT::i16, Action: Custom);
148 } else {
149 setOperationAction(Op: ISD::BSWAP, VT: GRLenVT, Action: Expand);
150 setOperationAction(Op: ISD::CTTZ, VT: GRLenVT, Action: Expand);
151 setOperationAction(Op: ISD::CTLZ, VT: GRLenVT, Action: Expand);
152 setOperationAction(Op: ISD::ROTR, VT: GRLenVT, Action: Expand);
153 setOperationAction(Op: ISD::SELECT, VT: GRLenVT, Action: Custom);
154 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
155 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
156 }
157
158 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
159 setOperationAction(Op: ISD::BR_CC, VT: GRLenVT, Action: Expand);
160 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
161 setOperationAction(Op: ISD::SELECT_CC, VT: GRLenVT, Action: Expand);
162 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
163 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: GRLenVT, Action: Expand);
164
165 setOperationAction(Op: ISD::FP_TO_UINT, VT: GRLenVT, Action: Custom);
166 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Expand);
167
168 // Set operations for LA64 only.
169
170 if (Subtarget.is64Bit()) {
171 setOperationAction(Op: ISD::ADD, VT: MVT::i32, Action: Custom);
172 setOperationAction(Op: ISD::SUB, VT: MVT::i32, Action: Custom);
173 setOperationAction(Op: ISD::SHL, VT: MVT::i32, Action: Custom);
174 setOperationAction(Op: ISD::SRA, VT: MVT::i32, Action: Custom);
175 setOperationAction(Op: ISD::SRL, VT: MVT::i32, Action: Custom);
176 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
177 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
178 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Custom);
179 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Custom);
180 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
181 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Custom);
182 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
183 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i32, Action: Custom);
184 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i32, Action: Custom);
185 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
186 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
187 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i32, Action: Custom);
188
189 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Custom);
190 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Custom);
191 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: MVT::i32,
192 Action: Custom);
193 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
194 }
195
196 // Set operations for LA32 only.
197
198 if (!Subtarget.is64Bit()) {
199 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
200 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
201 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i64, Action: Custom);
202 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
203 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i64, Action: Custom);
204 if (Subtarget.hasBasicD())
205 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
206 }
207
208 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
209
210 static const ISD::CondCode FPCCToExpand[] = {
211 ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
212 ISD::SETGE, ISD::SETNE, ISD::SETGT};
213
214 // Set operations for 'F' feature.
215
216 if (Subtarget.hasBasicF()) {
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
218 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
220 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
221 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
222
223 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
224 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
225 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
226 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Legal);
227 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
228 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
229 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
230 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
231 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
232 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
233 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
234 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Legal);
235 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
236 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
237 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
238 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
239 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
240 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32,
241 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
242 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32,
243 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
244 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
245 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
246 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
247 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
248 setOperationAction(Op: ISD::GET_ROUNDING, VT: GRLenVT, Action: Custom);
249
250 if (Subtarget.is64Bit())
251 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
252
253 if (!Subtarget.hasBasicD()) {
254 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
255 if (Subtarget.is64Bit()) {
256 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
257 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
258 }
259 }
260 }
261
262 // Set operations for 'D' feature.
263
264 if (Subtarget.hasBasicD()) {
265 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
266 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
267 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
268 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
269 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
271 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
272
273 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
274 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
275 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
276 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
277 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
278 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Legal);
279 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
280 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
281 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
282 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
283 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
284 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Legal);
285 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
286 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
287 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
288 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
289 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
290 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
291 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64,
292 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
293 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
294 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
295 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
296
297 if (Subtarget.is64Bit())
298 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
299 }
300
301 // Set operations for 'LSX' feature.
302
303 if (Subtarget.hasExtLSX()) {
304 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
305 // Expand all truncating stores and extending loads.
306 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
307 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
308 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
309 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
310 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
311 }
312 // By default everything must be expanded. Then we will selectively turn
313 // on ones that can be effectively codegen'd.
314 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
315 setOperationAction(Op, VT, Action: Expand);
316 }
317
318 for (MVT VT : LSXVTs) {
319 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
320 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
321 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
322
323 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
324 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
325 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
326
327 setOperationAction(Op: ISD::SETCC, VT, Action: Legal);
328 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
329 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
330 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
331 }
332 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
333 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
334 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
335 Action: Legal);
336 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
337 VT, Action: Legal);
338 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
339 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
340 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
341 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
342 setCondCodeAction(
343 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
344 Action: Expand);
345 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
346 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
347 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
348 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
349 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
350 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
351 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
352 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
353 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
354 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
355 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
356 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
357 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
358 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
359 }
360 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
361 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
362 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
363 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
364 for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
365 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Action: Legal);
366 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
367 }
368 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Custom);
369 for (MVT VT : {MVT::v4f32, MVT::v2f64}) {
370 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
371 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
372 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
373 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
374 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
375 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
376 ISD::SETUGE, ISD::SETUGT},
377 VT, Action: Expand);
378 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
379 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
380 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
381 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
382 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
383 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
384 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
385 }
386 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Legal);
387 setOperationAction(Ops: ISD::FCEIL, VTs: {MVT::f32, MVT::f64}, Action: Legal);
388 setOperationAction(Ops: ISD::FFLOOR, VTs: {MVT::f32, MVT::f64}, Action: Legal);
389 setOperationAction(Ops: ISD::FTRUNC, VTs: {MVT::f32, MVT::f64}, Action: Legal);
390 setOperationAction(Ops: ISD::FROUNDEVEN, VTs: {MVT::f32, MVT::f64}, Action: Legal);
391
392 for (MVT VT :
393 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
394 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
395 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
396 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
397 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
398 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
399 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
400 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
401 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
402 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
403 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
404 }
405 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Custom);
406 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
407 // We want to legalize this to an f64 load rather than an i64 load.
408 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Custom);
409 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
410 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
411 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
412 MVT::v16i64})
413 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Custom);
414 }
415
416 // Set operations for 'LASX' feature.
417
418 if (Subtarget.hasExtLASX()) {
419 for (MVT VT : LASXVTs) {
420 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
421 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
422 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
423
424 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
425 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
426 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
427 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
428 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT, Action: Legal);
429
430 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
431 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
432 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
433 }
434 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
435 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
436 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
437 Action: Legal);
438 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
439 VT, Action: Legal);
440 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
441 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
442 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
443 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
444 setCondCodeAction(
445 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
446 Action: Expand);
447 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
448 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
449 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
450 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
451 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
452 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
453 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
454 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
455 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
456 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
457 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
458 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
459 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
460 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
461 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
462 }
463 for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
464 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
465 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
466 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
467 for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
468 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
469 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Legal);
470 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
471 }
472 for (MVT VT : {MVT::v8f32, MVT::v4f64}) {
473 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
474 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
475 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
476 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
477 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
478 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
479 ISD::SETUGE, ISD::SETUGT},
480 VT, Action: Expand);
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
482 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
483 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
484 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
485 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
486 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
487 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
488 }
489 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v4f32, Action: Custom);
490 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f64, Action: Custom);
491 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
492 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Legal);
493 setOperationAction(Op: ISD::ZERO_EXTEND, VT, Action: Legal);
494 setOperationAction(Op: ISD::ANY_EXTEND, VT, Action: Custom);
495 }
496 for (MVT VT :
497 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
498 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Legal);
499 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Legal);
500 }
501 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
502 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Legal);
503 }
504
505 // Set DAG combine for LA32 and LA64.
506 if (Subtarget.hasBasicF()) {
507 setTargetDAGCombine(ISD::SINT_TO_FP);
508 }
509
510 setTargetDAGCombine(ISD::AND);
511 setTargetDAGCombine(ISD::OR);
512 setTargetDAGCombine(ISD::SRL);
513 setTargetDAGCombine(ISD::SETCC);
514
515 // On targets with the 32S feature, `select` is expanded into
516 // maskeqz + masknez + or (3 instructions), which is more expensive than
517 // on most other architectures where a single cmov-like instruction
518 // suffices. Enable a combine that can turn
519 // select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
520 // select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
521 // for binop in {add, or, xor, sub}, replacing the 3-insn select (plus
522 // the original binop) with a single mask instruction plus the binop.
523 if (Subtarget.has32S())
524 setTargetDAGCombine(ISD::SELECT);
525
526 // Set DAG combine for 'LSX' feature.
527
528 if (Subtarget.hasExtLSX()) {
529 setTargetDAGCombine(ISD::ADD);
530 setTargetDAGCombine(ISD::SUB);
531 setTargetDAGCombine(ISD::SHL);
532 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
533 setTargetDAGCombine(ISD::BITCAST);
534 setTargetDAGCombine(ISD::VSELECT);
535 setTargetDAGCombine(ISD::FP_TO_SINT);
536 setTargetDAGCombine(ISD::FP_TO_UINT);
537 setTargetDAGCombine(ISD::UINT_TO_FP);
538 setTargetDAGCombine(ISD::ZERO_EXTEND);
539 setTargetDAGCombine(ISD::SIGN_EXTEND);
540 }
541
542 // Set DAG combine for 'LASX' feature.
543 if (Subtarget.hasExtLASX()) {
544 setTargetDAGCombine(ISD::ANY_EXTEND);
545 setTargetDAGCombine(ISD::CONCAT_VECTORS);
546 }
547
548 // Compute derived properties from the register classes.
549 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
550
551 setStackPointerRegisterToSaveRestore(LoongArch::R3);
552
553 setBooleanContents(ZeroOrOneBooleanContent);
554 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
555
556 setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());
557
558 setMinCmpXchgSizeInBits(32);
559
560 // Function alignments.
561 setMinFunctionAlignment(Align(4));
562 // Set preferred alignments.
563 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
564 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
565 setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
566
567 // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
568 if (Subtarget.hasLAMCAS())
569 setMinCmpXchgSizeInBits(8);
570
571 if (Subtarget.hasSCQ()) {
572 setMaxAtomicSizeInBitsSupported(128);
573 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i128, Action: Custom);
574 }
575
576 // Disable strict node mutation.
577 IsStrictFPEnabled = true;
578}
579
580bool LoongArchTargetLowering::isOffsetFoldingLegal(
581 const GlobalAddressSDNode *GA) const {
582 // In order to maximise the opportunity for common subexpression elimination,
583 // keep a separate ADD node for the global address offset instead of folding
584 // it in the global address node. Later peephole optimisations may choose to
585 // fold it back in when profitable.
586 return false;
587}
588
589SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
590 SelectionDAG &DAG) const {
591 switch (Op.getOpcode()) {
592 case ISD::ATOMIC_FENCE:
593 return lowerATOMIC_FENCE(Op, DAG);
594 case ISD::EH_DWARF_CFA:
595 return lowerEH_DWARF_CFA(Op, DAG);
596 case ISD::GlobalAddress:
597 return lowerGlobalAddress(Op, DAG);
598 case ISD::GlobalTLSAddress:
599 return lowerGlobalTLSAddress(Op, DAG);
600 case ISD::INTRINSIC_WO_CHAIN:
601 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
602 case ISD::INTRINSIC_W_CHAIN:
603 return lowerINTRINSIC_W_CHAIN(Op, DAG);
604 case ISD::INTRINSIC_VOID:
605 return lowerINTRINSIC_VOID(Op, DAG);
606 case ISD::BlockAddress:
607 return lowerBlockAddress(Op, DAG);
608 case ISD::JumpTable:
609 return lowerJumpTable(Op, DAG);
610 case ISD::SHL_PARTS:
611 return lowerShiftLeftParts(Op, DAG);
612 case ISD::SRA_PARTS:
613 return lowerShiftRightParts(Op, DAG, IsSRA: true);
614 case ISD::SRL_PARTS:
615 return lowerShiftRightParts(Op, DAG, IsSRA: false);
616 case ISD::ConstantPool:
617 return lowerConstantPool(Op, DAG);
618 case ISD::FP_TO_SINT:
619 return lowerFP_TO_SINT(Op, DAG);
620 case ISD::FP_TO_UINT:
621 return lowerFP_TO_UINT(Op, DAG);
622 case ISD::BITCAST:
623 return lowerBITCAST(Op, DAG);
624 case ISD::UINT_TO_FP:
625 return lowerUINT_TO_FP(Op, DAG);
626 case ISD::SINT_TO_FP:
627 return lowerSINT_TO_FP(Op, DAG);
628 case ISD::VASTART:
629 return lowerVASTART(Op, DAG);
630 case ISD::FRAMEADDR:
631 return lowerFRAMEADDR(Op, DAG);
632 case ISD::RETURNADDR:
633 return lowerRETURNADDR(Op, DAG);
634 case ISD::SET_ROUNDING:
635 return lowerSET_ROUNDING(Op, DAG);
636 case ISD::GET_ROUNDING:
637 return lowerGET_ROUNDING(Op, DAG);
638 case ISD::WRITE_REGISTER:
639 return lowerWRITE_REGISTER(Op, DAG);
640 case ISD::INSERT_VECTOR_ELT:
641 return lowerINSERT_VECTOR_ELT(Op, DAG);
642 case ISD::EXTRACT_VECTOR_ELT:
643 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
644 case ISD::BUILD_VECTOR:
645 return lowerBUILD_VECTOR(Op, DAG);
646 case ISD::CONCAT_VECTORS:
647 return lowerCONCAT_VECTORS(Op, DAG);
648 case ISD::VECTOR_SHUFFLE:
649 return lowerVECTOR_SHUFFLE(Op, DAG);
650 case ISD::BITREVERSE:
651 return lowerBITREVERSE(Op, DAG);
652 case ISD::SCALAR_TO_VECTOR:
653 return lowerSCALAR_TO_VECTOR(Op, DAG);
654 case ISD::PREFETCH:
655 return lowerPREFETCH(Op, DAG);
656 case ISD::SELECT:
657 return lowerSELECT(Op, DAG);
658 case ISD::BRCOND:
659 return lowerBRCOND(Op, DAG);
660 case ISD::FP_TO_FP16:
661 return lowerFP_TO_FP16(Op, DAG);
662 case ISD::FP16_TO_FP:
663 return lowerFP16_TO_FP(Op, DAG);
664 case ISD::FP_TO_BF16:
665 return lowerFP_TO_BF16(Op, DAG);
666 case ISD::BF16_TO_FP:
667 return lowerBF16_TO_FP(Op, DAG);
668 case ISD::VECREDUCE_ADD:
669 return lowerVECREDUCE_ADD(Op, DAG);
670 case ISD::ROTL:
671 case ISD::ROTR:
672 return lowerRotate(Op, DAG);
673 case ISD::VECREDUCE_AND:
674 case ISD::VECREDUCE_OR:
675 case ISD::VECREDUCE_XOR:
676 case ISD::VECREDUCE_SMAX:
677 case ISD::VECREDUCE_SMIN:
678 case ISD::VECREDUCE_UMAX:
679 case ISD::VECREDUCE_UMIN:
680 return lowerVECREDUCE(Op, DAG);
681 case ISD::ConstantFP:
682 return lowerConstantFP(Op, DAG);
683 case ISD::SETCC:
684 return lowerSETCC(Op, DAG);
685 case ISD::FP_ROUND:
686 return lowerFP_ROUND(Op, DAG);
687 case ISD::FP_EXTEND:
688 return lowerFP_EXTEND(Op, DAG);
689 case ISD::SIGN_EXTEND_VECTOR_INREG:
690 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
691 case ISD::DYNAMIC_STACKALLOC:
692 return lowerDYNAMIC_STACKALLOC(Op, DAG);
693 case ISD::ANY_EXTEND:
694 return lowerANY_EXTEND(Op, DAG);
695 }
696 return SDValue();
697}
698
699// Helper to attempt to return a cheaper, bit-inverted version of \p V.
700static SDValue isNOT(SDValue V, SelectionDAG &DAG) {
701 // TODO: don't always ignore oneuse constraints.
702 V = peekThroughBitcasts(V);
703 EVT VT = V.getValueType();
704
705 // Match not(xor X, -1) -> X.
706 if (V.getOpcode() == ISD::XOR &&
707 (ISD::isBuildVectorAllOnes(N: V.getOperand(i: 1).getNode()) ||
708 isAllOnesConstant(V: V.getOperand(i: 1))))
709 return V.getOperand(i: 0);
710
711 // Match not(extract_subvector(not(X)) -> extract_subvector(X).
712 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
713 (isNullConstant(V: V.getOperand(i: 1)) || V.getOperand(i: 0).hasOneUse())) {
714 if (SDValue Not = isNOT(V: V.getOperand(i: 0), DAG)) {
715 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
716 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Not), VT, N1: Not,
717 N2: V.getOperand(i: 1));
718 }
719 }
720
721 // Match not(SplatVector(not(X)) -> SplatVector(X).
722 if (V.getOpcode() == ISD::BUILD_VECTOR) {
723 if (SDValue SplatValue =
724 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
725 if (!V->isOnlyUserOf(N: SplatValue.getNode()))
726 return SDValue();
727
728 if (SDValue Not = isNOT(V: SplatValue, DAG)) {
729 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
730 return DAG.getSplat(VT, DL: SDLoc(Not), Op: Not);
731 }
732 }
733 }
734
735 // Match not(or(not(X),not(Y))) -> and(X, Y).
736 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
737 V.getOperand(i: 0).hasOneUse() && V.getOperand(i: 1).hasOneUse()) {
738 // TODO: Handle cases with single NOT operand -> VANDN
739 if (SDValue Op1 = isNOT(V: V.getOperand(i: 1), DAG))
740 if (SDValue Op0 = isNOT(V: V.getOperand(i: 0), DAG))
741 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(V), VT, N1: DAG.getBitcast(VT, V: Op0),
742 N2: DAG.getBitcast(VT, V: Op1));
743 }
744
745 // TODO: Add more matching patterns. Such as,
746 // not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
747 // not(slt(C, X)) -> slt(X - 1, C)
748 return SDValue();
749}
750
751// Combine two ISD::FP_ROUND / LoongArchISD::VFCVT nodes with same type to
752// LoongArchISD::VFCVT. For example:
753// x1 = fp_round x, 0
754// y1 = fp_round y, 0
755// z = concat_vectors x1, y1
756// Or
757// x1 = LoongArch::VFCVT undef, x
758// y1 = LoongArch::VFCVT undef, y
759// z = LoongArchISD::VPACKEV y1, x1; or LoongArchISD::VPERMI y1, x1, 68
760// can be combined to:
761// z = LoongArch::VFCVT y, x
762static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG,
763 const LoongArchSubtarget &Subtarget) {
764 assert(((N->getOpcode() == ISD::CONCAT_VECTORS && N->getNumOperands() == 2) ||
765 (N->getOpcode() == LoongArchISD::VPACKEV) ||
766 (N->getOpcode() == LoongArchISD::VPERMI)) &&
767 "Invalid Node");
768
769 SDValue Op0 = peekThroughBitcasts(V: N->getOperand(Num: 0));
770 SDValue Op1 = peekThroughBitcasts(V: N->getOperand(Num: 1));
771 unsigned Opcode0 = Op0.getOpcode();
772 unsigned Opcode1 = Op1.getOpcode();
773 if (Opcode0 != Opcode1)
774 return SDValue();
775
776 if (Opcode0 != ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
777 return SDValue();
778
779 // Check if two nodes have only one use.
780 if (!Op0.hasOneUse() || !Op1.hasOneUse())
781 return SDValue();
782
783 EVT VT = N.getValueType();
784 EVT SVT0 = Op0.getValueType();
785 EVT SVT1 = Op1.getValueType();
786 // Check if two nodes have the same result type.
787 if (SVT0 != SVT1)
788 return SDValue();
789
790 // Check if two nodes have the same operand type.
791 EVT SSVT0 = Op0.getOperand(i: 0).getValueType();
792 EVT SSVT1 = Op1.getOperand(i: 0).getValueType();
793 if (SSVT0 != SSVT1)
794 return SDValue();
795
796 if (N->getOpcode() == ISD::CONCAT_VECTORS && Opcode0 == ISD::FP_ROUND) {
797 if (Subtarget.hasExtLASX() && VT.is256BitVector() && SVT0 == MVT::v4f32 &&
798 SSVT0 == MVT::v4f64) {
799 // A vector_shuffle is required in the final step, as xvfcvt instruction
800 // operates on each 128-bit segament as a lane.
801 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v8f32,
802 N1: Op1.getOperand(i: 0), N2: Op0.getOperand(i: 0));
803 SDValue Undef = DAG.getUNDEF(VT: Res.getValueType());
804 // After VFCVT, the high part of Res comes from the high parts of Op0 and
805 // Op1, and the low part comes from the low parts of Op0 and Op1. However,
806 // the desired order requires Op0 to fully occupy the lower half and Op1
807 // the upper half of Res. The Mask reorders the elements of Res to achieve
808 // this:
809 // - The first four elements (0, 1, 4, 5) come from Op0.
810 // - The next four elements (2, 3, 6, 7) come from Op1.
811 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
812 Res = DAG.getVectorShuffle(VT: Res.getValueType(), dl: DL, N1: Res, N2: Undef, Mask);
813 return DAG.getBitcast(VT, V: Res);
814 }
815 }
816
817 if ((N->getOpcode() == LoongArchISD::VPACKEV ||
818 N->getOpcode() == LoongArchISD::VPERMI) &&
819 Opcode0 == LoongArchISD::VFCVT) {
820 // For VPACKEV or VPERMI, check if the first operation of VFCVT is undef.
821 if (!Op0.getOperand(i: 0).isUndef() || !Op1.getOperand(i: 0).isUndef())
822 return SDValue();
823
824 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
825 return SDValue();
826
827 if (N->getOpcode() == LoongArchISD::VPACKEV &&
828 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
829 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32,
830 N1: Op0.getOperand(i: 1), N2: Op1.getOperand(i: 1));
831 return DAG.getBitcast(VT, V: Res);
832 }
833
834 if (N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
835 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
836 if (Imm != 68)
837 return SDValue();
838 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Op0.getOperand(i: 1),
839 N2: Op1.getOperand(i: 1));
840 }
841 }
842
843 return SDValue();
844}
845
846SDValue LoongArchTargetLowering::lowerFP_ROUND(SDValue Op,
847 SelectionDAG &DAG) const {
848 SDLoc DL(Op);
849 SDValue In = Op.getOperand(i: 0);
850 MVT VT = Op.getSimpleValueType();
851 MVT SVT = In.getSimpleValueType();
852
853 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
854 SDValue Lo, Hi;
855 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: In, DL);
856 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT, N1: Hi, N2: Lo);
857 }
858
859 return SDValue();
860}
861
862SDValue LoongArchTargetLowering::lowerFP_EXTEND(SDValue Op,
863 SelectionDAG &DAG) const {
864
865 SDLoc DL(Op);
866 EVT VT = Op.getValueType();
867 SDValue Src = Op->getOperand(Num: 0);
868 EVT SVT = Src.getValueType();
869
870 bool V2F32ToV2F64 =
871 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
872 bool V4F32ToV4F64 =
873 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
874 if (!V2F32ToV2F64 && !V4F32ToV4F64)
875 return SDValue();
876
877 // Check if Op is the high part of vector.
878 auto CheckVecHighPart = [](SDValue Op) {
879 Op = peekThroughBitcasts(V: Op);
880 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
881 SDValue SOp = Op.getOperand(i: 0);
882 EVT SVT = SOp.getValueType();
883 if (!SVT.isVector() || (SVT.getVectorNumElements() % 2 != 0))
884 return SDValue();
885
886 const uint64_t Imm = Op.getConstantOperandVal(i: 1);
887 if (Imm == SVT.getVectorNumElements() / 2)
888 return SOp;
889 return SDValue();
890 }
891 return SDValue();
892 };
893
894 unsigned Opcode;
895 SDValue VFCVTOp;
896 EVT WideOpVT = SVT.getSimpleVT().getDoubleNumVectorElementsVT();
897 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
898
899 // If the operand of ISD::FP_EXTEND comes from the high part of vector,
900 // generate LoongArchISD::VFCVTH, otherwise LoongArchISD::VFCVTL.
901 if (SDValue V = CheckVecHighPart(Src)) {
902 assert(V.getValueSizeInBits() == WideOpVT.getSizeInBits() &&
903 "Unexpected wide vector");
904 Opcode = LoongArchISD::VFCVTH;
905 VFCVTOp = DAG.getBitcast(VT: WideOpVT, V);
906 } else {
907 Opcode = LoongArchISD::VFCVTL;
908 VFCVTOp = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideOpVT,
909 N1: DAG.getUNDEF(VT: WideOpVT), N2: Src, N3: ZeroIdx);
910 }
911
912 // v2f64 = fp_extend v2f32
913 if (V2F32ToV2F64)
914 return DAG.getNode(Opcode, DL, VT, Operand: VFCVTOp);
915
916 // v4f64 = fp_extend v4f32
917 if (V4F32ToV4F64) {
918 // XVFCVT instruction operates on each 128-bit segment as a lane, so a
919 // vector_shuffle is required firstly.
920 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
921 SDValue Res = DAG.getVectorShuffle(VT: WideOpVT, dl: DL, N1: VFCVTOp,
922 N2: DAG.getUNDEF(VT: WideOpVT), Mask);
923 Res = DAG.getNode(Opcode, DL, VT, Operand: Res);
924 return Res;
925 }
926
927 return SDValue();
928}
929
930SDValue LoongArchTargetLowering::lowerConstantFP(SDValue Op,
931 SelectionDAG &DAG) const {
932 EVT VT = Op.getValueType();
933 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
934 const APFloat &FPVal = CFP->getValueAPF();
935 SDLoc DL(CFP);
936
937 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
938 (VT == MVT::f64 && Subtarget.hasBasicD()));
939
940 // If value is 0.0 or -0.0, just ignore it.
941 if (FPVal.isZero())
942 return SDValue();
943
944 // If lsx enabled, use cheaper 'vldi' instruction if possible.
945 if (isFPImmVLDILegal(Imm: FPVal, VT))
946 return SDValue();
947
948 // Construct as integer, and move to float register.
949 APInt INTVal = FPVal.bitcastToAPInt();
950
951 // If more than MaterializeFPImmInsNum instructions will be used to
952 // generate the INTVal and move it to float register, fallback to
953 // use floating point load from the constant pool.
954 auto Seq = LoongArchMatInt::generateInstSeq(Val: INTVal.getSExtValue());
955 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
956 if (InsNum > MaterializeFPImmInsNum && !FPVal.isOne())
957 return SDValue();
958
959 switch (VT.getSimpleVT().SimpleTy) {
960 default:
961 llvm_unreachable("Unexpected floating point type!");
962 break;
963 case MVT::f32: {
964 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i32);
965 if (Subtarget.is64Bit())
966 NewVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: NewVal);
967 return DAG.getNode(Opcode: Subtarget.is64Bit() ? LoongArchISD::MOVGR2FR_W_LA64
968 : LoongArchISD::MOVGR2FR_W,
969 DL, VT, Operand: NewVal);
970 }
971 case MVT::f64: {
972 if (Subtarget.is64Bit()) {
973 SDValue NewVal = DAG.getConstant(Val: INTVal, DL, VT: MVT::i64);
974 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D, DL, VT, Operand: NewVal);
975 }
976 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
977 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
978 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D_LO_HI, DL, VT, N1: Lo, N2: Hi);
979 }
980 }
981
982 return SDValue();
983}
984
985// Ensure SETCC result and operand have the same bit width; isel does not
986// support mismatched widths.
987SDValue LoongArchTargetLowering::lowerSETCC(SDValue Op,
988 SelectionDAG &DAG) const {
989 SDLoc DL(Op);
990 EVT ResultVT = Op.getValueType();
991 EVT OperandVT = Op.getOperand(i: 0).getValueType();
992
993 EVT SetCCResultVT =
994 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT);
995
996 if (ResultVT == SetCCResultVT)
997 return Op;
998
999 assert(Op.getOperand(0).getValueType() == Op.getOperand(1).getValueType() &&
1000 "SETCC operands must have the same type!");
1001
1002 SDValue SetCCNode =
1003 DAG.getNode(Opcode: ISD::SETCC, DL, VT: SetCCResultVT, N1: Op.getOperand(i: 0),
1004 N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2));
1005
1006 if (ResultVT.bitsGT(VT: SetCCResultVT))
1007 SetCCNode = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: ResultVT, Operand: SetCCNode);
1008 else if (ResultVT.bitsLT(VT: SetCCResultVT))
1009 SetCCNode = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: SetCCNode);
1010
1011 return SetCCNode;
1012}
1013
1014// Lower sext_invec using vslti instructions.
1015// For example:
1016// %b = sext <4 x i16> %a to <4 x i32>
1017// can be lowered to:
1018// VSLTI_H vr2, vr1, 0
1019// VILVL.H vr1, vr2, vr1
1020SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1021 SDValue Op, SelectionDAG &DAG) const {
1022 SDLoc DL(Op);
1023 SDValue Src = Op.getOperand(i: 0);
1024 MVT SrcVT = Src.getSimpleValueType();
1025 MVT DstVT = Op.getSimpleValueType();
1026
1027 if (!SrcVT.is128BitVector())
1028 return SDValue();
1029
1030 // lower to VSLTI + VILVL if extend could be done in single step.
1031 if (DstVT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits() == 2) {
1032 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
1033 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Src, N2: Zero,
1034 N3: DAG.getCondCode(Cond: ISD::SETLT));
1035 SDValue LoInterleaved =
1036 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Src);
1037
1038 return DAG.getBitcast(VT: DstVT, V: LoInterleaved);
1039 }
1040
1041 return SDValue();
1042}
1043
1044// ANY_EXTEND can be replaced by ZERO_EXTEND when LASX is enabled.
1045SDValue LoongArchTargetLowering::lowerANY_EXTEND(SDValue Op,
1046 SelectionDAG &DAG) const {
1047 assert(Subtarget.hasExtLASX());
1048 // We don't have corresponding instrunction for ANY_EXTEND, lowering it to
1049 // ZERO_EXTEND won't break its semantics, while avoid scalar extract/insert.
1050 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: Op.getValueType(),
1051 Operand: Op.getOperand(i: 0));
1052}
1053
1054// Lower vecreduce_add using vhaddw instructions.
1055// For Example:
1056// call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
1057// can be lowered to:
1058// VHADDW_D_W vr0, vr0, vr0
1059// VHADDW_Q_D vr0, vr0, vr0
1060// VPICKVE2GR_D a0, vr0, 0
1061// ADDI_W a0, a0, 0
1062SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
1063 SelectionDAG &DAG) const {
1064
1065 SDLoc DL(Op);
1066 MVT OpVT = Op.getSimpleValueType();
1067 SDValue Val = Op.getOperand(i: 0);
1068
1069 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1070 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1071 unsigned ResBits = OpVT.getScalarSizeInBits();
1072
1073 unsigned LegalVecSize = 128;
1074 bool isLASX256Vector =
1075 Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
1076
1077 // Ensure operand type legal or enable it legal.
1078 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1079 Val = DAG.WidenVector(N: Val, DL);
1080 }
1081
1082 // NumEles is designed for iterations count, v4i32 for LSX
1083 // and v8i32 for LASX should have the same count.
1084 if (isLASX256Vector) {
1085 NumEles /= 2;
1086 LegalVecSize = 256;
1087 }
1088
1089 EleBits *= 2;
1090 for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1091 EleBits = std::min(a: EleBits, b: 64u);
1092 MVT IntTy = MVT::getIntegerVT(BitWidth: EleBits);
1093 MVT VecTy = MVT::getVectorVT(VT: IntTy, NumElements: LegalVecSize / EleBits);
1094 Val = DAG.getNode(Opcode: LoongArchISD::VHADDW, DL, VT: VecTy, N1: Val, N2: Val);
1095 }
1096
1097 if (isLASX256Vector) {
1098 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: Val,
1099 N2: DAG.getConstant(Val: 2, DL, VT: Subtarget.getGRLenVT()));
1100 Val = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::v4i64, N1: Tmp, N2: Val);
1101 }
1102
1103 Val = DAG.getBitcast(VT: MVT::getVectorVT(VT: OpVT, NumElements: LegalVecSize / ResBits), V: Val);
1104 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1105 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
1106}
1107
1108// Lower vecreduce_and/or/xor/[s/u]max/[s/u]min.
1109// For Example:
1110// call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
1111// can be lowered to:
1112// VBSRL_V vr1, vr0, 8
1113// VMAX_W vr0, vr1, vr0
1114// VBSRL_V vr1, vr0, 4
1115// VMAX_W vr0, vr1, vr0
1116// VPICKVE2GR_W a0, vr0, 0
1117// For 256 bit vector, it is illegal and will be spilt into
1118// two 128 bit vector by default then processed by this.
1119SDValue LoongArchTargetLowering::lowerVECREDUCE(SDValue Op,
1120 SelectionDAG &DAG) const {
1121 SDLoc DL(Op);
1122
1123 MVT OpVT = Op.getSimpleValueType();
1124 SDValue Val = Op.getOperand(i: 0);
1125
1126 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1127 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1128
1129 // Ensure operand type legal or enable it legal.
1130 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1131 Val = DAG.WidenVector(N: Val, DL);
1132 }
1133
1134 unsigned Opcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
1135 MVT VecTy = Val.getSimpleValueType();
1136 MVT GRLenVT = Subtarget.getGRLenVT();
1137
1138 for (int i = NumEles; i > 1; i /= 2) {
1139 SDValue ShiftAmt = DAG.getConstant(Val: i * EleBits / 16, DL, VT: GRLenVT);
1140 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: VecTy, N1: Val, N2: ShiftAmt);
1141 Val = DAG.getNode(Opcode, DL, VT: VecTy, N1: Tmp, N2: Val);
1142 }
1143
1144 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1145 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
1146}
1147
1148SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
1149 SelectionDAG &DAG) const {
1150 unsigned IsData = Op.getConstantOperandVal(i: 4);
1151
1152 // We don't support non-data prefetch.
1153 // Just preserve the chain.
1154 if (!IsData)
1155 return Op.getOperand(i: 0);
1156
1157 return Op;
1158}
1159
1160SDValue LoongArchTargetLowering::lowerRotate(SDValue Op,
1161 SelectionDAG &DAG) const {
1162 MVT VT = Op.getSimpleValueType();
1163 assert(VT.isVector() && "Unexpected type");
1164
1165 SDLoc DL(Op);
1166 SDValue R = Op.getOperand(i: 0);
1167 SDValue Amt = Op.getOperand(i: 1);
1168 unsigned Opcode = Op.getOpcode();
1169 unsigned EltSizeInBits = VT.getScalarSizeInBits();
1170
1171 auto checkCstSplat = [](SDValue V, APInt &CstSplatValue) {
1172 if (V.getOpcode() != ISD::BUILD_VECTOR)
1173 return false;
1174 if (SDValue SplatValue =
1175 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
1176 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
1177 CstSplatValue = C->getAPIntValue();
1178 return true;
1179 }
1180 }
1181 return false;
1182 };
1183
1184 // Check for constant splat rotation amount.
1185 APInt CstSplatValue;
1186 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1187 bool isROTL = Opcode == ISD::ROTL;
1188
1189 // Check for splat rotate by zero.
1190 if (IsCstSplat && CstSplatValue.urem(RHS: EltSizeInBits) == 0)
1191 return R;
1192
1193 // LoongArch targets always prefer ISD::ROTR.
1194 if (isROTL) {
1195 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
1196 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: R,
1197 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: Amt));
1198 }
1199
1200 // Rotate by a immediate.
1201 if (IsCstSplat) {
1202 // ISD::ROTR: Attemp to rotate by a positive immediate.
1203 SDValue Bits = DAG.getConstant(Val: EltSizeInBits, DL, VT);
1204 if (SDValue Urem =
1205 DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL, VT, Ops: {Amt, Bits}))
1206 return DAG.getNode(Opcode, DL, VT, N1: R, N2: Urem);
1207 }
1208
1209 return Op;
1210}
1211
1212// Return true if Val is equal to (setcc LHS, RHS, CC).
1213// Return false if Val is the inverse of (setcc LHS, RHS, CC).
1214// Otherwise, return std::nullopt.
1215static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
1216 ISD::CondCode CC, SDValue Val) {
1217 assert(Val->getOpcode() == ISD::SETCC);
1218 SDValue LHS2 = Val.getOperand(i: 0);
1219 SDValue RHS2 = Val.getOperand(i: 1);
1220 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
1221
1222 if (LHS == LHS2 && RHS == RHS2) {
1223 if (CC == CC2)
1224 return true;
1225 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1226 return false;
1227 } else if (LHS == RHS2 && RHS == LHS2) {
1228 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
1229 if (CC == CC2)
1230 return true;
1231 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1232 return false;
1233 }
1234
1235 return std::nullopt;
1236}
1237
1238static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
1239 const LoongArchSubtarget &Subtarget) {
1240 SDValue CondV = N->getOperand(Num: 0);
1241 SDValue TrueV = N->getOperand(Num: 1);
1242 SDValue FalseV = N->getOperand(Num: 2);
1243 MVT VT = N->getSimpleValueType(ResNo: 0);
1244 SDLoc DL(N);
1245
1246 // (select c, -1, y) -> -c | y
1247 if (isAllOnesConstant(V: TrueV)) {
1248 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1249 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1250 }
1251 // (select c, y, -1) -> (c-1) | y
1252 if (isAllOnesConstant(V: FalseV)) {
1253 SDValue Neg =
1254 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1255 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1256 }
1257
1258 // (select c, 0, y) -> (c-1) & y
1259 if (isNullConstant(V: TrueV)) {
1260 SDValue Neg =
1261 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1262 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1263 }
1264 // (select c, y, 0) -> -c & y
1265 if (isNullConstant(V: FalseV)) {
1266 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1267 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1268 }
1269
1270 // select c, ~x, x --> xor -c, x
1271 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
1272 const APInt &TrueVal = TrueV->getAsAPIntVal();
1273 const APInt &FalseVal = FalseV->getAsAPIntVal();
1274 if (~TrueVal == FalseVal) {
1275 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1276 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
1277 }
1278 }
1279
1280 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
1281 // when both truev and falsev are also setcc.
1282 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
1283 FalseV.getOpcode() == ISD::SETCC) {
1284 SDValue LHS = CondV.getOperand(i: 0);
1285 SDValue RHS = CondV.getOperand(i: 1);
1286 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1287
1288 // (select x, x, y) -> x | y
1289 // (select !x, x, y) -> x & y
1290 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
1291 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
1292 N2: DAG.getFreeze(V: FalseV));
1293 }
1294 // (select x, y, x) -> x & y
1295 // (select !x, y, x) -> x | y
1296 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
1297 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
1298 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
1299 }
1300 }
1301
1302 return SDValue();
1303}
1304
1305// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
1306// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
1307// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
1308// being `0` or `-1`. In such cases we can replace `select` with `and`.
1309// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
1310// than `c0`?
1311static SDValue
1312foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
1313 const LoongArchSubtarget &Subtarget) {
1314 unsigned SelOpNo = 0;
1315 SDValue Sel = BO->getOperand(Num: 0);
1316 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1317 SelOpNo = 1;
1318 Sel = BO->getOperand(Num: 1);
1319 }
1320
1321 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1322 return SDValue();
1323
1324 unsigned ConstSelOpNo = 1;
1325 unsigned OtherSelOpNo = 2;
1326 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
1327 ConstSelOpNo = 2;
1328 OtherSelOpNo = 1;
1329 }
1330 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
1331 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
1332 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
1333 return SDValue();
1334
1335 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
1336 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
1337 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
1338 return SDValue();
1339
1340 SDLoc DL(Sel);
1341 EVT VT = BO->getValueType(ResNo: 0);
1342
1343 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1344 if (SelOpNo == 1)
1345 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
1346
1347 SDValue NewConstOp =
1348 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
1349 if (!NewConstOp)
1350 return SDValue();
1351
1352 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
1353 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
1354 return SDValue();
1355
1356 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
1357 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1358 if (SelOpNo == 1)
1359 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
1360 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
1361
1362 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1363 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1364 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
1365}
1366
1367// Changes the condition code and swaps operands if necessary, so the SetCC
1368// operation matches one of the comparisons supported directly by branches
1369// in the LoongArch ISA. May adjust compares to favor compare with 0 over
1370// compare with 1/-1.
1371static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1372 ISD::CondCode &CC, SelectionDAG &DAG) {
1373 // If this is a single bit test that can't be handled by ANDI, shift the
1374 // bit to be tested to the MSB and perform a signed compare with 0.
1375 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
1376 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1377 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
1378 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
1379 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
1380 unsigned ShAmt = 0;
1381 if (isPowerOf2_64(Value: Mask)) {
1382 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
1383 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
1384 } else {
1385 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
1386 }
1387
1388 LHS = LHS.getOperand(i: 0);
1389 if (ShAmt != 0)
1390 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
1391 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
1392 return;
1393 }
1394 }
1395
1396 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
1397 int64_t C = RHSC->getSExtValue();
1398 switch (CC) {
1399 default:
1400 break;
1401 case ISD::SETGT:
1402 // Convert X > -1 to X >= 0.
1403 if (C == -1) {
1404 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1405 CC = ISD::SETGE;
1406 return;
1407 }
1408 break;
1409 case ISD::SETLT:
1410 // Convert X < 1 to 0 >= X.
1411 if (C == 1) {
1412 RHS = LHS;
1413 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1414 CC = ISD::SETGE;
1415 return;
1416 }
1417 break;
1418 }
1419 }
1420
1421 switch (CC) {
1422 default:
1423 break;
1424 case ISD::SETGT:
1425 case ISD::SETLE:
1426 case ISD::SETUGT:
1427 case ISD::SETULE:
1428 CC = ISD::getSetCCSwappedOperands(Operation: CC);
1429 std::swap(a&: LHS, b&: RHS);
1430 break;
1431 }
1432}
1433
1434SDValue LoongArchTargetLowering::lowerSELECT(SDValue Op,
1435 SelectionDAG &DAG) const {
1436 SDValue CondV = Op.getOperand(i: 0);
1437 SDValue TrueV = Op.getOperand(i: 1);
1438 SDValue FalseV = Op.getOperand(i: 2);
1439 SDLoc DL(Op);
1440 MVT VT = Op.getSimpleValueType();
1441 MVT GRLenVT = Subtarget.getGRLenVT();
1442
1443 if (SDValue V = combineSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
1444 return V;
1445
1446 if (Op.hasOneUse()) {
1447 unsigned UseOpc = Op->user_begin()->getOpcode();
1448 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
1449 SDNode *BinOp = *Op->user_begin();
1450 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
1451 DAG, Subtarget)) {
1452 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
1453 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
1454 // may return a constant node and cause crash in lowerSELECT.
1455 if (NewSel.getOpcode() == ISD::SELECT)
1456 return lowerSELECT(Op: NewSel, DAG);
1457 return NewSel;
1458 }
1459 }
1460 }
1461
1462 // If the condition is not an integer SETCC which operates on GRLenVT, we need
1463 // to emit a LoongArchISD::SELECT_CC comparing the condition to zero. i.e.:
1464 // (select condv, truev, falsev)
1465 // -> (loongarchisd::select_cc condv, zero, setne, truev, falsev)
1466 if (CondV.getOpcode() != ISD::SETCC ||
1467 CondV.getOperand(i: 0).getSimpleValueType() != GRLenVT) {
1468 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: GRLenVT);
1469 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
1470
1471 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
1472
1473 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1474 }
1475
1476 // If the CondV is the output of a SETCC node which operates on GRLenVT
1477 // inputs, then merge the SETCC node into the lowered LoongArchISD::SELECT_CC
1478 // to take advantage of the integer compare+branch instructions. i.e.: (select
1479 // (setcc lhs, rhs, cc), truev, falsev)
1480 // -> (loongarchisd::select_cc lhs, rhs, cc, truev, falsev)
1481 SDValue LHS = CondV.getOperand(i: 0);
1482 SDValue RHS = CondV.getOperand(i: 1);
1483 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1484
1485 // Special case for a select of 2 constants that have a difference of 1.
1486 // Normally this is done by DAGCombine, but if the select is introduced by
1487 // type legalization or op legalization, we miss it. Restricting to SETLT
1488 // case for now because that is what signed saturating add/sub need.
1489 // FIXME: We don't need the condition to be SETLT or even a SETCC,
1490 // but we would probably want to swap the true/false values if the condition
1491 // is SETGE/SETLE to avoid an XORI.
1492 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
1493 CCVal == ISD::SETLT) {
1494 const APInt &TrueVal = TrueV->getAsAPIntVal();
1495 const APInt &FalseVal = FalseV->getAsAPIntVal();
1496 if (TrueVal - 1 == FalseVal)
1497 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
1498 if (TrueVal + 1 == FalseVal)
1499 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
1500 }
1501
1502 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1503 // 1 < x ? x : 1 -> 0 < x ? x : 1
1504 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
1505 RHS == TrueV && LHS == FalseV) {
1506 LHS = DAG.getConstant(Val: 0, DL, VT);
1507 // 0 <u x is the same as x != 0.
1508 if (CCVal == ISD::SETULT) {
1509 std::swap(a&: LHS, b&: RHS);
1510 CCVal = ISD::SETNE;
1511 }
1512 }
1513
1514 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
1515 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
1516 RHS == FalseV) {
1517 RHS = DAG.getConstant(Val: 0, DL, VT);
1518 }
1519
1520 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1521
1522 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
1523 // (select (setcc lhs, rhs, CC), constant, falsev)
1524 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
1525 std::swap(a&: TrueV, b&: FalseV);
1526 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
1527 }
1528
1529 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
1530 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1531}
1532
1533SDValue LoongArchTargetLowering::lowerBRCOND(SDValue Op,
1534 SelectionDAG &DAG) const {
1535 SDValue CondV = Op.getOperand(i: 1);
1536 SDLoc DL(Op);
1537 MVT GRLenVT = Subtarget.getGRLenVT();
1538
1539 if (CondV.getOpcode() == ISD::SETCC) {
1540 if (CondV.getOperand(i: 0).getValueType() == GRLenVT) {
1541 SDValue LHS = CondV.getOperand(i: 0);
1542 SDValue RHS = CondV.getOperand(i: 1);
1543 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
1544
1545 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1546
1547 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1548 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1549 N1: Op.getOperand(i: 0), N2: LHS, N3: RHS, N4: TargetCC,
1550 N5: Op.getOperand(i: 2));
1551 } else if (CondV.getOperand(i: 0).getValueType().isFloatingPoint()) {
1552 return DAG.getNode(Opcode: LoongArchISD::BRCOND, DL, VT: Op.getValueType(),
1553 N1: Op.getOperand(i: 0), N2: CondV, N3: Op.getOperand(i: 2));
1554 }
1555 }
1556
1557 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1558 N1: Op.getOperand(i: 0), N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT),
1559 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
1560}
1561
1562SDValue
1563LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
1564 SelectionDAG &DAG) const {
1565 SDLoc DL(Op);
1566 MVT OpVT = Op.getSimpleValueType();
1567
1568 SDValue Vector = DAG.getUNDEF(VT: OpVT);
1569 SDValue Val = Op.getOperand(i: 0);
1570 SDValue Idx = DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT());
1571
1572 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: OpVT, N1: Vector, N2: Val, N3: Idx);
1573}
1574
1575SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
1576 SelectionDAG &DAG) const {
1577 EVT ResTy = Op->getValueType(ResNo: 0);
1578 SDValue Src = Op->getOperand(Num: 0);
1579 SDLoc DL(Op);
1580
1581 // LoongArchISD::BITREV_8B is not supported on LA32.
1582 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1583 return SDValue();
1584
1585 EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
1586 unsigned int OrigEltNum = ResTy.getVectorNumElements();
1587 unsigned int NewEltNum = NewVT.getVectorNumElements();
1588
1589 SDValue NewSrc = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: NewVT, Operand: Src);
1590
1591 SmallVector<SDValue, 8> Ops;
1592 for (unsigned int i = 0; i < NewEltNum; i++) {
1593 SDValue Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: NewSrc,
1594 N2: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
1595 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1596 ? (unsigned)LoongArchISD::BITREV_8B
1597 : (unsigned)ISD::BITREVERSE;
1598 Ops.push_back(Elt: DAG.getNode(Opcode: RevOp, DL, VT: MVT::i64, Operand: Op));
1599 }
1600 SDValue Res =
1601 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResTy, Operand: DAG.getBuildVector(VT: NewVT, DL, Ops));
1602
1603 switch (ResTy.getSimpleVT().SimpleTy) {
1604 default:
1605 return SDValue();
1606 case MVT::v16i8:
1607 case MVT::v32i8:
1608 return Res;
1609 case MVT::v8i16:
1610 case MVT::v16i16:
1611 case MVT::v4i32:
1612 case MVT::v8i32: {
1613 SmallVector<int, 32> Mask;
1614 for (unsigned int i = 0; i < NewEltNum; i++)
1615 for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
1616 Mask.push_back(Elt: j + (OrigEltNum / NewEltNum) * i);
1617 return DAG.getVectorShuffle(VT: ResTy, dl: DL, N1: Res, N2: DAG.getUNDEF(VT: ResTy), Mask);
1618 }
1619 }
1620}
1621
1622// Widen element type to get a new mask value (if possible).
1623// For example:
1624// shufflevector <4 x i32> %a, <4 x i32> %b,
1625// <4 x i32> <i32 6, i32 7, i32 2, i32 3>
1626// is equivalent to:
1627// shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 1>
1628// can be lowered to:
1629// VPACKOD_D vr0, vr0, vr1
1630static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1631 SDValue V1, SDValue V2, SelectionDAG &DAG) {
1632 unsigned EltBits = VT.getScalarSizeInBits();
1633
1634 if (EltBits > 32 || EltBits == 1)
1635 return SDValue();
1636
1637 SmallVector<int, 8> NewMask;
1638 if (widenShuffleMaskElts(M: Mask, NewMask)) {
1639 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: EltBits * 2)
1640 : MVT::getIntegerVT(BitWidth: EltBits * 2);
1641 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
1642 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT)) {
1643 SDValue NewV1 = DAG.getBitcast(VT: NewVT, V: V1);
1644 SDValue NewV2 = DAG.getBitcast(VT: NewVT, V: V2);
1645 return DAG.getBitcast(
1646 VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: NewV1, N2: NewV2, Mask: NewMask));
1647 }
1648 }
1649
1650 return SDValue();
1651}
1652
1653/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
1654/// instruction.
1655// The funciton matches elements from one of the input vector shuffled to the
1656// left or right with zeroable elements 'shifted in'. It handles both the
1657// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
1658// lane.
1659// Mostly copied from X86.
1660static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
1661 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
1662 int MaskOffset, const APInt &Zeroable) {
1663 int Size = Mask.size();
1664 unsigned SizeInBits = Size * ScalarSizeInBits;
1665
1666 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
1667 for (int i = 0; i < Size; i += Scale)
1668 for (int j = 0; j < Shift; ++j)
1669 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
1670 return false;
1671
1672 return true;
1673 };
1674
1675 auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
1676 int Step = 1) {
1677 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
1678 if (!(Mask[i] == -1 || Mask[i] == Low))
1679 return false;
1680 return true;
1681 };
1682
1683 auto MatchShift = [&](int Shift, int Scale, bool Left) {
1684 for (int i = 0; i != Size; i += Scale) {
1685 unsigned Pos = Left ? i + Shift : i;
1686 unsigned Low = Left ? i : i + Shift;
1687 unsigned Len = Scale - Shift;
1688 if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
1689 return -1;
1690 }
1691
1692 int ShiftEltBits = ScalarSizeInBits * Scale;
1693 bool ByteShift = ShiftEltBits > 64;
1694 Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1695 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1696 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1697
1698 // Normalize the scale for byte shifts to still produce an i64 element
1699 // type.
1700 Scale = ByteShift ? Scale / 2 : Scale;
1701
1702 // We need to round trip through the appropriate type for the shift.
1703 MVT ShiftSVT = MVT::getIntegerVT(BitWidth: ScalarSizeInBits * Scale);
1704 ShiftVT = ByteShift ? MVT::getVectorVT(VT: MVT::i8, NumElements: SizeInBits / 8)
1705 : MVT::getVectorVT(VT: ShiftSVT, NumElements: Size / Scale);
1706 return (int)ShiftAmt;
1707 };
1708
1709 unsigned MaxWidth = 128;
1710 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1711 for (int Shift = 1; Shift != Scale; ++Shift)
1712 for (bool Left : {true, false})
1713 if (CheckZeros(Shift, Scale, Left)) {
1714 int ShiftAmt = MatchShift(Shift, Scale, Left);
1715 if (0 < ShiftAmt)
1716 return ShiftAmt;
1717 }
1718
1719 // no match
1720 return -1;
1721}
1722
1723/// Lower VECTOR_SHUFFLE as shift (if possible).
1724///
1725/// For example:
1726/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1727/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
1728/// is lowered to:
1729/// (VBSLL_V $v0, $v0, 4)
1730///
1731/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1732/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
1733/// is lowered to:
1734/// (VSLLI_D $v0, $v0, 32)
1735static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef<int> Mask,
1736 MVT VT, SDValue V1, SDValue V2,
1737 SelectionDAG &DAG,
1738 const LoongArchSubtarget &Subtarget,
1739 const APInt &Zeroable) {
1740 int Size = Mask.size();
1741 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
1742
1743 MVT ShiftVT;
1744 SDValue V = V1;
1745 unsigned Opcode;
1746
1747 // Try to match shuffle against V1 shift.
1748 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1749 Mask, MaskOffset: 0, Zeroable);
1750
1751 // If V1 failed, try to match shuffle against V2 shift.
1752 if (ShiftAmt < 0) {
1753 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1754 Mask, MaskOffset: Size, Zeroable);
1755 V = V2;
1756 }
1757
1758 if (ShiftAmt < 0)
1759 return SDValue();
1760
1761 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
1762 "Illegal integer vector type");
1763 V = DAG.getBitcast(VT: ShiftVT, V);
1764 V = DAG.getNode(Opcode, DL, VT: ShiftVT, N1: V,
1765 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: Subtarget.getGRLenVT()));
1766 return DAG.getBitcast(VT, V);
1767}
1768
1769/// Determine whether a range fits a regular pattern of values.
1770/// This function accounts for the possibility of jumping over the End iterator.
1771template <typename ValType>
1772static bool
1773fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,
1774 unsigned CheckStride,
1775 typename SmallVectorImpl<ValType>::const_iterator End,
1776 ValType ExpectedIndex, unsigned ExpectedIndexStride) {
1777 auto &I = Begin;
1778
1779 while (I != End) {
1780 if (*I != -1 && *I != ExpectedIndex)
1781 return false;
1782 ExpectedIndex += ExpectedIndexStride;
1783
1784 // Incrementing past End is undefined behaviour so we must increment one
1785 // step at a time and check for End at each step.
1786 for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
1787 ; // Empty loop body.
1788 }
1789 return true;
1790}
1791
1792/// Compute whether each element of a shuffle is zeroable.
1793///
1794/// A "zeroable" vector shuffle element is one which can be lowered to zero.
1795static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
1796 SDValue V2, APInt &KnownUndef,
1797 APInt &KnownZero) {
1798 int Size = Mask.size();
1799 KnownUndef = KnownZero = APInt::getZero(numBits: Size);
1800
1801 V1 = peekThroughBitcasts(V: V1);
1802 V2 = peekThroughBitcasts(V: V2);
1803
1804 bool V1IsZero = ISD::isBuildVectorAllZeros(N: V1.getNode());
1805 bool V2IsZero = ISD::isBuildVectorAllZeros(N: V2.getNode());
1806
1807 int VectorSizeInBits = V1.getValueSizeInBits();
1808 int ScalarSizeInBits = VectorSizeInBits / Size;
1809 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
1810 (void)ScalarSizeInBits;
1811
1812 for (int i = 0; i < Size; ++i) {
1813 int M = Mask[i];
1814 if (M < 0) {
1815 KnownUndef.setBit(i);
1816 continue;
1817 }
1818 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
1819 KnownZero.setBit(i);
1820 continue;
1821 }
1822 }
1823}
1824
1825/// Test whether a shuffle mask is equivalent within each sub-lane.
1826///
1827/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
1828/// non-trivial to compute in the face of undef lanes. The representation is
1829/// suitable for use with existing 128-bit shuffles as entries from the second
1830/// vector have been remapped to [LaneSize, 2*LaneSize).
1831static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
1832 ArrayRef<int> Mask,
1833 SmallVectorImpl<int> &RepeatedMask) {
1834 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
1835 RepeatedMask.assign(NumElts: LaneSize, Elt: -1);
1836 int Size = Mask.size();
1837 for (int i = 0; i < Size; ++i) {
1838 assert(Mask[i] == -1 || Mask[i] >= 0);
1839 if (Mask[i] < 0)
1840 continue;
1841 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
1842 // This entry crosses lanes, so there is no way to model this shuffle.
1843 return false;
1844
1845 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
1846 // Adjust second vector indices to start at LaneSize instead of Size.
1847 int LocalM =
1848 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1849 if (RepeatedMask[i % LaneSize] < 0)
1850 // This is the first non-undef entry in this slot of a 128-bit lane.
1851 RepeatedMask[i % LaneSize] = LocalM;
1852 else if (RepeatedMask[i % LaneSize] != LocalM)
1853 // Found a mismatch with the repeated mask.
1854 return false;
1855 }
1856 return true;
1857}
1858
1859/// Attempts to match vector shuffle as byte rotation.
1860static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
1861 ArrayRef<int> Mask) {
1862
1863 SDValue Lo, Hi;
1864 SmallVector<int, 16> RepeatedMask;
1865
1866 if (!isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask, RepeatedMask))
1867 return -1;
1868
1869 int NumElts = RepeatedMask.size();
1870 int Rotation = 0;
1871 int Scale = 16 / NumElts;
1872
1873 for (int i = 0; i < NumElts; ++i) {
1874 int M = RepeatedMask[i];
1875 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1876 "Unexpected mask index.");
1877 if (M < 0)
1878 continue;
1879
1880 // Determine where a rotated vector would have started.
1881 int StartIdx = i - (M % NumElts);
1882 if (StartIdx == 0)
1883 return -1;
1884
1885 // If we found the tail of a vector the rotation must be the missing
1886 // front. If we found the head of a vector, it must be how much of the
1887 // head.
1888 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1889
1890 if (Rotation == 0)
1891 Rotation = CandidateRotation;
1892 else if (Rotation != CandidateRotation)
1893 return -1;
1894
1895 // Compute which value this mask is pointing at.
1896 SDValue MaskV = M < NumElts ? V1 : V2;
1897
1898 // Compute which of the two target values this index should be assigned
1899 // to. This reflects whether the high elements are remaining or the low
1900 // elements are remaining.
1901 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
1902
1903 // Either set up this value if we've not encountered it before, or check
1904 // that it remains consistent.
1905 if (!TargetV)
1906 TargetV = MaskV;
1907 else if (TargetV != MaskV)
1908 return -1;
1909 }
1910
1911 // Check that we successfully analyzed the mask, and normalize the results.
1912 assert(Rotation != 0 && "Failed to locate a viable rotation!");
1913 assert((Lo || Hi) && "Failed to find a rotated input vector!");
1914 if (!Lo)
1915 Lo = Hi;
1916 else if (!Hi)
1917 Hi = Lo;
1918
1919 V1 = Lo;
1920 V2 = Hi;
1921
1922 return Rotation * Scale;
1923}
1924
1925/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
1926///
1927/// For example:
1928/// %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
1929/// <2 x i32> <i32 3, i32 0>
1930/// is lowered to:
1931/// (VBSRL_V $v1, $v1, 8)
1932/// (VBSLL_V $v0, $v0, 8)
1933/// (VOR_V $v0, $V0, $v1)
1934static SDValue
1935lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1936 SDValue V1, SDValue V2, SelectionDAG &DAG,
1937 const LoongArchSubtarget &Subtarget) {
1938
1939 SDValue Lo = V1, Hi = V2;
1940 int ByteRotation = matchShuffleAsByteRotate(VT, V1&: Lo, V2&: Hi, Mask);
1941 if (ByteRotation <= 0)
1942 return SDValue();
1943
1944 MVT ByteVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VT.getSizeInBits() / 8);
1945 Lo = DAG.getBitcast(VT: ByteVT, V: Lo);
1946 Hi = DAG.getBitcast(VT: ByteVT, V: Hi);
1947
1948 int LoByteShift = 16 - ByteRotation;
1949 int HiByteShift = ByteRotation;
1950 MVT GRLenVT = Subtarget.getGRLenVT();
1951
1952 SDValue LoShift = DAG.getNode(Opcode: LoongArchISD::VBSLL, DL, VT: ByteVT, N1: Lo,
1953 N2: DAG.getConstant(Val: LoByteShift, DL, VT: GRLenVT));
1954 SDValue HiShift = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: ByteVT, N1: Hi,
1955 N2: DAG.getConstant(Val: HiByteShift, DL, VT: GRLenVT));
1956 return DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::OR, DL, VT: ByteVT, N1: LoShift, N2: HiShift));
1957}
1958
1959/// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
1960///
1961/// For example:
1962/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1963/// <4 x i32> <i32 0, i32 4, i32 1, i32 4>
1964/// %3 = bitcast <4 x i32> %2 to <2 x i64>
1965/// is lowered to:
1966/// (VREPLI $v1, 0)
1967/// (VILVL $v0, $v1, $v0)
1968static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL,
1969 ArrayRef<int> Mask, MVT VT,
1970 SDValue V1, SDValue V2,
1971 SelectionDAG &DAG,
1972 const APInt &Zeroable) {
1973 int Bits = VT.getSizeInBits();
1974 int EltBits = VT.getScalarSizeInBits();
1975 int NumElements = VT.getVectorNumElements();
1976
1977 if (Zeroable.isAllOnes())
1978 return DAG.getConstant(Val: 0, DL, VT);
1979
1980 // Define a helper function to check a particular ext-scale and lower to it if
1981 // valid.
1982 auto Lower = [&](int Scale) -> SDValue {
1983 SDValue InputV;
1984 bool AnyExt = true;
1985 int Offset = 0;
1986 for (int i = 0; i < NumElements; i++) {
1987 int M = Mask[i];
1988 if (M < 0)
1989 continue;
1990 if (i % Scale != 0) {
1991 // Each of the extended elements need to be zeroable.
1992 if (!Zeroable[i])
1993 return SDValue();
1994
1995 AnyExt = false;
1996 continue;
1997 }
1998
1999 // Each of the base elements needs to be consecutive indices into the
2000 // same input vector.
2001 SDValue V = M < NumElements ? V1 : V2;
2002 M = M % NumElements;
2003 if (!InputV) {
2004 InputV = V;
2005 Offset = M - (i / Scale);
2006
2007 // These offset can't be handled
2008 if (Offset % (NumElements / Scale))
2009 return SDValue();
2010 } else if (InputV != V)
2011 return SDValue();
2012
2013 if (M != (Offset + (i / Scale)))
2014 return SDValue(); // Non-consecutive strided elements.
2015 }
2016
2017 // If we fail to find an input, we have a zero-shuffle which should always
2018 // have already been handled.
2019 if (!InputV)
2020 return SDValue();
2021
2022 do {
2023 unsigned VilVLoHi = LoongArchISD::VILVL;
2024 if (Offset >= (NumElements / 2)) {
2025 VilVLoHi = LoongArchISD::VILVH;
2026 Offset -= (NumElements / 2);
2027 }
2028
2029 MVT InputVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits), NumElements);
2030 SDValue Ext =
2031 AnyExt ? DAG.getFreeze(V: InputV) : DAG.getConstant(Val: 0, DL, VT: InputVT);
2032 InputV = DAG.getBitcast(VT: InputVT, V: InputV);
2033 InputV = DAG.getNode(Opcode: VilVLoHi, DL, VT: InputVT, N1: Ext, N2: InputV);
2034 Scale /= 2;
2035 EltBits *= 2;
2036 NumElements /= 2;
2037 } while (Scale > 1);
2038 return DAG.getBitcast(VT, V: InputV);
2039 };
2040
2041 // Each iteration, try extending the elements half as much, but into twice as
2042 // many elements.
2043 for (int NumExtElements = Bits / 64; NumExtElements < NumElements;
2044 NumExtElements *= 2) {
2045 if (SDValue V = Lower(NumElements / NumExtElements))
2046 return V;
2047 }
2048 return SDValue();
2049}
2050
2051/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
2052///
2053/// VREPLVEI performs vector broadcast based on an element specified by an
2054/// integer immediate, with its mask being similar to:
2055/// <x, x, x, ...>
2056/// where x is any valid index.
2057///
2058/// When undef's appear in the mask they are treated as if they were whatever
2059/// value is necessary in order to fit the above form.
2060static SDValue
2061lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2062 SDValue V1, SelectionDAG &DAG,
2063 const LoongArchSubtarget &Subtarget) {
2064 int SplatIndex = -1;
2065 for (const auto &M : Mask) {
2066 if (M != -1) {
2067 SplatIndex = M;
2068 break;
2069 }
2070 }
2071
2072 if (SplatIndex == -1)
2073 return DAG.getUNDEF(VT);
2074
2075 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2076 if (fitsRegularPattern<int>(Begin: Mask.begin(), CheckStride: 1, End: Mask.end(), ExpectedIndex: SplatIndex, ExpectedIndexStride: 0)) {
2077 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2078 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2079 }
2080
2081 return SDValue();
2082}
2083
2084/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
2085///
2086/// VSHUF4I splits the vector into blocks of four elements, then shuffles these
2087/// elements according to a <4 x i2> constant (encoded as an integer immediate).
2088///
2089/// It is therefore possible to lower into VSHUF4I when the mask takes the form:
2090/// <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2091/// When undef's appear they are treated as if they were whatever value is
2092/// necessary in order to fit the above forms.
2093///
2094/// For example:
2095/// %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2096/// <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2097/// i32 7, i32 6, i32 5, i32 4>
2098/// is lowered to:
2099/// (VSHUF4I_H $v0, $v1, 27)
2100/// where the 27 comes from:
2101/// 3 + (2 << 2) + (1 << 4) + (0 << 6)
2102static SDValue
2103lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2104 SDValue V1, SDValue V2, SelectionDAG &DAG,
2105 const LoongArchSubtarget &Subtarget) {
2106
2107 unsigned SubVecSize = 4;
2108 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2109 SubVecSize = 2;
2110
2111 int SubMask[4] = {-1, -1, -1, -1};
2112 for (unsigned i = 0; i < SubVecSize; ++i) {
2113 for (unsigned j = i; j < Mask.size(); j += SubVecSize) {
2114 int M = Mask[j];
2115
2116 // Convert from vector index to 4-element subvector index
2117 // If an index refers to an element outside of the subvector then give up
2118 if (M != -1) {
2119 M -= 4 * (j / SubVecSize);
2120 if (M < 0 || M >= 4)
2121 return SDValue();
2122 }
2123
2124 // If the mask has an undef, replace it with the current index.
2125 // Note that it might still be undef if the current index is also undef
2126 if (SubMask[i] == -1)
2127 SubMask[i] = M;
2128 // Check that non-undef values are the same as in the mask. If they
2129 // aren't then give up
2130 else if (M != -1 && M != SubMask[i])
2131 return SDValue();
2132 }
2133 }
2134
2135 // Calculate the immediate. Replace any remaining undefs with zero
2136 int Imm = 0;
2137 for (int i = SubVecSize - 1; i >= 0; --i) {
2138 int M = SubMask[i];
2139
2140 if (M == -1)
2141 M = 0;
2142
2143 Imm <<= 2;
2144 Imm |= M & 0x3;
2145 }
2146
2147 MVT GRLenVT = Subtarget.getGRLenVT();
2148
2149 // Return vshuf4i.d
2150 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2151 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2152 N3: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2153
2154 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT, N1: V1,
2155 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2156}
2157
2158/// Lower VECTOR_SHUFFLE whose result is the reversed source vector.
2159///
2160/// It is possible to do optimization for VECTOR_SHUFFLE performing vector
2161/// reverse whose mask likes:
2162/// <7, 6, 5, 4, 3, 2, 1, 0>
2163///
2164/// When undef's appear in the mask they are treated as if they were whatever
2165/// value is necessary in order to fit the above forms.
2166static SDValue
2167lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2168 SDValue V1, SelectionDAG &DAG,
2169 const LoongArchSubtarget &Subtarget) {
2170 // Only vectors with i8/i16 elements which cannot match other patterns
2171 // directly needs to do this.
2172 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2173 VT != MVT::v16i16)
2174 return SDValue();
2175
2176 if (!ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: Mask.size()))
2177 return SDValue();
2178
2179 int WidenNumElts = VT.getVectorNumElements() / 4;
2180 SmallVector<int, 16> WidenMask(WidenNumElts, -1);
2181 for (int i = 0; i < WidenNumElts; ++i)
2182 WidenMask[i] = WidenNumElts - 1 - i;
2183
2184 MVT WidenVT = MVT::getVectorVT(
2185 VT: VT.getVectorElementType() == MVT::i8 ? MVT::i32 : MVT::i64, NumElements: WidenNumElts);
2186 SDValue NewV1 = DAG.getBitcast(VT: WidenVT, V: V1);
2187 SDValue WidenRev = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: NewV1,
2188 N2: DAG.getUNDEF(VT: WidenVT), Mask: WidenMask);
2189
2190 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT,
2191 N1: DAG.getBitcast(VT, V: WidenRev),
2192 N2: DAG.getConstant(Val: 27, DL, VT: Subtarget.getGRLenVT()));
2193}
2194
2195/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).
2196///
2197/// VPACKEV interleaves the even elements from each vector.
2198///
2199/// It is possible to lower into VPACKEV when the mask consists of two of the
2200/// following forms interleaved:
2201/// <0, 2, 4, ...>
2202/// <n, n+2, n+4, ...>
2203/// where n is the number of elements in the vector.
2204/// For example:
2205/// <0, 0, 2, 2, 4, 4, ...>
2206/// <0, n, 2, n+2, 4, n+4, ...>
2207///
2208/// When undef's appear in the mask they are treated as if they were whatever
2209/// value is necessary in order to fit the above forms.
2210static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2211 MVT VT, SDValue V1, SDValue V2,
2212 SelectionDAG &DAG) {
2213
2214 const auto &Begin = Mask.begin();
2215 const auto &End = Mask.end();
2216 SDValue OriV1 = V1, OriV2 = V2;
2217
2218 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2219 V1 = OriV1;
2220 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2221 V1 = OriV2;
2222 else
2223 return SDValue();
2224
2225 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2226 V2 = OriV1;
2227 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2228 V2 = OriV2;
2229 else
2230 return SDValue();
2231
2232 return DAG.getNode(Opcode: LoongArchISD::VPACKEV, DL, VT, N1: V2, N2: V1);
2233}
2234
2235/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).
2236///
2237/// VPACKOD interleaves the odd elements from each vector.
2238///
2239/// It is possible to lower into VPACKOD when the mask consists of two of the
2240/// following forms interleaved:
2241/// <1, 3, 5, ...>
2242/// <n+1, n+3, n+5, ...>
2243/// where n is the number of elements in the vector.
2244/// For example:
2245/// <1, 1, 3, 3, 5, 5, ...>
2246/// <1, n+1, 3, n+3, 5, n+5, ...>
2247///
2248/// When undef's appear in the mask they are treated as if they were whatever
2249/// value is necessary in order to fit the above forms.
2250static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2251 MVT VT, SDValue V1, SDValue V2,
2252 SelectionDAG &DAG) {
2253
2254 const auto &Begin = Mask.begin();
2255 const auto &End = Mask.end();
2256 SDValue OriV1 = V1, OriV2 = V2;
2257
2258 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2259 V1 = OriV1;
2260 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2261 V1 = OriV2;
2262 else
2263 return SDValue();
2264
2265 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2266 V2 = OriV1;
2267 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2268 V2 = OriV2;
2269 else
2270 return SDValue();
2271
2272 return DAG.getNode(Opcode: LoongArchISD::VPACKOD, DL, VT, N1: V2, N2: V1);
2273}
2274
2275/// Lower VECTOR_SHUFFLE into VILVH (if possible).
2276///
2277/// VILVH interleaves consecutive elements from the left (highest-indexed) half
2278/// of each vector.
2279///
2280/// It is possible to lower into VILVH when the mask consists of two of the
2281/// following forms interleaved:
2282/// <x, x+1, x+2, ...>
2283/// <n+x, n+x+1, n+x+2, ...>
2284/// where n is the number of elements in the vector and x is half n.
2285/// For example:
2286/// <x, x, x+1, x+1, x+2, x+2, ...>
2287/// <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2288///
2289/// When undef's appear in the mask they are treated as if they were whatever
2290/// value is necessary in order to fit the above forms.
2291static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef<int> Mask,
2292 MVT VT, SDValue V1, SDValue V2,
2293 SelectionDAG &DAG) {
2294
2295 const auto &Begin = Mask.begin();
2296 const auto &End = Mask.end();
2297 unsigned HalfSize = Mask.size() / 2;
2298 SDValue OriV1 = V1, OriV2 = V2;
2299
2300 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2301 V1 = OriV1;
2302 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2303 V1 = OriV2;
2304 else
2305 return SDValue();
2306
2307 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2308 V2 = OriV1;
2309 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize,
2310 ExpectedIndexStride: 1))
2311 V2 = OriV2;
2312 else
2313 return SDValue();
2314
2315 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2316}
2317
2318/// Lower VECTOR_SHUFFLE into VILVL (if possible).
2319///
2320/// VILVL interleaves consecutive elements from the right (lowest-indexed) half
2321/// of each vector.
2322///
2323/// It is possible to lower into VILVL when the mask consists of two of the
2324/// following forms interleaved:
2325/// <0, 1, 2, ...>
2326/// <n, n+1, n+2, ...>
2327/// where n is the number of elements in the vector.
2328/// For example:
2329/// <0, 0, 1, 1, 2, 2, ...>
2330/// <0, n, 1, n+1, 2, n+2, ...>
2331///
2332/// When undef's appear in the mask they are treated as if they were whatever
2333/// value is necessary in order to fit the above forms.
2334static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef<int> Mask,
2335 MVT VT, SDValue V1, SDValue V2,
2336 SelectionDAG &DAG) {
2337
2338 const auto &Begin = Mask.begin();
2339 const auto &End = Mask.end();
2340 SDValue OriV1 = V1, OriV2 = V2;
2341
2342 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2343 V1 = OriV1;
2344 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2345 V1 = OriV2;
2346 else
2347 return SDValue();
2348
2349 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2350 V2 = OriV1;
2351 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2352 V2 = OriV2;
2353 else
2354 return SDValue();
2355
2356 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2357}
2358
2359/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).
2360///
2361/// VPICKEV copies the even elements of each vector into the result vector.
2362///
2363/// It is possible to lower into VPICKEV when the mask consists of two of the
2364/// following forms concatenated:
2365/// <0, 2, 4, ...>
2366/// <n, n+2, n+4, ...>
2367/// where n is the number of elements in the vector.
2368/// For example:
2369/// <0, 2, 4, ..., 0, 2, 4, ...>
2370/// <0, 2, 4, ..., n, n+2, n+4, ...>
2371///
2372/// When undef's appear in the mask they are treated as if they were whatever
2373/// value is necessary in order to fit the above forms.
2374static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2375 MVT VT, SDValue V1, SDValue V2,
2376 SelectionDAG &DAG) {
2377
2378 const auto &Begin = Mask.begin();
2379 const auto &Mid = Mask.begin() + Mask.size() / 2;
2380 const auto &End = Mask.end();
2381 SDValue OriV1 = V1, OriV2 = V2;
2382
2383 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2))
2384 V1 = OriV1;
2385 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2386 V1 = OriV2;
2387 else
2388 return SDValue();
2389
2390 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2391 V2 = OriV1;
2392 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2393 V2 = OriV2;
2394
2395 else
2396 return SDValue();
2397
2398 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2399}
2400
2401/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).
2402///
2403/// VPICKOD copies the odd elements of each vector into the result vector.
2404///
2405/// It is possible to lower into VPICKOD when the mask consists of two of the
2406/// following forms concatenated:
2407/// <1, 3, 5, ...>
2408/// <n+1, n+3, n+5, ...>
2409/// where n is the number of elements in the vector.
2410/// For example:
2411/// <1, 3, 5, ..., 1, 3, 5, ...>
2412/// <1, 3, 5, ..., n+1, n+3, n+5, ...>
2413///
2414/// When undef's appear in the mask they are treated as if they were whatever
2415/// value is necessary in order to fit the above forms.
2416static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2417 MVT VT, SDValue V1, SDValue V2,
2418 SelectionDAG &DAG) {
2419
2420 const auto &Begin = Mask.begin();
2421 const auto &Mid = Mask.begin() + Mask.size() / 2;
2422 const auto &End = Mask.end();
2423 SDValue OriV1 = V1, OriV2 = V2;
2424
2425 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2))
2426 V1 = OriV1;
2427 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2428 V1 = OriV2;
2429 else
2430 return SDValue();
2431
2432 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2433 V2 = OriV1;
2434 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2435 V2 = OriV2;
2436 else
2437 return SDValue();
2438
2439 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
2440}
2441
2442/// Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
2443///
2444/// VEXTRINS copies one element of a vector into any place of the result
2445/// vector and makes no change to the rest elements of the result vector.
2446///
2447/// It is possible to lower into VEXTRINS when the mask takes the form:
2448/// <0, 1, 2, ..., n+i, ..., n-1> or <n, n+1, n+2, ..., i, ..., 2n-1> or
2449/// <0, 1, 2, ..., i, ..., n-1> or <n, n+1, n+2, ..., n+i, ..., 2n-1>
2450/// where n is the number of elements in the vector and i is in [0, n).
2451/// For example:
2452/// <0, 1, 2, 3, 4, 5, 6, 8> , <2, 9, 10, 11, 12, 13, 14, 15> ,
2453/// <0, 1, 2, 6, 4, 5, 6, 7> , <8, 9, 10, 11, 12, 9, 14, 15>
2454///
2455/// When undef's appear in the mask they are treated as if they were whatever
2456/// value is necessary in order to fit the above forms.
2457static SDValue
2458lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2459 SDValue V1, SDValue V2, SelectionDAG &DAG,
2460 const LoongArchSubtarget &Subtarget) {
2461 unsigned NumElts = VT.getVectorNumElements();
2462 MVT EltVT = VT.getVectorElementType();
2463 MVT GRLenVT = Subtarget.getGRLenVT();
2464
2465 if (Mask.size() != NumElts)
2466 return SDValue();
2467
2468 auto tryLowerToExtrAndIns = [&](unsigned Base) -> SDValue {
2469 int DiffCount = 0;
2470 int DiffPos = -1;
2471 for (unsigned i = 0; i < NumElts; ++i) {
2472 if (Mask[i] == -1)
2473 continue;
2474 if (Mask[i] != int(Base + i)) {
2475 ++DiffCount;
2476 DiffPos = int(i);
2477 if (DiffCount > 1)
2478 return SDValue();
2479 }
2480 }
2481
2482 // Need exactly one differing element to lower into VEXTRINS.
2483 if (DiffCount != 1)
2484 return SDValue();
2485
2486 // DiffMask must be in [0, 2N).
2487 int DiffMask = Mask[DiffPos];
2488 if (DiffMask < 0 || DiffMask >= int(2 * NumElts))
2489 return SDValue();
2490
2491 // Determine source vector and source index.
2492 SDValue SrcVec;
2493 unsigned SrcIdx;
2494 if (unsigned(DiffMask) < NumElts) {
2495 SrcVec = V1;
2496 SrcIdx = unsigned(DiffMask);
2497 } else {
2498 SrcVec = V2;
2499 SrcIdx = unsigned(DiffMask) - NumElts;
2500 }
2501
2502 // Replace with EXTRACT_VECTOR_ELT + INSERT_VECTOR_ELT, it will match the
2503 // patterns of VEXTRINS in tablegen.
2504 SDValue Extracted = DAG.getNode(
2505 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT.isFloatingPoint() ? EltVT : GRLenVT,
2506 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdx, DL, VT: GRLenVT));
2507 SDValue Result =
2508 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: (Base == 0) ? V1 : V2,
2509 N2: Extracted, N3: DAG.getConstant(Val: DiffPos, DL, VT: GRLenVT));
2510
2511 return Result;
2512 };
2513
2514 // Try [0, n-1) insertion then [n, 2n-1) insertion.
2515 if (SDValue Result = tryLowerToExtrAndIns(0))
2516 return Result;
2517 return tryLowerToExtrAndIns(NumElts);
2518}
2519
2520// Check the Mask and then build SrcVec and MaskImm infos which will
2521// be used to build LoongArchISD nodes for VPERMI_W or XVPERMI_W.
2522// On success, return true. Otherwise, return false.
2523static bool buildVPERMIInfo(ArrayRef<int> Mask, SDValue V1, SDValue V2,
2524 SmallVectorImpl<SDValue> &SrcVec,
2525 unsigned &MaskImm) {
2526 unsigned MaskSize = Mask.size();
2527
2528 auto isValid = [&](int M, int Off) {
2529 return (M == -1) || (M >= Off && M < Off + 4);
2530 };
2531
2532 auto buildImm = [&](int MLo, int MHi, unsigned Off, unsigned I) {
2533 auto immPart = [&](int M, unsigned Off) {
2534 return (M == -1 ? 0 : (M - Off)) & 0x3;
2535 };
2536 MaskImm |= immPart(MLo, Off) << (I * 2);
2537 MaskImm |= immPart(MHi, Off) << ((I + 1) * 2);
2538 };
2539
2540 for (unsigned i = 0; i < 4; i += 2) {
2541 int MLo = Mask[i];
2542 int MHi = Mask[i + 1];
2543
2544 if (MaskSize == 8) { // Only v8i32/v8f32 need this check.
2545 auto isValid2 = [&](int &M, int M2) {
2546 // If high half index is undef, it's always valid.
2547 if (M2 == -1)
2548 return true;
2549 if (M == -1) {
2550 // If low half index is undef, use index from high half,
2551 // remapped to low half.
2552 if ((M2 % MaskSize) < 4)
2553 return false;
2554 M = M2 - 4;
2555 return true;
2556 }
2557 // Index in low half must be same as index in high half.
2558 return M2 == M + 4;
2559 };
2560 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2561 return false;
2562 }
2563
2564 if (isValid(MLo, 0) && isValid(MHi, 0)) {
2565 SrcVec.push_back(Elt: V1);
2566 buildImm(MLo, MHi, 0, i);
2567 } else if (isValid(MLo, MaskSize) && isValid(MHi, MaskSize)) {
2568 SrcVec.push_back(Elt: V2);
2569 buildImm(MLo, MHi, MaskSize, i);
2570 } else {
2571 return false;
2572 }
2573 }
2574
2575 return true;
2576}
2577
2578/// Lower VECTOR_SHUFFLE into VPERMI (if possible).
2579///
2580/// VPERMI selects two elements from each of the two vectors based on the
2581/// mask and places them in the corresponding positions of the result vector
2582/// in order. Only v4i32 and v4f32 types are allowed.
2583///
2584/// It is possible to lower into VPERMI when the mask consists of two of the
2585/// following forms concatenated:
2586/// <i, j, u, v>
2587/// <u, v, i, j>
2588/// where i,j are in [0,4) and u,v are in [4, 8).
2589/// For example:
2590/// <2, 3, 4, 5>
2591/// <5, 7, 0, 2>
2592///
2593/// When undef's appear in the mask they are treated as if they were whatever
2594/// value is necessary in order to fit the above forms.
2595static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef<int> Mask,
2596 MVT VT, SDValue V1, SDValue V2,
2597 SelectionDAG &DAG,
2598 const LoongArchSubtarget &Subtarget) {
2599 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2600 Mask.size() != VT.getVectorNumElements())
2601 return SDValue();
2602
2603 SmallVector<SDValue, 2> SrcVec;
2604 unsigned MaskImm = 0;
2605 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2606 return SDValue();
2607
2608 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2609 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2610}
2611
2612/// Lower VECTOR_SHUFFLE into VSHUF.
2613///
2614/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and
2615/// adding it as an operand to the resulting VSHUF.
2616static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef<int> Mask,
2617 MVT VT, SDValue V1, SDValue V2,
2618 SelectionDAG &DAG,
2619 const LoongArchSubtarget &Subtarget) {
2620
2621 SmallVector<SDValue, 16> Ops;
2622 for (auto M : Mask)
2623 Ops.push_back(Elt: DAG.getSignedConstant(Val: M, DL, VT: Subtarget.getGRLenVT()));
2624
2625 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
2626 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops);
2627
2628 // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.
2629 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>
2630 // VSHF concatenates the vectors in a bitwise fashion:
2631 // <0b00, 0b01> + <0b10, 0b11> ->
2632 // 0b0100 + 0b1110 -> 0b01001110
2633 // <0b10, 0b11, 0b00, 0b01>
2634 // We must therefore swap the operands to get the correct result.
2635 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
2636}
2637
2638/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
2639///
2640/// This routine breaks down the specific type of 128-bit shuffle and
2641/// dispatches to the lowering routines accordingly.
2642static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2643 SDValue V1, SDValue V2, SelectionDAG &DAG,
2644 const LoongArchSubtarget &Subtarget) {
2645 assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||
2646 VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||
2647 VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&
2648 "Vector type is unsupported for lsx!");
2649 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
2650 "Two operands have different types!");
2651 assert(VT.getVectorNumElements() == Mask.size() &&
2652 "Unexpected mask size for shuffle!");
2653 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
2654
2655 APInt KnownUndef, KnownZero;
2656 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
2657 APInt Zeroable = KnownUndef | KnownZero;
2658
2659 SDValue Result;
2660 // TODO: Add more comparison patterns.
2661 if (V2.isUndef()) {
2662 if ((Result =
2663 lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
2664 return Result;
2665 if ((Result =
2666 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2667 return Result;
2668 if ((Result =
2669 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
2670 return Result;
2671
2672 // TODO: This comment may be enabled in the future to better match the
2673 // pattern for instruction selection.
2674 /* V2 = V1; */
2675 }
2676
2677 // It is recommended not to change the pattern comparison order for better
2678 // performance.
2679 if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))
2680 return Result;
2681 if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))
2682 return Result;
2683 if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))
2684 return Result;
2685 if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))
2686 return Result;
2687 if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))
2688 return Result;
2689 if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
2690 return Result;
2691 if ((VT.SimpleTy == MVT::v2i64 || VT.SimpleTy == MVT::v2f64) &&
2692 (Result =
2693 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2694 return Result;
2695 if ((Result =
2696 lowerVECTOR_SHUFFLE_VEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2697 return Result;
2698 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
2699 Zeroable)))
2700 return Result;
2701 if ((Result =
2702 lowerVECTOR_SHUFFLE_VPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2703 return Result;
2704 if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
2705 Zeroable)))
2706 return Result;
2707 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
2708 Subtarget)))
2709 return Result;
2710 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
2711 return NewShuffle;
2712 if ((Result =
2713 lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2714 return Result;
2715 return SDValue();
2716}
2717
2718/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
2719///
2720/// It is a XVREPLVEI when the mask is:
2721/// <x, x, x, ..., x+n, x+n, x+n, ...>
2722/// where the number of x is equal to n and n is half the length of vector.
2723///
2724/// When undef's appear in the mask they are treated as if they were whatever
2725/// value is necessary in order to fit the above form.
2726static SDValue
2727lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2728 SDValue V1, SelectionDAG &DAG,
2729 const LoongArchSubtarget &Subtarget) {
2730 int SplatIndex = -1;
2731 for (const auto &M : Mask) {
2732 if (M != -1) {
2733 SplatIndex = M;
2734 break;
2735 }
2736 }
2737
2738 if (SplatIndex == -1)
2739 return DAG.getUNDEF(VT);
2740
2741 const auto &Begin = Mask.begin();
2742 const auto &End = Mask.end();
2743 int HalfSize = Mask.size() / 2;
2744
2745 if (SplatIndex >= HalfSize)
2746 return SDValue();
2747
2748 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2749 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: End - HalfSize, ExpectedIndex: SplatIndex, ExpectedIndexStride: 0) &&
2750 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 1, End, ExpectedIndex: SplatIndex + HalfSize,
2751 ExpectedIndexStride: 0)) {
2752 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2753 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2754 }
2755
2756 return SDValue();
2757}
2758
2759/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
2760static SDValue
2761lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2762 SDValue V1, SDValue V2, SelectionDAG &DAG,
2763 const LoongArchSubtarget &Subtarget) {
2764 // XVSHUF4I_D must be handled separately because it is different from other
2765 // types of [X]VSHUF4I instructions.
2766 if (Mask.size() == 4) {
2767 unsigned MaskImm = 0;
2768 for (int i = 1; i >= 0; --i) {
2769 int MLo = Mask[i];
2770 int MHi = Mask[i + 2];
2771 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2772 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2773 return SDValue();
2774 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2775 return SDValue();
2776
2777 MaskImm <<= 2;
2778 if (MLo != -1)
2779 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2780 else if (MHi != -1)
2781 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2782 }
2783
2784 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2785 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2786 }
2787
2788 return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget);
2789}
2790
2791/// Lower VECTOR_SHUFFLE into XVPERMI (if possible).
2792static SDValue
2793lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2794 SDValue V1, SDValue V2, SelectionDAG &DAG,
2795 const LoongArchSubtarget &Subtarget) {
2796 MVT GRLenVT = Subtarget.getGRLenVT();
2797 unsigned MaskSize = Mask.size();
2798 if (MaskSize != VT.getVectorNumElements())
2799 return SDValue();
2800
2801 // Consider XVPERMI_W.
2802 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2803 SmallVector<SDValue, 2> SrcVec;
2804 unsigned MaskImm = 0;
2805 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2806 return SDValue();
2807
2808 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2809 N3: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2810 }
2811
2812 // Consider XVPERMI_D.
2813 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2814 unsigned MaskImm = 0;
2815 for (unsigned i = 0; i < MaskSize; ++i) {
2816 if (Mask[i] == -1)
2817 continue;
2818 if (Mask[i] >= (int)MaskSize)
2819 return SDValue();
2820 MaskImm |= Mask[i] << (i * 2);
2821 }
2822
2823 return DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT, N1: V1,
2824 N2: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2825 }
2826
2827 return SDValue();
2828}
2829
2830/// Lower VECTOR_SHUFFLE into XVPERM (if possible).
2831static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef<int> Mask,
2832 MVT VT, SDValue V1, SelectionDAG &DAG,
2833 const LoongArchSubtarget &Subtarget) {
2834 // LoongArch LASX only have XVPERM_W.
2835 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2836 return SDValue();
2837
2838 unsigned NumElts = VT.getVectorNumElements();
2839 unsigned HalfSize = NumElts / 2;
2840 bool FrontLo = true, FrontHi = true;
2841 bool BackLo = true, BackHi = true;
2842
2843 auto inRange = [](int val, int low, int high) {
2844 return (val == -1) || (val >= low && val < high);
2845 };
2846
2847 for (unsigned i = 0; i < HalfSize; ++i) {
2848 int Fronti = Mask[i];
2849 int Backi = Mask[i + HalfSize];
2850
2851 FrontLo &= inRange(Fronti, 0, HalfSize);
2852 FrontHi &= inRange(Fronti, HalfSize, NumElts);
2853 BackLo &= inRange(Backi, 0, HalfSize);
2854 BackHi &= inRange(Backi, HalfSize, NumElts);
2855 }
2856
2857 // If both the lower and upper 128-bit parts access only one half of the
2858 // vector (either lower or upper), avoid using xvperm.w. The latency of
2859 // xvperm.w(3) is higher than using xvshuf(1) and xvori(1).
2860 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2861 return SDValue();
2862
2863 SmallVector<SDValue, 8> Masks;
2864 MVT GRLenVT = Subtarget.getGRLenVT();
2865 for (unsigned i = 0; i < NumElts; ++i)
2866 Masks.push_back(Elt: Mask[i] == -1 ? DAG.getUNDEF(VT: GRLenVT)
2867 : DAG.getConstant(Val: Mask[i], DL, VT: GRLenVT));
2868 SDValue MaskVec = DAG.getBuildVector(VT: MVT::v8i32, DL, Ops: Masks);
2869
2870 return DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT, N1: V1, N2: MaskVec);
2871}
2872
2873/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
2874static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2875 MVT VT, SDValue V1, SDValue V2,
2876 SelectionDAG &DAG) {
2877 return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);
2878}
2879
2880/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
2881static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2882 MVT VT, SDValue V1, SDValue V2,
2883 SelectionDAG &DAG) {
2884 return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);
2885}
2886
2887/// Lower VECTOR_SHUFFLE into XVILVH (if possible).
2888static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef<int> Mask,
2889 MVT VT, SDValue V1, SDValue V2,
2890 SelectionDAG &DAG) {
2891
2892 const auto &Begin = Mask.begin();
2893 const auto &End = Mask.end();
2894 unsigned HalfSize = Mask.size() / 2;
2895 unsigned LeftSize = HalfSize / 2;
2896 SDValue OriV1 = V1, OriV2 = V2;
2897
2898 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2899 ExpectedIndexStride: 1) &&
2900 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize, ExpectedIndexStride: 1))
2901 V1 = OriV1;
2902 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize,
2903 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2904 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2905 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2906 V1 = OriV2;
2907 else
2908 return SDValue();
2909
2910 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2911 ExpectedIndexStride: 1) &&
2912 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize,
2913 ExpectedIndexStride: 1))
2914 V2 = OriV1;
2915 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize,
2916 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
2917 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2918 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
2919 V2 = OriV2;
2920 else
2921 return SDValue();
2922
2923 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2924}
2925
2926/// Lower VECTOR_SHUFFLE into XVILVL (if possible).
2927static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef<int> Mask,
2928 MVT VT, SDValue V1, SDValue V2,
2929 SelectionDAG &DAG) {
2930
2931 const auto &Begin = Mask.begin();
2932 const auto &End = Mask.end();
2933 unsigned HalfSize = Mask.size() / 2;
2934 SDValue OriV1 = V1, OriV2 = V2;
2935
2936 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2937 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2938 V1 = OriV1;
2939 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1) &&
2940 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
2941 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2942 V1 = OriV2;
2943 else
2944 return SDValue();
2945
2946 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
2947 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2948 V2 = OriV1;
2949 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(),
2950 ExpectedIndexStride: 1) &&
2951 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
2952 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2953 V2 = OriV2;
2954 else
2955 return SDValue();
2956
2957 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2958}
2959
2960/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
2961static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2962 MVT VT, SDValue V1, SDValue V2,
2963 SelectionDAG &DAG) {
2964
2965 const auto &Begin = Mask.begin();
2966 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2967 const auto &Mid = Mask.begin() + Mask.size() / 2;
2968 const auto &RightMid = Mask.end() - Mask.size() / 4;
2969 const auto &End = Mask.end();
2970 unsigned HalfSize = Mask.size() / 2;
2971 SDValue OriV1 = V1, OriV2 = V2;
2972
2973 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
2974 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
2975 V1 = OriV1;
2976 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
2977 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
2978 V1 = OriV2;
2979 else
2980 return SDValue();
2981
2982 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
2983 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
2984 V2 = OriV1;
2985 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
2986 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
2987 V2 = OriV2;
2988
2989 else
2990 return SDValue();
2991
2992 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2993}
2994
2995/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
2996static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2997 MVT VT, SDValue V1, SDValue V2,
2998 SelectionDAG &DAG) {
2999
3000 const auto &Begin = Mask.begin();
3001 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3002 const auto &Mid = Mask.begin() + Mask.size() / 2;
3003 const auto &RightMid = Mask.end() - Mask.size() / 4;
3004 const auto &End = Mask.end();
3005 unsigned HalfSize = Mask.size() / 2;
3006 SDValue OriV1 = V1, OriV2 = V2;
3007
3008 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3009 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3010 V1 = OriV1;
3011 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3012 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize + 1,
3013 ExpectedIndexStride: 2))
3014 V1 = OriV2;
3015 else
3016 return SDValue();
3017
3018 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3019 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3020 V2 = OriV1;
3021 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3022 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize + 1,
3023 ExpectedIndexStride: 2))
3024 V2 = OriV2;
3025 else
3026 return SDValue();
3027
3028 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
3029}
3030
3031/// Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
3032static SDValue
3033lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3034 SDValue V1, SDValue V2, SelectionDAG &DAG,
3035 const LoongArchSubtarget &Subtarget) {
3036 int NumElts = VT.getVectorNumElements();
3037 int HalfSize = NumElts / 2;
3038 MVT EltVT = VT.getVectorElementType();
3039 MVT GRLenVT = Subtarget.getGRLenVT();
3040
3041 if ((int)Mask.size() != NumElts)
3042 return SDValue();
3043
3044 auto tryLowerToExtrAndIns = [&](int Base) -> SDValue {
3045 SmallVector<int> DiffPos;
3046 for (int i = 0; i < NumElts; ++i) {
3047 if (Mask[i] == -1)
3048 continue;
3049 if (Mask[i] != Base + i) {
3050 DiffPos.push_back(Elt: i);
3051 if (DiffPos.size() > 2)
3052 return SDValue();
3053 }
3054 }
3055
3056 // Need exactly two differing element to lower into XVEXTRINS.
3057 // If only one differing element, the element at a distance of
3058 // HalfSize from it must be undef.
3059 if (DiffPos.size() == 1) {
3060 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3061 DiffPos.push_back(Elt: DiffPos[0] + HalfSize);
3062 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3063 DiffPos.insert(I: DiffPos.begin(), Elt: DiffPos[0] - HalfSize);
3064 else
3065 return SDValue();
3066 }
3067 if (DiffPos.size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3068 return SDValue();
3069
3070 // DiffMask must be in its low or high part.
3071 int DiffMaskLo = Mask[DiffPos[0]];
3072 int DiffMaskHi = Mask[DiffPos[1]];
3073 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3074 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3075 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3076 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3077 return SDValue();
3078 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3079 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3080 return SDValue();
3081 if (DiffMaskHi != DiffMaskLo + HalfSize)
3082 return SDValue();
3083
3084 // Determine source vector and source index.
3085 SDValue SrcVec = (DiffMaskLo < HalfSize) ? V1 : V2;
3086 int SrcIdxLo =
3087 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3088 bool IsEltFP = EltVT.isFloatingPoint();
3089
3090 // Replace with 2*EXTRACT_VECTOR_ELT + 2*INSERT_VECTOR_ELT, it will match
3091 // the patterns of XVEXTRINS in tablegen.
3092 SDValue BaseVec = (Base == 0) ? V1 : V2;
3093 SDValue EltLo =
3094 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3095 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo, DL, VT: GRLenVT));
3096 SDValue InsLo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: BaseVec, N2: EltLo,
3097 N3: DAG.getConstant(Val: DiffPos[0], DL, VT: GRLenVT));
3098 SDValue EltHi =
3099 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3100 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo + HalfSize, DL, VT: GRLenVT));
3101 SDValue Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: InsLo, N2: EltHi,
3102 N3: DAG.getConstant(Val: DiffPos[1], DL, VT: GRLenVT));
3103
3104 return Result;
3105 };
3106
3107 // Try [0, n-1) insertion then [n, 2n-1) insertion.
3108 if (SDValue Result = tryLowerToExtrAndIns(0))
3109 return Result;
3110 return tryLowerToExtrAndIns(NumElts);
3111}
3112
3113/// Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
3114static SDValue
3115lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3116 SDValue V1, SDValue V2, SelectionDAG &DAG,
3117 const LoongArchSubtarget &Subtarget) {
3118 // LoongArch LASX only supports xvinsve0.{w/d}.
3119 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3120 VT != MVT::v4f64)
3121 return SDValue();
3122
3123 MVT GRLenVT = Subtarget.getGRLenVT();
3124 int MaskSize = Mask.size();
3125 assert(MaskSize == (int)VT.getVectorNumElements() && "Unexpected mask size");
3126
3127 // Check if exactly one element of the Mask is replaced by 'Replaced', while
3128 // all other elements are either 'Base + i' or undef (-1). On success, return
3129 // the index of the replaced element. Otherwise, just return -1.
3130 auto checkReplaceOne = [&](int Base, int Replaced) -> int {
3131 int Idx = -1;
3132 for (int i = 0; i < MaskSize; ++i) {
3133 if (Mask[i] == Base + i || Mask[i] == -1)
3134 continue;
3135 if (Mask[i] != Replaced)
3136 return -1;
3137 if (Idx == -1)
3138 Idx = i;
3139 else
3140 return -1;
3141 }
3142 return Idx;
3143 };
3144
3145 // Case 1: the lowest element of V2 replaces one element in V1.
3146 int Idx = checkReplaceOne(0, MaskSize);
3147 if (Idx != -1)
3148 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V1, N2: V2,
3149 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3150
3151 // Case 2: the lowest element of V1 replaces one element in V2.
3152 Idx = checkReplaceOne(MaskSize, 0);
3153 if (Idx != -1)
3154 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V2, N2: V1,
3155 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3156
3157 return SDValue();
3158}
3159
3160/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).
3161static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,
3162 MVT VT, SDValue V1, SDValue V2,
3163 SelectionDAG &DAG) {
3164
3165 int MaskSize = Mask.size();
3166 int HalfSize = Mask.size() / 2;
3167 const auto &Begin = Mask.begin();
3168 const auto &Mid = Mask.begin() + HalfSize;
3169 const auto &End = Mask.end();
3170
3171 // VECTOR_SHUFFLE concatenates the vectors:
3172 // <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>
3173 // shuffling ->
3174 // <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>
3175 //
3176 // XVSHUF concatenates the vectors:
3177 // <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>
3178 // shuffling ->
3179 // <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>
3180 SmallVector<SDValue, 8> MaskAlloc;
3181 for (auto it = Begin; it < Mid; it++) {
3182 if (*it < 0) // UNDEF
3183 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3184 else if ((*it >= 0 && *it < HalfSize) ||
3185 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3186 int M = *it < HalfSize ? *it : *it - HalfSize;
3187 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3188 } else
3189 return SDValue();
3190 }
3191 assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");
3192
3193 for (auto it = Mid; it < End; it++) {
3194 if (*it < 0) // UNDEF
3195 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3196 else if ((*it >= HalfSize && *it < MaskSize) ||
3197 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3198 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3199 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3200 } else
3201 return SDValue();
3202 }
3203 assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");
3204
3205 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
3206 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops: MaskAlloc);
3207 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
3208}
3209
3210/// Shuffle vectors by lane to generate more optimized instructions.
3211/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
3212///
3213/// Therefore, except for the following four cases, other cases are regarded
3214/// as cross-lane shuffles, where optimization is relatively limited.
3215///
3216/// - Shuffle high, low lanes of two inputs vector
3217/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
3218/// - Shuffle low, high lanes of two inputs vector
3219/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
3220/// - Shuffle low, low lanes of two inputs vector
3221/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
3222/// - Shuffle high, high lanes of two inputs vector
3223/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
3224///
3225/// The first case is the closest to LoongArch instructions and the other
3226/// cases need to be converted to it for processing.
3227///
3228/// This function will return true for the last three cases above and will
3229/// modify V1, V2 and Mask. Otherwise, return false for the first case and
3230/// cross-lane shuffle cases.
3231static bool canonicalizeShuffleVectorByLane(
3232 const SDLoc &DL, MutableArrayRef<int> Mask, MVT VT, SDValue &V1,
3233 SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
3234
3235 enum HalfMaskType { HighLaneTy, LowLaneTy, None };
3236
3237 int MaskSize = Mask.size();
3238 int HalfSize = Mask.size() / 2;
3239 MVT GRLenVT = Subtarget.getGRLenVT();
3240
3241 HalfMaskType preMask = None, postMask = None;
3242
3243 if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3244 return M < 0 || (M >= 0 && M < HalfSize) ||
3245 (M >= MaskSize && M < MaskSize + HalfSize);
3246 }))
3247 preMask = HighLaneTy;
3248 else if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3249 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3250 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3251 }))
3252 preMask = LowLaneTy;
3253
3254 if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3255 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3256 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3257 }))
3258 postMask = LowLaneTy;
3259 else if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3260 return M < 0 || (M >= 0 && M < HalfSize) ||
3261 (M >= MaskSize && M < MaskSize + HalfSize);
3262 }))
3263 postMask = HighLaneTy;
3264
3265 // The pre-half of mask is high lane type, and the post-half of mask
3266 // is low lane type, which is closest to the LoongArch instructions.
3267 //
3268 // Note: In the LoongArch architecture, the high lane of mask corresponds
3269 // to the lower 128-bit of vector register, and the low lane of mask
3270 // corresponds the higher 128-bit of vector register.
3271 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3272 return false;
3273 }
3274 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3275 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3276 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3277 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3278 V1 = DAG.getBitcast(VT, V: V1);
3279
3280 if (!V2.isUndef()) {
3281 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3282 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3283 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3284 V2 = DAG.getBitcast(VT, V: V2);
3285 }
3286
3287 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3288 *it = *it < 0 ? *it : *it - HalfSize;
3289 }
3290 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3291 *it = *it < 0 ? *it : *it + HalfSize;
3292 }
3293 } else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3294 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3295 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3296 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3297 V1 = DAG.getBitcast(VT, V: V1);
3298
3299 if (!V2.isUndef()) {
3300 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3301 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3302 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3303 V2 = DAG.getBitcast(VT, V: V2);
3304 }
3305
3306 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3307 *it = *it < 0 ? *it : *it - HalfSize;
3308 }
3309 } else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3310 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3311 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3312 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3313 V1 = DAG.getBitcast(VT, V: V1);
3314
3315 if (!V2.isUndef()) {
3316 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3317 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3318 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3319 V2 = DAG.getBitcast(VT, V: V2);
3320 }
3321
3322 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3323 *it = *it < 0 ? *it : *it + HalfSize;
3324 }
3325 } else { // cross-lane
3326 return false;
3327 }
3328
3329 return true;
3330}
3331
3332/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
3333/// Only for 256-bit vector.
3334///
3335/// For example:
3336/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
3337/// <4 x i64> <i32 0, i32 3, i32 2, i32 0>
3338/// is lowerded to:
3339/// (XVPERMI $xr2, $xr0, 78)
3340/// (XVSHUF $xr1, $xr2, $xr0)
3341/// (XVORI $xr0, $xr1, 0)
3342static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL,
3343 ArrayRef<int> Mask,
3344 MVT VT, SDValue V1,
3345 SDValue V2,
3346 SelectionDAG &DAG) {
3347 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
3348 int Size = Mask.size();
3349 int LaneSize = Size / 2;
3350
3351 bool LaneCrossing[2] = {false, false};
3352 for (int i = 0; i < Size; ++i)
3353 if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
3354 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
3355
3356 // Ensure that all lanes ared involved.
3357 if (!LaneCrossing[0] && !LaneCrossing[1])
3358 return SDValue();
3359
3360 SmallVector<int> InLaneMask;
3361 InLaneMask.assign(in_start: Mask.begin(), in_end: Mask.end());
3362 for (int i = 0; i < Size; ++i) {
3363 int &M = InLaneMask[i];
3364 if (M < 0)
3365 continue;
3366 if (((M % Size) / LaneSize) != (i / LaneSize))
3367 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
3368 }
3369
3370 SDValue Flipped = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3371 Flipped = DAG.getVectorShuffle(VT: MVT::v4i64, dl: DL, N1: Flipped,
3372 N2: DAG.getUNDEF(VT: MVT::v4i64), Mask: {2, 3, 0, 1});
3373 Flipped = DAG.getBitcast(VT, V: Flipped);
3374 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: Flipped, Mask: InLaneMask);
3375}
3376
3377/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
3378///
3379/// This routine breaks down the specific type of 256-bit shuffle and
3380/// dispatches to the lowering routines accordingly.
3381static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3382 SDValue V1, SDValue V2, SelectionDAG &DAG,
3383 const LoongArchSubtarget &Subtarget) {
3384 assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||
3385 VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||
3386 VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&
3387 "Vector type is unsupported for lasx!");
3388 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
3389 "Two operands have different types!");
3390 assert(VT.getVectorNumElements() == Mask.size() &&
3391 "Unexpected mask size for shuffle!");
3392 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
3393 assert(Mask.size() >= 4 && "Mask size is less than 4.");
3394
3395 APInt KnownUndef, KnownZero;
3396 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
3397 APInt Zeroable = KnownUndef | KnownZero;
3398
3399 SDValue Result;
3400 // TODO: Add more comparison patterns.
3401 if (V2.isUndef()) {
3402 if ((Result =
3403 lowerVECTOR_SHUFFLE_XVREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
3404 return Result;
3405 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG,
3406 Subtarget)))
3407 return Result;
3408 // Try to widen vectors to gain more optimization opportunities.
3409 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
3410 return NewShuffle;
3411 if ((Result =
3412 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3413 return Result;
3414 if ((Result = lowerVECTOR_SHUFFLE_XVPERM(DL, Mask, VT, V1, DAG, Subtarget)))
3415 return Result;
3416 if ((Result =
3417 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
3418 return Result;
3419
3420 // TODO: This comment may be enabled in the future to better match the
3421 // pattern for instruction selection.
3422 /* V2 = V1; */
3423 }
3424
3425 // It is recommended not to change the pattern comparison order for better
3426 // performance.
3427 if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, Mask, VT, V1, V2, DAG)))
3428 return Result;
3429 if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, Mask, VT, V1, V2, DAG)))
3430 return Result;
3431 if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, Mask, VT, V1, V2, DAG)))
3432 return Result;
3433 if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, Mask, VT, V1, V2, DAG)))
3434 return Result;
3435 if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, Mask, VT, V1, V2, DAG)))
3436 return Result;
3437 if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, Mask, VT, V1, V2, DAG)))
3438 return Result;
3439 if ((VT.SimpleTy == MVT::v4i64 || VT.SimpleTy == MVT::v4f64) &&
3440 (Result =
3441 lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3442 return Result;
3443 if ((Result =
3444 lowerVECTOR_SHUFFLE_XVEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3445 return Result;
3446 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
3447 Zeroable)))
3448 return Result;
3449 if ((Result =
3450 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3451 return Result;
3452 if ((Result =
3453 lowerVECTOR_SHUFFLE_XVINSVE0(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3454 return Result;
3455 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
3456 Subtarget)))
3457 return Result;
3458
3459 // canonicalize non cross-lane shuffle vector
3460 SmallVector<int> NewMask(Mask);
3461 if (canonicalizeShuffleVectorByLane(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget))
3462 return lower256BitShuffle(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget);
3463
3464 // FIXME: Handling the remaining cases earlier can degrade performance
3465 // in some situations. Further analysis is required to enable more
3466 // effective optimizations.
3467 if (V2.isUndef()) {
3468 if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, Mask: NewMask, VT,
3469 V1, V2, DAG)))
3470 return Result;
3471 }
3472
3473 if (SDValue NewShuffle = widenShuffleMask(DL, Mask: NewMask, VT, V1, V2, DAG))
3474 return NewShuffle;
3475 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, Mask: NewMask, VT, V1, V2, DAG)))
3476 return Result;
3477
3478 return SDValue();
3479}
3480
3481SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3482 SelectionDAG &DAG) const {
3483 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
3484 ArrayRef<int> OrigMask = SVOp->getMask();
3485 SDValue V1 = Op.getOperand(i: 0);
3486 SDValue V2 = Op.getOperand(i: 1);
3487 MVT VT = Op.getSimpleValueType();
3488 int NumElements = VT.getVectorNumElements();
3489 SDLoc DL(Op);
3490
3491 bool V1IsUndef = V1.isUndef();
3492 bool V2IsUndef = V2.isUndef();
3493 if (V1IsUndef && V2IsUndef)
3494 return DAG.getUNDEF(VT);
3495
3496 // When we create a shuffle node we put the UNDEF node to second operand,
3497 // but in some cases the first operand may be transformed to UNDEF.
3498 // In this case we should just commute the node.
3499 if (V1IsUndef)
3500 return DAG.getCommutedVectorShuffle(SV: *SVOp);
3501
3502 // Check for non-undef masks pointing at an undef vector and make the masks
3503 // undef as well. This makes it easier to match the shuffle based solely on
3504 // the mask.
3505 if (V2IsUndef &&
3506 any_of(Range&: OrigMask, P: [NumElements](int M) { return M >= NumElements; })) {
3507 SmallVector<int, 8> NewMask(OrigMask);
3508 for (int &M : NewMask)
3509 if (M >= NumElements)
3510 M = -1;
3511 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask: NewMask);
3512 }
3513
3514 // Check for illegal shuffle mask element index values.
3515 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
3516 (void)MaskUpperLimit;
3517 assert(llvm::all_of(OrigMask,
3518 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
3519 "Out of bounds shuffle index");
3520
3521 // For each vector width, delegate to a specialized lowering routine.
3522 if (VT.is128BitVector())
3523 return lower128BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3524
3525 if (VT.is256BitVector())
3526 return lower256BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3527
3528 return SDValue();
3529}
3530
3531SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
3532 SelectionDAG &DAG) const {
3533 // Custom lower to ensure the libcall return is passed in an FPR on hard
3534 // float ABIs.
3535 SDLoc DL(Op);
3536 MakeLibCallOptions CallOptions;
3537 SDValue Op0 = Op.getOperand(i: 0);
3538 SDValue Chain = SDValue();
3539 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
3540 SDValue Res;
3541 std::tie(args&: Res, args&: Chain) =
3542 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
3543 if (Subtarget.is64Bit())
3544 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3545 return DAG.getBitcast(VT: MVT::i32, V: Res);
3546}
3547
3548SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
3549 SelectionDAG &DAG) const {
3550 // Custom lower to ensure the libcall argument is passed in an FPR on hard
3551 // float ABIs.
3552 SDLoc DL(Op);
3553 MakeLibCallOptions CallOptions;
3554 SDValue Op0 = Op.getOperand(i: 0);
3555 SDValue Chain = SDValue();
3556 SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3557 DL, VT: MVT::f32, Operand: Op0)
3558 : DAG.getBitcast(VT: MVT::f32, V: Op0);
3559 SDValue Res;
3560 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
3561 CallOptions, dl: DL, Chain);
3562 return Res;
3563}
3564
3565SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
3566 SelectionDAG &DAG) const {
3567 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3568 SDLoc DL(Op);
3569 MakeLibCallOptions CallOptions;
3570 RTLIB::Libcall LC =
3571 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
3572 SDValue Res =
3573 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
3574 if (Subtarget.is64Bit())
3575 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3576 return DAG.getBitcast(VT: MVT::i32, V: Res);
3577}
3578
3579SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
3580 SelectionDAG &DAG) const {
3581 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3582 MVT VT = Op.getSimpleValueType();
3583 SDLoc DL(Op);
3584 Op = DAG.getNode(
3585 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
3586 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
3587 SDValue Res = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3588 DL, VT: MVT::f32, Operand: Op)
3589 : DAG.getBitcast(VT: MVT::f32, V: Op);
3590 if (VT != MVT::f32)
3591 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
3592 return Res;
3593}
3594
3595// Lower BUILD_VECTOR as broadcast load (if possible).
3596// For example:
3597// %a = load i8, ptr %ptr
3598// %b = build_vector %a, %a, %a, %a
3599// is lowered to :
3600// (VLDREPL_B $a0, 0)
3601static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp,
3602 const SDLoc &DL,
3603 SelectionDAG &DAG) {
3604 MVT VT = BVOp->getSimpleValueType(ResNo: 0);
3605 int NumOps = BVOp->getNumOperands();
3606
3607 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3608 "Unsupported vector type for broadcast.");
3609
3610 SDValue IdentitySrc;
3611 bool IsIdeneity = true;
3612
3613 for (int i = 0; i != NumOps; i++) {
3614 SDValue Op = BVOp->getOperand(Num: i);
3615 if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
3616 IsIdeneity = false;
3617 break;
3618 }
3619 IdentitySrc = BVOp->getOperand(Num: 0);
3620 }
3621
3622 // make sure that this load is valid and only has one user.
3623 if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(N: IdentitySrc.getNode()))
3624 return SDValue();
3625
3626 auto *LN = cast<LoadSDNode>(Val&: IdentitySrc);
3627 auto ExtType = LN->getExtensionType();
3628
3629 if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
3630 VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) {
3631 // Indexed loads and stores are not supported on LoongArch.
3632 assert(LN->isUnindexed() && "Unexpected indexed load.");
3633
3634 SDVTList Tys = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3635 // The offset operand of unindexed load is always undefined, so there is
3636 // no need to pass it to VLDREPL.
3637 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3638 SDValue BCast = DAG.getNode(Opcode: LoongArchISD::VLDREPL, DL, VTList: Tys, Ops);
3639 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: BCast.getValue(R: 1));
3640 return BCast;
3641 }
3642 return SDValue();
3643}
3644
3645// Sequentially insert elements from Ops into Vector, from low to high indices.
3646// Note: Ops can have fewer elements than Vector.
3647static void fillVector(ArrayRef<SDValue> Ops, SelectionDAG &DAG, SDLoc DL,
3648 const LoongArchSubtarget &Subtarget, SDValue &Vector,
3649 EVT ResTy) {
3650 assert(Ops.size() <= ResTy.getVectorNumElements());
3651
3652 SDValue Op0 = Ops[0];
3653 if (!Op0.isUndef())
3654 Vector = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: ResTy, Operand: Op0);
3655 for (unsigned i = 1; i < Ops.size(); ++i) {
3656 SDValue Opi = Ops[i];
3657 if (Opi.isUndef())
3658 continue;
3659 Vector = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Vector, N2: Opi,
3660 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3661 }
3662}
3663
3664// Build a ResTy subvector from Node, taking NumElts elements starting at index
3665// 'first'.
3666static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node,
3667 SelectionDAG &DAG, SDLoc DL,
3668 const LoongArchSubtarget &Subtarget,
3669 EVT ResTy, unsigned first) {
3670 unsigned NumElts = ResTy.getVectorNumElements();
3671
3672 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3673
3674 SmallVector<SDValue, 16> Ops(Node->op_begin() + first,
3675 Node->op_begin() + first + NumElts);
3676 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3677 fillVector(Ops, DAG, DL, Subtarget, Vector, ResTy);
3678 return Vector;
3679}
3680
3681SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
3682 SelectionDAG &DAG) const {
3683 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Val&: Op);
3684 MVT VT = Node->getSimpleValueType(ResNo: 0);
3685 EVT ResTy = Op->getValueType(ResNo: 0);
3686 unsigned NumElts = ResTy.getVectorNumElements();
3687 SDLoc DL(Op);
3688 APInt SplatValue, SplatUndef;
3689 unsigned SplatBitSize;
3690 bool HasAnyUndefs;
3691 bool IsConstant = false;
3692 bool UseSameConstant = true;
3693 SDValue ConstantValue;
3694 bool Is128Vec = ResTy.is128BitVector();
3695 bool Is256Vec = ResTy.is256BitVector();
3696
3697 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3698 (!Subtarget.hasExtLASX() || !Is256Vec))
3699 return SDValue();
3700
3701 if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(BVOp: Node, DL, DAG))
3702 return Result;
3703
3704 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3705 /*MinSplatBits=*/8) &&
3706 SplatBitSize <= 64) {
3707 // We can only cope with 8, 16, 32, or 64-bit elements.
3708 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3709 SplatBitSize != 64)
3710 return SDValue();
3711
3712 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3713 // We can only handle 64-bit elements that are within
3714 // the signed 10-bit range or match vldi patterns on 32-bit targets.
3715 // See the BUILD_VECTOR case in LoongArchDAGToDAGISel::Select().
3716 if (!SplatValue.isSignedIntN(N: 10) &&
3717 !isImmVLDILegalForMode1(SplatValue, SplatBitSize).first)
3718 return SDValue();
3719 if ((Is128Vec && ResTy == MVT::v4i32) ||
3720 (Is256Vec && ResTy == MVT::v8i32))
3721 return Op;
3722 }
3723
3724 EVT ViaVecTy;
3725
3726 switch (SplatBitSize) {
3727 default:
3728 return SDValue();
3729 case 8:
3730 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3731 break;
3732 case 16:
3733 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3734 break;
3735 case 32:
3736 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3737 break;
3738 case 64:
3739 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3740 break;
3741 }
3742
3743 // SelectionDAG::getConstant will promote SplatValue appropriately.
3744 SDValue Result = DAG.getConstant(Val: SplatValue, DL, VT: ViaVecTy);
3745
3746 // Bitcast to the type we originally wanted.
3747 if (ViaVecTy != ResTy)
3748 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Node), VT: ResTy, Operand: Result);
3749
3750 return Result;
3751 }
3752
3753 if (DAG.isSplatValue(V: Op, /*AllowUndefs=*/false))
3754 return Op;
3755
3756 for (unsigned i = 0; i < NumElts; ++i) {
3757 SDValue Opi = Node->getOperand(Num: i);
3758 if (isIntOrFPConstant(V: Opi)) {
3759 IsConstant = true;
3760 if (!ConstantValue.getNode())
3761 ConstantValue = Opi;
3762 else if (ConstantValue != Opi)
3763 UseSameConstant = false;
3764 }
3765 }
3766
3767 // If the type of BUILD_VECTOR is v2f64, custom legalizing it has no benefits.
3768 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3769 SDValue Result = DAG.getSplatBuildVector(VT: ResTy, DL, Op: ConstantValue);
3770 for (unsigned i = 0; i < NumElts; ++i) {
3771 SDValue Opi = Node->getOperand(Num: i);
3772 if (!isIntOrFPConstant(V: Opi))
3773 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Result, N2: Opi,
3774 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3775 }
3776 return Result;
3777 }
3778
3779 if (!IsConstant) {
3780 // If the BUILD_VECTOR has a repeated pattern, use INSERT_VECTOR_ELT to fill
3781 // the sub-sequence of the vector and then broadcast the sub-sequence.
3782 //
3783 // TODO: If the BUILD_VECTOR contains undef elements, consider falling
3784 // back to use INSERT_VECTOR_ELT to materialize the vector, because it
3785 // generates worse code in some cases. This could be further optimized
3786 // with more consideration.
3787 SmallVector<SDValue> Sequence;
3788 BitVector UndefElements;
3789 if (Node->getRepeatedSequence(Sequence, UndefElements: &UndefElements) &&
3790 UndefElements.count() == 0) {
3791 // Using LSX instructions to fill the sub-sequence of 256-bits vector,
3792 // because the high part can be simply treated as undef.
3793 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3794 EVT FillTy = Is256Vec
3795 ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext())
3796 : ResTy;
3797 SDValue FillVec =
3798 Is256Vec ? DAG.getExtractSubvector(DL, VT: FillTy, Vec: Vector, Idx: 0) : Vector;
3799
3800 fillVector(Ops: Sequence, DAG, DL, Subtarget, Vector&: FillVec, ResTy: FillTy);
3801
3802 unsigned SeqLen = Sequence.size();
3803 unsigned SplatLen = NumElts / SeqLen;
3804 MVT SplatEltTy = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits() * SeqLen);
3805 MVT SplatTy = MVT::getVectorVT(VT: SplatEltTy, NumElements: SplatLen);
3806
3807 // If size of the sub-sequence is half of a 256-bits vector, bitcast the
3808 // vector to v4i64 type in order to match the pattern of XVREPLVE0Q.
3809 if (SplatEltTy == MVT::i128)
3810 SplatTy = MVT::v4i64;
3811
3812 SDValue SplatVec;
3813 SDValue SrcVec = DAG.getBitcast(
3814 VT: SplatTy,
3815 V: Is256Vec ? DAG.getInsertSubvector(DL, Vec: Vector, SubVec: FillVec, Idx: 0) : FillVec);
3816 if (Is256Vec) {
3817 SplatVec =
3818 DAG.getNode(Opcode: (SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
3819 : LoongArchISD::XVREPLVE0,
3820 DL, VT: SplatTy, Operand: SrcVec);
3821 } else {
3822 SplatVec = DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT: SplatTy, N1: SrcVec,
3823 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
3824 }
3825
3826 return DAG.getBitcast(VT: ResTy, V: SplatVec);
3827 }
3828
3829 // Use INSERT_VECTOR_ELT operations rather than expand to stores, because
3830 // using memory operations is much lower.
3831 //
3832 // For 256-bit vectors, normally split into two halves and concatenate.
3833 // Special case: for v8i32/v8f32/v4i64/v4f64, if the upper half has only
3834 // one non-undef element, skip spliting to avoid a worse result.
3835 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
3836 ResTy == MVT::v4f64) {
3837 unsigned NonUndefCount = 0;
3838 for (unsigned i = NumElts / 2; i < NumElts; ++i) {
3839 if (!Node->getOperand(Num: i).isUndef()) {
3840 ++NonUndefCount;
3841 if (NonUndefCount > 1)
3842 break;
3843 }
3844 }
3845 if (NonUndefCount == 1)
3846 return fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy, first: 0);
3847 }
3848
3849 EVT VecTy =
3850 Is256Vec ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext()) : ResTy;
3851 SDValue Vector =
3852 fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy: VecTy, first: 0);
3853
3854 if (Is128Vec)
3855 return Vector;
3856
3857 SDValue VectorHi = fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget,
3858 ResTy: VecTy, first: NumElts / 2);
3859
3860 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResTy, N1: Vector, N2: VectorHi);
3861 }
3862
3863 return SDValue();
3864}
3865
3866SDValue LoongArchTargetLowering::lowerCONCAT_VECTORS(SDValue Op,
3867 SelectionDAG &DAG) const {
3868 SDLoc DL(Op);
3869 MVT ResVT = Op.getSimpleValueType();
3870 assert(ResVT.is256BitVector() && Op.getNumOperands() == 2);
3871
3872 if (Op.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
3873 Op.getOperand(i: 1).getOpcode() == ISD::TRUNCATE)
3874 return Op;
3875
3876 unsigned NumOperands = Op.getNumOperands();
3877 unsigned NumFreezeUndef = 0;
3878 unsigned NumZero = 0;
3879 unsigned NumNonZero = 0;
3880 unsigned NonZeros = 0;
3881 SmallSet<SDValue, 4> Undefs;
3882 for (unsigned i = 0; i != NumOperands; ++i) {
3883 SDValue SubVec = Op.getOperand(i);
3884 if (SubVec.isUndef())
3885 continue;
3886 if (ISD::isFreezeUndef(N: SubVec.getNode())) {
3887 // If the freeze(undef) has multiple uses then we must fold to zero.
3888 if (SubVec.hasOneUse()) {
3889 ++NumFreezeUndef;
3890 } else {
3891 ++NumZero;
3892 Undefs.insert(V: SubVec);
3893 }
3894 } else if (ISD::isBuildVectorAllZeros(N: SubVec.getNode()))
3895 ++NumZero;
3896 else {
3897 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
3898 NonZeros |= 1 << i;
3899 ++NumNonZero;
3900 }
3901 }
3902
3903 // If we have more than 2 non-zeros, build each half separately.
3904 if (NumNonZero > 2) {
3905 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
3906 ArrayRef<SDUse> Ops = Op->ops();
3907 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3908 Ops: Ops.slice(N: 0, M: NumOperands / 2));
3909 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
3910 Ops: Ops.slice(N: NumOperands / 2));
3911 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
3912 }
3913
3914 // Otherwise, build it up through insert_subvectors.
3915 SDValue Vec = NumZero ? DAG.getConstant(Val: 0, DL, VT: ResVT)
3916 : (NumFreezeUndef ? DAG.getFreeze(V: DAG.getUNDEF(VT: ResVT))
3917 : DAG.getUNDEF(VT: ResVT));
3918
3919 // Replace Undef operands with ZeroVector.
3920 for (SDValue U : Undefs)
3921 DAG.ReplaceAllUsesWith(From: U, To: DAG.getConstant(Val: 0, DL, VT: U.getSimpleValueType()));
3922
3923 MVT SubVT = Op.getOperand(i: 0).getSimpleValueType();
3924 unsigned NumSubElems = SubVT.getVectorNumElements();
3925 for (unsigned i = 0; i != NumOperands; ++i) {
3926 if ((NonZeros & (1 << i)) == 0)
3927 continue;
3928
3929 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ResVT, N1: Vec, N2: Op.getOperand(i),
3930 N3: DAG.getVectorIdxConstant(Val: i * NumSubElems, DL));
3931 }
3932
3933 return Vec;
3934}
3935
3936SDValue
3937LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
3938 SelectionDAG &DAG) const {
3939 MVT EltVT = Op.getSimpleValueType();
3940 SDValue Vec = Op->getOperand(Num: 0);
3941 EVT VecTy = Vec->getValueType(ResNo: 0);
3942 SDValue Idx = Op->getOperand(Num: 1);
3943 SDLoc DL(Op);
3944 MVT GRLenVT = Subtarget.getGRLenVT();
3945
3946 assert(VecTy.is256BitVector() && "Unexpected EXTRACT_VECTOR_ELT vector type");
3947
3948 if (isa<ConstantSDNode>(Val: Idx))
3949 return Op;
3950
3951 switch (VecTy.getSimpleVT().SimpleTy) {
3952 default:
3953 llvm_unreachable("Unexpected type");
3954 case MVT::v32i8:
3955 case MVT::v16i16:
3956 case MVT::v4i64:
3957 case MVT::v4f64: {
3958 // Extract the high half subvector and place it to the low half of a new
3959 // vector. It doesn't matter what the high half of the new vector is.
3960 EVT HalfTy = VecTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3961 SDValue VecHi =
3962 DAG.getExtractSubvector(DL, VT: HalfTy, Vec, Idx: HalfTy.getVectorNumElements());
3963 SDValue TmpVec =
3964 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: VecTy, N1: DAG.getUNDEF(VT: VecTy),
3965 N2: VecHi, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3966
3967 // Shuffle the origin Vec and the TmpVec using MaskVec, the lowest element
3968 // of MaskVec is Idx, the rest do not matter. ResVec[0] will hold the
3969 // desired element.
3970 SDValue IdxCp =
3971 Subtarget.is64Bit()
3972 ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Idx)
3973 : DAG.getBitcast(VT: MVT::f32, V: Idx);
3974 SDValue IdxVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v8f32, Operand: IdxCp);
3975 SDValue MaskVec =
3976 DAG.getBitcast(VT: (VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, V: IdxVec);
3977 SDValue ResVec =
3978 DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT: VecTy, N1: MaskVec, N2: TmpVec, N3: Vec);
3979
3980 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: ResVec,
3981 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3982 }
3983 case MVT::v8i32:
3984 case MVT::v8f32: {
3985 SDValue SplatIdx = DAG.getSplatBuildVector(VT: MVT::v8i32, DL, Op: Idx);
3986 SDValue SplatValue =
3987 DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT: VecTy, N1: Vec, N2: SplatIdx);
3988
3989 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: SplatValue,
3990 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
3991 }
3992 }
3993}
3994
3995SDValue
3996LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
3997 SelectionDAG &DAG) const {
3998 MVT VT = Op.getSimpleValueType();
3999 MVT EltVT = VT.getVectorElementType();
4000 unsigned NumElts = VT.getVectorNumElements();
4001 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
4002 SDLoc DL(Op);
4003 SDValue Op0 = Op.getOperand(i: 0);
4004 SDValue Op1 = Op.getOperand(i: 1);
4005 SDValue Op2 = Op.getOperand(i: 2);
4006
4007 if (isa<ConstantSDNode>(Val: Op2))
4008 return Op;
4009
4010 MVT IdxTy = MVT::getIntegerVT(BitWidth: EltSizeInBits);
4011 MVT IdxVTy = MVT::getVectorVT(VT: IdxTy, NumElements: NumElts);
4012
4013 if (!isTypeLegal(VT) || !isTypeLegal(VT: IdxVTy))
4014 return SDValue();
4015
4016 SDValue SplatElt = DAG.getSplatBuildVector(VT, DL, Op: Op1);
4017 SmallVector<SDValue, 32> RawIndices;
4018 SDValue SplatIdx;
4019 SDValue Indices;
4020
4021 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4022 MVT PairVTy = MVT::getVectorVT(VT: MVT::i32, NumElements: NumElts * 2);
4023 for (unsigned i = 0; i < NumElts; ++i) {
4024 RawIndices.push_back(Elt: Op2);
4025 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4026 }
4027 SplatIdx = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4028 SplatIdx = DAG.getBitcast(VT: IdxVTy, V: SplatIdx);
4029
4030 RawIndices.clear();
4031 for (unsigned i = 0; i < NumElts; ++i) {
4032 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: MVT::i32));
4033 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4034 }
4035 Indices = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4036 Indices = DAG.getBitcast(VT: IdxVTy, V: Indices);
4037 } else {
4038 SplatIdx = DAG.getSplatBuildVector(VT: IdxVTy, DL, Op: Op2);
4039
4040 for (unsigned i = 0; i < NumElts; ++i)
4041 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
4042 Indices = DAG.getBuildVector(VT: IdxVTy, DL, Ops: RawIndices);
4043 }
4044
4045 // insert vec, elt, idx
4046 // =>
4047 // select (splatidx == {0,1,2...}) ? splatelt : vec
4048 SDValue SelectCC =
4049 DAG.getSetCC(DL, VT: IdxVTy, LHS: SplatIdx, RHS: Indices, Cond: ISD::CondCode::SETEQ);
4050 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectCC, N2: SplatElt, N3: Op0);
4051}
4052
4053SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4054 SelectionDAG &DAG) const {
4055 SDLoc DL(Op);
4056 SyncScope::ID FenceSSID =
4057 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4058
4059 // singlethread fences only synchronize with signal handlers on the same
4060 // thread and thus only need to preserve instruction order, not actually
4061 // enforce memory ordering.
4062 if (FenceSSID == SyncScope::SingleThread)
4063 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4064 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
4065
4066 return Op;
4067}
4068
4069static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL,
4070 MVT GRLenVT, SDValue RMValue) {
4071 // LLVM rounding mode encoding differs from LoongArch FCSR encoding:
4072 // LLVM: 0=RTZ, 1=RNE, 2=RUP, 3=RDN
4073 // FCSR: 0=RNE, 1=RZ, 2=RP, 3=RN
4074 //
4075 // The conversion swaps encodings 0 and 1 while preserving 2 and 3.
4076 // Since the transformation is self-inverse, it applies in both directions:
4077 // LLVM RM <-> LoongArch FCSR RM
4078 //
4079 // Transformation: RM ^ (~(RM >> 1) & 1)
4080 SDValue ShiftRight1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4081 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4082
4083 SDValue SwapMask = DAG.getNode(Opcode: ISD::AND, DL, VT: GRLenVT,
4084 N1: DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: ShiftRight1,
4085 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT)),
4086 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4087
4088 return DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: RMValue, N2: SwapMask);
4089}
4090
4091SDValue LoongArchTargetLowering::lowerSET_ROUNDING(SDValue Op,
4092 SelectionDAG &DAG) const {
4093 MVT GRLenVT = Subtarget.getGRLenVT();
4094 SDLoc DL(Op);
4095 SDValue Chain = Op.getOperand(i: 0);
4096 SDValue RMValue = Op.getOperand(i: 1);
4097
4098 if (auto *CVal = dyn_cast<ConstantSDNode>(Val&: RMValue)) {
4099 uint64_t RM = CVal->getZExtValue();
4100 if (RM > 3) {
4101 MachineFunction &MF = DAG.getMachineFunction();
4102 LLVMContext &C = MF.getFunction().getContext();
4103 C.diagnose(DI: DiagnosticInfoUnsupported(
4104 MF.getFunction(),
4105 "rounding mode is not supported by LoongArch hardware",
4106 DiagnosticLocation(DL.getDebugLoc()), DS_Error));
4107 return Chain;
4108 }
4109 }
4110
4111 RMValue = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: RMValue);
4112 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4113
4114 // The RM field in FCSR is at bits [9:8]. Shift the rounding mode value
4115 // into position before writing via WRFCSR.
4116 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: RMValue,
4117 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4118
4119 // FCSR3 is an alias of the RM field; writing it avoids clobbering
4120 // unrelated fields in FCSR0.
4121 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4122 MachineSDNode *RN = DAG.getMachineNode(Opcode: LoongArch::WRFCSR, dl: DL, VT: MVT::Other,
4123 Op1: FCSRNo, Op2: RMValue, Op3: Chain);
4124 return SDValue(RN, 0);
4125}
4126
4127SDValue LoongArchTargetLowering::lowerGET_ROUNDING(SDValue Op,
4128 SelectionDAG &DAG) const {
4129 MVT GRLenVT = Subtarget.getGRLenVT();
4130 SDLoc DL(Op);
4131 SDValue Chain = Op->getOperand(Num: 0);
4132
4133 // FCSR3 is an alias of the RM field.
4134 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4135 MachineSDNode *FCSR = DAG.getMachineNode(Opcode: LoongArch::RDFCSR, dl: DL, VT1: GRLenVT,
4136 VT2: MVT::Other, Op1: FCSRNo, Op2: Chain);
4137 SDValue RMValue = SDValue(FCSR, 0);
4138 Chain = SDValue(FCSR, 1);
4139
4140 // The RM field in FCSR is at bits [9:8].
4141 RMValue = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4142 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4143 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4144
4145 SDValue RetVal = DAG.getZExtOrTrunc(Op: RMValue, DL, VT: Op.getValueType());
4146 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl: DL);
4147}
4148
4149SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
4150 SelectionDAG &DAG) const {
4151
4152 if (Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i32) {
4153 DAG.getContext()->emitError(
4154 ErrorStr: "On LA64, only 64-bit registers can be written.");
4155 return Op.getOperand(i: 0);
4156 }
4157
4158 if (!Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i64) {
4159 DAG.getContext()->emitError(
4160 ErrorStr: "On LA32, only 32-bit registers can be written.");
4161 return Op.getOperand(i: 0);
4162 }
4163
4164 return Op;
4165}
4166
4167SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
4168 SelectionDAG &DAG) const {
4169 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 0))) {
4170 DAG.getContext()->emitError(ErrorStr: "argument to '__builtin_frame_address' must "
4171 "be a constant integer");
4172 return SDValue();
4173 }
4174
4175 MachineFunction &MF = DAG.getMachineFunction();
4176 MF.getFrameInfo().setFrameAddressIsTaken(true);
4177 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4178 EVT VT = Op.getValueType();
4179 SDLoc DL(Op);
4180 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
4181 unsigned Depth = Op.getConstantOperandVal(i: 0);
4182 int GRLenInBytes = Subtarget.getGRLen() / 8;
4183
4184 while (Depth--) {
4185 int Offset = -(GRLenInBytes * 2);
4186 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
4187 N2: DAG.getSignedConstant(Val: Offset, DL, VT));
4188 FrameAddr =
4189 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
4190 }
4191 return FrameAddr;
4192}
4193
4194SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
4195 SelectionDAG &DAG) const {
4196 // Currently only support lowering return address for current frame.
4197 if (Op.getConstantOperandVal(i: 0) != 0) {
4198 DAG.getContext()->emitError(
4199 ErrorStr: "return address can only be determined for the current frame");
4200 return SDValue();
4201 }
4202
4203 MachineFunction &MF = DAG.getMachineFunction();
4204 MF.getFrameInfo().setReturnAddressIsTaken(true);
4205 MVT GRLenVT = Subtarget.getGRLenVT();
4206
4207 // Return the value of the return address register, marking it an implicit
4208 // live-in.
4209 Register Reg = MF.addLiveIn(PReg: Subtarget.getRegisterInfo()->getRARegister(),
4210 RC: getRegClassFor(VT: GRLenVT));
4211 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg, VT: GRLenVT);
4212}
4213
4214SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
4215 SelectionDAG &DAG) const {
4216 MachineFunction &MF = DAG.getMachineFunction();
4217 auto Size = Subtarget.getGRLen() / 8;
4218 auto FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: 0, IsImmutable: false);
4219 return DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4220}
4221
4222SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,
4223 SelectionDAG &DAG) const {
4224 MachineFunction &MF = DAG.getMachineFunction();
4225 auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();
4226
4227 SDLoc DL(Op);
4228 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4229 VT: getPointerTy(DL: MF.getDataLayout()));
4230
4231 // vastart just stores the address of the VarArgsFrameIndex slot into the
4232 // memory location argument.
4233 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4234 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
4235 PtrInfo: MachinePointerInfo(SV));
4236}
4237
4238SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
4239 SelectionDAG &DAG) const {
4240 SDLoc DL(Op);
4241 SDValue Op0 = Op.getOperand(i: 0);
4242 EVT VT = Op.getValueType();
4243 EVT Op0VT = Op0.getValueType();
4244
4245 if (VT.isVector()) {
4246 if (VT.getScalarSizeInBits() != Op0VT.getScalarSizeInBits())
4247 return SDValue();
4248 return Op;
4249 }
4250
4251 if ((DAG.SignBitIsZero(Op: Op0) || Op->getFlags().hasNonNeg()) &&
4252 !isOperationLegal(Op: ISD::UINT_TO_FP, VT: Op0VT) &&
4253 isOperationLegal(Op: ISD::SINT_TO_FP, VT: Op0VT))
4254 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: Op0);
4255
4256 // We can't do uint64 -> double -> float because of double-rounding issue.
4257 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4258 Op0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i64, Operand: Op0);
4259 SDValue Conv = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::v2f64, Operand: Op0);
4260 Conv = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f64, N1: Conv,
4261 N2: DAG.getIntPtrConstant(Val: 0, DL));
4262 return Conv;
4263 }
4264
4265 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4266 return SDValue();
4267
4268 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4269 !Subtarget.hasBasicD() && "unexpected target features");
4270
4271 if (Op0->getOpcode() == ISD::AND) {
4272 auto *C = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
4273 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4274 return Op;
4275 }
4276
4277 if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&
4278 Op0.getConstantOperandVal(i: 1) < UINT64_C(0X1F) &&
4279 Op0.getConstantOperandVal(i: 2) == UINT64_C(0))
4280 return Op;
4281
4282 if (Op0.getOpcode() == ISD::AssertZext &&
4283 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLT(VT: MVT::i32))
4284 return Op;
4285
4286 EVT OpVT = Op0.getValueType();
4287 EVT RetVT = Op.getValueType();
4288 RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
4289 MakeLibCallOptions CallOptions;
4290 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4291 SDValue Chain = SDValue();
4292 SDValue Result;
4293 std::tie(args&: Result, args&: Chain) =
4294 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4295 return Result;
4296}
4297
4298SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
4299 SelectionDAG &DAG) const {
4300 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4301 !Subtarget.hasBasicD() && "unexpected target features");
4302
4303 SDLoc DL(Op);
4304 SDValue Op0 = Op.getOperand(i: 0);
4305
4306 if ((Op0.getOpcode() == ISD::AssertSext ||
4307 Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&
4308 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLE(VT: MVT::i32))
4309 return Op;
4310
4311 EVT OpVT = Op0.getValueType();
4312 EVT RetVT = Op.getValueType();
4313 RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
4314 MakeLibCallOptions CallOptions;
4315 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4316 SDValue Chain = SDValue();
4317 SDValue Result;
4318 std::tie(args&: Result, args&: Chain) =
4319 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4320 return Result;
4321}
4322
4323SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
4324 SelectionDAG &DAG) const {
4325
4326 SDLoc DL(Op);
4327 EVT VT = Op.getValueType();
4328 SDValue Op0 = Op.getOperand(i: 0);
4329 EVT Op0VT = Op0.getValueType();
4330
4331 if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4332 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4333 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4334 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: NewOp0);
4335 }
4336 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4337 SDValue Lo, Hi;
4338 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
4339 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
4340 }
4341 return Op;
4342}
4343
4344SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
4345 SelectionDAG &DAG) const {
4346
4347 SDLoc DL(Op);
4348 SDValue Op0 = Op.getOperand(i: 0);
4349
4350 if (Op0.getValueType() == MVT::f16)
4351 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
4352
4353 if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4354 !Subtarget.hasBasicD()) {
4355 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f32, Operand: Op0);
4356 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Dst);
4357 }
4358
4359 EVT FPTy = EVT::getFloatingPointVT(BitWidth: Op.getValueSizeInBits());
4360 SDValue Trunc = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FPTy, Operand: Op0);
4361 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op.getValueType(), Operand: Trunc);
4362}
4363
4364SDValue LoongArchTargetLowering::lowerFP_TO_UINT(SDValue Op,
4365 SelectionDAG &DAG) const {
4366 if (!Subtarget.hasExtLSX())
4367 return SDValue();
4368
4369 SDLoc DL(Op);
4370 SDValue Src = Op.getOperand(i: 0);
4371 EVT VT = Op.getValueType();
4372 EVT SrcVT = Src.getValueType();
4373
4374 if (VT != MVT::i64)
4375 return SDValue();
4376
4377 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4378 return SDValue();
4379
4380 if (SrcVT == MVT::f32)
4381 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Src);
4382 Src = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2f64, Operand: Src);
4383 SDValue Conv = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: MVT::v2i64, Operand: Src);
4384 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Conv,
4385 N2: DAG.getIntPtrConstant(Val: 0, DL));
4386}
4387
4388static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
4389 SelectionDAG &DAG, unsigned Flags) {
4390 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
4391}
4392
4393static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
4394 SelectionDAG &DAG, unsigned Flags) {
4395 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
4396 TargetFlags: Flags);
4397}
4398
4399static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
4400 SelectionDAG &DAG, unsigned Flags) {
4401 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
4402 Offset: N->getOffset(), TargetFlags: Flags);
4403}
4404
4405static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
4406 SelectionDAG &DAG, unsigned Flags) {
4407 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
4408}
4409
4410template <class NodeTy>
4411SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4412 CodeModel::Model M,
4413 bool IsLocal) const {
4414 SDLoc DL(N);
4415 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4416 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
4417 SDValue Load;
4418
4419 switch (M) {
4420 default:
4421 report_fatal_error(reason: "Unsupported code model");
4422
4423 case CodeModel::Large: {
4424 assert(Subtarget.is64Bit() && "Large code model requires LA64");
4425
4426 // This is not actually used, but is necessary for successfully matching
4427 // the PseudoLA_*_LARGE nodes.
4428 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4429 if (IsLocal) {
4430 // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that
4431 // eventually becomes the desired 5-insn code sequence.
4432 Load = SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL_LARGE, dl: DL, VT: Ty,
4433 Op1: Tmp, Op2: Addr),
4434 0);
4435 } else {
4436 // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that
4437 // eventually becomes the desired 5-insn code sequence.
4438 Load = SDValue(
4439 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT_LARGE, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr),
4440 0);
4441 }
4442 break;
4443 }
4444
4445 case CodeModel::Small:
4446 case CodeModel::Medium:
4447 if (IsLocal) {
4448 // This generates the pattern (PseudoLA_PCREL sym), which
4449 //
4450 // for la32r expands to:
4451 // (addi.w (pcaddu12i %pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4452 //
4453 // for la32s and la64 expands to:
4454 // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).
4455 Load = SDValue(
4456 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL, dl: DL, VT: Ty, Op1: Addr), 0);
4457 } else {
4458 // This generates the pattern (PseudoLA_GOT sym), which
4459 //
4460 // for la32r expands to:
4461 // (ld.w (pcaddu12i %got_pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4462 //
4463 // for la32s and la64 expands to:
4464 // (ld.w/d (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).
4465 Load =
4466 SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT, dl: DL, VT: Ty, Op1: Addr), 0);
4467 }
4468 }
4469
4470 if (!IsLocal) {
4471 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4472 MachineFunction &MF = DAG.getMachineFunction();
4473 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4474 PtrInfo: MachinePointerInfo::getGOT(MF),
4475 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4476 MachineMemOperand::MOInvariant,
4477 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4478 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
4479 }
4480
4481 return Load;
4482}
4483
4484SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,
4485 SelectionDAG &DAG) const {
4486 return getAddr(N: cast<BlockAddressSDNode>(Val&: Op), DAG,
4487 M: DAG.getTarget().getCodeModel());
4488}
4489
4490SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,
4491 SelectionDAG &DAG) const {
4492 return getAddr(N: cast<JumpTableSDNode>(Val&: Op), DAG,
4493 M: DAG.getTarget().getCodeModel());
4494}
4495
4496SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,
4497 SelectionDAG &DAG) const {
4498 return getAddr(N: cast<ConstantPoolSDNode>(Val&: Op), DAG,
4499 M: DAG.getTarget().getCodeModel());
4500}
4501
4502SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,
4503 SelectionDAG &DAG) const {
4504 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4505 assert(N->getOffset() == 0 && "unexpected offset in global node");
4506 auto CM = DAG.getTarget().getCodeModel();
4507 const GlobalValue *GV = N->getGlobal();
4508
4509 if (GV->isDSOLocal() && isa<GlobalVariable>(Val: GV)) {
4510 if (auto GCM = dyn_cast<GlobalVariable>(Val: GV)->getCodeModel())
4511 CM = *GCM;
4512 }
4513
4514 return getAddr(N, DAG, M: CM, IsLocal: GV->isDSOLocal());
4515}
4516
4517SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
4518 SelectionDAG &DAG,
4519 unsigned Opc, bool UseGOT,
4520 bool Large) const {
4521 SDLoc DL(N);
4522 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4523 MVT GRLenVT = Subtarget.getGRLenVT();
4524
4525 // This is not actually used, but is necessary for successfully matching the
4526 // PseudoLA_*_LARGE nodes.
4527 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4528 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4529
4530 // Only IE needs an extra argument for large code model.
4531 SDValue Offset = Opc == LoongArch::PseudoLA_TLS_IE_LARGE
4532 ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4533 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4534
4535 // If it is LE for normal/medium code model, the add tp operation will occur
4536 // during the pseudo-instruction expansion.
4537 if (Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4538 return Offset;
4539
4540 if (UseGOT) {
4541 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4542 MachineFunction &MF = DAG.getMachineFunction();
4543 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4544 PtrInfo: MachinePointerInfo::getGOT(MF),
4545 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4546 MachineMemOperand::MOInvariant,
4547 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4548 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Offset.getNode()), NewMemRefs: {MemOp});
4549 }
4550
4551 // Add the thread pointer.
4552 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Offset,
4553 N2: DAG.getRegister(Reg: LoongArch::R2, VT: GRLenVT));
4554}
4555
4556SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
4557 SelectionDAG &DAG,
4558 unsigned Opc,
4559 bool Large) const {
4560 SDLoc DL(N);
4561 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4562 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
4563
4564 // This is not actually used, but is necessary for successfully matching the
4565 // PseudoLA_*_LARGE nodes.
4566 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4567
4568 // Use a PC-relative addressing mode to access the dynamic GOT address.
4569 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4570 SDValue Load = Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4571 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4572
4573 // Prepare argument list to generate call.
4574 ArgListTy Args;
4575 Args.emplace_back(args&: Load, args&: CallTy);
4576
4577 // Setup call to __tls_get_addr.
4578 TargetLowering::CallLoweringInfo CLI(DAG);
4579 CLI.setDebugLoc(DL)
4580 .setChain(DAG.getEntryNode())
4581 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
4582 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
4583 ArgsList: std::move(Args));
4584
4585 return LowerCallTo(CLI).first;
4586}
4587
4588SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
4589 SelectionDAG &DAG, unsigned Opc,
4590 bool Large) const {
4591 SDLoc DL(N);
4592 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4593 const GlobalValue *GV = N->getGlobal();
4594
4595 // This is not actually used, but is necessary for successfully matching the
4596 // PseudoLA_*_LARGE nodes.
4597 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4598
4599 // Use a PC-relative addressing mode to access the global dynamic GOT address.
4600 // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).
4601 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
4602 return Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4603 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4604}
4605
4606SDValue
4607LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,
4608 SelectionDAG &DAG) const {
4609 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4610 CallingConv::GHC)
4611 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4612
4613 bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;
4614 assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");
4615
4616 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4617 assert(N->getOffset() == 0 && "unexpected offset in global node");
4618
4619 if (DAG.getTarget().useEmulatedTLS())
4620 reportFatalUsageError(reason: "the emulated TLS is prohibited");
4621
4622 bool IsDesc = DAG.getTarget().useTLSDESC();
4623
4624 switch (getTargetMachine().getTLSModel(GV: N->getGlobal())) {
4625 case TLSModel::GeneralDynamic:
4626 // In this model, application code calls the dynamic linker function
4627 // __tls_get_addr to locate TLS offsets into the dynamic thread vector at
4628 // runtime.
4629 if (!IsDesc)
4630 return getDynamicTLSAddr(N, DAG,
4631 Opc: Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4632 : LoongArch::PseudoLA_TLS_GD,
4633 Large);
4634 break;
4635 case TLSModel::LocalDynamic:
4636 // Same as GeneralDynamic, except for assembly modifiers and relocation
4637 // records.
4638 if (!IsDesc)
4639 return getDynamicTLSAddr(N, DAG,
4640 Opc: Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4641 : LoongArch::PseudoLA_TLS_LD,
4642 Large);
4643 break;
4644 case TLSModel::InitialExec:
4645 // This model uses the GOT to resolve TLS offsets.
4646 return getStaticTLSAddr(N, DAG,
4647 Opc: Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4648 : LoongArch::PseudoLA_TLS_IE,
4649 /*UseGOT=*/true, Large);
4650 case TLSModel::LocalExec:
4651 // This model is used when static linking as the TLS offsets are resolved
4652 // during program linking.
4653 //
4654 // This node doesn't need an extra argument for the large code model.
4655 return getStaticTLSAddr(N, DAG, Opc: LoongArch::PseudoLA_TLS_LE,
4656 /*UseGOT=*/false, Large);
4657 }
4658
4659 return getTLSDescAddr(N, DAG,
4660 Opc: Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
4661 : LoongArch::PseudoLA_TLS_DESC,
4662 Large);
4663}
4664
4665template <unsigned N>
4666static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp,
4667 SelectionDAG &DAG, bool IsSigned = false) {
4668 auto *CImm = cast<ConstantSDNode>(Val: Op->getOperand(Num: ImmOp));
4669 // Check the ImmArg.
4670 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
4671 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
4672 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) +
4673 ": argument out of range.");
4674 return DAG.getNode(Opcode: ISD::UNDEF, DL: SDLoc(Op), VT: Op.getValueType());
4675 }
4676 return SDValue();
4677}
4678
4679SDValue
4680LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
4681 SelectionDAG &DAG) const {
4682 switch (Op.getConstantOperandVal(i: 0)) {
4683 default:
4684 return SDValue(); // Don't custom lower most intrinsics.
4685 case Intrinsic::thread_pointer: {
4686 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4687 return DAG.getRegister(Reg: LoongArch::R2, VT: PtrVT);
4688 }
4689 case Intrinsic::loongarch_lsx_vpickve2gr_d:
4690 case Intrinsic::loongarch_lsx_vpickve2gr_du:
4691 case Intrinsic::loongarch_lsx_vreplvei_d:
4692 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
4693 return checkIntrinsicImmArg<1>(Op, ImmOp: 2, DAG);
4694 case Intrinsic::loongarch_lsx_vreplvei_w:
4695 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
4696 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
4697 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
4698 case Intrinsic::loongarch_lasx_xvpickve_d:
4699 case Intrinsic::loongarch_lasx_xvpickve_d_f:
4700 return checkIntrinsicImmArg<2>(Op, ImmOp: 2, DAG);
4701 case Intrinsic::loongarch_lasx_xvinsve0_d:
4702 return checkIntrinsicImmArg<2>(Op, ImmOp: 3, DAG);
4703 case Intrinsic::loongarch_lsx_vsat_b:
4704 case Intrinsic::loongarch_lsx_vsat_bu:
4705 case Intrinsic::loongarch_lsx_vrotri_b:
4706 case Intrinsic::loongarch_lsx_vsllwil_h_b:
4707 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
4708 case Intrinsic::loongarch_lsx_vsrlri_b:
4709 case Intrinsic::loongarch_lsx_vsrari_b:
4710 case Intrinsic::loongarch_lsx_vreplvei_h:
4711 case Intrinsic::loongarch_lasx_xvsat_b:
4712 case Intrinsic::loongarch_lasx_xvsat_bu:
4713 case Intrinsic::loongarch_lasx_xvrotri_b:
4714 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
4715 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
4716 case Intrinsic::loongarch_lasx_xvsrlri_b:
4717 case Intrinsic::loongarch_lasx_xvsrari_b:
4718 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
4719 case Intrinsic::loongarch_lasx_xvpickve_w:
4720 case Intrinsic::loongarch_lasx_xvpickve_w_f:
4721 return checkIntrinsicImmArg<3>(Op, ImmOp: 2, DAG);
4722 case Intrinsic::loongarch_lasx_xvinsve0_w:
4723 return checkIntrinsicImmArg<3>(Op, ImmOp: 3, DAG);
4724 case Intrinsic::loongarch_lsx_vsat_h:
4725 case Intrinsic::loongarch_lsx_vsat_hu:
4726 case Intrinsic::loongarch_lsx_vrotri_h:
4727 case Intrinsic::loongarch_lsx_vsllwil_w_h:
4728 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
4729 case Intrinsic::loongarch_lsx_vsrlri_h:
4730 case Intrinsic::loongarch_lsx_vsrari_h:
4731 case Intrinsic::loongarch_lsx_vreplvei_b:
4732 case Intrinsic::loongarch_lasx_xvsat_h:
4733 case Intrinsic::loongarch_lasx_xvsat_hu:
4734 case Intrinsic::loongarch_lasx_xvrotri_h:
4735 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
4736 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
4737 case Intrinsic::loongarch_lasx_xvsrlri_h:
4738 case Intrinsic::loongarch_lasx_xvsrari_h:
4739 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
4740 return checkIntrinsicImmArg<4>(Op, ImmOp: 2, DAG);
4741 case Intrinsic::loongarch_lsx_vsrlni_b_h:
4742 case Intrinsic::loongarch_lsx_vsrani_b_h:
4743 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
4744 case Intrinsic::loongarch_lsx_vsrarni_b_h:
4745 case Intrinsic::loongarch_lsx_vssrlni_b_h:
4746 case Intrinsic::loongarch_lsx_vssrani_b_h:
4747 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
4748 case Intrinsic::loongarch_lsx_vssrani_bu_h:
4749 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
4750 case Intrinsic::loongarch_lsx_vssrarni_b_h:
4751 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
4752 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
4753 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
4754 case Intrinsic::loongarch_lasx_xvsrani_b_h:
4755 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
4756 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
4757 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
4758 case Intrinsic::loongarch_lasx_xvssrani_b_h:
4759 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
4760 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
4761 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
4762 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
4763 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
4764 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
4765 return checkIntrinsicImmArg<4>(Op, ImmOp: 3, DAG);
4766 case Intrinsic::loongarch_lsx_vsat_w:
4767 case Intrinsic::loongarch_lsx_vsat_wu:
4768 case Intrinsic::loongarch_lsx_vrotri_w:
4769 case Intrinsic::loongarch_lsx_vsllwil_d_w:
4770 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
4771 case Intrinsic::loongarch_lsx_vsrlri_w:
4772 case Intrinsic::loongarch_lsx_vsrari_w:
4773 case Intrinsic::loongarch_lsx_vslei_bu:
4774 case Intrinsic::loongarch_lsx_vslei_hu:
4775 case Intrinsic::loongarch_lsx_vslei_wu:
4776 case Intrinsic::loongarch_lsx_vslei_du:
4777 case Intrinsic::loongarch_lsx_vslti_bu:
4778 case Intrinsic::loongarch_lsx_vslti_hu:
4779 case Intrinsic::loongarch_lsx_vslti_wu:
4780 case Intrinsic::loongarch_lsx_vslti_du:
4781 case Intrinsic::loongarch_lsx_vbsll_v:
4782 case Intrinsic::loongarch_lsx_vbsrl_v:
4783 case Intrinsic::loongarch_lasx_xvsat_w:
4784 case Intrinsic::loongarch_lasx_xvsat_wu:
4785 case Intrinsic::loongarch_lasx_xvrotri_w:
4786 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
4787 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
4788 case Intrinsic::loongarch_lasx_xvsrlri_w:
4789 case Intrinsic::loongarch_lasx_xvsrari_w:
4790 case Intrinsic::loongarch_lasx_xvslei_bu:
4791 case Intrinsic::loongarch_lasx_xvslei_hu:
4792 case Intrinsic::loongarch_lasx_xvslei_wu:
4793 case Intrinsic::loongarch_lasx_xvslei_du:
4794 case Intrinsic::loongarch_lasx_xvslti_bu:
4795 case Intrinsic::loongarch_lasx_xvslti_hu:
4796 case Intrinsic::loongarch_lasx_xvslti_wu:
4797 case Intrinsic::loongarch_lasx_xvslti_du:
4798 case Intrinsic::loongarch_lasx_xvbsll_v:
4799 case Intrinsic::loongarch_lasx_xvbsrl_v:
4800 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG);
4801 case Intrinsic::loongarch_lsx_vseqi_b:
4802 case Intrinsic::loongarch_lsx_vseqi_h:
4803 case Intrinsic::loongarch_lsx_vseqi_w:
4804 case Intrinsic::loongarch_lsx_vseqi_d:
4805 case Intrinsic::loongarch_lsx_vslei_b:
4806 case Intrinsic::loongarch_lsx_vslei_h:
4807 case Intrinsic::loongarch_lsx_vslei_w:
4808 case Intrinsic::loongarch_lsx_vslei_d:
4809 case Intrinsic::loongarch_lsx_vslti_b:
4810 case Intrinsic::loongarch_lsx_vslti_h:
4811 case Intrinsic::loongarch_lsx_vslti_w:
4812 case Intrinsic::loongarch_lsx_vslti_d:
4813 case Intrinsic::loongarch_lasx_xvseqi_b:
4814 case Intrinsic::loongarch_lasx_xvseqi_h:
4815 case Intrinsic::loongarch_lasx_xvseqi_w:
4816 case Intrinsic::loongarch_lasx_xvseqi_d:
4817 case Intrinsic::loongarch_lasx_xvslei_b:
4818 case Intrinsic::loongarch_lasx_xvslei_h:
4819 case Intrinsic::loongarch_lasx_xvslei_w:
4820 case Intrinsic::loongarch_lasx_xvslei_d:
4821 case Intrinsic::loongarch_lasx_xvslti_b:
4822 case Intrinsic::loongarch_lasx_xvslti_h:
4823 case Intrinsic::loongarch_lasx_xvslti_w:
4824 case Intrinsic::loongarch_lasx_xvslti_d:
4825 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG, /*IsSigned=*/true);
4826 case Intrinsic::loongarch_lsx_vsrlni_h_w:
4827 case Intrinsic::loongarch_lsx_vsrani_h_w:
4828 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
4829 case Intrinsic::loongarch_lsx_vsrarni_h_w:
4830 case Intrinsic::loongarch_lsx_vssrlni_h_w:
4831 case Intrinsic::loongarch_lsx_vssrani_h_w:
4832 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
4833 case Intrinsic::loongarch_lsx_vssrani_hu_w:
4834 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
4835 case Intrinsic::loongarch_lsx_vssrarni_h_w:
4836 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
4837 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
4838 case Intrinsic::loongarch_lsx_vfrstpi_b:
4839 case Intrinsic::loongarch_lsx_vfrstpi_h:
4840 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
4841 case Intrinsic::loongarch_lasx_xvsrani_h_w:
4842 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
4843 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
4844 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
4845 case Intrinsic::loongarch_lasx_xvssrani_h_w:
4846 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
4847 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
4848 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
4849 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
4850 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
4851 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
4852 case Intrinsic::loongarch_lasx_xvfrstpi_b:
4853 case Intrinsic::loongarch_lasx_xvfrstpi_h:
4854 return checkIntrinsicImmArg<5>(Op, ImmOp: 3, DAG);
4855 case Intrinsic::loongarch_lsx_vsat_d:
4856 case Intrinsic::loongarch_lsx_vsat_du:
4857 case Intrinsic::loongarch_lsx_vrotri_d:
4858 case Intrinsic::loongarch_lsx_vsrlri_d:
4859 case Intrinsic::loongarch_lsx_vsrari_d:
4860 case Intrinsic::loongarch_lasx_xvsat_d:
4861 case Intrinsic::loongarch_lasx_xvsat_du:
4862 case Intrinsic::loongarch_lasx_xvrotri_d:
4863 case Intrinsic::loongarch_lasx_xvsrlri_d:
4864 case Intrinsic::loongarch_lasx_xvsrari_d:
4865 return checkIntrinsicImmArg<6>(Op, ImmOp: 2, DAG);
4866 case Intrinsic::loongarch_lsx_vsrlni_w_d:
4867 case Intrinsic::loongarch_lsx_vsrani_w_d:
4868 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
4869 case Intrinsic::loongarch_lsx_vsrarni_w_d:
4870 case Intrinsic::loongarch_lsx_vssrlni_w_d:
4871 case Intrinsic::loongarch_lsx_vssrani_w_d:
4872 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
4873 case Intrinsic::loongarch_lsx_vssrani_wu_d:
4874 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
4875 case Intrinsic::loongarch_lsx_vssrarni_w_d:
4876 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
4877 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
4878 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
4879 case Intrinsic::loongarch_lasx_xvsrani_w_d:
4880 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
4881 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
4882 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
4883 case Intrinsic::loongarch_lasx_xvssrani_w_d:
4884 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
4885 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
4886 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
4887 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
4888 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
4889 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
4890 return checkIntrinsicImmArg<6>(Op, ImmOp: 3, DAG);
4891 case Intrinsic::loongarch_lsx_vsrlni_d_q:
4892 case Intrinsic::loongarch_lsx_vsrani_d_q:
4893 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
4894 case Intrinsic::loongarch_lsx_vsrarni_d_q:
4895 case Intrinsic::loongarch_lsx_vssrlni_d_q:
4896 case Intrinsic::loongarch_lsx_vssrani_d_q:
4897 case Intrinsic::loongarch_lsx_vssrlni_du_q:
4898 case Intrinsic::loongarch_lsx_vssrani_du_q:
4899 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
4900 case Intrinsic::loongarch_lsx_vssrarni_d_q:
4901 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
4902 case Intrinsic::loongarch_lsx_vssrarni_du_q:
4903 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
4904 case Intrinsic::loongarch_lasx_xvsrani_d_q:
4905 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
4906 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
4907 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
4908 case Intrinsic::loongarch_lasx_xvssrani_d_q:
4909 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
4910 case Intrinsic::loongarch_lasx_xvssrani_du_q:
4911 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
4912 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
4913 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
4914 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
4915 return checkIntrinsicImmArg<7>(Op, ImmOp: 3, DAG);
4916 case Intrinsic::loongarch_lsx_vnori_b:
4917 case Intrinsic::loongarch_lsx_vshuf4i_b:
4918 case Intrinsic::loongarch_lsx_vshuf4i_h:
4919 case Intrinsic::loongarch_lsx_vshuf4i_w:
4920 case Intrinsic::loongarch_lasx_xvnori_b:
4921 case Intrinsic::loongarch_lasx_xvshuf4i_b:
4922 case Intrinsic::loongarch_lasx_xvshuf4i_h:
4923 case Intrinsic::loongarch_lasx_xvshuf4i_w:
4924 case Intrinsic::loongarch_lasx_xvpermi_d:
4925 return checkIntrinsicImmArg<8>(Op, ImmOp: 2, DAG);
4926 case Intrinsic::loongarch_lsx_vshuf4i_d:
4927 case Intrinsic::loongarch_lsx_vpermi_w:
4928 case Intrinsic::loongarch_lsx_vbitseli_b:
4929 case Intrinsic::loongarch_lsx_vextrins_b:
4930 case Intrinsic::loongarch_lsx_vextrins_h:
4931 case Intrinsic::loongarch_lsx_vextrins_w:
4932 case Intrinsic::loongarch_lsx_vextrins_d:
4933 case Intrinsic::loongarch_lasx_xvshuf4i_d:
4934 case Intrinsic::loongarch_lasx_xvpermi_w:
4935 case Intrinsic::loongarch_lasx_xvpermi_q:
4936 case Intrinsic::loongarch_lasx_xvbitseli_b:
4937 case Intrinsic::loongarch_lasx_xvextrins_b:
4938 case Intrinsic::loongarch_lasx_xvextrins_h:
4939 case Intrinsic::loongarch_lasx_xvextrins_w:
4940 case Intrinsic::loongarch_lasx_xvextrins_d:
4941 return checkIntrinsicImmArg<8>(Op, ImmOp: 3, DAG);
4942 case Intrinsic::loongarch_lsx_vrepli_b:
4943 case Intrinsic::loongarch_lsx_vrepli_h:
4944 case Intrinsic::loongarch_lsx_vrepli_w:
4945 case Intrinsic::loongarch_lsx_vrepli_d:
4946 case Intrinsic::loongarch_lasx_xvrepli_b:
4947 case Intrinsic::loongarch_lasx_xvrepli_h:
4948 case Intrinsic::loongarch_lasx_xvrepli_w:
4949 case Intrinsic::loongarch_lasx_xvrepli_d:
4950 return checkIntrinsicImmArg<10>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4951 case Intrinsic::loongarch_lsx_vldi:
4952 case Intrinsic::loongarch_lasx_xvldi:
4953 return checkIntrinsicImmArg<13>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
4954 }
4955}
4956
4957// Helper function that emits error message for intrinsics with chain and return
4958// merge values of a UNDEF and the chain.
4959static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op,
4960 StringRef ErrorMsg,
4961 SelectionDAG &DAG) {
4962 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
4963 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: Op.getValueType()), Op.getOperand(i: 0)},
4964 dl: SDLoc(Op));
4965}
4966
4967SDValue
4968LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
4969 SelectionDAG &DAG) const {
4970 SDLoc DL(Op);
4971 MVT GRLenVT = Subtarget.getGRLenVT();
4972 EVT VT = Op.getValueType();
4973 SDValue Chain = Op.getOperand(i: 0);
4974 const StringRef ErrorMsgOOR = "argument out of range";
4975 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
4976 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
4977
4978 switch (Op.getConstantOperandVal(i: 1)) {
4979 default:
4980 return Op;
4981 case Intrinsic::loongarch_crc_w_b_w:
4982 case Intrinsic::loongarch_crc_w_h_w:
4983 case Intrinsic::loongarch_crc_w_w_w:
4984 case Intrinsic::loongarch_crc_w_d_w:
4985 case Intrinsic::loongarch_crcc_w_b_w:
4986 case Intrinsic::loongarch_crcc_w_h_w:
4987 case Intrinsic::loongarch_crcc_w_w_w:
4988 case Intrinsic::loongarch_crcc_w_d_w:
4989 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
4990 case Intrinsic::loongarch_csrrd_w:
4991 case Intrinsic::loongarch_csrrd_d: {
4992 unsigned Imm = Op.getConstantOperandVal(i: 2);
4993 return !isUInt<14>(x: Imm)
4994 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
4995 : DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
4996 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
4997 }
4998 case Intrinsic::loongarch_csrwr_w:
4999 case Intrinsic::loongarch_csrwr_d: {
5000 unsigned Imm = Op.getConstantOperandVal(i: 3);
5001 return !isUInt<14>(x: Imm)
5002 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5003 : DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5004 Ops: {Chain, Op.getOperand(i: 2),
5005 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5006 }
5007 case Intrinsic::loongarch_csrxchg_w:
5008 case Intrinsic::loongarch_csrxchg_d: {
5009 unsigned Imm = Op.getConstantOperandVal(i: 4);
5010 return !isUInt<14>(x: Imm)
5011 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5012 : DAG.getNode(Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5013 Ops: {Chain, Op.getOperand(i: 2), Op.getOperand(i: 3),
5014 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5015 }
5016 case Intrinsic::loongarch_iocsrrd_d: {
5017 return DAG.getNode(
5018 Opcode: LoongArchISD::IOCSRRD_D, DL, ResultTys: {GRLenVT, MVT::Other},
5019 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op.getOperand(i: 2))});
5020 }
5021#define IOCSRRD_CASE(NAME, NODE) \
5022 case Intrinsic::loongarch_##NAME: { \
5023 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5024 {Chain, Op.getOperand(2)}); \
5025 }
5026 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5027 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5028 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5029#undef IOCSRRD_CASE
5030 case Intrinsic::loongarch_cpucfg: {
5031 return DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5032 Ops: {Chain, Op.getOperand(i: 2)});
5033 }
5034 case Intrinsic::loongarch_lddir_d: {
5035 unsigned Imm = Op.getConstantOperandVal(i: 3);
5036 return !isUInt<8>(x: Imm)
5037 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5038 : Op;
5039 }
5040 case Intrinsic::loongarch_movfcsr2gr: {
5041 if (!Subtarget.hasBasicF())
5042 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5043 unsigned Imm = Op.getConstantOperandVal(i: 2);
5044 return !isUInt<2>(x: Imm)
5045 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5046 : DAG.getNode(Opcode: LoongArchISD::MOVFCSR2GR, DL, ResultTys: {VT, MVT::Other},
5047 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5048 }
5049 case Intrinsic::loongarch_lsx_vld:
5050 case Intrinsic::loongarch_lsx_vldrepl_b:
5051 case Intrinsic::loongarch_lasx_xvld:
5052 case Intrinsic::loongarch_lasx_xvldrepl_b:
5053 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5054 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5055 : SDValue();
5056 case Intrinsic::loongarch_lsx_vldrepl_h:
5057 case Intrinsic::loongarch_lasx_xvldrepl_h:
5058 return !isShiftedInt<11, 1>(
5059 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5060 ? emitIntrinsicWithChainErrorMessage(
5061 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5062 : SDValue();
5063 case Intrinsic::loongarch_lsx_vldrepl_w:
5064 case Intrinsic::loongarch_lasx_xvldrepl_w:
5065 return !isShiftedInt<10, 2>(
5066 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5067 ? emitIntrinsicWithChainErrorMessage(
5068 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5069 : SDValue();
5070 case Intrinsic::loongarch_lsx_vldrepl_d:
5071 case Intrinsic::loongarch_lasx_xvldrepl_d:
5072 return !isShiftedInt<9, 3>(
5073 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5074 ? emitIntrinsicWithChainErrorMessage(
5075 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5076 : SDValue();
5077 }
5078}
5079
5080// Helper function that emits error message for intrinsics with void return
5081// value and return the chain.
5082static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg,
5083 SelectionDAG &DAG) {
5084
5085 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5086 return Op.getOperand(i: 0);
5087}
5088
5089SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
5090 SelectionDAG &DAG) const {
5091 SDLoc DL(Op);
5092 MVT GRLenVT = Subtarget.getGRLenVT();
5093 SDValue Chain = Op.getOperand(i: 0);
5094 uint64_t IntrinsicEnum = Op.getConstantOperandVal(i: 1);
5095 SDValue Op2 = Op.getOperand(i: 2);
5096 const StringRef ErrorMsgOOR = "argument out of range";
5097 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5098 const StringRef ErrorMsgReqLA32 = "requires loongarch32";
5099 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5100
5101 switch (IntrinsicEnum) {
5102 default:
5103 // TODO: Add more Intrinsics.
5104 return SDValue();
5105 case Intrinsic::loongarch_cacop_d:
5106 case Intrinsic::loongarch_cacop_w: {
5107 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5108 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5109 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5110 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA32, DAG);
5111 // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)
5112 unsigned Imm1 = Op2->getAsZExtVal();
5113 int Imm2 = cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue();
5114 if (!isUInt<5>(x: Imm1) || !isInt<12>(x: Imm2))
5115 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG);
5116 return Op;
5117 }
5118 case Intrinsic::loongarch_dbar: {
5119 unsigned Imm = Op2->getAsZExtVal();
5120 return !isUInt<15>(x: Imm)
5121 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5122 : DAG.getNode(Opcode: LoongArchISD::DBAR, DL, VT: MVT::Other, N1: Chain,
5123 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5124 }
5125 case Intrinsic::loongarch_ibar: {
5126 unsigned Imm = Op2->getAsZExtVal();
5127 return !isUInt<15>(x: Imm)
5128 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5129 : DAG.getNode(Opcode: LoongArchISD::IBAR, DL, VT: MVT::Other, N1: Chain,
5130 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5131 }
5132 case Intrinsic::loongarch_break: {
5133 unsigned Imm = Op2->getAsZExtVal();
5134 return !isUInt<15>(x: Imm)
5135 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5136 : DAG.getNode(Opcode: LoongArchISD::BREAK, DL, VT: MVT::Other, N1: Chain,
5137 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5138 }
5139 case Intrinsic::loongarch_movgr2fcsr: {
5140 if (!Subtarget.hasBasicF())
5141 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5142 unsigned Imm = Op2->getAsZExtVal();
5143 return !isUInt<2>(x: Imm)
5144 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5145 : DAG.getNode(Opcode: LoongArchISD::MOVGR2FCSR, DL, VT: MVT::Other, N1: Chain,
5146 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT),
5147 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT,
5148 Operand: Op.getOperand(i: 3)));
5149 }
5150 case Intrinsic::loongarch_syscall: {
5151 unsigned Imm = Op2->getAsZExtVal();
5152 return !isUInt<15>(x: Imm)
5153 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5154 : DAG.getNode(Opcode: LoongArchISD::SYSCALL, DL, VT: MVT::Other, N1: Chain,
5155 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5156 }
5157#define IOCSRWR_CASE(NAME, NODE) \
5158 case Intrinsic::loongarch_##NAME: { \
5159 SDValue Op3 = Op.getOperand(3); \
5160 return Subtarget.is64Bit() \
5161 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5162 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5163 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5164 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5165 Op3); \
5166 }
5167 IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);
5168 IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);
5169 IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);
5170#undef IOCSRWR_CASE
5171 case Intrinsic::loongarch_iocsrwr_d: {
5172 return !Subtarget.is64Bit()
5173 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5174 : DAG.getNode(Opcode: LoongArchISD::IOCSRWR_D, DL, VT: MVT::Other, N1: Chain,
5175 N2: Op2,
5176 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64,
5177 Operand: Op.getOperand(i: 3)));
5178 }
5179#define ASRT_LE_GT_CASE(NAME) \
5180 case Intrinsic::loongarch_##NAME: { \
5181 return !Subtarget.is64Bit() \
5182 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5183 : Op; \
5184 }
5185 ASRT_LE_GT_CASE(asrtle_d)
5186 ASRT_LE_GT_CASE(asrtgt_d)
5187#undef ASRT_LE_GT_CASE
5188 case Intrinsic::loongarch_ldpte_d: {
5189 unsigned Imm = Op.getConstantOperandVal(i: 3);
5190 return !Subtarget.is64Bit()
5191 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5192 : !isUInt<8>(x: Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5193 : Op;
5194 }
5195 case Intrinsic::loongarch_lsx_vst:
5196 case Intrinsic::loongarch_lasx_xvst:
5197 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue())
5198 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5199 : SDValue();
5200 case Intrinsic::loongarch_lasx_xvstelm_b:
5201 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5202 !isUInt<5>(x: Op.getConstantOperandVal(i: 5)))
5203 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5204 : SDValue();
5205 case Intrinsic::loongarch_lsx_vstelm_b:
5206 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5207 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5208 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5209 : SDValue();
5210 case Intrinsic::loongarch_lasx_xvstelm_h:
5211 return (!isShiftedInt<8, 1>(
5212 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5213 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5214 ? emitIntrinsicErrorMessage(
5215 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5216 : SDValue();
5217 case Intrinsic::loongarch_lsx_vstelm_h:
5218 return (!isShiftedInt<8, 1>(
5219 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5220 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5221 ? emitIntrinsicErrorMessage(
5222 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5223 : SDValue();
5224 case Intrinsic::loongarch_lasx_xvstelm_w:
5225 return (!isShiftedInt<8, 2>(
5226 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5227 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5228 ? emitIntrinsicErrorMessage(
5229 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5230 : SDValue();
5231 case Intrinsic::loongarch_lsx_vstelm_w:
5232 return (!isShiftedInt<8, 2>(
5233 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5234 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5235 ? emitIntrinsicErrorMessage(
5236 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5237 : SDValue();
5238 case Intrinsic::loongarch_lasx_xvstelm_d:
5239 return (!isShiftedInt<8, 3>(
5240 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5241 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5242 ? emitIntrinsicErrorMessage(
5243 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5244 : SDValue();
5245 case Intrinsic::loongarch_lsx_vstelm_d:
5246 return (!isShiftedInt<8, 3>(
5247 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5248 !isUInt<1>(x: Op.getConstantOperandVal(i: 5)))
5249 ? emitIntrinsicErrorMessage(
5250 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5251 : SDValue();
5252 }
5253}
5254
5255SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,
5256 SelectionDAG &DAG) const {
5257 SDLoc DL(Op);
5258 SDValue Lo = Op.getOperand(i: 0);
5259 SDValue Hi = Op.getOperand(i: 1);
5260 SDValue Shamt = Op.getOperand(i: 2);
5261 EVT VT = Lo.getValueType();
5262
5263 // if Shamt-GRLen < 0: // Shamt < GRLen
5264 // Lo = Lo << Shamt
5265 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))
5266 // else:
5267 // Lo = 0
5268 // Hi = Lo << (Shamt-GRLen)
5269
5270 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5271 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5272 SDValue MinusGRLen =
5273 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5274 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5275 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5276 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5277
5278 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
5279 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
5280 SDValue ShiftRightLo =
5281 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: GRLenMinus1Shamt);
5282 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
5283 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
5284 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusGRLen);
5285
5286 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5287
5288 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
5289 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5290
5291 SDValue Parts[2] = {Lo, Hi};
5292 return DAG.getMergeValues(Ops: Parts, dl: DL);
5293}
5294
5295SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,
5296 SelectionDAG &DAG,
5297 bool IsSRA) const {
5298 SDLoc DL(Op);
5299 SDValue Lo = Op.getOperand(i: 0);
5300 SDValue Hi = Op.getOperand(i: 1);
5301 SDValue Shamt = Op.getOperand(i: 2);
5302 EVT VT = Lo.getValueType();
5303
5304 // SRA expansion:
5305 // if Shamt-GRLen < 0: // Shamt < GRLen
5306 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5307 // Hi = Hi >>s Shamt
5308 // else:
5309 // Lo = Hi >>s (Shamt-GRLen);
5310 // Hi = Hi >>s (GRLen-1)
5311 //
5312 // SRL expansion:
5313 // if Shamt-GRLen < 0: // Shamt < GRLen
5314 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5315 // Hi = Hi >>u Shamt
5316 // else:
5317 // Lo = Hi >>u (Shamt-GRLen);
5318 // Hi = 0;
5319
5320 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
5321
5322 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5323 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5324 SDValue MinusGRLen =
5325 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5326 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5327 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5328 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5329
5330 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
5331 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
5332 SDValue ShiftLeftHi =
5333 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: GRLenMinus1Shamt);
5334 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
5335 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
5336 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusGRLen);
5337 SDValue HiFalse =
5338 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: GRLenMinus1) : Zero;
5339
5340 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5341
5342 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
5343 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5344
5345 SDValue Parts[2] = {Lo, Hi};
5346 return DAG.getMergeValues(Ops: Parts, dl: DL);
5347}
5348
5349// Returns the opcode of the target-specific SDNode that implements the 32-bit
5350// form of the given Opcode.
5351static unsigned getLoongArchWOpcode(unsigned Opcode) {
5352 switch (Opcode) {
5353 default:
5354 llvm_unreachable("Unexpected opcode");
5355 case ISD::SDIV:
5356 return LoongArchISD::DIV_W;
5357 case ISD::UDIV:
5358 return LoongArchISD::DIV_WU;
5359 case ISD::SREM:
5360 return LoongArchISD::MOD_W;
5361 case ISD::UREM:
5362 return LoongArchISD::MOD_WU;
5363 case ISD::SHL:
5364 return LoongArchISD::SLL_W;
5365 case ISD::SRA:
5366 return LoongArchISD::SRA_W;
5367 case ISD::SRL:
5368 return LoongArchISD::SRL_W;
5369 case ISD::ROTL:
5370 case ISD::ROTR:
5371 return LoongArchISD::ROTR_W;
5372 case ISD::CTTZ:
5373 return LoongArchISD::CTZ_W;
5374 case ISD::CTLZ:
5375 return LoongArchISD::CLZ_W;
5376 }
5377}
5378
5379// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5380// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
5381// otherwise be promoted to i64, making it difficult to select the
5382// SLL_W/.../*W later one because the fact the operation was originally of
5383// type i8/i16/i32 is lost.
5384static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
5385 unsigned ExtOpc = ISD::ANY_EXTEND) {
5386 SDLoc DL(N);
5387 unsigned WOpcode = getLoongArchWOpcode(Opcode: N->getOpcode());
5388 SDValue NewOp0, NewRes;
5389
5390 switch (NumOp) {
5391 default:
5392 llvm_unreachable("Unexpected NumOp");
5393 case 1: {
5394 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5395 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, Operand: NewOp0);
5396 break;
5397 }
5398 case 2: {
5399 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5400 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5401 if (N->getOpcode() == ISD::ROTL) {
5402 SDValue TmpOp = DAG.getConstant(Val: 32, DL, VT: MVT::i64);
5403 NewOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: TmpOp, N2: NewOp1);
5404 }
5405 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5406 break;
5407 }
5408 // TODO:Handle more NumOp.
5409 }
5410
5411 // ReplaceNodeResults requires we maintain the same type for the return
5412 // value.
5413 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
5414}
5415
5416// Converts the given 32-bit operation to a i64 operation with signed extension
5417// semantic to reduce the signed extension instructions.
5418static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
5419 SDLoc DL(N);
5420 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5421 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5422 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5423 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
5424 N2: DAG.getValueType(MVT::i32));
5425 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
5426}
5427
5428// Helper function that emits error message for intrinsics with/without chain
5429// and return a UNDEF or and the chain as the results.
5430static void emitErrorAndReplaceIntrinsicResults(
5431 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
5432 StringRef ErrorMsg, bool WithChain = true) {
5433 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5434 Results.push_back(Elt: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)));
5435 if (!WithChain)
5436 return;
5437 Results.push_back(Elt: N->getOperand(Num: 0));
5438}
5439
5440template <unsigned N>
5441static void
5442replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl<SDValue> &Results,
5443 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget,
5444 unsigned ResOp) {
5445 const StringRef ErrorMsgOOR = "argument out of range";
5446 unsigned Imm = Node->getConstantOperandVal(Num: 2);
5447 if (!isUInt<N>(Imm)) {
5448 emitErrorAndReplaceIntrinsicResults(N: Node, Results, DAG, ErrorMsg: ErrorMsgOOR,
5449 /*WithChain=*/false);
5450 return;
5451 }
5452 SDLoc DL(Node);
5453 SDValue Vec = Node->getOperand(Num: 1);
5454
5455 SDValue PickElt =
5456 DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), N1: Vec,
5457 N2: DAG.getConstant(Val: Imm, DL, VT: Subtarget.getGRLenVT()),
5458 N3: DAG.getValueType(Vec.getValueType().getVectorElementType()));
5459 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Node->getValueType(ResNo: 0),
5460 Operand: PickElt.getValue(R: 0)));
5461}
5462
5463static void replaceVecCondBranchResults(SDNode *N,
5464 SmallVectorImpl<SDValue> &Results,
5465 SelectionDAG &DAG,
5466 const LoongArchSubtarget &Subtarget,
5467 unsigned ResOp) {
5468 SDLoc DL(N);
5469 SDValue Vec = N->getOperand(Num: 1);
5470
5471 SDValue CB = DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), Operand: Vec);
5472 Results.push_back(
5473 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: CB.getValue(R: 0)));
5474}
5475
5476static void
5477replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
5478 SelectionDAG &DAG,
5479 const LoongArchSubtarget &Subtarget) {
5480 switch (N->getConstantOperandVal(Num: 0)) {
5481 default:
5482 llvm_unreachable("Unexpected Intrinsic.");
5483 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5484 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5485 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5486 break;
5487 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5488 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5489 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5490 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5491 break;
5492 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5493 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5494 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5495 break;
5496 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5497 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5498 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5499 break;
5500 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5501 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5502 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5503 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5504 break;
5505 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5506 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5507 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5508 break;
5509 case Intrinsic::loongarch_lsx_bz_b:
5510 case Intrinsic::loongarch_lsx_bz_h:
5511 case Intrinsic::loongarch_lsx_bz_w:
5512 case Intrinsic::loongarch_lsx_bz_d:
5513 case Intrinsic::loongarch_lasx_xbz_b:
5514 case Intrinsic::loongarch_lasx_xbz_h:
5515 case Intrinsic::loongarch_lasx_xbz_w:
5516 case Intrinsic::loongarch_lasx_xbz_d:
5517 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5518 ResOp: LoongArchISD::VALL_ZERO);
5519 break;
5520 case Intrinsic::loongarch_lsx_bz_v:
5521 case Intrinsic::loongarch_lasx_xbz_v:
5522 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5523 ResOp: LoongArchISD::VANY_ZERO);
5524 break;
5525 case Intrinsic::loongarch_lsx_bnz_b:
5526 case Intrinsic::loongarch_lsx_bnz_h:
5527 case Intrinsic::loongarch_lsx_bnz_w:
5528 case Intrinsic::loongarch_lsx_bnz_d:
5529 case Intrinsic::loongarch_lasx_xbnz_b:
5530 case Intrinsic::loongarch_lasx_xbnz_h:
5531 case Intrinsic::loongarch_lasx_xbnz_w:
5532 case Intrinsic::loongarch_lasx_xbnz_d:
5533 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5534 ResOp: LoongArchISD::VALL_NONZERO);
5535 break;
5536 case Intrinsic::loongarch_lsx_bnz_v:
5537 case Intrinsic::loongarch_lasx_xbnz_v:
5538 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5539 ResOp: LoongArchISD::VANY_NONZERO);
5540 break;
5541 }
5542}
5543
5544static void replaceCMP_XCHG_128Results(SDNode *N,
5545 SmallVectorImpl<SDValue> &Results,
5546 SelectionDAG &DAG) {
5547 assert(N->getValueType(0) == MVT::i128 &&
5548 "AtomicCmpSwap on types less than 128 should be legal");
5549 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
5550
5551 unsigned Opcode;
5552 switch (MemOp->getMergedOrdering()) {
5553 case AtomicOrdering::Acquire:
5554 case AtomicOrdering::AcquireRelease:
5555 case AtomicOrdering::SequentiallyConsistent:
5556 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5557 break;
5558 case AtomicOrdering::Monotonic:
5559 case AtomicOrdering::Release:
5560 Opcode = LoongArch::PseudoCmpXchg128;
5561 break;
5562 default:
5563 llvm_unreachable("Unexpected ordering!");
5564 }
5565
5566 SDLoc DL(N);
5567 auto CmpVal = DAG.SplitScalar(N: N->getOperand(Num: 2), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5568 auto NewVal = DAG.SplitScalar(N: N->getOperand(Num: 3), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5569 SDValue Ops[] = {N->getOperand(Num: 1), CmpVal.first, CmpVal.second,
5570 NewVal.first, NewVal.second, N->getOperand(Num: 0)};
5571
5572 SDNode *CmpSwap = DAG.getMachineNode(
5573 Opcode, dl: SDLoc(N), VTs: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::i64, VT4: MVT::Other),
5574 Ops);
5575 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
5576 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
5577 N1: SDValue(CmpSwap, 0), N2: SDValue(CmpSwap, 1)));
5578 Results.push_back(Elt: SDValue(CmpSwap, 3));
5579}
5580
5581void LoongArchTargetLowering::ReplaceNodeResults(
5582 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
5583 SDLoc DL(N);
5584 EVT VT = N->getValueType(ResNo: 0);
5585 switch (N->getOpcode()) {
5586 default:
5587 llvm_unreachable("Don't know how to legalize this operation");
5588 case ISD::ADD:
5589 case ISD::SUB:
5590 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5591 "Unexpected custom legalisation");
5592 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
5593 break;
5594 case ISD::SDIV:
5595 case ISD::UDIV:
5596 case ISD::SREM:
5597 case ISD::UREM:
5598 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5599 "Unexpected custom legalisation");
5600 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2,
5601 ExtOpc: Subtarget.hasDiv32() && VT == MVT::i32
5602 ? ISD::ANY_EXTEND
5603 : ISD::SIGN_EXTEND));
5604 break;
5605 case ISD::SHL:
5606 case ISD::SRA:
5607 case ISD::SRL:
5608 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5609 "Unexpected custom legalisation");
5610 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
5611 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5612 break;
5613 }
5614 break;
5615 case ISD::ROTL:
5616 case ISD::ROTR:
5617 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5618 "Unexpected custom legalisation");
5619 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5620 break;
5621 case ISD::LOAD: {
5622 // Use an f64 load and a scalar_to_vector for v2f32 loads. This avoids
5623 // scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
5624 // cast since type legalization will try to use an i64 load.
5625 MVT VT = N->getSimpleValueType(ResNo: 0);
5626 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5627 "Unexpected custom legalisation");
5628 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
5629 "Unexpected type action!");
5630 if (!ISD::isNON_EXTLoad(N))
5631 return;
5632 auto *Ld = cast<LoadSDNode>(Val: N);
5633 SDValue Res = DAG.getLoad(VT: MVT::f64, dl: DL, Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5634 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getBaseAlign(),
5635 MMOFlags: Ld->getMemOperand()->getFlags());
5636 SDValue Chain = Res.getValue(R: 1);
5637 MVT VecVT = MVT::getVectorVT(VT: MVT::f64, NumElements: 2);
5638 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecVT, Operand: Res);
5639 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5640 Res = DAG.getBitcast(VT: WideVT, V: Res);
5641 Results.push_back(Elt: Res);
5642 Results.push_back(Elt: Chain);
5643 break;
5644 }
5645 case ISD::FP_TO_SINT: {
5646 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5647 "Unexpected custom legalisation");
5648 SDValue Src = N->getOperand(Num: 0);
5649 EVT FVT = EVT::getFloatingPointVT(BitWidth: N->getValueSizeInBits(ResNo: 0));
5650 if (getTypeAction(Context&: *DAG.getContext(), VT: Src.getValueType()) !=
5651 TargetLowering::TypeSoftenFloat) {
5652 if (!isTypeLegal(VT: Src.getValueType()))
5653 return;
5654 if (Src.getValueType() == MVT::f16)
5655 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
5656 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FVT, Operand: Src);
5657 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Dst));
5658 return;
5659 }
5660 // If the FP type needs to be softened, emit a library call using the 'si'
5661 // version. If we left it to default legalization we'd end up with 'di'.
5662 RTLIB::Libcall LC;
5663 LC = RTLIB::getFPTOSINT(OpVT: Src.getValueType(), RetVT: VT);
5664 MakeLibCallOptions CallOptions;
5665 EVT OpVT = Src.getValueType();
5666 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: VT);
5667 SDValue Chain = SDValue();
5668 SDValue Result;
5669 std::tie(args&: Result, args&: Chain) =
5670 makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl: DL, Chain);
5671 Results.push_back(Elt: Result);
5672 break;
5673 }
5674 case ISD::BITCAST: {
5675 SDValue Src = N->getOperand(Num: 0);
5676 EVT SrcVT = Src.getValueType();
5677 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
5678 Subtarget.hasBasicF()) {
5679 SDValue Dst =
5680 DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Src);
5681 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Dst));
5682 } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
5683 SDValue NewReg = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
5684 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Src);
5685 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64,
5686 N1: NewReg.getValue(R: 0), N2: NewReg.getValue(R: 1));
5687 Results.push_back(Elt: RetReg);
5688 }
5689 break;
5690 }
5691 case ISD::FP_TO_UINT: {
5692 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5693 "Unexpected custom legalisation");
5694 auto &TLI = DAG.getTargetLoweringInfo();
5695 SDValue Tmp1, Tmp2;
5696 TLI.expandFP_TO_UINT(N, Result&: Tmp1, Chain&: Tmp2, DAG);
5697 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Tmp1));
5698 break;
5699 }
5700 case ISD::FP_ROUND: {
5701 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5702 "Unexpected custom legalisation");
5703 // On LSX platforms, rounding from v2f64 to v4f32 (after legalization from
5704 // v2f32) is scalarized. Add a customized v2f32 widening to convert it into
5705 // a target-specific LoongArchISD::VFCVT to optimize it.
5706 SDValue Op0 = N->getOperand(Num: 0);
5707 EVT OpVT = Op0.getValueType();
5708 if (OpVT == MVT::v2f64) {
5709 SDValue Undef = DAG.getUNDEF(VT: OpVT);
5710 SDValue Dst =
5711 DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Undef, N2: Op0);
5712 Results.push_back(Elt: Dst);
5713 }
5714 break;
5715 }
5716 case ISD::BSWAP: {
5717 SDValue Src = N->getOperand(Num: 0);
5718 assert((VT == MVT::i16 || VT == MVT::i32) &&
5719 "Unexpected custom legalization");
5720 MVT GRLenVT = Subtarget.getGRLenVT();
5721 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5722 SDValue Tmp;
5723 switch (VT.getSizeInBits()) {
5724 default:
5725 llvm_unreachable("Unexpected operand width");
5726 case 16:
5727 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2H, DL, VT: GRLenVT, Operand: NewSrc);
5728 break;
5729 case 32:
5730 // Only LA64 will get to here due to the size mismatch between VT and
5731 // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
5732 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2W, DL, VT: GRLenVT, Operand: NewSrc);
5733 break;
5734 }
5735 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5736 break;
5737 }
5738 case ISD::BITREVERSE: {
5739 SDValue Src = N->getOperand(Num: 0);
5740 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
5741 "Unexpected custom legalization");
5742 MVT GRLenVT = Subtarget.getGRLenVT();
5743 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
5744 SDValue Tmp;
5745 switch (VT.getSizeInBits()) {
5746 default:
5747 llvm_unreachable("Unexpected operand width");
5748 case 8:
5749 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL, VT: GRLenVT, Operand: NewSrc);
5750 break;
5751 case 32:
5752 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_W, DL, VT: GRLenVT, Operand: NewSrc);
5753 break;
5754 }
5755 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
5756 break;
5757 }
5758 case ISD::CTLZ:
5759 case ISD::CTTZ: {
5760 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5761 "Unexpected custom legalisation");
5762 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 1));
5763 break;
5764 }
5765 case ISD::INTRINSIC_W_CHAIN: {
5766 SDValue Chain = N->getOperand(Num: 0);
5767 SDValue Op2 = N->getOperand(Num: 2);
5768 MVT GRLenVT = Subtarget.getGRLenVT();
5769 const StringRef ErrorMsgOOR = "argument out of range";
5770 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5771 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5772
5773 switch (N->getConstantOperandVal(Num: 1)) {
5774 default:
5775 llvm_unreachable("Unexpected Intrinsic.");
5776 case Intrinsic::loongarch_movfcsr2gr: {
5777 if (!Subtarget.hasBasicF()) {
5778 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqF);
5779 return;
5780 }
5781 unsigned Imm = Op2->getAsZExtVal();
5782 if (!isUInt<2>(x: Imm)) {
5783 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5784 return;
5785 }
5786 SDValue MOVFCSR2GRResults = DAG.getNode(
5787 Opcode: LoongArchISD::MOVFCSR2GR, DL: SDLoc(N), ResultTys: {MVT::i64, MVT::Other},
5788 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5789 Results.push_back(
5790 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: MOVFCSR2GRResults.getValue(R: 0)));
5791 Results.push_back(Elt: MOVFCSR2GRResults.getValue(R: 1));
5792 break;
5793 }
5794#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
5795 case Intrinsic::loongarch_##NAME: { \
5796 SDValue NODE = DAG.getNode( \
5797 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5798 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5799 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5800 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5801 Results.push_back(NODE.getValue(1)); \
5802 break; \
5803 }
5804 CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
5805 CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)
5806 CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)
5807 CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)
5808 CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)
5809 CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)
5810#undef CRC_CASE_EXT_BINARYOP
5811
5812#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
5813 case Intrinsic::loongarch_##NAME: { \
5814 SDValue NODE = DAG.getNode( \
5815 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5816 {Chain, Op2, \
5817 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5818 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5819 Results.push_back(NODE.getValue(1)); \
5820 break; \
5821 }
5822 CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
5823 CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)
5824#undef CRC_CASE_EXT_UNARYOP
5825#define CSR_CASE(ID) \
5826 case Intrinsic::loongarch_##ID: { \
5827 if (!Subtarget.is64Bit()) \
5828 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
5829 break; \
5830 }
5831 CSR_CASE(csrrd_d);
5832 CSR_CASE(csrwr_d);
5833 CSR_CASE(csrxchg_d);
5834 CSR_CASE(iocsrrd_d);
5835#undef CSR_CASE
5836 case Intrinsic::loongarch_csrrd_w: {
5837 unsigned Imm = Op2->getAsZExtVal();
5838 if (!isUInt<14>(x: Imm)) {
5839 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5840 return;
5841 }
5842 SDValue CSRRDResults =
5843 DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
5844 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5845 Results.push_back(
5846 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRRDResults.getValue(R: 0)));
5847 Results.push_back(Elt: CSRRDResults.getValue(R: 1));
5848 break;
5849 }
5850 case Intrinsic::loongarch_csrwr_w: {
5851 unsigned Imm = N->getConstantOperandVal(Num: 3);
5852 if (!isUInt<14>(x: Imm)) {
5853 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5854 return;
5855 }
5856 SDValue CSRWRResults =
5857 DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5858 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5859 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5860 Results.push_back(
5861 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRWRResults.getValue(R: 0)));
5862 Results.push_back(Elt: CSRWRResults.getValue(R: 1));
5863 break;
5864 }
5865 case Intrinsic::loongarch_csrxchg_w: {
5866 unsigned Imm = N->getConstantOperandVal(Num: 4);
5867 if (!isUInt<14>(x: Imm)) {
5868 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
5869 return;
5870 }
5871 SDValue CSRXCHGResults = DAG.getNode(
5872 Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5873 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
5874 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 3)),
5875 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5876 Results.push_back(
5877 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRXCHGResults.getValue(R: 0)));
5878 Results.push_back(Elt: CSRXCHGResults.getValue(R: 1));
5879 break;
5880 }
5881#define IOCSRRD_CASE(NAME, NODE) \
5882 case Intrinsic::loongarch_##NAME: { \
5883 SDValue IOCSRRDResults = \
5884 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5885 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
5886 Results.push_back( \
5887 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
5888 Results.push_back(IOCSRRDResults.getValue(1)); \
5889 break; \
5890 }
5891 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5892 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5893 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5894#undef IOCSRRD_CASE
5895 case Intrinsic::loongarch_cpucfg: {
5896 SDValue CPUCFGResults =
5897 DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5898 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2)});
5899 Results.push_back(
5900 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CPUCFGResults.getValue(R: 0)));
5901 Results.push_back(Elt: CPUCFGResults.getValue(R: 1));
5902 break;
5903 }
5904 case Intrinsic::loongarch_lddir_d: {
5905 if (!Subtarget.is64Bit()) {
5906 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqLA64);
5907 return;
5908 }
5909 break;
5910 }
5911 }
5912 break;
5913 }
5914 case ISD::READ_REGISTER: {
5915 if (Subtarget.is64Bit())
5916 DAG.getContext()->emitError(
5917 ErrorStr: "On LA64, only 64-bit registers can be read.");
5918 else
5919 DAG.getContext()->emitError(
5920 ErrorStr: "On LA32, only 32-bit registers can be read.");
5921 Results.push_back(Elt: DAG.getUNDEF(VT));
5922 Results.push_back(Elt: N->getOperand(Num: 0));
5923 break;
5924 }
5925 case ISD::INTRINSIC_WO_CHAIN: {
5926 replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);
5927 break;
5928 }
5929 case ISD::LROUND: {
5930 SDValue Op0 = N->getOperand(Num: 0);
5931 EVT OpVT = Op0.getValueType();
5932 RTLIB::Libcall LC =
5933 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
5934 MakeLibCallOptions CallOptions;
5935 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
5936 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
5937 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
5938 Results.push_back(Elt: Result);
5939 break;
5940 }
5941 case ISD::ATOMIC_CMP_SWAP: {
5942 replaceCMP_XCHG_128Results(N, Results, DAG);
5943 break;
5944 }
5945 case ISD::TRUNCATE: {
5946 MVT VT = N->getSimpleValueType(ResNo: 0);
5947 if (getTypeAction(Context&: *DAG.getContext(), VT) != TypeWidenVector)
5948 return;
5949
5950 MVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT).getSimpleVT();
5951 SDValue In = N->getOperand(Num: 0);
5952 EVT InVT = In.getValueType();
5953 EVT InEltVT = InVT.getVectorElementType();
5954 EVT EltVT = VT.getVectorElementType();
5955 unsigned MinElts = VT.getVectorNumElements();
5956 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5957 unsigned InBits = InVT.getSizeInBits();
5958
5959 // v8i64 -> (v8i32) -> v8i8
5960 if (InVT == MVT::v8i64 && WidenVT.is128BitVector()) {
5961 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 256 / MinElts), NumElements: MinElts);
5962 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5963 InBits = 256;
5964 }
5965
5966 // v8i32 -> v8i8 / v4i64 -> v4i16 / v4i64 -> v4i8
5967 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
5968 WidenVT.is128BitVector()) {
5969 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / MinElts), NumElements: MinElts);
5970 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
5971 InBits = 128;
5972 InEltVT = InVT.getVectorElementType();
5973 }
5974
5975 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
5976 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
5977 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
5978 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
5979 for (unsigned I = 0; I < MinElts; ++I)
5980 TruncMask[I] = Scale * I;
5981
5982 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
5983 MVT SVT = In.getSimpleValueType().getScalarType();
5984 MVT VT = MVT::getVectorVT(VT: SVT, NumElements: WidenNumElts);
5985 SDValue WidenIn =
5986 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT), N2: In,
5987 N3: DAG.getVectorIdxConstant(Val: 0, DL));
5988 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
5989 "Illegal vector type in truncation");
5990 WidenIn = DAG.getBitcast(VT: WidenVT, V: WidenIn);
5991 Results.push_back(
5992 Elt: DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: WidenIn, N2: WidenIn, Mask: TruncMask));
5993 return;
5994 }
5995 }
5996
5997 break;
5998 }
5999 case ISD::SIGN_EXTEND: {
6000 // LASX has native VEXT2XV_* for sign extension.
6001 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
6002 return;
6003
6004 EVT DstVT = N->getValueType(ResNo: 0);
6005 SDValue Src = N->getOperand(Num: 0);
6006 MVT SrcVT = Src.getSimpleValueType();
6007
6008 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6009 unsigned DstEltBits = DstVT.getScalarSizeInBits();
6010 unsigned NumElts = DstVT.getVectorNumElements();
6011
6012 if (SrcVT.getSizeInBits() > 128)
6013 return;
6014
6015 if (!DstVT.isVector() || DstVT.getSizeInBits() <= 128)
6016 return;
6017
6018 // Legalize and extend the src to 128-bit first.
6019 if (SrcVT.getSizeInBits() < 128) {
6020 unsigned WidenSrcElts = 128 / SrcEltBits;
6021 MVT WidenSrcVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: WidenSrcElts);
6022 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WidenSrcVT,
6023 N1: DAG.getUNDEF(VT: WidenSrcVT), N2: Src,
6024 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6025 SrcVT = WidenSrcVT;
6026
6027 unsigned FirstStageEltBits = 128 / NumElts;
6028 MVT FirstStageEltVT = MVT::getIntegerVT(BitWidth: FirstStageEltBits);
6029 MVT FirstStageVT = MVT::getVectorVT(VT: FirstStageEltVT, NumElements: NumElts);
6030 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: FirstStageVT, Operand: Src);
6031 SrcVT = FirstStageVT;
6032 SrcEltBits = FirstStageEltBits;
6033 }
6034
6035 SmallVector<SDValue, 8> Blocks;
6036 Blocks.push_back(Elt: Src);
6037
6038 // Sign-extend the src by using SLTI + VILVL + VILVH recursively.
6039 while (SrcEltBits < DstEltBits) {
6040 unsigned NextEltBits = SrcEltBits * 2;
6041 MVT NextEltVT = MVT::getIntegerVT(BitWidth: NextEltBits);
6042 unsigned CurEltsPerBlock = SrcVT.getVectorNumElements();
6043 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6044 MVT NextBlockVT = MVT::getVectorVT(VT: NextEltVT, NumElements: NextEltsPerBlock);
6045
6046 SmallVector<SDValue, 8> NextBlocks;
6047 NextBlocks.reserve(N: Blocks.size() * 2);
6048 for (SDValue Block : Blocks) {
6049 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
6050 SDValue Mask = DAG.getNode(Opcode: ISD::SETCC, DL, VT: SrcVT, N1: Block, N2: Zero,
6051 N3: DAG.getCondCode(Cond: ISD::SETLT));
6052 SDValue LoInterleaved =
6053 DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Mask, N2: Block);
6054 SDValue HiInterleaved =
6055 DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT: SrcVT, N1: Mask, N2: Block);
6056
6057 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: LoInterleaved));
6058 NextBlocks.push_back(Elt: DAG.getBitcast(VT: NextBlockVT, V: HiInterleaved));
6059 }
6060
6061 Blocks = std::move(NextBlocks);
6062 SrcVT = NextBlockVT;
6063 SrcEltBits = NextEltBits;
6064 }
6065
6066 Results.push_back(Elt: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Blocks));
6067 break;
6068 }
6069 case ISD::FP_EXTEND:
6070 // FP_EXTEND may reach here due to the Custom action for v2f32 results, but
6071 // no target-specific lowering is required. Leave it unchanged and rely on
6072 // the default type legalization.
6073 break;
6074 }
6075}
6076
6077/// Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
6078static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL,
6079 SelectionDAG &DAG) {
6080 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDN");
6081
6082 MVT VT = N->getSimpleValueType(ResNo: 0);
6083 if (!VT.is128BitVector() && !VT.is256BitVector())
6084 return SDValue();
6085
6086 SDValue X, Y;
6087 SDValue N0 = N->getOperand(Num: 0);
6088 SDValue N1 = N->getOperand(Num: 1);
6089
6090 if (SDValue Not = isNOT(V: N0, DAG)) {
6091 X = Not;
6092 Y = N1;
6093 } else if (SDValue Not = isNOT(V: N1, DAG)) {
6094 X = Not;
6095 Y = N0;
6096 } else
6097 return SDValue();
6098
6099 X = DAG.getBitcast(VT, V: X);
6100 Y = DAG.getBitcast(VT, V: Y);
6101 return DAG.getNode(Opcode: LoongArchISD::VANDN, DL, VT, N1: X, N2: Y);
6102}
6103
6104static bool isConstantSplatVector(SDValue N, APInt &SplatValue,
6105 unsigned MinSizeInBits) {
6106 N = peekThroughBitcasts(V: N);
6107 BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(Val&: N);
6108
6109 if (!Node)
6110 return false;
6111
6112 APInt SplatUndef;
6113 unsigned SplatBitSize;
6114 bool HasAnyUndefs;
6115
6116 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6117 HasAnyUndefs, MinSplatBits: MinSizeInBits,
6118 /*IsBigEndian=*/isBigEndian: false);
6119}
6120
6121static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex) {
6122 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6123 if (!BV)
6124 return SDValue();
6125
6126 SDValue Src;
6127 int Start = -1;
6128
6129 for (unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6130 SDValue Op = BV->getOperand(Num: i);
6131 if (Op.isUndef())
6132 continue;
6133 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6134 return SDValue();
6135
6136 auto *IdxC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6137 if (!IdxC)
6138 return SDValue();
6139
6140 unsigned EltIdx = IdxC->getZExtValue();
6141 if (Start < 0)
6142 Start = (int)EltIdx - (int)(i * 2);
6143 if (Start < 0 || Start > 1 || EltIdx != (unsigned)(Start + (int)(i * 2)))
6144 return SDValue();
6145
6146 SDValue CurSrc = Op.getOperand(i: 0);
6147 if (!Src)
6148 Src = CurSrc;
6149 else if (Src != CurSrc)
6150 return SDValue();
6151 }
6152
6153 if (!Src || Start < 0)
6154 return SDValue();
6155
6156 StartIndex = (unsigned)Start;
6157 return Src;
6158}
6159
6160static SDValue
6161performHorizWideningCombine(SDNode *N, SelectionDAG &DAG,
6162 const LoongArchSubtarget &Subtarget) {
6163 if (!Subtarget.hasExtLSX())
6164 return SDValue();
6165
6166 unsigned Opc = N->getOpcode();
6167 assert((Opc == ISD::ADD || Opc == ISD::SUB) && "Unexpected opcode");
6168
6169 EVT VT = N->getValueType(ResNo: 0);
6170 SDLoc DL(N);
6171
6172 SDValue LHS = N->getOperand(Num: 0);
6173 SDValue RHS = N->getOperand(Num: 1);
6174
6175 bool isSigned;
6176 unsigned ExtOpc = LHS.getOpcode();
6177 if (ExtOpc == ISD::SIGN_EXTEND)
6178 isSigned = true;
6179 else if (ExtOpc == ISD::ZERO_EXTEND)
6180 isSigned = false;
6181 else
6182 return SDValue();
6183
6184 if (ExtOpc != RHS.getOpcode())
6185 return SDValue();
6186
6187 if (!LHS.hasOneUse() || !RHS.hasOneUse())
6188 return SDValue();
6189
6190 unsigned OddIdx, EvenIdx;
6191 SDValue LHSVec = matchDeinterleaveBuildVector(N: LHS.getOperand(i: 0), StartIndex&: OddIdx);
6192 SDValue RHSVec = matchDeinterleaveBuildVector(N: RHS.getOperand(i: 0), StartIndex&: EvenIdx);
6193
6194 if (!LHSVec || !RHSVec)
6195 return SDValue();
6196 if (OddIdx != 1 || EvenIdx != 0)
6197 return SDValue();
6198 if (LHSVec.getValueType() != RHSVec.getValueType())
6199 return SDValue();
6200
6201 EVT SrcVT = LHSVec.getValueType();
6202 EVT SrcEltVT = SrcVT.getVectorElementType();
6203 EVT DstEltVT = VT.getVectorElementType();
6204 auto &TLI = DAG.getTargetLoweringInfo();
6205
6206 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
6207 return SDValue();
6208 if (!SrcVT.isVector() || !VT.isVector())
6209 return SDValue();
6210 if (SrcVT.getSizeInBits() != VT.getSizeInBits())
6211 return SDValue();
6212 if (DstEltVT.getSizeInBits() != SrcEltVT.getSizeInBits() * 2)
6213 return SDValue();
6214 if (!SrcEltVT.isInteger() || SrcEltVT.getSizeInBits() > 32)
6215 return SDValue();
6216
6217 unsigned TargetOpc;
6218 if (Opc == ISD::ADD)
6219 TargetOpc = isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6220 else
6221 TargetOpc = isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6222
6223 return DAG.getNode(Opcode: TargetOpc, DL, VT, N1: LHSVec, N2: RHSVec);
6224}
6225
6226static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
6227 TargetLowering::DAGCombinerInfo &DCI,
6228 const LoongArchSubtarget &Subtarget) {
6229 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6230 return V;
6231
6232 if (DCI.isBeforeLegalizeOps())
6233 return SDValue();
6234
6235 EVT VT = N->getValueType(ResNo: 0);
6236 if (!VT.isVector())
6237 return SDValue();
6238
6239 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6240 return SDValue();
6241
6242 EVT EltVT = VT.getVectorElementType();
6243 if (!EltVT.isInteger())
6244 return SDValue();
6245
6246 // match:
6247 //
6248 // add
6249 // (and
6250 // (srl X, shift-1) / X
6251 // 1)
6252 // (srl/sra X, shift)
6253
6254 SDValue Add0 = N->getOperand(Num: 0);
6255 SDValue Add1 = N->getOperand(Num: 1);
6256 SDValue And;
6257 SDValue Shr;
6258
6259 if (Add0.getOpcode() == ISD::AND) {
6260 And = Add0;
6261 Shr = Add1;
6262 } else if (Add1.getOpcode() == ISD::AND) {
6263 And = Add1;
6264 Shr = Add0;
6265 } else {
6266 return SDValue();
6267 }
6268
6269 // match:
6270 //
6271 // srl/sra X, shift
6272
6273 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
6274 return SDValue();
6275
6276 SDValue X = Shr.getOperand(i: 0);
6277 SDValue Shift = Shr.getOperand(i: 1);
6278 APInt ShiftVal;
6279
6280 if (!isConstantSplatVector(N: Shift, SplatValue&: ShiftVal, MinSizeInBits: EltVT.getSizeInBits()))
6281 return SDValue();
6282
6283 if (ShiftVal == 0)
6284 return SDValue();
6285
6286 // match:
6287 //
6288 // and
6289 // (srl X, shift-1) / X
6290 // 1
6291
6292 SDValue One = And.getOperand(i: 1);
6293 APInt SplatVal;
6294
6295 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6296 return SDValue();
6297
6298 if (SplatVal != 1)
6299 return SDValue();
6300
6301 if (And.getOperand(i: 0) == X) {
6302 // match:
6303 //
6304 // shift == 1
6305
6306 if (ShiftVal != 1)
6307 return SDValue();
6308 } else {
6309 // match:
6310 //
6311 // srl X, shift-1
6312
6313 SDValue Srl = And.getOperand(i: 0);
6314
6315 if (Srl.getOpcode() != ISD::SRL)
6316 return SDValue();
6317
6318 if (Srl.getOperand(i: 0) != X)
6319 return SDValue();
6320
6321 // match:
6322 //
6323 // shift-1
6324
6325 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
6326
6327 if (!isConstantSplatVector(N: ShiftMinus1, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6328 return SDValue();
6329
6330 if (ShiftVal != (SplatVal + 1))
6331 return SDValue();
6332 }
6333
6334 // We matched a rounded right shift pattern and can lower it
6335 // to a single vector rounded shift instruction.
6336
6337 SDLoc DL(N);
6338 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
6339 : LoongArchISD::VSRAR,
6340 DL, VT, N1: X, N2: Shift);
6341}
6342
6343static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
6344 TargetLowering::DAGCombinerInfo &DCI,
6345 const LoongArchSubtarget &Subtarget) {
6346 if (DCI.isBeforeLegalizeOps())
6347 return SDValue();
6348
6349 SDValue FirstOperand = N->getOperand(Num: 0);
6350 SDValue SecondOperand = N->getOperand(Num: 1);
6351 unsigned FirstOperandOpc = FirstOperand.getOpcode();
6352 EVT ValTy = N->getValueType(ResNo: 0);
6353 SDLoc DL(N);
6354 uint64_t lsb, msb;
6355 unsigned SMIdx, SMLen;
6356 ConstantSDNode *CN;
6357 SDValue NewOperand;
6358 MVT GRLenVT = Subtarget.getGRLenVT();
6359
6360 if (SDValue R = combineAndNotIntoVANDN(N, DL, DAG))
6361 return R;
6362
6363 // BSTRPICK requires the 32S feature.
6364 if (!Subtarget.has32S())
6365 return SDValue();
6366
6367 // Op's second operand must be a shifted mask.
6368 if (!(CN = dyn_cast<ConstantSDNode>(Val&: SecondOperand)) ||
6369 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx&: SMIdx, MaskLen&: SMLen))
6370 return SDValue();
6371
6372 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
6373 // Pattern match BSTRPICK.
6374 // $dst = and ((sra or srl) $src , lsb), (2**len - 1)
6375 // => BSTRPICK $dst, $src, msb, lsb
6376 // where msb = lsb + len - 1
6377
6378 // The second operand of the shift must be an immediate.
6379 if (!(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))))
6380 return SDValue();
6381
6382 lsb = CN->getZExtValue();
6383
6384 // Return if the shifted mask does not start at bit 0 or the sum of its
6385 // length and lsb exceeds the word's size.
6386 if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())
6387 return SDValue();
6388
6389 NewOperand = FirstOperand.getOperand(i: 0);
6390 } else {
6391 // Pattern match BSTRPICK.
6392 // $dst = and $src, (2**len- 1) , if len > 12
6393 // => BSTRPICK $dst, $src, msb, lsb
6394 // where lsb = 0 and msb = len - 1
6395
6396 // If the mask is <= 0xfff, andi can be used instead.
6397 if (CN->getZExtValue() <= 0xfff)
6398 return SDValue();
6399
6400 // Return if the MSB exceeds.
6401 if (SMIdx + SMLen > ValTy.getSizeInBits())
6402 return SDValue();
6403
6404 if (SMIdx > 0) {
6405 // Omit if the constant has more than 2 uses. This a conservative
6406 // decision. Whether it is a win depends on the HW microarchitecture.
6407 // However it should always be better for 1 and 2 uses.
6408 if (CN->use_size() > 2)
6409 return SDValue();
6410 // Return if the constant can be composed by a single LU12I.W.
6411 if ((CN->getZExtValue() & 0xfff) == 0)
6412 return SDValue();
6413 // Return if the constand can be composed by a single ADDI with
6414 // the zero register.
6415 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)
6416 return SDValue();
6417 }
6418
6419 lsb = SMIdx;
6420 NewOperand = FirstOperand;
6421 }
6422
6423 msb = lsb + SMLen - 1;
6424 SDValue NR0 = DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy, N1: NewOperand,
6425 N2: DAG.getConstant(Val: msb, DL, VT: GRLenVT),
6426 N3: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6427 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)
6428 return NR0;
6429 // Try to optimize to
6430 // bstrpick $Rd, $Rs, msb, lsb
6431 // slli $Rd, $Rd, lsb
6432 return DAG.getNode(Opcode: ISD::SHL, DL, VT: ValTy, N1: NR0,
6433 N2: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6434}
6435
6436// Return the original source vector if N consists of the half
6437// of each 128-bit lane.
6438static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow) {
6439 N = peekThroughBitcasts(V: N);
6440
6441 EVT DstVT = N.getValueType();
6442 if (!DstVT.isVector())
6443 return SDValue();
6444
6445 unsigned NumElts = DstVT.getVectorNumElements();
6446
6447 // LSX canonical form:
6448 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6449 SDValue Src = N.getOperand(i: 0);
6450 EVT SrcVT = Src.getValueType();
6451
6452 if (!SrcVT.isVector() || !SrcVT.is128BitVector())
6453 return SDValue();
6454 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6455 return SDValue();
6456 if (SrcVT.getVectorNumElements() != NumElts * 2)
6457 return SDValue();
6458 if (N.getConstantOperandVal(i: 1) != (isLow ? 0 : NumElts))
6459 return SDValue();
6460
6461 return Src;
6462 }
6463
6464 // LASX canonical form:
6465 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6466 if (!BV)
6467 return SDValue();
6468
6469 if (NumElts % 2 != 0)
6470 return SDValue();
6471
6472 SDValue Src;
6473 EVT SrcVT;
6474
6475 for (unsigned I = 0; I != NumElts; ++I) {
6476 SDValue Elt = BV->getOperand(Num: I);
6477 if (Elt.isUndef())
6478 continue;
6479 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6480 return SDValue();
6481
6482 SDValue ThisSrc = Elt.getOperand(i: 0);
6483 SDValue Idx = Elt.getOperand(i: 1);
6484 auto *CI = dyn_cast<ConstantSDNode>(Val&: Idx);
6485 if (!CI)
6486 return SDValue();
6487
6488 if (!Src) {
6489 Src = ThisSrc;
6490 SrcVT = Src.getValueType();
6491 if (!SrcVT.isVector())
6492 return SDValue();
6493
6494 if (!SrcVT.is256BitVector())
6495 return SDValue();
6496 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6497 return SDValue();
6498 if (SrcVT.getVectorNumElements() != NumElts * 2)
6499 return SDValue();
6500 } else if (ThisSrc != Src) {
6501 return SDValue();
6502 }
6503
6504 unsigned Half = NumElts / 2;
6505 unsigned ExpectedIdx = (I < Half) ? I : (I + Half);
6506 ExpectedIdx += isLow ? 0 : Half;
6507
6508 if (CI->getZExtValue() != ExpectedIdx)
6509 return SDValue();
6510 }
6511
6512 return Src;
6513}
6514
6515static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
6516 TargetLowering::DAGCombinerInfo &DCI,
6517 const LoongArchSubtarget &Subtarget) {
6518 assert(N->getOpcode() == ISD::SHL && "Unexpected opcode");
6519
6520 EVT VT = N->getValueType(ResNo: 0);
6521 SDLoc DL(N);
6522
6523 SDValue LHS = N->getOperand(Num: 0);
6524 SDValue RHS = N->getOperand(Num: 1);
6525
6526 bool isSigned;
6527 unsigned ExtOpc = LHS.getOpcode();
6528 if (ExtOpc == ISD::SIGN_EXTEND)
6529 isSigned = true;
6530 else if (ExtOpc == ISD::ZERO_EXTEND)
6531 isSigned = false;
6532 else
6533 return SDValue();
6534
6535 if (!LHS.hasOneUse())
6536 return SDValue();
6537
6538 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
6539 N->getValueSizeInBits(ResNo: 0) != LHS->getOperand(Num: 0).getValueSizeInBits() * 2)
6540 return SDValue();
6541
6542 SDValue Vec = matchHalfOf128BitLanes(N: LHS.getOperand(i: 0), /*isLow=*/true);
6543 if (!Vec)
6544 return SDValue();
6545
6546 EVT SrcVT = Vec.getValueType();
6547 EVT SrcEltVT = SrcVT.getVectorElementType();
6548 EVT DstEltVT = VT.getVectorElementType();
6549 APInt Imm;
6550 if (!isConstantSplatVector(N: RHS, SplatValue&: Imm, MinSizeInBits: DstEltVT.getSizeInBits()))
6551 return SDValue();
6552 if (!Imm.ult(RHS: SrcEltVT.getSizeInBits()))
6553 return SDValue();
6554
6555 unsigned Opc = isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
6556 SDValue Sht = DAG.getConstant(Val: Imm.getZExtValue(), DL, VT: Subtarget.getGRLenVT());
6557 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Sht);
6558}
6559
6560static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
6561 TargetLowering::DAGCombinerInfo &DCI,
6562 const LoongArchSubtarget &Subtarget) {
6563 // BSTRPICK requires the 32S feature.
6564 if (!Subtarget.has32S())
6565 return SDValue();
6566
6567 if (DCI.isBeforeLegalizeOps())
6568 return SDValue();
6569
6570 // $dst = srl (and $src, Mask), Shamt
6571 // =>
6572 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt
6573 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1
6574 //
6575
6576 SDValue FirstOperand = N->getOperand(Num: 0);
6577 ConstantSDNode *CN;
6578 EVT ValTy = N->getValueType(ResNo: 0);
6579 SDLoc DL(N);
6580 MVT GRLenVT = Subtarget.getGRLenVT();
6581 unsigned MaskIdx, MaskLen;
6582 uint64_t Shamt;
6583
6584 // The first operand must be an AND and the second operand of the AND must be
6585 // a shifted mask.
6586 if (FirstOperand.getOpcode() != ISD::AND ||
6587 !(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))) ||
6588 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx, MaskLen))
6589 return SDValue();
6590
6591 // The second operand (shift amount) must be an immediate.
6592 if (!(CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))))
6593 return SDValue();
6594
6595 Shamt = CN->getZExtValue();
6596 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
6597 return DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy,
6598 N1: FirstOperand->getOperand(Num: 0),
6599 N2: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
6600 N3: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6601
6602 return SDValue();
6603}
6604
6605static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
6606 TargetLowering::DAGCombinerInfo &DCI,
6607 const LoongArchSubtarget &Subtarget) {
6608 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6609 return V;
6610
6611 return SDValue();
6612}
6613
6614// Helper to peek through bitops/trunc/setcc to determine size of source vector.
6615// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
6616static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
6617 unsigned Depth) {
6618 // Limit recursion.
6619 if (Depth >= SelectionDAG::MaxRecursionDepth)
6620 return false;
6621 switch (Src.getOpcode()) {
6622 case ISD::SETCC:
6623 case ISD::TRUNCATE:
6624 return Src.getOperand(i: 0).getValueSizeInBits() == Size;
6625 case ISD::FREEZE:
6626 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1);
6627 case ISD::AND:
6628 case ISD::XOR:
6629 case ISD::OR:
6630 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1) &&
6631 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1);
6632 case ISD::SELECT:
6633 case ISD::VSELECT:
6634 return Src.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
6635 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1) &&
6636 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 2), Size, Depth: Depth + 1);
6637 case ISD::BUILD_VECTOR:
6638 return ISD::isBuildVectorAllZeros(N: Src.getNode()) ||
6639 ISD::isBuildVectorAllOnes(N: Src.getNode());
6640 }
6641 return false;
6642}
6643
6644// Helper to push sign extension of vXi1 SETCC result through bitops.
6645static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
6646 SDValue Src, const SDLoc &DL) {
6647 switch (Src.getOpcode()) {
6648 case ISD::SETCC:
6649 case ISD::FREEZE:
6650 case ISD::TRUNCATE:
6651 case ISD::BUILD_VECTOR:
6652 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6653 case ISD::AND:
6654 case ISD::XOR:
6655 case ISD::OR:
6656 return DAG.getNode(
6657 Opcode: Src.getOpcode(), DL, VT: SExtVT,
6658 N1: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 0), DL),
6659 N2: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL));
6660 case ISD::SELECT:
6661 case ISD::VSELECT:
6662 return DAG.getSelect(
6663 DL, VT: SExtVT, Cond: Src.getOperand(i: 0),
6664 LHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL),
6665 RHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 2), DL));
6666 }
6667 llvm_unreachable("Unexpected node type for vXi1 sign extension");
6668}
6669
6670static SDValue
6671performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
6672 TargetLowering::DAGCombinerInfo &DCI,
6673 const LoongArchSubtarget &Subtarget) {
6674 SDLoc DL(N);
6675 EVT VT = N->getValueType(ResNo: 0);
6676 SDValue Src = N->getOperand(Num: 0);
6677 EVT SrcVT = Src.getValueType();
6678
6679 if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse())
6680 return SDValue();
6681
6682 bool UseLASX;
6683 unsigned Opc = ISD::DELETED_NODE;
6684 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6685 EVT EltVT = CmpVT.getVectorElementType();
6686
6687 if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128)
6688 UseLASX = false;
6689 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
6690 CmpVT.getSizeInBits() == 256)
6691 UseLASX = true;
6692 else
6693 return SDValue();
6694
6695 SDValue SrcN1 = Src.getOperand(i: 1);
6696 switch (cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get()) {
6697 default:
6698 break;
6699 case ISD::SETEQ:
6700 // x == 0 => not (vmsknez.b x)
6701 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6702 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
6703 break;
6704 case ISD::SETGT:
6705 // x > -1 => vmskgez.b x
6706 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) && EltVT == MVT::i8)
6707 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6708 break;
6709 case ISD::SETGE:
6710 // x >= 0 => vmskgez.b x
6711 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6712 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6713 break;
6714 case ISD::SETLT:
6715 // x < 0 => vmskltz.{b,h,w,d} x
6716 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) &&
6717 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6718 EltVT == MVT::i64))
6719 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6720 break;
6721 case ISD::SETLE:
6722 // x <= -1 => vmskltz.{b,h,w,d} x
6723 if (ISD::isBuildVectorAllOnes(N: SrcN1.getNode()) &&
6724 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6725 EltVT == MVT::i64))
6726 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6727 break;
6728 case ISD::SETNE:
6729 // x != 0 => vmsknez.b x
6730 if (ISD::isBuildVectorAllZeros(N: SrcN1.getNode()) && EltVT == MVT::i8)
6731 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
6732 break;
6733 }
6734
6735 if (Opc == ISD::DELETED_NODE)
6736 return SDValue();
6737
6738 SDValue V = DAG.getNode(Opcode: Opc, DL, VT: Subtarget.getGRLenVT(), Operand: Src.getOperand(i: 0));
6739 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6740 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6741 return DAG.getBitcast(VT, V);
6742}
6743
6744static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG,
6745 TargetLowering::DAGCombinerInfo &DCI,
6746 const LoongArchSubtarget &Subtarget) {
6747 SDLoc DL(N);
6748 EVT VT = N->getValueType(ResNo: 0);
6749 SDValue Src = N->getOperand(Num: 0);
6750 EVT SrcVT = Src.getValueType();
6751 MVT GRLenVT = Subtarget.getGRLenVT();
6752
6753 if (!DCI.isBeforeLegalizeOps())
6754 return SDValue();
6755
6756 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
6757 return SDValue();
6758
6759 // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
6760 SDValue Res = performSETCC_BITCASTCombine(N, DAG, DCI, Subtarget);
6761 if (Res)
6762 return Res;
6763
6764 // Generate vXi1 using [X]VMSKLTZ
6765 MVT SExtVT;
6766 unsigned Opc;
6767 bool UseLASX = false;
6768 bool PropagateSExt = false;
6769
6770 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
6771 EVT CmpVT = Src.getOperand(i: 0).getValueType();
6772 if (CmpVT.getSizeInBits() > 256)
6773 return SDValue();
6774 }
6775
6776 switch (SrcVT.getSimpleVT().SimpleTy) {
6777 default:
6778 return SDValue();
6779 case MVT::v2i1:
6780 SExtVT = MVT::v2i64;
6781 break;
6782 case MVT::v4i1:
6783 SExtVT = MVT::v4i32;
6784 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6785 SExtVT = MVT::v4i64;
6786 UseLASX = true;
6787 PropagateSExt = true;
6788 }
6789 break;
6790 case MVT::v8i1:
6791 SExtVT = MVT::v8i16;
6792 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6793 SExtVT = MVT::v8i32;
6794 UseLASX = true;
6795 PropagateSExt = true;
6796 }
6797 break;
6798 case MVT::v16i1:
6799 SExtVT = MVT::v16i8;
6800 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
6801 SExtVT = MVT::v16i16;
6802 UseLASX = true;
6803 PropagateSExt = true;
6804 }
6805 break;
6806 case MVT::v32i1:
6807 SExtVT = MVT::v32i8;
6808 UseLASX = true;
6809 break;
6810 };
6811 Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
6812 : DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
6813
6814 SDValue V;
6815 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
6816 if (Src.getSimpleValueType() == MVT::v32i8) {
6817 SDValue Lo, Hi;
6818 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Src, DL);
6819 Lo = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Lo);
6820 Hi = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Hi);
6821 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: Hi,
6822 N2: DAG.getShiftAmountConstant(Val: 16, VT: GRLenVT, DL));
6823 V = DAG.getNode(Opcode: ISD::OR, DL, VT: GRLenVT, N1: Lo, N2: Hi);
6824 } else if (UseLASX) {
6825 return SDValue();
6826 }
6827 }
6828
6829 if (!V) {
6830 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6831 V = DAG.getNode(Opcode: Opc, DL, VT: GRLenVT, Operand: Src);
6832 }
6833
6834 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
6835 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
6836 return DAG.getBitcast(VT, V);
6837}
6838
6839static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
6840 TargetLowering::DAGCombinerInfo &DCI,
6841 const LoongArchSubtarget &Subtarget) {
6842 MVT GRLenVT = Subtarget.getGRLenVT();
6843 EVT ValTy = N->getValueType(ResNo: 0);
6844 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
6845 ConstantSDNode *CN0, *CN1;
6846 SDLoc DL(N);
6847 unsigned ValBits = ValTy.getSizeInBits();
6848 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
6849 unsigned Shamt;
6850 bool SwapAndRetried = false;
6851
6852 // BSTRPICK requires the 32S feature.
6853 if (!Subtarget.has32S())
6854 return SDValue();
6855
6856 if (DCI.isBeforeLegalizeOps())
6857 return SDValue();
6858
6859 if (ValBits != 32 && ValBits != 64)
6860 return SDValue();
6861
6862Retry:
6863 // 1st pattern to match BSTRINS:
6864 // R = or (and X, mask0), (and (shl Y, lsb), mask1)
6865 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1
6866 // =>
6867 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6868 if (N0.getOpcode() == ISD::AND &&
6869 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6870 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6871 N1.getOpcode() == ISD::AND && N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
6872 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6873 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6874 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
6875 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6876 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6877 (MaskIdx0 + MaskLen0 <= ValBits)) {
6878 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");
6879 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6880 N2: N1.getOperand(i: 0).getOperand(i: 0),
6881 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6882 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6883 }
6884
6885 // 2nd pattern to match BSTRINS:
6886 // R = or (and X, mask0), (shl (and Y, mask1), lsb)
6887 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)
6888 // =>
6889 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
6890 if (N0.getOpcode() == ISD::AND &&
6891 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6892 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6893 N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
6894 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6895 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
6896 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6897 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
6898 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
6899 (MaskIdx0 + MaskLen0 <= ValBits)) {
6900 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");
6901 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6902 N2: N1.getOperand(i: 0).getOperand(i: 0),
6903 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
6904 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6905 }
6906
6907 // 3rd pattern to match BSTRINS:
6908 // R = or (and X, mask0), (and Y, mask1)
6909 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0
6910 // =>
6911 // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb
6912 // where msb = lsb + size - 1
6913 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
6914 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6915 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6916 (MaskIdx0 + MaskLen0 <= 64) &&
6917 (CN1 = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) &&
6918 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6919 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");
6920 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6921 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0), N1,
6922 N2: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT)),
6923 N3: DAG.getConstant(Val: ValBits == 32
6924 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6925 : (MaskIdx0 + MaskLen0 - 1),
6926 DL, VT: GRLenVT),
6927 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6928 }
6929
6930 // 4th pattern to match BSTRINS:
6931 // R = or (and X, mask), (shl Y, shamt)
6932 // where mask = (2**shamt - 1)
6933 // =>
6934 // R = BSTRINS X, Y, ValBits - 1, shamt
6935 // where ValBits = 32 or 64
6936 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&
6937 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6938 isShiftedMask_64(Value: CN0->getZExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6939 MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6940 (Shamt = CN1->getZExtValue()) == MaskLen0 &&
6941 (MaskIdx0 + MaskLen0 <= ValBits)) {
6942 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");
6943 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6944 N2: N1.getOperand(i: 0),
6945 N3: DAG.getConstant(Val: (ValBits - 1), DL, VT: GRLenVT),
6946 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6947 }
6948
6949 // 5th pattern to match BSTRINS:
6950 // R = or (and X, mask), const
6951 // where ~mask = (2**size - 1) << lsb, mask & const = 0
6952 // =>
6953 // R = BSTRINS X, (const >> lsb), msb, lsb
6954 // where msb = lsb + size - 1
6955 if (N0.getOpcode() == ISD::AND &&
6956 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
6957 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
6958 (CN1 = dyn_cast<ConstantSDNode>(Val&: N1)) &&
6959 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
6960 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");
6961 return DAG.getNode(
6962 Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
6963 N2: DAG.getSignedConstant(Val: CN1->getSExtValue() >> MaskIdx0, DL, VT: ValTy),
6964 N3: DAG.getConstant(Val: ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6965 : (MaskIdx0 + MaskLen0 - 1),
6966 DL, VT: GRLenVT),
6967 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
6968 }
6969
6970 // 6th pattern.
6971 // a = b | ((c & mask) << shamt), where all positions in b to be overwritten
6972 // by the incoming bits are known to be zero.
6973 // =>
6974 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt
6975 //
6976 // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th
6977 // pattern is more common than the 1st. So we put the 1st before the 6th in
6978 // order to match as many nodes as possible.
6979 ConstantSDNode *CNMask, *CNShamt;
6980 unsigned MaskIdx, MaskLen;
6981 if (N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
6982 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
6983 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
6984 MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
6985 CNShamt->getZExtValue() + MaskLen <= ValBits) {
6986 Shamt = CNShamt->getZExtValue();
6987 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);
6988 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
6989 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");
6990 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
6991 N2: N1.getOperand(i: 0).getOperand(i: 0),
6992 N3: DAG.getConstant(Val: Shamt + MaskLen - 1, DL, VT: GRLenVT),
6993 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
6994 }
6995 }
6996
6997 // 7th pattern.
6998 // a = b | ((c << shamt) & shifted_mask), where all positions in b to be
6999 // overwritten by the incoming bits are known to be zero.
7000 // =>
7001 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx
7002 //
7003 // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd
7004 // before the 7th in order to match as many nodes as possible.
7005 if (N1.getOpcode() == ISD::AND &&
7006 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7007 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7008 N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
7009 (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7010 CNShamt->getZExtValue() == MaskIdx) {
7011 APInt ShMask(ValBits, CNMask->getZExtValue());
7012 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7013 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");
7014 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7015 N2: N1.getOperand(i: 0).getOperand(i: 0),
7016 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7017 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7018 }
7019 }
7020
7021 // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.
7022 if (!SwapAndRetried) {
7023 std::swap(a&: N0, b&: N1);
7024 SwapAndRetried = true;
7025 goto Retry;
7026 }
7027
7028 SwapAndRetried = false;
7029Retry2:
7030 // 8th pattern.
7031 // a = b | (c & shifted_mask), where all positions in b to be overwritten by
7032 // the incoming bits are known to be zero.
7033 // =>
7034 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx
7035 //
7036 // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So
7037 // we put it here in order to match as many nodes as possible or generate less
7038 // instructions.
7039 if (N1.getOpcode() == ISD::AND &&
7040 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7041 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen)) {
7042 APInt ShMask(ValBits, CNMask->getZExtValue());
7043 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7044 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");
7045 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7046 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0),
7047 N1: N1->getOperand(Num: 0),
7048 N2: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT)),
7049 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7050 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7051 }
7052 }
7053 // Swap N0/N1 and retry.
7054 if (!SwapAndRetried) {
7055 std::swap(a&: N0, b&: N1);
7056 SwapAndRetried = true;
7057 goto Retry2;
7058 }
7059
7060 return SDValue();
7061}
7062
7063static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
7064 ExtType = ISD::NON_EXTLOAD;
7065
7066 switch (V.getNode()->getOpcode()) {
7067 case ISD::LOAD: {
7068 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: V.getNode());
7069 if ((LoadNode->getMemoryVT() == MVT::i8) ||
7070 (LoadNode->getMemoryVT() == MVT::i16)) {
7071 ExtType = LoadNode->getExtensionType();
7072 return true;
7073 }
7074 return false;
7075 }
7076 case ISD::AssertSext: {
7077 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7078 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7079 ExtType = ISD::SEXTLOAD;
7080 return true;
7081 }
7082 return false;
7083 }
7084 case ISD::AssertZext: {
7085 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7086 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7087 ExtType = ISD::ZEXTLOAD;
7088 return true;
7089 }
7090 return false;
7091 }
7092 default:
7093 return false;
7094 }
7095
7096 return false;
7097}
7098
7099// Eliminate redundant truncation and zero-extension nodes.
7100// * Case 1:
7101// +------------+ +------------+ +------------+
7102// | Input1 | | Input2 | | CC |
7103// +------------+ +------------+ +------------+
7104// | | |
7105// V V +----+
7106// +------------+ +------------+ |
7107// | TRUNCATE | | TRUNCATE | |
7108// +------------+ +------------+ |
7109// | | |
7110// V V |
7111// +------------+ +------------+ |
7112// | ZERO_EXT | | ZERO_EXT | |
7113// +------------+ +------------+ |
7114// | | |
7115// | +-------------+ |
7116// V V | |
7117// +----------------+ | |
7118// | AND | | |
7119// +----------------+ | |
7120// | | |
7121// +---------------+ | |
7122// | | |
7123// V V V
7124// +-------------+
7125// | CMP |
7126// +-------------+
7127// * Case 2:
7128// +------------+ +------------+ +-------------+ +------------+ +------------+
7129// | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
7130// +------------+ +------------+ +-------------+ +------------+ +------------+
7131// | | | | |
7132// V | | | |
7133// +------------+ | | | |
7134// | XOR |<---------------------+ | |
7135// +------------+ | | |
7136// | | | |
7137// V V +---------------+ |
7138// +------------+ +------------+ | |
7139// | TRUNCATE | | TRUNCATE | | +-------------------------+
7140// +------------+ +------------+ | |
7141// | | | |
7142// V V | |
7143// +------------+ +------------+ | |
7144// | ZERO_EXT | | ZERO_EXT | | |
7145// +------------+ +------------+ | |
7146// | | | |
7147// V V | |
7148// +----------------+ | |
7149// | AND | | |
7150// +----------------+ | |
7151// | | |
7152// +---------------+ | |
7153// | | |
7154// V V V
7155// +-------------+
7156// | CMP |
7157// +-------------+
7158static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
7159 TargetLowering::DAGCombinerInfo &DCI,
7160 const LoongArchSubtarget &Subtarget) {
7161 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
7162
7163 SDNode *AndNode = N->getOperand(Num: 0).getNode();
7164 if (AndNode->getOpcode() != ISD::AND)
7165 return SDValue();
7166
7167 SDValue AndInputValue2 = AndNode->getOperand(Num: 1);
7168 if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
7169 return SDValue();
7170
7171 SDValue CmpInputValue = N->getOperand(Num: 1);
7172 SDValue AndInputValue1 = AndNode->getOperand(Num: 0);
7173 if (AndInputValue1.getOpcode() == ISD::XOR) {
7174 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7175 return SDValue();
7176 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: AndInputValue1.getOperand(i: 1));
7177 if (!CN || !CN->isAllOnes())
7178 return SDValue();
7179 CN = dyn_cast<ConstantSDNode>(Val&: CmpInputValue);
7180 if (!CN || !CN->isZero())
7181 return SDValue();
7182 AndInputValue1 = AndInputValue1.getOperand(i: 0);
7183 if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
7184 return SDValue();
7185 } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
7186 if (AndInputValue2 != CmpInputValue)
7187 return SDValue();
7188 } else {
7189 return SDValue();
7190 }
7191
7192 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(Num: 0);
7193 if (TruncValue1.getOpcode() != ISD::TRUNCATE)
7194 return SDValue();
7195
7196 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(Num: 0);
7197 if (TruncValue2.getOpcode() != ISD::TRUNCATE)
7198 return SDValue();
7199
7200 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(Num: 0);
7201 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(Num: 0);
7202 ISD::LoadExtType ExtType1;
7203 ISD::LoadExtType ExtType2;
7204
7205 if (!checkValueWidth(V: TruncInputValue1, ExtType&: ExtType1) ||
7206 !checkValueWidth(V: TruncInputValue2, ExtType&: ExtType2))
7207 return SDValue();
7208
7209 if (TruncInputValue1->getValueType(ResNo: 0) != TruncInputValue2->getValueType(ResNo: 0) ||
7210 AndNode->getValueType(ResNo: 0) != TruncInputValue1->getValueType(ResNo: 0))
7211 return SDValue();
7212
7213 if ((ExtType2 != ISD::ZEXTLOAD) &&
7214 ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
7215 return SDValue();
7216
7217 // These truncation and zero-extension nodes are not necessary, remove them.
7218 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: AndNode->getValueType(ResNo: 0),
7219 N1: TruncInputValue1, N2: TruncInputValue2);
7220 SDValue NewSetCC =
7221 DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: NewAnd, RHS: TruncInputValue2, Cond: CC);
7222 DAG.ReplaceAllUsesWith(From: N, To: NewSetCC.getNode());
7223 return SDValue(N, 0);
7224}
7225
7226// Strip a single outer ISD::SIGN_EXTEND_INREG from \p V, if present, and
7227// return the inner value together with the narrow VT it was extending
7228// from. If no such node is present, returns \p V unchanged and an invalid
7229// EVT.
7230//
7231// i32 (and other sub-GRLen) arithmetic is legalized to operate on the full
7232// GRLen-width register, with a `sign_extend_inreg` re-normalizing the
7233// result back into the narrow type's range afterwards (see e.g. the
7234// `add i32` -> `add` + `sign_extend_inreg ..., i32` legalization). Any
7235// combine that reassociates such a binop must track and reapply this
7236// extension, otherwise the transformed code can produce a value whose
7237// high bits no longer match the narrow-type semantics.
7238static std::pair<SDValue, EVT> stripSignExtendInReg(SDValue V) {
7239 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
7240 return {V.getOperand(i: 0), cast<VTSDNode>(Val: V.getOperand(i: 1))->getVT()};
7241 return {V, EVT()};
7242}
7243
7244// Try to match \p BinV (after optionally stripping an outer
7245// sign_extend_inreg) as a supported binary operation that has \p X as one
7246// of its operands, returning the matched opcode, the other operand (the
7247// "delta"), and the narrow VT of the sign_extend_inreg that was stripped
7248// (invalid EVT if none was present).
7249//
7250// For commutative ops (add/or/xor), \p X may be either operand, since
7251// `binop(X, Y) == binop(Y, X)` and the identity element (0) works on
7252// either side.
7253//
7254// For `sub`, the operation is NOT commutative: `sub(X, Y) != sub(Y, X)`.
7255// Only `sub(X, Y)` (i.e. \p X is the *minuend*, the first operand) can be
7256// rewritten using the identity `X - 0 == X`. If \p X were the *subtrahend*
7257// (second operand, i.e. the pattern is actually `sub(Y, X)`), there is no
7258// way to express `cond ? (Y - X) : X` (or the symmetric case) as
7259// `X op (select ...)` without introducing an extra negation, so that case
7260// must be rejected instead of "optimized" into worse code.
7261static std::tuple<unsigned, SDValue, EVT>
7262matchBinOpWithSharedOperand(SDValue BinV, SDValue X) {
7263 auto [Inner, ExtVT] = stripSignExtendInReg(V: BinV);
7264
7265 unsigned Opc = Inner.getOpcode();
7266 switch (Opc) {
7267 case ISD::ADD:
7268 case ISD::OR:
7269 case ISD::XOR:
7270 if (Inner.getOperand(i: 0) == X)
7271 return {Opc, Inner.getOperand(i: 1), ExtVT};
7272 if (Inner.getOperand(i: 1) == X)
7273 return {Opc, Inner.getOperand(i: 0), ExtVT};
7274 return {0, SDValue(), EVT()};
7275 case ISD::SUB:
7276 // Only accept X as the minuend (first operand); see comment above.
7277 if (Inner.getOperand(i: 0) == X)
7278 return {Opc, Inner.getOperand(i: 1), ExtVT};
7279 return {0, SDValue(), EVT()};
7280 default:
7281 return {0, SDValue(), EVT()};
7282 }
7283}
7284
7285// Try to combine:
7286// select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
7287// select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
7288// for binop in {add, or, xor, sub}, where 0 is the identity element of the
7289// respective operation, additionally handling the common legalized form
7290// where the binop result is wrapped in a `sign_extend_inreg` (as happens
7291// for sub-GRLen types such as i32 on a 64-bit GRLen target). See
7292// matchBinOpWithSharedOperand() for the restrictions applied to
7293// non-commutative operations (currently only `sub`).
7294static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
7295 TargetLowering::DAGCombinerInfo &DCI,
7296 const LoongArchSubtarget &Subtarget) {
7297 if (DCI.isBeforeLegalizeOps())
7298 return SDValue();
7299
7300 EVT VT = N->getValueType(ResNo: 0);
7301 // Restrict to the scalar GRLen integer type that maskeqz/masknez operate
7302 // on; this also naturally excludes float and vector selects.
7303 if (VT != Subtarget.getGRLenVT())
7304 return SDValue();
7305
7306 SDValue Cond = N->getOperand(Num: 0);
7307 SDValue TrueV = N->getOperand(Num: 1);
7308 SDValue FalseV = N->getOperand(Num: 2);
7309 SDLoc DL(N);
7310
7311 auto TryFold = [&](SDValue BinV, SDValue SharedV,
7312 bool BinIsTrueArm) -> SDValue {
7313 auto [Opc, Delta, ExtVT] = matchBinOpWithSharedOperand(BinV, X: SharedV);
7314 if (!Opc)
7315 return SDValue();
7316
7317 // Avoid infinite combine loops: bail out if Delta is trivially the
7318 // same node we would otherwise be selecting on (shouldn't normally
7319 // happen, but guards against degenerate/self-referential IR).
7320 if (Delta.getNode() == N)
7321 return SDValue();
7322
7323 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
7324 SDValue NewSel = BinIsTrueArm ? DAG.getSelect(DL, VT, Cond, LHS: Delta, RHS: Zero)
7325 : DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: Delta);
7326 SDValue NewBin = DAG.getNode(Opcode: Opc, DL, VT, N1: SharedV, N2: NewSel);
7327
7328 // If the original binop result was normalized back into a narrower
7329 // type via sign_extend_inreg (e.g. i32 arithmetic on a 64-bit GRLen
7330 // target), the new binop must be re-normalized the same way: SharedV
7331 // is already known-sign-extended for that narrow type, but NewSel
7332 // (Delta or 0, selected) combined with SharedV via Opc can still
7333 // produce a 64-bit result whose high bits don't match the narrow
7334 // type's sign-extended representation.
7335 if (ExtVT != EVT())
7336 NewBin = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: NewBin,
7337 N2: DAG.getValueType(ExtVT));
7338
7339 return NewBin;
7340 };
7341
7342 if (SDValue R = TryFold(TrueV, FalseV, /*BinIsTrueArm=*/true))
7343 return R;
7344 if (SDValue R = TryFold(FalseV, TrueV, /*BinIsTrueArm=*/false))
7345 return R;
7346
7347 return SDValue();
7348}
7349
7350// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
7351static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
7352 TargetLowering::DAGCombinerInfo &DCI,
7353 const LoongArchSubtarget &Subtarget) {
7354 if (DCI.isBeforeLegalizeOps())
7355 return SDValue();
7356
7357 SDValue Src = N->getOperand(Num: 0);
7358 if (Src.getOpcode() != LoongArchISD::REVB_2W)
7359 return SDValue();
7360
7361 return DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
7362 Operand: Src.getOperand(i: 0));
7363}
7364
7365// Perform common combines for BR_CC and SELECT_CC conditions.
7366static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
7367 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
7368 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7369
7370 // As far as arithmetic right shift always saves the sign,
7371 // shift can be omitted.
7372 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
7373 // setge (sra X, N), 0 -> setge X, 0
7374 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
7375 LHS.getOpcode() == ISD::SRA) {
7376 LHS = LHS.getOperand(i: 0);
7377 return true;
7378 }
7379
7380 if (!ISD::isIntEqualitySetCC(Code: CCVal))
7381 return false;
7382
7383 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
7384 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
7385 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
7386 LHS.getOperand(i: 0).getValueType() == Subtarget.getGRLenVT()) {
7387 // If we're looking for eq 0 instead of ne 0, we need to invert the
7388 // condition.
7389 bool Invert = CCVal == ISD::SETEQ;
7390 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
7391 if (Invert)
7392 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7393
7394 RHS = LHS.getOperand(i: 1);
7395 LHS = LHS.getOperand(i: 0);
7396 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
7397
7398 CC = DAG.getCondCode(Cond: CCVal);
7399 return true;
7400 }
7401
7402 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, GRLen-1-C), 0, ge/lt)
7403 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
7404 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
7405 SDValue LHS0 = LHS.getOperand(i: 0);
7406 if (LHS0.getOpcode() == ISD::AND &&
7407 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7408 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
7409 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
7410 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
7411 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
7412 CC = DAG.getCondCode(Cond: CCVal);
7413
7414 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
7415 LHS = LHS0.getOperand(i: 0);
7416 if (ShAmt != 0)
7417 LHS =
7418 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
7419 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
7420 return true;
7421 }
7422 }
7423 }
7424
7425 // (X, 1, setne) -> (X, 0, seteq) if we can prove X is 0/1.
7426 // This can occur when legalizing some floating point comparisons.
7427 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
7428 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
7429 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
7430 CC = DAG.getCondCode(Cond: CCVal);
7431 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
7432 return true;
7433 }
7434
7435 // Fold ((shl (extract_vector_elt X, I), GRLen - EleBits)), 0, eq/ne) ->
7436 // ((extract_vector_elt X, I), 0, eq/ne)
7437 if (isNullConstant(V: RHS) && (CCVal == ISD::SETEQ || CCVal == ISD::SETNE) &&
7438 LHS.getOpcode() == ISD::SHL && LHS.hasOneUse() &&
7439 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
7440 SDValue Ext = LHS.getOperand(i: 0);
7441 unsigned Sht = LHS.getConstantOperandVal(i: 1);
7442 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
7443 SDValue Vec = Ext.getOperand(i: 0);
7444 unsigned EleBits = Vec.getScalarValueSizeInBits();
7445 if ((EleBits + Sht) == Subtarget.getGRLen()) {
7446 LHS = Ext;
7447 return true;
7448 }
7449 }
7450 }
7451
7452 // Fold (C1, C2, cond) -> (0, 0, seteq/setne)
7453 if (isa<ConstantSDNode>(Val: LHS) && isa<ConstantSDNode>(Val: RHS)) {
7454 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
7455 EVT VT = LHS.getValueType();
7456 EVT SetCCResVT =
7457 TLI->getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7458 if (SDValue Folded = DAG.FoldSetCC(VT: SetCCResVT, N1: LHS, N2: RHS, Cond: CCVal, dl: DL)) {
7459 LHS = DAG.getConstant(Val: 0, DL, VT);
7460 RHS = DAG.getConstant(Val: 0, DL, VT);
7461 CC = DAG.getCondCode(Cond: !isNullConstant(V: Folded) ? ISD::SETEQ : ISD::SETNE);
7462 return true;
7463 }
7464 }
7465
7466 return false;
7467}
7468
7469static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG,
7470 TargetLowering::DAGCombinerInfo &DCI,
7471 const LoongArchSubtarget &Subtarget) {
7472 SDValue LHS = N->getOperand(Num: 1);
7473 SDValue RHS = N->getOperand(Num: 2);
7474 SDValue CC = N->getOperand(Num: 3);
7475 SDLoc DL(N);
7476
7477 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7478 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
7479 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
7480
7481 return SDValue();
7482}
7483
7484static SDValue performSELECT_CCCombine(SDNode *N, SelectionDAG &DAG,
7485 TargetLowering::DAGCombinerInfo &DCI,
7486 const LoongArchSubtarget &Subtarget) {
7487 // Transform
7488 SDValue LHS = N->getOperand(Num: 0);
7489 SDValue RHS = N->getOperand(Num: 1);
7490 SDValue CC = N->getOperand(Num: 2);
7491 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
7492 SDValue TrueV = N->getOperand(Num: 3);
7493 SDValue FalseV = N->getOperand(Num: 4);
7494 SDLoc DL(N);
7495 EVT VT = N->getValueType(ResNo: 0);
7496
7497 // If the True and False values are the same, we don't need a select_cc.
7498 if (TrueV == FalseV)
7499 return TrueV;
7500
7501 // (select (x < 0), y, z) -> x >> (GRLEN - 1) & (y - z) + z
7502 // (select (x >= 0), y, z) -> x >> (GRLEN - 1) & (z - y) + y
7503 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
7504 isNullConstant(V: RHS) &&
7505 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
7506 if (CCVal == ISD::CondCode::SETGE)
7507 std::swap(a&: TrueV, b&: FalseV);
7508
7509 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
7510 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
7511 // Only handle simm12, if it is not in this range, it can be considered as
7512 // register.
7513 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
7514 isInt<12>(x: TrueSImm - FalseSImm)) {
7515 SDValue SRA =
7516 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
7517 N2: DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT));
7518 SDValue AND =
7519 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
7520 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
7521 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
7522 }
7523
7524 if (CCVal == ISD::CondCode::SETGE)
7525 std::swap(a&: TrueV, b&: FalseV);
7526 }
7527
7528 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
7529 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
7530 Ops: {LHS, RHS, CC, TrueV, FalseV});
7531
7532 return SDValue();
7533}
7534
7535template <unsigned N>
7536static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp,
7537 SelectionDAG &DAG,
7538 const LoongArchSubtarget &Subtarget,
7539 bool IsSigned = false) {
7540 SDLoc DL(Node);
7541 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7542 // Check the ImmArg.
7543 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7544 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7545 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7546 ": argument out of range.");
7547 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: Subtarget.getGRLenVT());
7548 }
7549 return DAG.getConstant(Val: CImm->getZExtValue(), DL, VT: Subtarget.getGRLenVT());
7550}
7551
7552template <unsigned N>
7553static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,
7554 SelectionDAG &DAG, bool IsSigned = false) {
7555 SDLoc DL(Node);
7556 EVT ResTy = Node->getValueType(ResNo: 0);
7557 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
7558
7559 // Check the ImmArg.
7560 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
7561 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
7562 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7563 ": argument out of range.");
7564 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7565 }
7566 return DAG.getConstant(
7567 Val: APInt(ResTy.getScalarType().getSizeInBits(),
7568 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
7569 DL, VT: ResTy);
7570}
7571
7572static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG) {
7573 SDLoc DL(Node);
7574 EVT ResTy = Node->getValueType(ResNo: 0);
7575 SDValue Vec = Node->getOperand(Num: 2);
7576 SDValue Mask = DAG.getConstant(Val: Vec.getScalarValueSizeInBits() - 1, DL, VT: ResTy);
7577 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Vec, N2: Mask);
7578}
7579
7580static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG) {
7581 SDLoc DL(Node);
7582 EVT ResTy = Node->getValueType(ResNo: 0);
7583 SDValue One = DAG.getConstant(Val: 1, DL, VT: ResTy);
7584 SDValue Bit =
7585 DAG.getNode(Opcode: ISD::SHL, DL, VT: ResTy, N1: One, N2: truncateVecElts(Node, DAG));
7586
7587 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1),
7588 N2: DAG.getNOT(DL, Val: Bit, VT: ResTy));
7589}
7590
7591template <unsigned N>
7592static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG) {
7593 SDLoc DL(Node);
7594 EVT ResTy = Node->getValueType(ResNo: 0);
7595 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7596 // Check the unsigned ImmArg.
7597 if (!isUInt<N>(CImm->getZExtValue())) {
7598 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7599 ": argument out of range.");
7600 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7601 }
7602
7603 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7604 SDValue Mask = DAG.getConstant(Val: ~BitImm, DL, VT: ResTy);
7605
7606 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: Mask);
7607}
7608
7609template <unsigned N>
7610static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG) {
7611 SDLoc DL(Node);
7612 EVT ResTy = Node->getValueType(ResNo: 0);
7613 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7614 // Check the unsigned ImmArg.
7615 if (!isUInt<N>(CImm->getZExtValue())) {
7616 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7617 ": argument out of range.");
7618 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7619 }
7620
7621 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7622 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7623 return DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7624}
7625
7626template <unsigned N>
7627static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG) {
7628 SDLoc DL(Node);
7629 EVT ResTy = Node->getValueType(ResNo: 0);
7630 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
7631 // Check the unsigned ImmArg.
7632 if (!isUInt<N>(CImm->getZExtValue())) {
7633 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
7634 ": argument out of range.");
7635 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
7636 }
7637
7638 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
7639 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
7640 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
7641}
7642
7643template <unsigned W>
7644static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG,
7645 unsigned ResOp) {
7646 unsigned Imm = N->getConstantOperandVal(Num: 2);
7647 if (!isUInt<W>(Imm)) {
7648 const StringRef ErrorMsg = "argument out of range";
7649 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
7650 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
7651 }
7652 SDLoc DL(N);
7653 SDValue Vec = N->getOperand(Num: 1);
7654 SDValue Idx = DAG.getConstant(Val: Imm, DL, VT: MVT::i32);
7655 SDValue EltVT = DAG.getValueType(Vec.getValueType().getVectorElementType());
7656 return DAG.getNode(Opcode: ResOp, DL, VT: N->getValueType(ResNo: 0), N1: Vec, N2: Idx, N3: EltVT);
7657}
7658
7659static SDValue
7660performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
7661 TargetLowering::DAGCombinerInfo &DCI,
7662 const LoongArchSubtarget &Subtarget) {
7663 SDLoc DL(N);
7664 switch (N->getConstantOperandVal(Num: 0)) {
7665 default:
7666 break;
7667 case Intrinsic::loongarch_lsx_vadd_b:
7668 case Intrinsic::loongarch_lsx_vadd_h:
7669 case Intrinsic::loongarch_lsx_vadd_w:
7670 case Intrinsic::loongarch_lsx_vadd_d:
7671 case Intrinsic::loongarch_lasx_xvadd_b:
7672 case Intrinsic::loongarch_lasx_xvadd_h:
7673 case Intrinsic::loongarch_lasx_xvadd_w:
7674 case Intrinsic::loongarch_lasx_xvadd_d:
7675 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7676 N2: N->getOperand(Num: 2));
7677 case Intrinsic::loongarch_lsx_vaddi_bu:
7678 case Intrinsic::loongarch_lsx_vaddi_hu:
7679 case Intrinsic::loongarch_lsx_vaddi_wu:
7680 case Intrinsic::loongarch_lsx_vaddi_du:
7681 case Intrinsic::loongarch_lasx_xvaddi_bu:
7682 case Intrinsic::loongarch_lasx_xvaddi_hu:
7683 case Intrinsic::loongarch_lasx_xvaddi_wu:
7684 case Intrinsic::loongarch_lasx_xvaddi_du:
7685 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7686 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7687 case Intrinsic::loongarch_lsx_vsub_b:
7688 case Intrinsic::loongarch_lsx_vsub_h:
7689 case Intrinsic::loongarch_lsx_vsub_w:
7690 case Intrinsic::loongarch_lsx_vsub_d:
7691 case Intrinsic::loongarch_lasx_xvsub_b:
7692 case Intrinsic::loongarch_lasx_xvsub_h:
7693 case Intrinsic::loongarch_lasx_xvsub_w:
7694 case Intrinsic::loongarch_lasx_xvsub_d:
7695 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7696 N2: N->getOperand(Num: 2));
7697 case Intrinsic::loongarch_lsx_vsubi_bu:
7698 case Intrinsic::loongarch_lsx_vsubi_hu:
7699 case Intrinsic::loongarch_lsx_vsubi_wu:
7700 case Intrinsic::loongarch_lsx_vsubi_du:
7701 case Intrinsic::loongarch_lasx_xvsubi_bu:
7702 case Intrinsic::loongarch_lasx_xvsubi_hu:
7703 case Intrinsic::loongarch_lasx_xvsubi_wu:
7704 case Intrinsic::loongarch_lasx_xvsubi_du:
7705 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7706 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7707 case Intrinsic::loongarch_lsx_vneg_b:
7708 case Intrinsic::loongarch_lsx_vneg_h:
7709 case Intrinsic::loongarch_lsx_vneg_w:
7710 case Intrinsic::loongarch_lsx_vneg_d:
7711 case Intrinsic::loongarch_lasx_xvneg_b:
7712 case Intrinsic::loongarch_lasx_xvneg_h:
7713 case Intrinsic::loongarch_lasx_xvneg_w:
7714 case Intrinsic::loongarch_lasx_xvneg_d:
7715 return DAG.getNode(
7716 Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0),
7717 N1: DAG.getConstant(
7718 Val: APInt(N->getValueType(ResNo: 0).getScalarType().getSizeInBits(), 0,
7719 /*isSigned=*/true),
7720 DL: SDLoc(N), VT: N->getValueType(ResNo: 0)),
7721 N2: N->getOperand(Num: 1));
7722 case Intrinsic::loongarch_lsx_vmax_b:
7723 case Intrinsic::loongarch_lsx_vmax_h:
7724 case Intrinsic::loongarch_lsx_vmax_w:
7725 case Intrinsic::loongarch_lsx_vmax_d:
7726 case Intrinsic::loongarch_lasx_xvmax_b:
7727 case Intrinsic::loongarch_lasx_xvmax_h:
7728 case Intrinsic::loongarch_lasx_xvmax_w:
7729 case Intrinsic::loongarch_lasx_xvmax_d:
7730 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7731 N2: N->getOperand(Num: 2));
7732 case Intrinsic::loongarch_lsx_vmax_bu:
7733 case Intrinsic::loongarch_lsx_vmax_hu:
7734 case Intrinsic::loongarch_lsx_vmax_wu:
7735 case Intrinsic::loongarch_lsx_vmax_du:
7736 case Intrinsic::loongarch_lasx_xvmax_bu:
7737 case Intrinsic::loongarch_lasx_xvmax_hu:
7738 case Intrinsic::loongarch_lasx_xvmax_wu:
7739 case Intrinsic::loongarch_lasx_xvmax_du:
7740 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7741 N2: N->getOperand(Num: 2));
7742 case Intrinsic::loongarch_lsx_vmaxi_b:
7743 case Intrinsic::loongarch_lsx_vmaxi_h:
7744 case Intrinsic::loongarch_lsx_vmaxi_w:
7745 case Intrinsic::loongarch_lsx_vmaxi_d:
7746 case Intrinsic::loongarch_lasx_xvmaxi_b:
7747 case Intrinsic::loongarch_lasx_xvmaxi_h:
7748 case Intrinsic::loongarch_lasx_xvmaxi_w:
7749 case Intrinsic::loongarch_lasx_xvmaxi_d:
7750 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7751 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7752 case Intrinsic::loongarch_lsx_vmaxi_bu:
7753 case Intrinsic::loongarch_lsx_vmaxi_hu:
7754 case Intrinsic::loongarch_lsx_vmaxi_wu:
7755 case Intrinsic::loongarch_lsx_vmaxi_du:
7756 case Intrinsic::loongarch_lasx_xvmaxi_bu:
7757 case Intrinsic::loongarch_lasx_xvmaxi_hu:
7758 case Intrinsic::loongarch_lasx_xvmaxi_wu:
7759 case Intrinsic::loongarch_lasx_xvmaxi_du:
7760 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7761 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7762 case Intrinsic::loongarch_lsx_vmin_b:
7763 case Intrinsic::loongarch_lsx_vmin_h:
7764 case Intrinsic::loongarch_lsx_vmin_w:
7765 case Intrinsic::loongarch_lsx_vmin_d:
7766 case Intrinsic::loongarch_lasx_xvmin_b:
7767 case Intrinsic::loongarch_lasx_xvmin_h:
7768 case Intrinsic::loongarch_lasx_xvmin_w:
7769 case Intrinsic::loongarch_lasx_xvmin_d:
7770 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7771 N2: N->getOperand(Num: 2));
7772 case Intrinsic::loongarch_lsx_vmin_bu:
7773 case Intrinsic::loongarch_lsx_vmin_hu:
7774 case Intrinsic::loongarch_lsx_vmin_wu:
7775 case Intrinsic::loongarch_lsx_vmin_du:
7776 case Intrinsic::loongarch_lasx_xvmin_bu:
7777 case Intrinsic::loongarch_lasx_xvmin_hu:
7778 case Intrinsic::loongarch_lasx_xvmin_wu:
7779 case Intrinsic::loongarch_lasx_xvmin_du:
7780 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7781 N2: N->getOperand(Num: 2));
7782 case Intrinsic::loongarch_lsx_vmini_b:
7783 case Intrinsic::loongarch_lsx_vmini_h:
7784 case Intrinsic::loongarch_lsx_vmini_w:
7785 case Intrinsic::loongarch_lsx_vmini_d:
7786 case Intrinsic::loongarch_lasx_xvmini_b:
7787 case Intrinsic::loongarch_lasx_xvmini_h:
7788 case Intrinsic::loongarch_lasx_xvmini_w:
7789 case Intrinsic::loongarch_lasx_xvmini_d:
7790 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7791 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
7792 case Intrinsic::loongarch_lsx_vmini_bu:
7793 case Intrinsic::loongarch_lsx_vmini_hu:
7794 case Intrinsic::loongarch_lsx_vmini_wu:
7795 case Intrinsic::loongarch_lsx_vmini_du:
7796 case Intrinsic::loongarch_lasx_xvmini_bu:
7797 case Intrinsic::loongarch_lasx_xvmini_hu:
7798 case Intrinsic::loongarch_lasx_xvmini_wu:
7799 case Intrinsic::loongarch_lasx_xvmini_du:
7800 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7801 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7802 case Intrinsic::loongarch_lsx_vmul_b:
7803 case Intrinsic::loongarch_lsx_vmul_h:
7804 case Intrinsic::loongarch_lsx_vmul_w:
7805 case Intrinsic::loongarch_lsx_vmul_d:
7806 case Intrinsic::loongarch_lasx_xvmul_b:
7807 case Intrinsic::loongarch_lasx_xvmul_h:
7808 case Intrinsic::loongarch_lasx_xvmul_w:
7809 case Intrinsic::loongarch_lasx_xvmul_d:
7810 return DAG.getNode(Opcode: ISD::MUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7811 N2: N->getOperand(Num: 2));
7812 case Intrinsic::loongarch_lsx_vmadd_b:
7813 case Intrinsic::loongarch_lsx_vmadd_h:
7814 case Intrinsic::loongarch_lsx_vmadd_w:
7815 case Intrinsic::loongarch_lsx_vmadd_d:
7816 case Intrinsic::loongarch_lasx_xvmadd_b:
7817 case Intrinsic::loongarch_lasx_xvmadd_h:
7818 case Intrinsic::loongarch_lasx_xvmadd_w:
7819 case Intrinsic::loongarch_lasx_xvmadd_d: {
7820 EVT ResTy = N->getValueType(ResNo: 0);
7821 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7822 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7823 N2: N->getOperand(Num: 3)));
7824 }
7825 case Intrinsic::loongarch_lsx_vmsub_b:
7826 case Intrinsic::loongarch_lsx_vmsub_h:
7827 case Intrinsic::loongarch_lsx_vmsub_w:
7828 case Intrinsic::loongarch_lsx_vmsub_d:
7829 case Intrinsic::loongarch_lasx_xvmsub_b:
7830 case Intrinsic::loongarch_lasx_xvmsub_h:
7831 case Intrinsic::loongarch_lasx_xvmsub_w:
7832 case Intrinsic::loongarch_lasx_xvmsub_d: {
7833 EVT ResTy = N->getValueType(ResNo: 0);
7834 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
7835 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
7836 N2: N->getOperand(Num: 3)));
7837 }
7838 case Intrinsic::loongarch_lsx_vdiv_b:
7839 case Intrinsic::loongarch_lsx_vdiv_h:
7840 case Intrinsic::loongarch_lsx_vdiv_w:
7841 case Intrinsic::loongarch_lsx_vdiv_d:
7842 case Intrinsic::loongarch_lasx_xvdiv_b:
7843 case Intrinsic::loongarch_lasx_xvdiv_h:
7844 case Intrinsic::loongarch_lasx_xvdiv_w:
7845 case Intrinsic::loongarch_lasx_xvdiv_d:
7846 return DAG.getNode(Opcode: ISD::SDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7847 N2: N->getOperand(Num: 2));
7848 case Intrinsic::loongarch_lsx_vdiv_bu:
7849 case Intrinsic::loongarch_lsx_vdiv_hu:
7850 case Intrinsic::loongarch_lsx_vdiv_wu:
7851 case Intrinsic::loongarch_lsx_vdiv_du:
7852 case Intrinsic::loongarch_lasx_xvdiv_bu:
7853 case Intrinsic::loongarch_lasx_xvdiv_hu:
7854 case Intrinsic::loongarch_lasx_xvdiv_wu:
7855 case Intrinsic::loongarch_lasx_xvdiv_du:
7856 return DAG.getNode(Opcode: ISD::UDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7857 N2: N->getOperand(Num: 2));
7858 case Intrinsic::loongarch_lsx_vmod_b:
7859 case Intrinsic::loongarch_lsx_vmod_h:
7860 case Intrinsic::loongarch_lsx_vmod_w:
7861 case Intrinsic::loongarch_lsx_vmod_d:
7862 case Intrinsic::loongarch_lasx_xvmod_b:
7863 case Intrinsic::loongarch_lasx_xvmod_h:
7864 case Intrinsic::loongarch_lasx_xvmod_w:
7865 case Intrinsic::loongarch_lasx_xvmod_d:
7866 return DAG.getNode(Opcode: ISD::SREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7867 N2: N->getOperand(Num: 2));
7868 case Intrinsic::loongarch_lsx_vmod_bu:
7869 case Intrinsic::loongarch_lsx_vmod_hu:
7870 case Intrinsic::loongarch_lsx_vmod_wu:
7871 case Intrinsic::loongarch_lsx_vmod_du:
7872 case Intrinsic::loongarch_lasx_xvmod_bu:
7873 case Intrinsic::loongarch_lasx_xvmod_hu:
7874 case Intrinsic::loongarch_lasx_xvmod_wu:
7875 case Intrinsic::loongarch_lasx_xvmod_du:
7876 return DAG.getNode(Opcode: ISD::UREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7877 N2: N->getOperand(Num: 2));
7878 case Intrinsic::loongarch_lsx_vand_v:
7879 case Intrinsic::loongarch_lasx_xvand_v:
7880 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7881 N2: N->getOperand(Num: 2));
7882 case Intrinsic::loongarch_lsx_vor_v:
7883 case Intrinsic::loongarch_lasx_xvor_v:
7884 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7885 N2: N->getOperand(Num: 2));
7886 case Intrinsic::loongarch_lsx_vxor_v:
7887 case Intrinsic::loongarch_lasx_xvxor_v:
7888 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7889 N2: N->getOperand(Num: 2));
7890 case Intrinsic::loongarch_lsx_vnor_v:
7891 case Intrinsic::loongarch_lasx_xvnor_v: {
7892 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7893 N2: N->getOperand(Num: 2));
7894 return DAG.getNOT(DL, Val: Res, VT: Res->getValueType(ResNo: 0));
7895 }
7896 case Intrinsic::loongarch_lsx_vandi_b:
7897 case Intrinsic::loongarch_lasx_xvandi_b:
7898 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7899 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7900 case Intrinsic::loongarch_lsx_vori_b:
7901 case Intrinsic::loongarch_lasx_xvori_b:
7902 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7903 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7904 case Intrinsic::loongarch_lsx_vxori_b:
7905 case Intrinsic::loongarch_lasx_xvxori_b:
7906 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7907 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
7908 case Intrinsic::loongarch_lsx_vsll_b:
7909 case Intrinsic::loongarch_lsx_vsll_h:
7910 case Intrinsic::loongarch_lsx_vsll_w:
7911 case Intrinsic::loongarch_lsx_vsll_d:
7912 case Intrinsic::loongarch_lasx_xvsll_b:
7913 case Intrinsic::loongarch_lasx_xvsll_h:
7914 case Intrinsic::loongarch_lasx_xvsll_w:
7915 case Intrinsic::loongarch_lasx_xvsll_d:
7916 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7917 N2: truncateVecElts(Node: N, DAG));
7918 case Intrinsic::loongarch_lsx_vslli_b:
7919 case Intrinsic::loongarch_lasx_xvslli_b:
7920 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7921 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7922 case Intrinsic::loongarch_lsx_vslli_h:
7923 case Intrinsic::loongarch_lasx_xvslli_h:
7924 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7925 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7926 case Intrinsic::loongarch_lsx_vslli_w:
7927 case Intrinsic::loongarch_lasx_xvslli_w:
7928 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7929 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7930 case Intrinsic::loongarch_lsx_vslli_d:
7931 case Intrinsic::loongarch_lasx_xvslli_d:
7932 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7933 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7934 case Intrinsic::loongarch_lsx_vsrl_b:
7935 case Intrinsic::loongarch_lsx_vsrl_h:
7936 case Intrinsic::loongarch_lsx_vsrl_w:
7937 case Intrinsic::loongarch_lsx_vsrl_d:
7938 case Intrinsic::loongarch_lasx_xvsrl_b:
7939 case Intrinsic::loongarch_lasx_xvsrl_h:
7940 case Intrinsic::loongarch_lasx_xvsrl_w:
7941 case Intrinsic::loongarch_lasx_xvsrl_d:
7942 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7943 N2: truncateVecElts(Node: N, DAG));
7944 case Intrinsic::loongarch_lsx_vsrli_b:
7945 case Intrinsic::loongarch_lasx_xvsrli_b:
7946 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7947 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7948 case Intrinsic::loongarch_lsx_vsrli_h:
7949 case Intrinsic::loongarch_lasx_xvsrli_h:
7950 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7951 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7952 case Intrinsic::loongarch_lsx_vsrli_w:
7953 case Intrinsic::loongarch_lasx_xvsrli_w:
7954 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7955 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7956 case Intrinsic::loongarch_lsx_vsrli_d:
7957 case Intrinsic::loongarch_lasx_xvsrli_d:
7958 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7959 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7960 case Intrinsic::loongarch_lsx_vsra_b:
7961 case Intrinsic::loongarch_lsx_vsra_h:
7962 case Intrinsic::loongarch_lsx_vsra_w:
7963 case Intrinsic::loongarch_lsx_vsra_d:
7964 case Intrinsic::loongarch_lasx_xvsra_b:
7965 case Intrinsic::loongarch_lasx_xvsra_h:
7966 case Intrinsic::loongarch_lasx_xvsra_w:
7967 case Intrinsic::loongarch_lasx_xvsra_d:
7968 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7969 N2: truncateVecElts(Node: N, DAG));
7970 case Intrinsic::loongarch_lsx_vsrai_b:
7971 case Intrinsic::loongarch_lasx_xvsrai_b:
7972 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7973 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
7974 case Intrinsic::loongarch_lsx_vsrai_h:
7975 case Intrinsic::loongarch_lasx_xvsrai_h:
7976 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7977 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
7978 case Intrinsic::loongarch_lsx_vsrai_w:
7979 case Intrinsic::loongarch_lasx_xvsrai_w:
7980 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7981 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
7982 case Intrinsic::loongarch_lsx_vsrai_d:
7983 case Intrinsic::loongarch_lasx_xvsrai_d:
7984 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
7985 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
7986 case Intrinsic::loongarch_lsx_vclz_b:
7987 case Intrinsic::loongarch_lsx_vclz_h:
7988 case Intrinsic::loongarch_lsx_vclz_w:
7989 case Intrinsic::loongarch_lsx_vclz_d:
7990 case Intrinsic::loongarch_lasx_xvclz_b:
7991 case Intrinsic::loongarch_lasx_xvclz_h:
7992 case Intrinsic::loongarch_lasx_xvclz_w:
7993 case Intrinsic::loongarch_lasx_xvclz_d:
7994 return DAG.getNode(Opcode: ISD::CTLZ, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
7995 case Intrinsic::loongarch_lsx_vpcnt_b:
7996 case Intrinsic::loongarch_lsx_vpcnt_h:
7997 case Intrinsic::loongarch_lsx_vpcnt_w:
7998 case Intrinsic::loongarch_lsx_vpcnt_d:
7999 case Intrinsic::loongarch_lasx_xvpcnt_b:
8000 case Intrinsic::loongarch_lasx_xvpcnt_h:
8001 case Intrinsic::loongarch_lasx_xvpcnt_w:
8002 case Intrinsic::loongarch_lasx_xvpcnt_d:
8003 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
8004 case Intrinsic::loongarch_lsx_vbitclr_b:
8005 case Intrinsic::loongarch_lsx_vbitclr_h:
8006 case Intrinsic::loongarch_lsx_vbitclr_w:
8007 case Intrinsic::loongarch_lsx_vbitclr_d:
8008 case Intrinsic::loongarch_lasx_xvbitclr_b:
8009 case Intrinsic::loongarch_lasx_xvbitclr_h:
8010 case Intrinsic::loongarch_lasx_xvbitclr_w:
8011 case Intrinsic::loongarch_lasx_xvbitclr_d:
8012 return lowerVectorBitClear(Node: N, DAG);
8013 case Intrinsic::loongarch_lsx_vbitclri_b:
8014 case Intrinsic::loongarch_lasx_xvbitclri_b:
8015 return lowerVectorBitClearImm<3>(Node: N, DAG);
8016 case Intrinsic::loongarch_lsx_vbitclri_h:
8017 case Intrinsic::loongarch_lasx_xvbitclri_h:
8018 return lowerVectorBitClearImm<4>(Node: N, DAG);
8019 case Intrinsic::loongarch_lsx_vbitclri_w:
8020 case Intrinsic::loongarch_lasx_xvbitclri_w:
8021 return lowerVectorBitClearImm<5>(Node: N, DAG);
8022 case Intrinsic::loongarch_lsx_vbitclri_d:
8023 case Intrinsic::loongarch_lasx_xvbitclri_d:
8024 return lowerVectorBitClearImm<6>(Node: N, DAG);
8025 case Intrinsic::loongarch_lsx_vbitset_b:
8026 case Intrinsic::loongarch_lsx_vbitset_h:
8027 case Intrinsic::loongarch_lsx_vbitset_w:
8028 case Intrinsic::loongarch_lsx_vbitset_d:
8029 case Intrinsic::loongarch_lasx_xvbitset_b:
8030 case Intrinsic::loongarch_lasx_xvbitset_h:
8031 case Intrinsic::loongarch_lasx_xvbitset_w:
8032 case Intrinsic::loongarch_lasx_xvbitset_d: {
8033 EVT VecTy = N->getValueType(ResNo: 0);
8034 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8035 return DAG.getNode(
8036 Opcode: ISD::OR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8037 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8038 }
8039 case Intrinsic::loongarch_lsx_vbitseti_b:
8040 case Intrinsic::loongarch_lasx_xvbitseti_b:
8041 return lowerVectorBitSetImm<3>(Node: N, DAG);
8042 case Intrinsic::loongarch_lsx_vbitseti_h:
8043 case Intrinsic::loongarch_lasx_xvbitseti_h:
8044 return lowerVectorBitSetImm<4>(Node: N, DAG);
8045 case Intrinsic::loongarch_lsx_vbitseti_w:
8046 case Intrinsic::loongarch_lasx_xvbitseti_w:
8047 return lowerVectorBitSetImm<5>(Node: N, DAG);
8048 case Intrinsic::loongarch_lsx_vbitseti_d:
8049 case Intrinsic::loongarch_lasx_xvbitseti_d:
8050 return lowerVectorBitSetImm<6>(Node: N, DAG);
8051 case Intrinsic::loongarch_lsx_vbitrev_b:
8052 case Intrinsic::loongarch_lsx_vbitrev_h:
8053 case Intrinsic::loongarch_lsx_vbitrev_w:
8054 case Intrinsic::loongarch_lsx_vbitrev_d:
8055 case Intrinsic::loongarch_lasx_xvbitrev_b:
8056 case Intrinsic::loongarch_lasx_xvbitrev_h:
8057 case Intrinsic::loongarch_lasx_xvbitrev_w:
8058 case Intrinsic::loongarch_lasx_xvbitrev_d: {
8059 EVT VecTy = N->getValueType(ResNo: 0);
8060 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8061 return DAG.getNode(
8062 Opcode: ISD::XOR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8063 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8064 }
8065 case Intrinsic::loongarch_lsx_vbitrevi_b:
8066 case Intrinsic::loongarch_lasx_xvbitrevi_b:
8067 return lowerVectorBitRevImm<3>(Node: N, DAG);
8068 case Intrinsic::loongarch_lsx_vbitrevi_h:
8069 case Intrinsic::loongarch_lasx_xvbitrevi_h:
8070 return lowerVectorBitRevImm<4>(Node: N, DAG);
8071 case Intrinsic::loongarch_lsx_vbitrevi_w:
8072 case Intrinsic::loongarch_lasx_xvbitrevi_w:
8073 return lowerVectorBitRevImm<5>(Node: N, DAG);
8074 case Intrinsic::loongarch_lsx_vbitrevi_d:
8075 case Intrinsic::loongarch_lasx_xvbitrevi_d:
8076 return lowerVectorBitRevImm<6>(Node: N, DAG);
8077 case Intrinsic::loongarch_lsx_vfadd_s:
8078 case Intrinsic::loongarch_lsx_vfadd_d:
8079 case Intrinsic::loongarch_lasx_xvfadd_s:
8080 case Intrinsic::loongarch_lasx_xvfadd_d:
8081 return DAG.getNode(Opcode: ISD::FADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8082 N2: N->getOperand(Num: 2));
8083 case Intrinsic::loongarch_lsx_vfsub_s:
8084 case Intrinsic::loongarch_lsx_vfsub_d:
8085 case Intrinsic::loongarch_lasx_xvfsub_s:
8086 case Intrinsic::loongarch_lasx_xvfsub_d:
8087 return DAG.getNode(Opcode: ISD::FSUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8088 N2: N->getOperand(Num: 2));
8089 case Intrinsic::loongarch_lsx_vfmul_s:
8090 case Intrinsic::loongarch_lsx_vfmul_d:
8091 case Intrinsic::loongarch_lasx_xvfmul_s:
8092 case Intrinsic::loongarch_lasx_xvfmul_d:
8093 return DAG.getNode(Opcode: ISD::FMUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8094 N2: N->getOperand(Num: 2));
8095 case Intrinsic::loongarch_lsx_vfdiv_s:
8096 case Intrinsic::loongarch_lsx_vfdiv_d:
8097 case Intrinsic::loongarch_lasx_xvfdiv_s:
8098 case Intrinsic::loongarch_lasx_xvfdiv_d:
8099 return DAG.getNode(Opcode: ISD::FDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8100 N2: N->getOperand(Num: 2));
8101 case Intrinsic::loongarch_lsx_vfmadd_s:
8102 case Intrinsic::loongarch_lsx_vfmadd_d:
8103 case Intrinsic::loongarch_lasx_xvfmadd_s:
8104 case Intrinsic::loongarch_lasx_xvfmadd_d:
8105 return DAG.getNode(Opcode: ISD::FMA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8106 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
8107 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
8108 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8109 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8110 N3: legalizeIntrinsicImmArg<4>(Node: N, ImmOp: 3, DAG, Subtarget));
8111 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
8112 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
8113 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8114 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8115 N3: legalizeIntrinsicImmArg<3>(Node: N, ImmOp: 3, DAG, Subtarget));
8116 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
8117 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
8118 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8119 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8120 N3: legalizeIntrinsicImmArg<2>(Node: N, ImmOp: 3, DAG, Subtarget));
8121 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
8122 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8123 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8124 N3: legalizeIntrinsicImmArg<1>(Node: N, ImmOp: 3, DAG, Subtarget));
8125 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
8126 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
8127 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
8128 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
8129 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
8130 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
8131 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
8132 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
8133 return DAG.getNode(Opcode: LoongArchISD::VREPLGR2VR, DL, VT: N->getValueType(ResNo: 0),
8134 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8135 Operand: N->getOperand(Num: 1)));
8136 case Intrinsic::loongarch_lsx_vreplve_b:
8137 case Intrinsic::loongarch_lsx_vreplve_h:
8138 case Intrinsic::loongarch_lsx_vreplve_w:
8139 case Intrinsic::loongarch_lsx_vreplve_d:
8140 case Intrinsic::loongarch_lasx_xvreplve_b:
8141 case Intrinsic::loongarch_lasx_xvreplve_h:
8142 case Intrinsic::loongarch_lasx_xvreplve_w:
8143 case Intrinsic::loongarch_lasx_xvreplve_d:
8144 return DAG.getNode(Opcode: LoongArchISD::VREPLVE, DL, VT: N->getValueType(ResNo: 0),
8145 N1: N->getOperand(Num: 1),
8146 N2: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8147 Operand: N->getOperand(Num: 2)));
8148 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8149 if (!Subtarget.is64Bit())
8150 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8151 break;
8152 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8153 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8154 if (!Subtarget.is64Bit())
8155 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8156 break;
8157 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8158 if (!Subtarget.is64Bit())
8159 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8160 break;
8161 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8162 if (!Subtarget.is64Bit())
8163 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8164 break;
8165 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8166 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8167 if (!Subtarget.is64Bit())
8168 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8169 break;
8170 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8171 if (!Subtarget.is64Bit())
8172 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8173 break;
8174 case Intrinsic::loongarch_lsx_bz_b:
8175 case Intrinsic::loongarch_lsx_bz_h:
8176 case Intrinsic::loongarch_lsx_bz_w:
8177 case Intrinsic::loongarch_lsx_bz_d:
8178 case Intrinsic::loongarch_lasx_xbz_b:
8179 case Intrinsic::loongarch_lasx_xbz_h:
8180 case Intrinsic::loongarch_lasx_xbz_w:
8181 case Intrinsic::loongarch_lasx_xbz_d:
8182 if (!Subtarget.is64Bit())
8183 return DAG.getNode(Opcode: LoongArchISD::VALL_ZERO, DL, VT: N->getValueType(ResNo: 0),
8184 Operand: N->getOperand(Num: 1));
8185 break;
8186 case Intrinsic::loongarch_lsx_bz_v:
8187 case Intrinsic::loongarch_lasx_xbz_v:
8188 if (!Subtarget.is64Bit())
8189 return DAG.getNode(Opcode: LoongArchISD::VANY_ZERO, DL, VT: N->getValueType(ResNo: 0),
8190 Operand: N->getOperand(Num: 1));
8191 break;
8192 case Intrinsic::loongarch_lsx_bnz_b:
8193 case Intrinsic::loongarch_lsx_bnz_h:
8194 case Intrinsic::loongarch_lsx_bnz_w:
8195 case Intrinsic::loongarch_lsx_bnz_d:
8196 case Intrinsic::loongarch_lasx_xbnz_b:
8197 case Intrinsic::loongarch_lasx_xbnz_h:
8198 case Intrinsic::loongarch_lasx_xbnz_w:
8199 case Intrinsic::loongarch_lasx_xbnz_d:
8200 if (!Subtarget.is64Bit())
8201 return DAG.getNode(Opcode: LoongArchISD::VALL_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8202 Operand: N->getOperand(Num: 1));
8203 break;
8204 case Intrinsic::loongarch_lsx_bnz_v:
8205 case Intrinsic::loongarch_lasx_xbnz_v:
8206 if (!Subtarget.is64Bit())
8207 return DAG.getNode(Opcode: LoongArchISD::VANY_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8208 Operand: N->getOperand(Num: 1));
8209 break;
8210 case Intrinsic::loongarch_lasx_concat_128_s:
8211 case Intrinsic::loongarch_lasx_concat_128_d:
8212 case Intrinsic::loongarch_lasx_concat_128:
8213 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
8214 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2));
8215 }
8216 return SDValue();
8217}
8218
8219static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG,
8220 TargetLowering::DAGCombinerInfo &DCI,
8221 const LoongArchSubtarget &Subtarget) {
8222 // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
8223 // conversion is unnecessary and can be replaced with the
8224 // MOVFR2GR_S_LA64 operand.
8225 SDValue Op0 = N->getOperand(Num: 0);
8226 if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8227 return Op0.getOperand(i: 0);
8228 return SDValue();
8229}
8230
8231static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG,
8232 TargetLowering::DAGCombinerInfo &DCI,
8233 const LoongArchSubtarget &Subtarget) {
8234 // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
8235 // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
8236 // operand.
8237 SDValue Op0 = N->getOperand(Num: 0);
8238 if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8239 assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
8240 "Unexpected value type!");
8241 return Op0.getOperand(i: 0);
8242 }
8243 return SDValue();
8244}
8245
8246static SDValue
8247performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG,
8248 TargetLowering::DAGCombinerInfo &DCI) {
8249 MVT VT = N->getSimpleValueType(ResNo: 0);
8250 unsigned NumBits = VT.getScalarSizeInBits();
8251
8252 // Simplify the inputs.
8253 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8254 APInt DemandedMask(APInt::getAllOnes(numBits: NumBits));
8255 if (TLI.SimplifyDemandedBits(Op: SDValue(N, 0), DemandedBits: DemandedMask, DCI))
8256 return SDValue(N, 0);
8257
8258 return SDValue();
8259}
8260
8261static SDValue
8262performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG,
8263 TargetLowering::DAGCombinerInfo &DCI,
8264 const LoongArchSubtarget &Subtarget) {
8265 SDValue Op0 = N->getOperand(Num: 0);
8266 SDLoc DL(N);
8267
8268 // If the input to SplitPairF64 is just BuildPairF64 then the operation is
8269 // redundant. Instead, use BuildPairF64's operands directly.
8270 if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8271 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
8272
8273 if (Op0->isUndef()) {
8274 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
8275 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
8276 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8277 }
8278
8279 // It's cheaper to materialise two 32-bit integers than to load a double
8280 // from the constant pool and transfer it to integer registers through the
8281 // stack.
8282 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
8283 APInt V = C->getValueAPF().bitcastToAPInt();
8284 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
8285 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
8286 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8287 }
8288
8289 return SDValue();
8290}
8291
8292/// Do target-specific dag combines on LoongArchISD::VANDN nodes.
8293static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG,
8294 TargetLowering::DAGCombinerInfo &DCI,
8295 const LoongArchSubtarget &Subtarget) {
8296 SDValue N0 = N->getOperand(Num: 0);
8297 SDValue N1 = N->getOperand(Num: 1);
8298 MVT VT = N->getSimpleValueType(ResNo: 0);
8299 SDLoc DL(N);
8300
8301 // VANDN(undef, x) -> 0
8302 // VANDN(x, undef) -> 0
8303 if (N0.isUndef() || N1.isUndef())
8304 return DAG.getConstant(Val: 0, DL, VT);
8305
8306 // VANDN(0, x) -> x
8307 if (ISD::isBuildVectorAllZeros(N: N0.getNode()))
8308 return N1;
8309
8310 // VANDN(x, 0) -> 0
8311 if (ISD::isBuildVectorAllZeros(N: N1.getNode()))
8312 return DAG.getConstant(Val: 0, DL, VT);
8313
8314 // VANDN(x, -1) -> NOT(x) -> XOR(x, -1)
8315 if (ISD::isBuildVectorAllOnes(N: N1.getNode()))
8316 return DAG.getNOT(DL, Val: N0, VT);
8317
8318 // Turn VANDN back to AND if input is inverted.
8319 if (SDValue Not = isNOT(V: N0, DAG))
8320 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: DAG.getBitcast(VT, V: Not), N2: N1);
8321
8322 // Folds for better commutativity:
8323 if (N1->hasOneUse()) {
8324 // VANDN(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
8325 if (SDValue Not = isNOT(V: N1, DAG))
8326 return DAG.getNOT(
8327 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)), VT);
8328
8329 // VANDN(x, SplatVector(Imm)) -> AND(NOT(x), NOT(SplatVector(~Imm)))
8330 // -> NOT(OR(x, SplatVector(-Imm))
8331 // Combination is performed only when VT is v16i8/v32i8, using `vnori.b` to
8332 // gain benefits.
8333 if (!DCI.isBeforeLegalizeOps() && (VT == MVT::v16i8 || VT == MVT::v32i8) &&
8334 N1.getOpcode() == ISD::BUILD_VECTOR) {
8335 if (SDValue SplatValue =
8336 cast<BuildVectorSDNode>(Val: N1.getNode())->getSplatValue()) {
8337 if (!N1->isOnlyUserOf(N: SplatValue.getNode()))
8338 return SDValue();
8339
8340 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
8341 uint8_t NCVal = static_cast<uint8_t>(~(C->getSExtValue()));
8342 SDValue Not =
8343 DAG.getSplat(VT, DL, Op: DAG.getTargetConstant(Val: NCVal, DL, VT: MVT::i8));
8344 return DAG.getNOT(
8345 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)),
8346 VT);
8347 }
8348 }
8349 }
8350 }
8351
8352 return SDValue();
8353}
8354
8355static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp) {
8356 SDLoc DL(N);
8357 EVT VT = N->getValueType(ResNo: 0);
8358 SDValue Src = N->getOperand(Num: 0);
8359 EVT SrcVT = Src.getValueType();
8360
8361 unsigned DstElts = VT.getVectorNumElements();
8362 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8363 unsigned DstEltBits = VT.getScalarSizeInBits();
8364
8365 if (SrcEltBits >= DstEltBits)
8366 return SDValue();
8367
8368 MVT WidenEltVT = MVT::getIntegerVT(BitWidth: DstEltBits);
8369 MVT WidenSrcVT = MVT::getVectorVT(VT: WidenEltVT, NumElements: DstElts);
8370
8371 SDValue Extend = DAG.getNode(Opcode: ExtendOp, DL, VT: WidenSrcVT, Operand: Src);
8372 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Extend);
8373}
8374
8375// Merge two 64 to 32 convert instructions into one,
8376// e.g.
8377// vffint.s.l $vr0, $vr1, $vr2
8378// will convert 4 si64 into 4 float at once.
8379// or
8380// vftintrz.w.d $vr0, $vr1, $vr2
8381// which will convert 4 double into 4 si32 at once.
8382// also deal with their 256-bits LASX version.
8383static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode,
8384 unsigned BlockBits) {
8385 SDLoc DL(N);
8386 MVT DstVT = N->getSimpleValueType(ResNo: 0);
8387 SDValue Src = N->getOperand(Num: 0);
8388 MVT SrcVT = Src.getSimpleValueType();
8389 unsigned SrcBits = SrcVT.getSizeInBits();
8390
8391 SmallVector<SDValue, 4> Blocks;
8392 unsigned BlockNumElts = BlockBits / SrcVT.getScalarSizeInBits();
8393 MVT BlockVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: BlockNumElts);
8394 if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
8395 Src.getOperand(i: 0).getValueType() == BlockVT) {
8396 for (unsigned i = 0; i < Src.getNumOperands(); ++i)
8397 Blocks.push_back(Elt: Src.getOperand(i));
8398 } else if (SrcBits > BlockBits) {
8399 // Wider than one register: extract each BlockBits-wide sub-vector.
8400 for (unsigned i = 0; i < SrcBits / BlockBits; ++i)
8401 Blocks.push_back(
8402 Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: BlockVT, N1: Src,
8403 N2: DAG.getVectorIdxConstant(Val: i * BlockNumElts, DL)));
8404 } else {
8405 BlockBits = SrcBits;
8406 Blocks.push_back(Elt: Src);
8407 }
8408
8409 MVT NativeVecVT = MVT::getVectorVT(VT: DstVT.getScalarType(),
8410 NumElements: BlockBits / DstVT.getScalarSizeInBits());
8411 SmallVector<SDValue, 4> Parts;
8412 for (unsigned i = 0; i < Blocks.size(); i += 2) {
8413 SDValue Lo = Blocks[i];
8414 SDValue Hi = Blocks.size() > 1 ? Blocks[i + 1] : Lo;
8415 SDValue Res = DAG.getNode(Opcode, DL, VT: NativeVecVT, N1: Hi, N2: Lo);
8416
8417 if (BlockBits == 256) {
8418 SDValue Undef = DAG.getUNDEF(VT: NativeVecVT);
8419 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
8420 Res = DAG.getVectorShuffle(VT: NativeVecVT, dl: DL, N1: Res, N2: Undef, Mask);
8421 Res = DAG.getBitcast(VT: NativeVecVT, V: Res);
8422 }
8423
8424 Parts.push_back(Elt: Res);
8425 }
8426
8427 if (Blocks.size() == 1)
8428 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: DstVT, N1: Parts[0],
8429 N2: DAG.getVectorIdxConstant(Val: 0, DL));
8430 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Parts);
8431}
8432
8433static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8434 TargetLowering::DAGCombinerInfo &DCI,
8435 const LoongArchSubtarget &Subtarget) {
8436 SDLoc DL(N);
8437 EVT VT = N->getValueType(ResNo: 0);
8438 SDValue Src = N->getOperand(Num: 0);
8439 EVT SrcVT = Src.getValueType();
8440
8441 if (VT.isVector()) {
8442 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8443 unsigned DstEltBits = VT.getScalarSizeInBits();
8444 unsigned NumElts = VT.getVectorNumElements();
8445 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8446
8447 // Sign-extend src to avoid scalarization.
8448 if (SrcEltBits <= DstEltBits)
8449 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::SIGN_EXTEND);
8450
8451 if (SrcEltBits != 64 || DstEltBits != 32 || !isPowerOf2_32(Value: NumElts))
8452 return SDValue();
8453
8454 if (!SrcVT.isSimple() || !VT.isSimple())
8455 return SDValue();
8456
8457 // Combine [x]vffint.s.l for vector si64 to float conversion.
8458 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFFINT, BlockBits);
8459 }
8460
8461 if (VT != MVT::f32 && VT != MVT::f64)
8462 return SDValue();
8463 if (VT == MVT::f32 && !Subtarget.hasBasicF())
8464 return SDValue();
8465 if (VT == MVT::f64 && !Subtarget.hasBasicD())
8466 return SDValue();
8467
8468 // Only optimize when the source and destination types have the same width.
8469 if (VT.getSizeInBits() != N->getOperand(Num: 0).getValueSizeInBits())
8470 return SDValue();
8471
8472 // If the result of an integer load is only used by an integer-to-float
8473 // conversion, use a fp load instead. This eliminates an integer-to-float-move
8474 // (movgr2fr) instruction.
8475 if (ISD::isNormalLoad(N: Src.getNode()) && Src.hasOneUse() &&
8476 // Do not change the width of a volatile load. This condition check is
8477 // inspired by AArch64.
8478 !cast<LoadSDNode>(Val&: Src)->isVolatile()) {
8479 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: Src);
8480 SDValue Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
8481 PtrInfo: LN0->getPointerInfo(), Alignment: LN0->getAlign(),
8482 MMOFlags: LN0->getMemOperand()->getFlags());
8483
8484 // Make sure successors of the original load stay after it by updating them
8485 // to use the new Chain.
8486 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: Load.getValue(R: 1));
8487 return DAG.getNode(Opcode: LoongArchISD::SITOF, DL: SDLoc(N), VT, Operand: Load);
8488 }
8489
8490 return SDValue();
8491}
8492
8493static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
8494 TargetLowering::DAGCombinerInfo &DCI,
8495 const LoongArchSubtarget &Subtarget) {
8496 SDLoc DL(N);
8497 EVT VT = N->getValueType(ResNo: 0);
8498
8499 // Zero-extend src to avoid scalarization.
8500 if (VT.isVector())
8501 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::ZERO_EXTEND);
8502
8503 return SDValue();
8504}
8505
8506// Using [X]VFTINTRZ_W_D for double to signed 32-bit integer conversion.
8507// For example:
8508// v4i32 = fp_to_sint (concat_vectors v2f64, v2f64)
8509// Can be combined into:
8510// v4i32 = VFTINTRZ_W_D v2f64. v2f64
8511static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG,
8512 TargetLowering::DAGCombinerInfo &DCI,
8513 const LoongArchSubtarget &Subtarget) {
8514 if (!Subtarget.hasExtLSX())
8515 return SDValue();
8516
8517 SDLoc DL(N);
8518 EVT DstVT = N->getValueType(ResNo: 0);
8519 SDValue Src = N->getOperand(Num: 0);
8520 EVT SrcVT = Src.getValueType();
8521 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
8522
8523 if (!DstVT.isVector() || !DstVT.isSimple() || !SrcVT.isSimple())
8524 return SDValue();
8525
8526 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8527 unsigned SrcBits = SrcVT.getSizeInBits();
8528 unsigned DstEltBits = DstVT.getScalarSizeInBits();
8529 unsigned NumElts = DstVT.getVectorNumElements();
8530 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8531
8532 if (!isPowerOf2_32(Value: NumElts) || !isPowerOf2_32(Value: DstEltBits))
8533 return SDValue();
8534
8535 if (SrcBits % BlockBits != 0 && SrcBits != 128)
8536 return SDValue();
8537
8538 if (DstEltBits < 32) {
8539 MVT PromoteVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 32), NumElements: NumElts);
8540 SDValue Conv = DAG.getNode(Opcode: N->getOpcode(), DL, VT: PromoteVT, Operand: Src);
8541 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Conv);
8542 }
8543
8544 if (SrcEltBits != 64 || DstEltBits != 32)
8545 return SDValue();
8546
8547 if (!IsSigned) {
8548 // LASX already has pattern for double convert to uint32.
8549 if (Subtarget.hasExtLASX())
8550 return SDValue();
8551 MVT TmpVT = MVT::getVectorVT(VT: MVT::i64, NumElements: NumElts);
8552 SDValue Tmp = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: TmpVT, Operand: Src);
8553 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Tmp);
8554 }
8555
8556 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFTINTRZ, BlockBits);
8557}
8558
8559// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
8560// logical operations, like in the example below.
8561// or (and (truncate x, truncate y)),
8562// (xor (truncate z, build_vector (constants)))
8563// Given a target type \p VT, we generate
8564// or (and x, y), (xor z, zext(build_vector (constants)))
8565// given x, y and z are of type \p VT. We can do so, if operands are either
8566// truncates from VT types, the second operand is a vector of constants, can
8567// be recursively promoted or is an existing extension we can extend further.
8568static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
8569 SelectionDAG &DAG,
8570 const LoongArchSubtarget &Subtarget,
8571 unsigned Depth) {
8572 // Limit recursion to avoid excessive compile times.
8573 if (Depth >= SelectionDAG::MaxRecursionDepth)
8574 return SDValue();
8575
8576 if (!ISD::isBitwiseLogicOp(Opcode: N.getOpcode()))
8577 return SDValue();
8578
8579 SDValue N0 = N.getOperand(i: 0);
8580 SDValue N1 = N.getOperand(i: 1);
8581
8582 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8583 if (!TLI.isOperationLegalOrPromote(Op: N.getOpcode(), VT))
8584 return SDValue();
8585
8586 if (SDValue NN0 =
8587 PromoteMaskArithmetic(N: N0, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8588 N0 = NN0;
8589 else {
8590 // The left side has to be a 'trunc'.
8591 bool LHSTrunc = N0.getOpcode() == ISD::TRUNCATE &&
8592 N0.getOperand(i: 0).getValueType() == VT;
8593 if (LHSTrunc)
8594 N0 = N0.getOperand(i: 0);
8595 else
8596 return SDValue();
8597 }
8598
8599 if (SDValue NN1 =
8600 PromoteMaskArithmetic(N: N1, DL, VT, DAG, Subtarget, Depth: Depth + 1))
8601 N1 = NN1;
8602 else {
8603 // The right side has to be a 'trunc', a (foldable) constant or an
8604 // existing extension we can extend further.
8605 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
8606 N1.getOperand(i: 0).getValueType() == VT;
8607 if (RHSTrunc)
8608 N1 = N1.getOperand(i: 0);
8609 else if (ISD::isExtVecInRegOpcode(Opcode: N1.getOpcode()) && VT.is256BitVector() &&
8610 Subtarget.hasExtLASX() && N1.hasOneUse())
8611 N1 = DAG.getNode(Opcode: N1.getOpcode(), DL, VT, Operand: N1.getOperand(i: 0));
8612 // On 32-bit platform, i64 is an illegal integer scalar type, and
8613 // FoldConstantArithmetic will fail for v4i64. This may be optimized in the
8614 // future.
8615 else if (SDValue Cst =
8616 DAG.FoldConstantArithmetic(Opcode: ISD::ZERO_EXTEND, DL, VT, Ops: {N1}))
8617 N1 = Cst;
8618 else
8619 return SDValue();
8620 }
8621
8622 return DAG.getNode(Opcode: N.getOpcode(), DL, VT, N1: N0, N2: N1);
8623}
8624
8625// On LASX the type v4i1/v8i1/v16i1 may be legalized to v4i32/v8i16/v16i8, which
8626// is LSX-sized register. In most cases we actually compare or select LASX-sized
8627// registers and mixing the two types creates horrible code. This method
8628// optimizes some of the transition sequences.
8629static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
8630 SelectionDAG &DAG,
8631 const LoongArchSubtarget &Subtarget) {
8632 EVT VT = N.getValueType();
8633 assert(VT.isVector() && "Expected vector type");
8634 assert((N.getOpcode() == ISD::ANY_EXTEND ||
8635 N.getOpcode() == ISD::ZERO_EXTEND ||
8636 N.getOpcode() == ISD::SIGN_EXTEND) &&
8637 "Invalid Node");
8638
8639 if (!Subtarget.hasExtLASX() || !VT.is256BitVector())
8640 return SDValue();
8641
8642 SDValue Narrow = N.getOperand(i: 0);
8643 EVT NarrowVT = Narrow.getValueType();
8644
8645 // Generate the wide operation.
8646 SDValue Op = PromoteMaskArithmetic(N: Narrow, DL, VT, DAG, Subtarget, Depth: 0);
8647 if (!Op)
8648 return SDValue();
8649 switch (N.getOpcode()) {
8650 default:
8651 llvm_unreachable("Unexpected opcode");
8652 case ISD::ANY_EXTEND:
8653 return Op;
8654 case ISD::ZERO_EXTEND:
8655 return DAG.getZeroExtendInReg(Op, DL, VT: NarrowVT);
8656 case ISD::SIGN_EXTEND:
8657 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
8658 N2: DAG.getValueType(NarrowVT));
8659 }
8660}
8661
8662static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG,
8663 TargetLowering::DAGCombinerInfo &DCI,
8664 const LoongArchSubtarget &Subtarget) {
8665 EVT VT = N->getValueType(ResNo: 0);
8666 SDLoc DL(N);
8667
8668 if (VT.isVector()) {
8669 if (SDValue R = PromoteMaskArithmetic(N: SDValue(N, 0), DL, DAG, Subtarget))
8670 return R;
8671
8672 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
8673 N->getValueSizeInBits(ResNo: 0) != N->getOperand(Num: 0).getValueSizeInBits() * 2)
8674 return SDValue();
8675
8676 if (SDValue R = matchHalfOf128BitLanes(N: N->getOperand(Num: 0), /*isLow=*/false)) {
8677 if (N->getOpcode() == ISD::SIGN_EXTEND)
8678 return DAG.getNode(Opcode: LoongArchISD::VEXTH, DL, VT, Operand: R);
8679 if (N->getOpcode() == ISD::ZERO_EXTEND)
8680 return DAG.getNode(Opcode: LoongArchISD::VEXTH_U, DL, VT, Operand: R);
8681 }
8682 }
8683
8684 return SDValue();
8685}
8686
8687static SDValue
8688performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
8689 TargetLowering::DAGCombinerInfo &DCI,
8690 const LoongArchSubtarget &Subtarget) {
8691 SDLoc DL(N);
8692 EVT VT = N->getValueType(ResNo: 0);
8693
8694 if (VT.isVector() && N->getNumOperands() == 2)
8695 if (SDValue R = combineFP_ROUND(N: SDValue(N, 0), DL, DAG, Subtarget))
8696 return R;
8697
8698 return SDValue();
8699}
8700
8701static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
8702 TargetLowering::DAGCombinerInfo &DCI,
8703 const LoongArchSubtarget &Subtarget) {
8704 if (DCI.isBeforeLegalizeOps())
8705 return SDValue();
8706
8707 EVT VT = N->getValueType(ResNo: 0);
8708 if (!VT.isVector())
8709 return SDValue();
8710
8711 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
8712 return SDValue();
8713
8714 EVT EltVT = VT.getVectorElementType();
8715 if (!EltVT.isInteger())
8716 return SDValue();
8717
8718 SDValue Cond = N->getOperand(Num: 0);
8719 SDValue TrueVal = N->getOperand(Num: 1);
8720 SDValue FalseVal = N->getOperand(Num: 2);
8721
8722 // match:
8723 //
8724 // vselect (setcc shift, 0, seteq),
8725 // x,
8726 // rounded_shift
8727
8728 if (Cond.getOpcode() != ISD::SETCC)
8729 return SDValue();
8730
8731 if (!ISD::isConstantSplatVectorAllZeros(N: Cond.getOperand(i: 1).getNode()))
8732 return SDValue();
8733
8734 auto *CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2));
8735 if (CC->get() != ISD::SETEQ)
8736 return SDValue();
8737
8738 SDValue Shift = Cond.getOperand(i: 0);
8739
8740 // True branch must be original value:
8741 //
8742 // vselect cond, x, ...
8743
8744 SDValue X = TrueVal;
8745
8746 // Now match rounded shift pattern:
8747 //
8748 // add
8749 // (and
8750 // (srl X, shift-1)
8751 // 1)
8752 // (srl/sra X, shift)
8753
8754 if (FalseVal.getOpcode() != ISD::ADD)
8755 return SDValue();
8756
8757 SDValue Add0 = FalseVal.getOperand(i: 0);
8758 SDValue Add1 = FalseVal.getOperand(i: 1);
8759 SDValue And;
8760 SDValue Shr;
8761
8762 if (Add0.getOpcode() == ISD::AND) {
8763 And = Add0;
8764 Shr = Add1;
8765 } else if (Add1.getOpcode() == ISD::AND) {
8766 And = Add1;
8767 Shr = Add0;
8768 } else {
8769 return SDValue();
8770 }
8771
8772 // match:
8773 //
8774 // srl/sra X, shift
8775
8776 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
8777 return SDValue();
8778
8779 if (Shr.getOperand(i: 0) != X)
8780 return SDValue();
8781
8782 if (Shr.getOperand(i: 1) != Shift)
8783 return SDValue();
8784
8785 // match:
8786 //
8787 // and
8788 // (srl X, shift-1)
8789 // 1
8790
8791 SDValue Srl = And.getOperand(i: 0);
8792 SDValue One = And.getOperand(i: 1);
8793 APInt SplatVal;
8794
8795 if (Srl.getOpcode() != ISD::SRL)
8796 return SDValue();
8797
8798 One = peekThroughBitcasts(V: One);
8799 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
8800 return SDValue();
8801
8802 if (SplatVal != 1)
8803 return SDValue();
8804
8805 if (Srl.getOperand(i: 0) != X)
8806 return SDValue();
8807
8808 // match:
8809 //
8810 // shift-1
8811
8812 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
8813
8814 if (ShiftMinus1.getOpcode() != ISD::ADD)
8815 return SDValue();
8816
8817 if (ShiftMinus1.getOperand(i: 0) != Shift)
8818 return SDValue();
8819
8820 if (!ISD::isConstantSplatVectorAllOnes(N: ShiftMinus1.getOperand(i: 1).getNode()))
8821 return SDValue();
8822
8823 // We matched a rounded right shift pattern and can lower it
8824 // to a single vector rounded shift instruction.
8825
8826 SDLoc DL(N);
8827 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
8828 : LoongArchISD::VSRAR,
8829 DL, VT, N1: X, N2: Shift);
8830}
8831
8832SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
8833 DAGCombinerInfo &DCI) const {
8834 SelectionDAG &DAG = DCI.DAG;
8835 switch (N->getOpcode()) {
8836 default:
8837 break;
8838 case ISD::ADD:
8839 return performADDCombine(N, DAG, DCI, Subtarget);
8840 case ISD::AND:
8841 return performANDCombine(N, DAG, DCI, Subtarget);
8842 case ISD::OR:
8843 return performORCombine(N, DAG, DCI, Subtarget);
8844 case ISD::SETCC:
8845 return performSETCCCombine(N, DAG, DCI, Subtarget);
8846 case ISD::SELECT:
8847 return performSELECTCombine(N, DAG, DCI, Subtarget);
8848 case ISD::SHL:
8849 return performSHLCombine(N, DAG, DCI, Subtarget);
8850 case ISD::SRL:
8851 return performSRLCombine(N, DAG, DCI, Subtarget);
8852 case ISD::SUB:
8853 return performSUBCombine(N, DAG, DCI, Subtarget);
8854 case ISD::BITCAST:
8855 return performBITCASTCombine(N, DAG, DCI, Subtarget);
8856 case ISD::ANY_EXTEND:
8857 case ISD::ZERO_EXTEND:
8858 case ISD::SIGN_EXTEND:
8859 return performEXTENDCombine(N, DAG, DCI, Subtarget);
8860 case ISD::SINT_TO_FP:
8861 return performSINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8862 case ISD::UINT_TO_FP:
8863 return performUINT_TO_FPCombine(N, DAG, DCI, Subtarget);
8864 case ISD::FP_TO_SINT:
8865 case ISD::FP_TO_UINT:
8866 return performFP_TO_INTCombine(N, DAG, DCI, Subtarget);
8867 case LoongArchISD::BITREV_W:
8868 return performBITREV_WCombine(N, DAG, DCI, Subtarget);
8869 case LoongArchISD::BR_CC:
8870 return performBR_CCCombine(N, DAG, DCI, Subtarget);
8871 case LoongArchISD::SELECT_CC:
8872 return performSELECT_CCCombine(N, DAG, DCI, Subtarget);
8873 case ISD::INTRINSIC_WO_CHAIN:
8874 return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
8875 case LoongArchISD::MOVGR2FR_W_LA64:
8876 return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
8877 case LoongArchISD::MOVFR2GR_S_LA64:
8878 return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
8879 case LoongArchISD::CRC_W_B_W:
8880 case LoongArchISD::CRC_W_H_W:
8881 case LoongArchISD::CRCC_W_B_W:
8882 case LoongArchISD::CRCC_W_H_W:
8883 case LoongArchISD::VMSKLTZ:
8884 case LoongArchISD::XVMSKLTZ:
8885 return performDemandedBitsCombine(N, DAG, DCI);
8886 case LoongArchISD::SPLIT_PAIR_F64:
8887 return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
8888 case LoongArchISD::VANDN:
8889 return performVANDNCombine(N, DAG, DCI, Subtarget);
8890 case ISD::CONCAT_VECTORS:
8891 return performCONCAT_VECTORSCombine(N, DAG, DCI, Subtarget);
8892 case ISD::VSELECT:
8893 return performVSELECTCombine(N, DAG, DCI, Subtarget);
8894 case LoongArchISD::VPACKEV:
8895 case LoongArchISD::VPERMI:
8896 if (SDValue Result =
8897 combineFP_ROUND(N: SDValue(N, 0), DL: SDLoc(N), DAG, Subtarget))
8898 return Result;
8899 }
8900 return SDValue();
8901}
8902
8903static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI,
8904 MachineBasicBlock *MBB) {
8905 if (!ZeroDivCheck)
8906 return MBB;
8907
8908 // Build instructions:
8909 // MBB:
8910 // div(or mod) $dst, $dividend, $divisor
8911 // bne $divisor, $zero, SinkMBB
8912 // BreakMBB:
8913 // break 7 // BRK_DIVZERO
8914 // SinkMBB:
8915 // fallthrough
8916 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8917 MachineFunction::iterator It = ++MBB->getIterator();
8918 MachineFunction *MF = MBB->getParent();
8919 auto BreakMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8920 auto SinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
8921 MF->insert(MBBI: It, MBB: BreakMBB);
8922 MF->insert(MBBI: It, MBB: SinkMBB);
8923
8924 // Transfer the remainder of MBB and its successor edges to SinkMBB.
8925 SinkMBB->splice(Where: SinkMBB->end(), Other: MBB, From: std::next(x: MI.getIterator()), To: MBB->end());
8926 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
8927
8928 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
8929 DebugLoc DL = MI.getDebugLoc();
8930 MachineOperand &Divisor = MI.getOperand(i: 2);
8931 Register DivisorReg = Divisor.getReg();
8932
8933 // MBB:
8934 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BNE))
8935 .addReg(RegNo: DivisorReg, Flags: getKillRegState(B: Divisor.isKill()))
8936 .addReg(RegNo: LoongArch::R0)
8937 .addMBB(MBB: SinkMBB);
8938 MBB->addSuccessor(Succ: BreakMBB);
8939 MBB->addSuccessor(Succ: SinkMBB);
8940
8941 // BreakMBB:
8942 // See linux header file arch/loongarch/include/uapi/asm/break.h for the
8943 // definition of BRK_DIVZERO.
8944 BuildMI(BB: BreakMBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BREAK)).addImm(Val: 7 /*BRK_DIVZERO*/);
8945 BreakMBB->addSuccessor(Succ: SinkMBB);
8946
8947 // Clear Divisor's kill flag.
8948 Divisor.setIsKill(false);
8949
8950 return SinkMBB;
8951}
8952
8953static MachineBasicBlock *
8954emitVecCondBranchPseudo(MachineInstr &MI, MachineBasicBlock *BB,
8955 const LoongArchSubtarget &Subtarget) {
8956 unsigned CondOpc;
8957 switch (MI.getOpcode()) {
8958 default:
8959 llvm_unreachable("Unexpected opcode");
8960 case LoongArch::PseudoVBZ:
8961 CondOpc = LoongArch::VSETEQZ_V;
8962 break;
8963 case LoongArch::PseudoVBZ_B:
8964 CondOpc = LoongArch::VSETANYEQZ_B;
8965 break;
8966 case LoongArch::PseudoVBZ_H:
8967 CondOpc = LoongArch::VSETANYEQZ_H;
8968 break;
8969 case LoongArch::PseudoVBZ_W:
8970 CondOpc = LoongArch::VSETANYEQZ_W;
8971 break;
8972 case LoongArch::PseudoVBZ_D:
8973 CondOpc = LoongArch::VSETANYEQZ_D;
8974 break;
8975 case LoongArch::PseudoVBNZ:
8976 CondOpc = LoongArch::VSETNEZ_V;
8977 break;
8978 case LoongArch::PseudoVBNZ_B:
8979 CondOpc = LoongArch::VSETALLNEZ_B;
8980 break;
8981 case LoongArch::PseudoVBNZ_H:
8982 CondOpc = LoongArch::VSETALLNEZ_H;
8983 break;
8984 case LoongArch::PseudoVBNZ_W:
8985 CondOpc = LoongArch::VSETALLNEZ_W;
8986 break;
8987 case LoongArch::PseudoVBNZ_D:
8988 CondOpc = LoongArch::VSETALLNEZ_D;
8989 break;
8990 case LoongArch::PseudoXVBZ:
8991 CondOpc = LoongArch::XVSETEQZ_V;
8992 break;
8993 case LoongArch::PseudoXVBZ_B:
8994 CondOpc = LoongArch::XVSETANYEQZ_B;
8995 break;
8996 case LoongArch::PseudoXVBZ_H:
8997 CondOpc = LoongArch::XVSETANYEQZ_H;
8998 break;
8999 case LoongArch::PseudoXVBZ_W:
9000 CondOpc = LoongArch::XVSETANYEQZ_W;
9001 break;
9002 case LoongArch::PseudoXVBZ_D:
9003 CondOpc = LoongArch::XVSETANYEQZ_D;
9004 break;
9005 case LoongArch::PseudoXVBNZ:
9006 CondOpc = LoongArch::XVSETNEZ_V;
9007 break;
9008 case LoongArch::PseudoXVBNZ_B:
9009 CondOpc = LoongArch::XVSETALLNEZ_B;
9010 break;
9011 case LoongArch::PseudoXVBNZ_H:
9012 CondOpc = LoongArch::XVSETALLNEZ_H;
9013 break;
9014 case LoongArch::PseudoXVBNZ_W:
9015 CondOpc = LoongArch::XVSETALLNEZ_W;
9016 break;
9017 case LoongArch::PseudoXVBNZ_D:
9018 CondOpc = LoongArch::XVSETALLNEZ_D;
9019 break;
9020 }
9021
9022 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9023 const BasicBlock *LLVM_BB = BB->getBasicBlock();
9024 DebugLoc DL = MI.getDebugLoc();
9025 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9026 MachineFunction::iterator It = ++BB->getIterator();
9027
9028 MachineFunction *F = BB->getParent();
9029 MachineBasicBlock *FalseBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9030 MachineBasicBlock *TrueBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9031 MachineBasicBlock *SinkBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9032
9033 F->insert(MBBI: It, MBB: FalseBB);
9034 F->insert(MBBI: It, MBB: TrueBB);
9035 F->insert(MBBI: It, MBB: SinkBB);
9036
9037 // Transfer the remainder of MBB and its successor edges to Sink.
9038 SinkBB->splice(Where: SinkBB->end(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
9039 SinkBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
9040
9041 // Insert the real instruction to BB.
9042 Register FCC = MRI.createVirtualRegister(RegClass: &LoongArch::CFRRegClass);
9043 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: CondOpc), DestReg: FCC).addReg(RegNo: MI.getOperand(i: 1).getReg());
9044
9045 // Insert branch.
9046 BuildMI(BB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::BCNEZ)).addReg(RegNo: FCC).addMBB(MBB: TrueBB);
9047 BB->addSuccessor(Succ: FalseBB);
9048 BB->addSuccessor(Succ: TrueBB);
9049
9050 // FalseBB.
9051 Register RD1 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9052 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD1)
9053 .addReg(RegNo: LoongArch::R0)
9054 .addImm(Val: 0);
9055 BuildMI(BB: FalseBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::PseudoBR)).addMBB(MBB: SinkBB);
9056 FalseBB->addSuccessor(Succ: SinkBB);
9057
9058 // TrueBB.
9059 Register RD2 = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9060 BuildMI(BB: TrueBB, MIMD: DL, MCID: TII->get(Opcode: LoongArch::ADDI_W), DestReg: RD2)
9061 .addReg(RegNo: LoongArch::R0)
9062 .addImm(Val: 1);
9063 TrueBB->addSuccessor(Succ: SinkBB);
9064
9065 // SinkBB: merge the results.
9066 BuildMI(BB&: *SinkBB, I: SinkBB->begin(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::PHI),
9067 DestReg: MI.getOperand(i: 0).getReg())
9068 .addReg(RegNo: RD1)
9069 .addMBB(MBB: FalseBB)
9070 .addReg(RegNo: RD2)
9071 .addMBB(MBB: TrueBB);
9072
9073 // The pseudo instruction is gone now.
9074 MI.eraseFromParent();
9075 return SinkBB;
9076}
9077
9078static MachineBasicBlock *
9079emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
9080 const LoongArchSubtarget &Subtarget) {
9081 unsigned InsOp;
9082 unsigned BroadcastOp;
9083 unsigned HalfSize;
9084 switch (MI.getOpcode()) {
9085 default:
9086 llvm_unreachable("Unexpected opcode");
9087 case LoongArch::PseudoXVINSGR2VR_B:
9088 HalfSize = 16;
9089 BroadcastOp = LoongArch::XVREPLGR2VR_B;
9090 InsOp = LoongArch::XVEXTRINS_B;
9091 break;
9092 case LoongArch::PseudoXVINSGR2VR_H:
9093 HalfSize = 8;
9094 BroadcastOp = LoongArch::XVREPLGR2VR_H;
9095 InsOp = LoongArch::XVEXTRINS_H;
9096 break;
9097 }
9098 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9099 const TargetRegisterClass *RC = &LoongArch::LASX256RegClass;
9100 const TargetRegisterClass *SubRC = &LoongArch::LSX128RegClass;
9101 DebugLoc DL = MI.getDebugLoc();
9102 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9103 // XDst = vector_insert XSrc, Elt, Idx
9104 Register XDst = MI.getOperand(i: 0).getReg();
9105 Register XSrc = MI.getOperand(i: 1).getReg();
9106 Register Elt = MI.getOperand(i: 2).getReg();
9107 unsigned Idx = MI.getOperand(i: 3).getImm();
9108
9109 if (XSrc.isVirtual() && MRI.getVRegDef(Reg: XSrc)->isImplicitDef() &&
9110 Idx < HalfSize) {
9111 Register ScratchSubReg1 = MRI.createVirtualRegister(RegClass: SubRC);
9112 Register ScratchSubReg2 = MRI.createVirtualRegister(RegClass: SubRC);
9113
9114 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::COPY), DestReg: ScratchSubReg1)
9115 .addReg(RegNo: XSrc, Flags: {}, SubReg: LoongArch::sub_128);
9116 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9117 MCID: TII->get(Opcode: HalfSize == 8 ? LoongArch::VINSGR2VR_H
9118 : LoongArch::VINSGR2VR_B),
9119 DestReg: ScratchSubReg2)
9120 .addReg(RegNo: ScratchSubReg1)
9121 .addReg(RegNo: Elt)
9122 .addImm(Val: Idx);
9123
9124 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::SUBREG_TO_REG), DestReg: XDst)
9125 .addReg(RegNo: ScratchSubReg2)
9126 .addImm(Val: LoongArch::sub_128);
9127 } else {
9128 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9129 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9130
9131 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Elt);
9132
9133 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPERMI_Q), DestReg: ScratchReg2)
9134 .addReg(RegNo: ScratchReg1)
9135 .addReg(RegNo: XSrc)
9136 .addImm(Val: Idx >= HalfSize ? 48 : 18);
9137
9138 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InsOp), DestReg: XDst)
9139 .addReg(RegNo: XSrc)
9140 .addReg(RegNo: ScratchReg2)
9141 .addImm(Val: (Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
9142 }
9143
9144 MI.eraseFromParent();
9145 return BB;
9146}
9147
9148static MachineBasicBlock *emitPseudoCTPOP(MachineInstr &MI,
9149 MachineBasicBlock *BB,
9150 const LoongArchSubtarget &Subtarget) {
9151 assert(Subtarget.hasExtLSX());
9152 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9153 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9154 DebugLoc DL = MI.getDebugLoc();
9155 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9156 Register Dst = MI.getOperand(i: 0).getReg();
9157 Register Src = MI.getOperand(i: 1).getReg();
9158
9159 unsigned BroadcastOp, CTOp, PickOp;
9160 switch (MI.getOpcode()) {
9161 default:
9162 llvm_unreachable("Unexpected opcode");
9163 case LoongArch::PseudoCTPOP_B:
9164 BroadcastOp = LoongArch::VREPLGR2VR_B;
9165 CTOp = LoongArch::VPCNT_B;
9166 PickOp = LoongArch::VPICKVE2GR_B;
9167 break;
9168 case LoongArch::PseudoCTPOP_H:
9169 case LoongArch::PseudoCTPOP_H_LA32:
9170 BroadcastOp = LoongArch::VREPLGR2VR_H;
9171 CTOp = LoongArch::VPCNT_H;
9172 PickOp = LoongArch::VPICKVE2GR_H;
9173 break;
9174 case LoongArch::PseudoCTPOP_W:
9175 case LoongArch::PseudoCTPOP_W_LA32:
9176 BroadcastOp = LoongArch::VREPLGR2VR_W;
9177 CTOp = LoongArch::VPCNT_W;
9178 PickOp = LoongArch::VPICKVE2GR_W;
9179 break;
9180 case LoongArch::PseudoCTPOP_D:
9181 BroadcastOp = LoongArch::VREPLGR2VR_D;
9182 CTOp = LoongArch::VPCNT_D;
9183 PickOp = LoongArch::VPICKVE2GR_D;
9184 break;
9185 }
9186
9187 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9188 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9189 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Src);
9190 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: CTOp), DestReg: ScratchReg2).addReg(RegNo: ScratchReg1);
9191 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PickOp), DestReg: Dst).addReg(RegNo: ScratchReg2).addImm(Val: 0);
9192
9193 MI.eraseFromParent();
9194 return BB;
9195}
9196
9197static MachineBasicBlock *
9198emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
9199 const LoongArchSubtarget &Subtarget) {
9200 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9201 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9202 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
9203 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9204 Register Dst = MI.getOperand(i: 0).getReg();
9205 Register Src = MI.getOperand(i: 1).getReg();
9206 DebugLoc DL = MI.getDebugLoc();
9207 unsigned EleBits = 8;
9208 unsigned NotOpc = 0;
9209 unsigned MskOpc;
9210
9211 switch (MI.getOpcode()) {
9212 default:
9213 llvm_unreachable("Unexpected opcode");
9214 case LoongArch::PseudoVMSKLTZ_B:
9215 MskOpc = LoongArch::VMSKLTZ_B;
9216 break;
9217 case LoongArch::PseudoVMSKLTZ_H:
9218 MskOpc = LoongArch::VMSKLTZ_H;
9219 EleBits = 16;
9220 break;
9221 case LoongArch::PseudoVMSKLTZ_W:
9222 MskOpc = LoongArch::VMSKLTZ_W;
9223 EleBits = 32;
9224 break;
9225 case LoongArch::PseudoVMSKLTZ_D:
9226 MskOpc = LoongArch::VMSKLTZ_D;
9227 EleBits = 64;
9228 break;
9229 case LoongArch::PseudoVMSKGEZ_B:
9230 MskOpc = LoongArch::VMSKGEZ_B;
9231 break;
9232 case LoongArch::PseudoVMSKEQZ_B:
9233 MskOpc = LoongArch::VMSKNZ_B;
9234 NotOpc = LoongArch::VNOR_V;
9235 break;
9236 case LoongArch::PseudoVMSKNEZ_B:
9237 MskOpc = LoongArch::VMSKNZ_B;
9238 break;
9239 case LoongArch::PseudoXVMSKLTZ_B:
9240 MskOpc = LoongArch::XVMSKLTZ_B;
9241 RC = &LoongArch::LASX256RegClass;
9242 break;
9243 case LoongArch::PseudoXVMSKLTZ_H:
9244 MskOpc = LoongArch::XVMSKLTZ_H;
9245 RC = &LoongArch::LASX256RegClass;
9246 EleBits = 16;
9247 break;
9248 case LoongArch::PseudoXVMSKLTZ_W:
9249 MskOpc = LoongArch::XVMSKLTZ_W;
9250 RC = &LoongArch::LASX256RegClass;
9251 EleBits = 32;
9252 break;
9253 case LoongArch::PseudoXVMSKLTZ_D:
9254 MskOpc = LoongArch::XVMSKLTZ_D;
9255 RC = &LoongArch::LASX256RegClass;
9256 EleBits = 64;
9257 break;
9258 case LoongArch::PseudoXVMSKGEZ_B:
9259 MskOpc = LoongArch::XVMSKGEZ_B;
9260 RC = &LoongArch::LASX256RegClass;
9261 break;
9262 case LoongArch::PseudoXVMSKEQZ_B:
9263 MskOpc = LoongArch::XVMSKNZ_B;
9264 NotOpc = LoongArch::XVNOR_V;
9265 RC = &LoongArch::LASX256RegClass;
9266 break;
9267 case LoongArch::PseudoXVMSKNEZ_B:
9268 MskOpc = LoongArch::XVMSKNZ_B;
9269 RC = &LoongArch::LASX256RegClass;
9270 break;
9271 }
9272
9273 Register Msk = MRI.createVirtualRegister(RegClass: RC);
9274 if (NotOpc) {
9275 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
9276 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Tmp).addReg(RegNo: Src);
9277 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: NotOpc), DestReg: Msk)
9278 .addReg(RegNo: Tmp, Flags: RegState::Kill)
9279 .addReg(RegNo: Tmp, Flags: RegState::Kill);
9280 } else {
9281 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Msk).addReg(RegNo: Src);
9282 }
9283
9284 if (TRI->getRegSizeInBits(RC: *RC) > 128) {
9285 Register Lo = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9286 Register Hi = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
9287 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Lo)
9288 .addReg(RegNo: Msk)
9289 .addImm(Val: 0);
9290 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Hi)
9291 .addReg(RegNo: Msk, Flags: RegState::Kill)
9292 .addImm(Val: 4);
9293 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9294 MCID: TII->get(Opcode: Subtarget.is64Bit() ? LoongArch::BSTRINS_D
9295 : LoongArch::BSTRINS_W),
9296 DestReg: Dst)
9297 .addReg(RegNo: Lo, Flags: RegState::Kill)
9298 .addReg(RegNo: Hi, Flags: RegState::Kill)
9299 .addImm(Val: 256 / EleBits - 1)
9300 .addImm(Val: 128 / EleBits);
9301 } else {
9302 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::VPICKVE2GR_HU), DestReg: Dst)
9303 .addReg(RegNo: Msk, Flags: RegState::Kill)
9304 .addImm(Val: 0);
9305 }
9306
9307 MI.eraseFromParent();
9308 return BB;
9309}
9310
9311static MachineBasicBlock *
9312emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9313 const LoongArchSubtarget &Subtarget) {
9314 assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
9315 "Unexpected instruction");
9316
9317 MachineFunction &MF = *BB->getParent();
9318 DebugLoc DL = MI.getDebugLoc();
9319 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9320 Register LoReg = MI.getOperand(i: 0).getReg();
9321 Register HiReg = MI.getOperand(i: 1).getReg();
9322 Register SrcReg = MI.getOperand(i: 2).getReg();
9323
9324 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFR2GR_S_64), DestReg: LoReg).addReg(RegNo: SrcReg);
9325 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFRH2GR_S), DestReg: HiReg)
9326 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9327 MI.eraseFromParent(); // The pseudo instruction is gone now.
9328 return BB;
9329}
9330
9331static MachineBasicBlock *
9332emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
9333 const LoongArchSubtarget &Subtarget) {
9334 assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
9335 "Unexpected instruction");
9336
9337 MachineFunction &MF = *BB->getParent();
9338 DebugLoc DL = MI.getDebugLoc();
9339 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
9340 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9341 Register TmpReg = MRI.createVirtualRegister(RegClass: &LoongArch::FPR64RegClass);
9342 Register DstReg = MI.getOperand(i: 0).getReg();
9343 Register LoReg = MI.getOperand(i: 1).getReg();
9344 Register HiReg = MI.getOperand(i: 2).getReg();
9345
9346 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FR_W_64), DestReg: TmpReg)
9347 .addReg(RegNo: LoReg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()));
9348 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FRH_W), DestReg: DstReg)
9349 .addReg(RegNo: TmpReg, Flags: RegState::Kill)
9350 .addReg(RegNo: HiReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
9351 MI.eraseFromParent(); // The pseudo instruction is gone now.
9352 return BB;
9353}
9354
9355static bool isSelectPseudo(MachineInstr &MI) {
9356 switch (MI.getOpcode()) {
9357 default:
9358 return false;
9359 case LoongArch::Select_GPR_Using_CC_GPR:
9360 return true;
9361 }
9362}
9363
9364static MachineBasicBlock *
9365emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB,
9366 const LoongArchSubtarget &Subtarget) {
9367 // To "insert" Select_* instructions, we actually have to insert the triangle
9368 // control-flow pattern. The incoming instructions know the destination vreg
9369 // to set, the condition code register to branch on, the true/false values to
9370 // select between, and the condcode to use to select the appropriate branch.
9371 //
9372 // We produce the following control flow:
9373 // HeadMBB
9374 // | \
9375 // | IfFalseMBB
9376 // | /
9377 // TailMBB
9378 //
9379 // When we find a sequence of selects we attempt to optimize their emission
9380 // by sharing the control flow. Currently we only handle cases where we have
9381 // multiple selects with the exact same condition (same LHS, RHS and CC).
9382 // The selects may be interleaved with other instructions if the other
9383 // instructions meet some requirements we deem safe:
9384 // - They are not pseudo instructions.
9385 // - They are debug instructions. Otherwise,
9386 // - They do not have side-effects, do not access memory and their inputs do
9387 // not depend on the results of the select pseudo-instructions.
9388 // The TrueV/FalseV operands of the selects cannot depend on the result of
9389 // previous selects in the sequence.
9390 // These conditions could be further relaxed. See the X86 target for a
9391 // related approach and more information.
9392
9393 Register LHS = MI.getOperand(i: 1).getReg();
9394 Register RHS;
9395 if (MI.getOperand(i: 2).isReg())
9396 RHS = MI.getOperand(i: 2).getReg();
9397 auto CC = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
9398
9399 SmallVector<MachineInstr *, 4> SelectDebugValues;
9400 SmallSet<Register, 4> SelectDests;
9401 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
9402
9403 MachineInstr *LastSelectPseudo = &MI;
9404 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
9405 SequenceMBBI != E; ++SequenceMBBI) {
9406 if (SequenceMBBI->isDebugInstr())
9407 continue;
9408 if (isSelectPseudo(MI&: *SequenceMBBI)) {
9409 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
9410 !SequenceMBBI->getOperand(i: 2).isReg() ||
9411 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
9412 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
9413 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
9414 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
9415 break;
9416 LastSelectPseudo = &*SequenceMBBI;
9417 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
9418 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
9419 continue;
9420 }
9421 if (SequenceMBBI->hasUnmodeledSideEffects() ||
9422 SequenceMBBI->mayLoadOrStore() ||
9423 SequenceMBBI->usesCustomInsertionHook())
9424 break;
9425 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
9426 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
9427 }))
9428 break;
9429 }
9430
9431 const LoongArchInstrInfo &TII = *Subtarget.getInstrInfo();
9432 const BasicBlock *LLVM_BB = BB->getBasicBlock();
9433 DebugLoc DL = MI.getDebugLoc();
9434 MachineFunction::iterator I = ++BB->getIterator();
9435
9436 MachineBasicBlock *HeadMBB = BB;
9437 MachineFunction *F = BB->getParent();
9438 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9439 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
9440
9441 F->insert(MBBI: I, MBB: IfFalseMBB);
9442 F->insert(MBBI: I, MBB: TailMBB);
9443
9444 // Set the call frame size on entry to the new basic blocks.
9445 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
9446 IfFalseMBB->setCallFrameSize(CallFrameSize);
9447 TailMBB->setCallFrameSize(CallFrameSize);
9448
9449 // Transfer debug instructions associated with the selects to TailMBB.
9450 for (MachineInstr *DebugInstr : SelectDebugValues) {
9451 TailMBB->push_back(MI: DebugInstr->removeFromParent());
9452 }
9453
9454 // Move all instructions after the sequence to TailMBB.
9455 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
9456 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
9457 // Update machine-CFG edges by transferring all successors of the current
9458 // block to the new block which will contain the Phi nodes for the selects.
9459 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
9460 // Set the successors for HeadMBB.
9461 HeadMBB->addSuccessor(Succ: IfFalseMBB);
9462 HeadMBB->addSuccessor(Succ: TailMBB);
9463
9464 // Insert appropriate branch.
9465 if (MI.getOperand(i: 2).isImm())
9466 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC))
9467 .addReg(RegNo: LHS)
9468 .addImm(Val: MI.getOperand(i: 2).getImm())
9469 .addMBB(MBB: TailMBB);
9470 else
9471 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: TailMBB);
9472
9473 // IfFalseMBB just falls through to TailMBB.
9474 IfFalseMBB->addSuccessor(Succ: TailMBB);
9475
9476 // Create PHIs for all of the select pseudo-instructions.
9477 auto SelectMBBI = MI.getIterator();
9478 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
9479 auto InsertionPoint = TailMBB->begin();
9480 while (SelectMBBI != SelectEnd) {
9481 auto Next = std::next(x: SelectMBBI);
9482 if (isSelectPseudo(MI&: *SelectMBBI)) {
9483 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
9484 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
9485 MCID: TII.get(Opcode: LoongArch::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
9486 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
9487 .addMBB(MBB: HeadMBB)
9488 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
9489 .addMBB(MBB: IfFalseMBB);
9490 SelectMBBI->eraseFromParent();
9491 }
9492 SelectMBBI = Next;
9493 }
9494
9495 F->getProperties().resetNoPHIs();
9496 return TailMBB;
9497}
9498
9499MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
9500 MachineInstr &MI, MachineBasicBlock *BB) const {
9501 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9502 DebugLoc DL = MI.getDebugLoc();
9503
9504 switch (MI.getOpcode()) {
9505 default:
9506 llvm_unreachable("Unexpected instr type to insert");
9507 case LoongArch::DIV_W:
9508 case LoongArch::DIV_WU:
9509 case LoongArch::MOD_W:
9510 case LoongArch::MOD_WU:
9511 case LoongArch::DIV_D:
9512 case LoongArch::DIV_DU:
9513 case LoongArch::MOD_D:
9514 case LoongArch::MOD_DU:
9515 return insertDivByZeroTrap(MI, MBB: BB);
9516 break;
9517 case LoongArch::WRFCSR: {
9518 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVGR2FCSR),
9519 DestReg: LoongArch::FCSR0 + MI.getOperand(i: 0).getImm())
9520 .addReg(RegNo: MI.getOperand(i: 1).getReg());
9521 MI.eraseFromParent();
9522 return BB;
9523 }
9524 case LoongArch::RDFCSR: {
9525 MachineInstr *ReadFCSR =
9526 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVFCSR2GR),
9527 DestReg: MI.getOperand(i: 0).getReg())
9528 .addReg(RegNo: LoongArch::FCSR0 + MI.getOperand(i: 1).getImm());
9529 ReadFCSR->getOperand(i: 1).setIsUndef();
9530 MI.eraseFromParent();
9531 return BB;
9532 }
9533 case LoongArch::Select_GPR_Using_CC_GPR:
9534 return emitSelectPseudo(MI, BB, Subtarget);
9535 case LoongArch::BuildPairF64Pseudo:
9536 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
9537 case LoongArch::SplitPairF64Pseudo:
9538 return emitSplitPairF64Pseudo(MI, BB, Subtarget);
9539 case LoongArch::PseudoVBZ:
9540 case LoongArch::PseudoVBZ_B:
9541 case LoongArch::PseudoVBZ_H:
9542 case LoongArch::PseudoVBZ_W:
9543 case LoongArch::PseudoVBZ_D:
9544 case LoongArch::PseudoVBNZ:
9545 case LoongArch::PseudoVBNZ_B:
9546 case LoongArch::PseudoVBNZ_H:
9547 case LoongArch::PseudoVBNZ_W:
9548 case LoongArch::PseudoVBNZ_D:
9549 case LoongArch::PseudoXVBZ:
9550 case LoongArch::PseudoXVBZ_B:
9551 case LoongArch::PseudoXVBZ_H:
9552 case LoongArch::PseudoXVBZ_W:
9553 case LoongArch::PseudoXVBZ_D:
9554 case LoongArch::PseudoXVBNZ:
9555 case LoongArch::PseudoXVBNZ_B:
9556 case LoongArch::PseudoXVBNZ_H:
9557 case LoongArch::PseudoXVBNZ_W:
9558 case LoongArch::PseudoXVBNZ_D:
9559 return emitVecCondBranchPseudo(MI, BB, Subtarget);
9560 case LoongArch::PseudoXVINSGR2VR_B:
9561 case LoongArch::PseudoXVINSGR2VR_H:
9562 return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
9563 case LoongArch::PseudoCTPOP_B:
9564 case LoongArch::PseudoCTPOP_H:
9565 case LoongArch::PseudoCTPOP_W:
9566 case LoongArch::PseudoCTPOP_D:
9567 case LoongArch::PseudoCTPOP_H_LA32:
9568 case LoongArch::PseudoCTPOP_W_LA32:
9569 return emitPseudoCTPOP(MI, BB, Subtarget);
9570 case LoongArch::PseudoVMSKLTZ_B:
9571 case LoongArch::PseudoVMSKLTZ_H:
9572 case LoongArch::PseudoVMSKLTZ_W:
9573 case LoongArch::PseudoVMSKLTZ_D:
9574 case LoongArch::PseudoVMSKGEZ_B:
9575 case LoongArch::PseudoVMSKEQZ_B:
9576 case LoongArch::PseudoVMSKNEZ_B:
9577 case LoongArch::PseudoXVMSKLTZ_B:
9578 case LoongArch::PseudoXVMSKLTZ_H:
9579 case LoongArch::PseudoXVMSKLTZ_W:
9580 case LoongArch::PseudoXVMSKLTZ_D:
9581 case LoongArch::PseudoXVMSKGEZ_B:
9582 case LoongArch::PseudoXVMSKEQZ_B:
9583 case LoongArch::PseudoXVMSKNEZ_B:
9584 return emitPseudoVMSKCOND(MI, BB, Subtarget);
9585 case TargetOpcode::STATEPOINT:
9586 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
9587 // while bl call instruction (where statepoint will be lowered at the
9588 // end) has implicit def. This def is early-clobber as it will be set at
9589 // the moment of the call and earlier than any use is read.
9590 // Add this implicit dead def here as a workaround.
9591 MI.addOperand(MF&: *MI.getMF(),
9592 Op: MachineOperand::CreateReg(
9593 Reg: LoongArch::R1, /*isDef*/ true,
9594 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
9595 /*isUndef*/ false, /*isEarlyClobber*/ true));
9596 if (!Subtarget.is64Bit())
9597 report_fatal_error(reason: "STATEPOINT is only supported on 64-bit targets");
9598 return emitPatchPoint(MI, MBB: BB);
9599 case LoongArch::PROBED_STACKALLOC_DYN:
9600 return emitDynamicProbedAlloc(MI, MBB: BB);
9601 }
9602}
9603
9604bool LoongArchTargetLowering::allowsMisalignedMemoryAccesses(
9605 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
9606 unsigned *Fast) const {
9607 if (!Subtarget.hasUAL())
9608 return false;
9609
9610 // TODO: set reasonable speed number.
9611 if (Fast)
9612 *Fast = 1;
9613 return true;
9614}
9615
9616//===----------------------------------------------------------------------===//
9617// Calling Convention Implementation
9618//===----------------------------------------------------------------------===//
9619
9620// Eight general-purpose registers a0-a7 used for passing integer arguments,
9621// with a0-a1 reused to return values. Generally, the GPRs are used to pass
9622// fixed-point arguments, and floating-point arguments when no FPR is available
9623// or with soft float ABI.
9624const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6,
9625 LoongArch::R7, LoongArch::R8, LoongArch::R9,
9626 LoongArch::R10, LoongArch::R11};
9627
9628// PreserveNone calling convention:
9629// Arguments may be passed in any general-purpose registers except:
9630// - R1 : return address register
9631// - R22 : frame pointer
9632// - R31 : base pointer
9633//
9634// All general-purpose registers are treated as caller-saved,
9635// except R1 (RA) and R22 (FP).
9636//
9637// Non-volatile registers are allocated first so that a function
9638// can call normal functions without having to spill and reload
9639// argument registers.
9640const MCPhysReg PreserveNoneArgGPRs[] = {
9641 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
9642 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
9643 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
9644 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
9645 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
9646 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
9647 LoongArch::R20};
9648
9649// Eight floating-point registers fa0-fa7 used for passing floating-point
9650// arguments, and fa0-fa1 are also used to return values.
9651const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2,
9652 LoongArch::F3, LoongArch::F4, LoongArch::F5,
9653 LoongArch::F6, LoongArch::F7};
9654// FPR32 and FPR64 alias each other.
9655const MCPhysReg ArgFPR64s[] = {
9656 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
9657 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
9658
9659const MCPhysReg ArgVRs[] = {LoongArch::VR0, LoongArch::VR1, LoongArch::VR2,
9660 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
9661 LoongArch::VR6, LoongArch::VR7};
9662
9663const MCPhysReg ArgXRs[] = {LoongArch::XR0, LoongArch::XR1, LoongArch::XR2,
9664 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
9665 LoongArch::XR6, LoongArch::XR7};
9666
9667static Register allocateArgGPR(CCState &State) {
9668 switch (State.getCallingConv()) {
9669 case CallingConv::PreserveNone:
9670 if (!State.isVarArg())
9671 return State.AllocateReg(Regs: PreserveNoneArgGPRs);
9672 [[fallthrough]];
9673 default:
9674 return State.AllocateReg(Regs: ArgGPRs);
9675 }
9676}
9677
9678// Pass a 2*GRLen argument that has been split into two GRLen values through
9679// registers or the stack as necessary.
9680static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
9681 CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1,
9682 unsigned ValNo2, MVT ValVT2, MVT LocVT2,
9683 ISD::ArgFlagsTy ArgFlags2) {
9684 unsigned GRLenInBytes = GRLen / 8;
9685 if (Register Reg = allocateArgGPR(State)) {
9686 // At least one half can be passed via register.
9687 State.addLoc(V: CCValAssign::getReg(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(), Reg,
9688 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9689 } else {
9690 // Both halves must be passed on the stack, with proper alignment.
9691 Align StackAlign =
9692 std::max(a: Align(GRLenInBytes), b: ArgFlags1.getNonZeroOrigAlign());
9693 State.addLoc(
9694 V: CCValAssign::getMem(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(),
9695 Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: StackAlign),
9696 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
9697 State.addLoc(V: CCValAssign::getMem(
9698 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9699 LocVT: LocVT2, HTP: CCValAssign::Full));
9700 return false;
9701 }
9702 if (Register Reg = allocateArgGPR(State)) {
9703 // The second half can also be passed via register.
9704 State.addLoc(
9705 V: CCValAssign::getReg(ValNo: ValNo2, ValVT: ValVT2, Reg, LocVT: LocVT2, HTP: CCValAssign::Full));
9706 } else {
9707 // The second half is passed via the stack, without additional alignment.
9708 State.addLoc(V: CCValAssign::getMem(
9709 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
9710 LocVT: LocVT2, HTP: CCValAssign::Full));
9711 }
9712 return false;
9713}
9714
9715// Implements the LoongArch calling convention. Returns true upon failure.
9716static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
9717 unsigned ValNo, MVT ValVT,
9718 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
9719 CCState &State, bool IsRet, Type *OrigTy) {
9720 unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
9721 assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
9722 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
9723 MVT LocVT = ValVT;
9724
9725 // Any return value split into more than two values can't be returned
9726 // directly.
9727 if (IsRet && ValNo > 1)
9728 return true;
9729
9730 // If passing a variadic argument, or if no FPR is available.
9731 bool UseGPRForFloat = true;
9732
9733 switch (ABI) {
9734 default:
9735 llvm_unreachable("Unexpected ABI");
9736 break;
9737 case LoongArchABI::ABI_ILP32F:
9738 case LoongArchABI::ABI_LP64F:
9739 case LoongArchABI::ABI_ILP32D:
9740 case LoongArchABI::ABI_LP64D:
9741 UseGPRForFloat = ArgFlags.isVarArg();
9742 break;
9743 case LoongArchABI::ABI_ILP32S:
9744 case LoongArchABI::ABI_LP64S:
9745 break;
9746 }
9747
9748 // If this is a variadic argument, the LoongArch calling convention requires
9749 // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
9750 // byte alignment. An aligned register should be used regardless of whether
9751 // the original argument was split during legalisation or not. The argument
9752 // will not be passed by registers if the original type is larger than
9753 // 2*GRLen, so the register alignment rule does not apply.
9754 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
9755 if (ArgFlags.isVarArg() &&
9756 ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
9757 DL.getTypeAllocSize(Ty: OrigTy) == TwoGRLenInBytes) {
9758 unsigned RegIdx = State.getFirstUnallocated(Regs: ArgGPRs);
9759 // Skip 'odd' register if necessary.
9760 if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1)
9761 State.AllocateReg(Regs: ArgGPRs);
9762 }
9763
9764 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
9765 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
9766 State.getPendingArgFlags();
9767
9768 assert(PendingLocs.size() == PendingArgFlags.size() &&
9769 "PendingLocs and PendingArgFlags out of sync");
9770
9771 // FPR32 and FPR64 alias each other.
9772 if (State.getFirstUnallocated(Regs: ArgFPR32s) == std::size(ArgFPR32s))
9773 UseGPRForFloat = true;
9774
9775 if (UseGPRForFloat && ValVT == MVT::f32) {
9776 LocVT = GRLenVT;
9777 LocInfo = CCValAssign::BCvt;
9778 } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
9779 LocVT = MVT::i64;
9780 LocInfo = CCValAssign::BCvt;
9781 } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
9782 // Handle passing f64 on LA32D with a soft float ABI or when floating point
9783 // registers are exhausted.
9784 assert(PendingLocs.empty() && "Can't lower f64 if it is split");
9785 // Depending on available argument GPRS, f64 may be passed in a pair of
9786 // GPRs, split between a GPR and the stack, or passed completely on the
9787 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
9788 // cases.
9789 MCRegister Reg = allocateArgGPR(State);
9790 if (!Reg) {
9791 int64_t StackOffset = State.AllocateStack(Size: 8, Alignment: Align(8));
9792 State.addLoc(
9793 V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9794 return false;
9795 }
9796 LocVT = MVT::i32;
9797 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9798 MCRegister HiReg = allocateArgGPR(State);
9799 if (HiReg) {
9800 State.addLoc(
9801 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: HiReg, LocVT, HTP: LocInfo));
9802 } else {
9803 int64_t StackOffset = State.AllocateStack(Size: 4, Alignment: Align(4));
9804 State.addLoc(
9805 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9806 }
9807 return false;
9808 }
9809
9810 // Split arguments might be passed indirectly, so keep track of the pending
9811 // values.
9812 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
9813 LocVT = GRLenVT;
9814 LocInfo = CCValAssign::Indirect;
9815 PendingLocs.push_back(
9816 Elt: CCValAssign::getPending(ValNo, ValVT, LocVT, HTP: LocInfo));
9817 PendingArgFlags.push_back(Elt: ArgFlags);
9818 if (!ArgFlags.isSplitEnd()) {
9819 return false;
9820 }
9821 }
9822
9823 // If the split argument only had two elements, it should be passed directly
9824 // in registers or on the stack.
9825 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
9826 PendingLocs.size() <= 2) {
9827 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
9828 // Apply the normal calling convention rules to the first half of the
9829 // split argument.
9830 CCValAssign VA = PendingLocs[0];
9831 ISD::ArgFlagsTy AF = PendingArgFlags[0];
9832 PendingLocs.clear();
9833 PendingArgFlags.clear();
9834 return CC_LoongArchAssign2GRLen(GRLen, State, VA1: VA, ArgFlags1: AF, ValNo2: ValNo, ValVT2: ValVT, LocVT2: LocVT,
9835 ArgFlags2: ArgFlags);
9836 }
9837
9838 // Allocate to a register if possible, or else a stack slot.
9839 Register Reg;
9840 unsigned StoreSizeBytes = GRLen / 8;
9841 Align StackAlign = Align(GRLen / 8);
9842
9843 if (ValVT == MVT::f32 && !UseGPRForFloat) {
9844 Reg = State.AllocateReg(Regs: ArgFPR32s);
9845 } else if (ValVT == MVT::f64 && !UseGPRForFloat) {
9846 Reg = State.AllocateReg(Regs: ArgFPR64s);
9847 } else if (ValVT.is128BitVector()) {
9848 Reg = State.AllocateReg(Regs: ArgVRs);
9849 UseGPRForFloat = false;
9850 StoreSizeBytes = 16;
9851 StackAlign = Align(16);
9852 } else if (ValVT.is256BitVector()) {
9853 Reg = State.AllocateReg(Regs: ArgXRs);
9854 UseGPRForFloat = false;
9855 StoreSizeBytes = 32;
9856 StackAlign = Align(32);
9857 } else {
9858 Reg = allocateArgGPR(State);
9859 }
9860
9861 unsigned StackOffset =
9862 Reg ? 0 : State.AllocateStack(Size: StoreSizeBytes, Alignment: StackAlign);
9863
9864 // If we reach this point and PendingLocs is non-empty, we must be at the
9865 // end of a split argument that must be passed indirectly.
9866 if (!PendingLocs.empty()) {
9867 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
9868 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
9869 for (auto &It : PendingLocs) {
9870 if (Reg)
9871 It.convertToReg(Reg);
9872 else
9873 It.convertToMem(Offset: StackOffset);
9874 State.addLoc(V: It);
9875 }
9876 PendingLocs.clear();
9877 PendingArgFlags.clear();
9878 return false;
9879 }
9880 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
9881 "Expected an GRLenVT at this stage");
9882
9883 if (Reg) {
9884 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
9885 return false;
9886 }
9887
9888 // When a floating-point value is passed on the stack, no bit-cast is needed.
9889 if (ValVT.isFloatingPoint()) {
9890 LocVT = ValVT;
9891 LocInfo = CCValAssign::Full;
9892 }
9893
9894 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
9895 return false;
9896}
9897
9898void LoongArchTargetLowering::analyzeInputArgs(
9899 MachineFunction &MF, CCState &CCInfo,
9900 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9901 LoongArchCCAssignFn Fn) const {
9902 FunctionType *FType = MF.getFunction().getFunctionType();
9903 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
9904 MVT ArgVT = Ins[i].VT;
9905 Type *ArgTy = nullptr;
9906 if (IsRet)
9907 ArgTy = FType->getReturnType();
9908 else if (Ins[i].isOrigArg())
9909 ArgTy = FType->getParamType(i: Ins[i].getOrigArgIndex());
9910 LoongArchABI::ABI ABI =
9911 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9912 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
9913 CCInfo, IsRet, ArgTy)) {
9914 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
9915 << '\n');
9916 llvm_unreachable("");
9917 }
9918 }
9919}
9920
9921void LoongArchTargetLowering::analyzeOutputArgs(
9922 MachineFunction &MF, CCState &CCInfo,
9923 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9924 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const {
9925 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9926 MVT ArgVT = Outs[i].VT;
9927 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9928 LoongArchABI::ABI ABI =
9929 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
9930 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
9931 CCInfo, IsRet, OrigTy)) {
9932 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
9933 << "\n");
9934 llvm_unreachable("");
9935 }
9936 }
9937}
9938
9939// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9940// values.
9941static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9942 const CCValAssign &VA, const SDLoc &DL) {
9943 switch (VA.getLocInfo()) {
9944 default:
9945 llvm_unreachable("Unexpected CCValAssign::LocInfo");
9946 case CCValAssign::Full:
9947 case CCValAssign::Indirect:
9948 break;
9949 case CCValAssign::BCvt:
9950 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9951 Val = DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Val);
9952 else
9953 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
9954 break;
9955 }
9956 return Val;
9957}
9958
9959static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9960 const CCValAssign &VA, const SDLoc &DL,
9961 const ISD::InputArg &In,
9962 const LoongArchTargetLowering &TLI) {
9963 MachineFunction &MF = DAG.getMachineFunction();
9964 MachineRegisterInfo &RegInfo = MF.getRegInfo();
9965 EVT LocVT = VA.getLocVT();
9966 SDValue Val;
9967 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
9968 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
9969 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
9970 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
9971
9972 // If input is sign extended from 32 bits, note it for the OptW pass.
9973 if (In.isOrigArg()) {
9974 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
9975 if (OrigArg->getType()->isIntegerTy()) {
9976 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
9977 // An input zero extended from i31 can also be considered sign extended.
9978 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
9979 (BitWidth < 32 && In.Flags.isZExt())) {
9980 LoongArchMachineFunctionInfo *LAFI =
9981 MF.getInfo<LoongArchMachineFunctionInfo>();
9982 LAFI->addSExt32Register(Reg: VReg);
9983 }
9984 }
9985 }
9986
9987 return convertLocVTToValVT(DAG, Val, VA, DL);
9988}
9989
9990// The caller is responsible for loading the full value if the argument is
9991// passed with CCValAssign::Indirect.
9992static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9993 const CCValAssign &VA, const SDLoc &DL) {
9994 MachineFunction &MF = DAG.getMachineFunction();
9995 MachineFrameInfo &MFI = MF.getFrameInfo();
9996 EVT ValVT = VA.getValVT();
9997 int FI = MFI.CreateFixedObject(Size: ValVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
9998 /*IsImmutable=*/true);
9999 SDValue FIN = DAG.getFrameIndex(
10000 FI, VT: MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0)));
10001
10002 ISD::LoadExtType ExtType;
10003 switch (VA.getLocInfo()) {
10004 default:
10005 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10006 case CCValAssign::Full:
10007 case CCValAssign::Indirect:
10008 case CCValAssign::BCvt:
10009 ExtType = ISD::NON_EXTLOAD;
10010 break;
10011 }
10012 return DAG.getExtLoad(
10013 ExtType, dl: DL, VT: VA.getLocVT(), Chain, Ptr: FIN,
10014 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT: ValVT);
10015}
10016
10017static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain,
10018 const CCValAssign &VA,
10019 const CCValAssign &HiVA,
10020 const SDLoc &DL) {
10021 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
10022 "Unexpected VA");
10023 MachineFunction &MF = DAG.getMachineFunction();
10024 MachineFrameInfo &MFI = MF.getFrameInfo();
10025 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10026
10027 assert(VA.isRegLoc() && "Expected register VA assignment");
10028
10029 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10030 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
10031 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
10032 SDValue Hi;
10033 if (HiVA.isMemLoc()) {
10034 // Second half of f64 is passed on the stack.
10035 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
10036 /*IsImmutable=*/true);
10037 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
10038 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
10039 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10040 } else {
10041 // Second half of f64 is passed in another GPR.
10042 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10043 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
10044 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
10045 }
10046 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
10047}
10048
10049static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
10050 const CCValAssign &VA, const SDLoc &DL) {
10051 EVT LocVT = VA.getLocVT();
10052
10053 switch (VA.getLocInfo()) {
10054 default:
10055 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10056 case CCValAssign::Full:
10057 break;
10058 case CCValAssign::BCvt:
10059 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10060 Val = DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Val);
10061 else
10062 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
10063 break;
10064 }
10065 return Val;
10066}
10067
10068static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
10069 CCValAssign::LocInfo LocInfo,
10070 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
10071 CCState &State) {
10072 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
10073 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
10074 // s0 s1 s2 s3 s4 s5 s6 s7 s8
10075 static const MCPhysReg GPRList[] = {
10076 LoongArch::R23, LoongArch::R24, LoongArch::R25,
10077 LoongArch::R26, LoongArch::R27, LoongArch::R28,
10078 LoongArch::R29, LoongArch::R30, LoongArch::R31};
10079 if (MCRegister Reg = State.AllocateReg(Regs: GPRList)) {
10080 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10081 return false;
10082 }
10083 }
10084
10085 if (LocVT == MVT::f32) {
10086 // Pass in STG registers: F1, F2, F3, F4
10087 // fs0,fs1,fs2,fs3
10088 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
10089 LoongArch::F26, LoongArch::F27};
10090 if (MCRegister Reg = State.AllocateReg(Regs: FPR32List)) {
10091 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10092 return false;
10093 }
10094 }
10095
10096 if (LocVT == MVT::f64) {
10097 // Pass in STG registers: D1, D2, D3, D4
10098 // fs4,fs5,fs6,fs7
10099 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
10100 LoongArch::F30_64, LoongArch::F31_64};
10101 if (MCRegister Reg = State.AllocateReg(Regs: FPR64List)) {
10102 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10103 return false;
10104 }
10105 }
10106
10107 report_fatal_error(reason: "No registers left in GHC calling convention");
10108 return true;
10109}
10110
10111// Transform physical registers into virtual registers.
10112SDValue LoongArchTargetLowering::LowerFormalArguments(
10113 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10114 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
10115 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
10116
10117 MachineFunction &MF = DAG.getMachineFunction();
10118
10119 switch (CallConv) {
10120 default:
10121 llvm_unreachable("Unsupported calling convention");
10122 case CallingConv::C:
10123 case CallingConv::Fast:
10124 case CallingConv::PreserveNone:
10125 case CallingConv::PreserveMost:
10126 break;
10127 case CallingConv::GHC:
10128 if (!MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicF) ||
10129 !MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicD))
10130 report_fatal_error(
10131 reason: "GHC calling convention requires the F and D extensions");
10132 }
10133
10134 const Function &Func = MF.getFunction();
10135 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10136 MVT GRLenVT = Subtarget.getGRLenVT();
10137 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
10138
10139 // Check if this function has any musttail calls. If so, incoming indirect
10140 // arg pointers must be saved in virtual registers so they survive across
10141 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
10142 // when needed to avoid adding register pressure to non-musttail functions.
10143 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
10144 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
10145 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
10146 return CI->isMustTailCall();
10147 return false;
10148 });
10149 });
10150 // Used with varargs to acumulate store chains.
10151 std::vector<SDValue> OutChains;
10152
10153 // Assign locations to all of the incoming arguments.
10154 SmallVector<CCValAssign> ArgLocs;
10155 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10156
10157 if (CallConv == CallingConv::GHC)
10158 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_LoongArch_GHC);
10159 else
10160 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, Fn: CC_LoongArch);
10161
10162 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10163 CCValAssign &VA = ArgLocs[i];
10164 SDValue ArgValue;
10165 // Passing f64 on LA32D with a soft float ABI must be handled as a special
10166 // case.
10167 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10168 assert(VA.needsCustom());
10169 ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
10170 } else if (VA.isRegLoc())
10171 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
10172 else
10173 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
10174 if (VA.getLocInfo() == CCValAssign::Indirect) {
10175 // If the original argument was split and passed by reference, we need to
10176 // load all parts of it here (using the same address).
10177 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
10178 PtrInfo: MachinePointerInfo()));
10179 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
10180 if (HasMusttail) {
10181 LoongArchMachineFunctionInfo *LAFI =
10182 MF.getInfo<LoongArchMachineFunctionInfo>();
10183 Register VReg =
10184 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10185 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
10186 LAFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
10187 }
10188 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
10189 assert(ArgPartOffset == 0);
10190 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
10191 CCValAssign &PartVA = ArgLocs[i + 1];
10192 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
10193 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10194 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
10195 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
10196 PtrInfo: MachinePointerInfo()));
10197 ++i;
10198 ++InsIdx;
10199 }
10200 continue;
10201 }
10202 InVals.push_back(Elt: ArgValue);
10203 }
10204
10205 if (IsVarArg) {
10206 ArrayRef<MCPhysReg> ArgRegs = ArrayRef(ArgGPRs);
10207 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
10208 const TargetRegisterClass *RC = &LoongArch::GPRRegClass;
10209 MachineFrameInfo &MFI = MF.getFrameInfo();
10210 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10211 auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>();
10212
10213 // Offset of the first variable argument from stack pointer, and size of
10214 // the vararg save area. For now, the varargs save area is either zero or
10215 // large enough to hold a0-a7.
10216 int VaArgOffset, VarArgsSaveSize;
10217
10218 // If all registers are allocated, then all varargs must be passed on the
10219 // stack and we don't need to save any argregs.
10220 if (ArgRegs.size() == Idx) {
10221 VaArgOffset = CCInfo.getStackSize();
10222 VarArgsSaveSize = 0;
10223 } else {
10224 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx);
10225 VaArgOffset = -VarArgsSaveSize;
10226 }
10227
10228 // Record the frame index of the first variable argument
10229 // which is a value necessary to VASTART.
10230 int FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10231 LoongArchFI->setVarArgsFrameIndex(FI);
10232
10233 // If saving an odd number of registers then create an extra stack slot to
10234 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures
10235 // offsets to even-numbered registered remain 2*GRLen-aligned.
10236 if (Idx % 2) {
10237 MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset - (int)GRLenInBytes,
10238 IsImmutable: true);
10239 VarArgsSaveSize += GRLenInBytes;
10240 }
10241
10242 // Copy the integer registers that may have been used for passing varargs
10243 // to the vararg save area.
10244 for (unsigned I = Idx; I < ArgRegs.size();
10245 ++I, VaArgOffset += GRLenInBytes) {
10246 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
10247 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
10248 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: GRLenVT);
10249 FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
10250 SDValue PtrOff = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10251 SDValue Store = DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: PtrOff,
10252 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10253 cast<StoreSDNode>(Val: Store.getNode())
10254 ->getMemOperand()
10255 ->setValue((Value *)nullptr);
10256 OutChains.push_back(x: Store);
10257 }
10258 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
10259 }
10260
10261 // All stores are grouped in one node to allow the matching between
10262 // the size of Ins and InVals. This only happens for vararg functions.
10263 if (!OutChains.empty()) {
10264 OutChains.push_back(x: Chain);
10265 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
10266 }
10267
10268 return Chain;
10269}
10270
10271bool LoongArchTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10272 return CI->isTailCall();
10273}
10274
10275// Check if the return value is used as only a return value, as otherwise
10276// we can't perform a tail-call.
10277bool LoongArchTargetLowering::isUsedByReturnOnly(SDNode *N,
10278 SDValue &Chain) const {
10279 if (N->getNumValues() != 1)
10280 return false;
10281 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
10282 return false;
10283
10284 SDNode *Copy = *N->user_begin();
10285 if (Copy->getOpcode() != ISD::CopyToReg)
10286 return false;
10287
10288 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
10289 // isn't safe to perform a tail call.
10290 if (Copy->getGluedNode())
10291 return false;
10292
10293 // The copy must be used by a LoongArchISD::RET, and nothing else.
10294 bool HasRet = false;
10295 for (SDNode *Node : Copy->users()) {
10296 if (Node->getOpcode() != LoongArchISD::RET)
10297 return false;
10298 HasRet = true;
10299 }
10300
10301 if (!HasRet)
10302 return false;
10303
10304 Chain = Copy->getOperand(Num: 0);
10305 return true;
10306}
10307
10308// Check whether the call is eligible for tail call optimization.
10309bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
10310 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
10311 const SmallVectorImpl<CCValAssign> &ArgLocs) const {
10312
10313 auto CalleeCC = CLI.CallConv;
10314 auto &Outs = CLI.Outs;
10315 auto &Caller = MF.getFunction();
10316 auto CallerCC = Caller.getCallingConv();
10317
10318 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
10319
10320 // Byval parameters hand the function a pointer directly into the stack area
10321 // we want to reuse during a tail call. Working around this *is* possible
10322 // but less efficient and uglier in LowerCall. For musttail, there is no
10323 // workaround today: a byval arg requires a local copy that becomes invalid
10324 // after the tail call deallocates the caller's frame, so rejecting here
10325 // (and triggering reportFatalInternalError in LowerCall) is safer than
10326 // miscompiling.
10327 for (auto &Arg : Outs)
10328 if (Arg.Flags.isByVal())
10329 return false;
10330
10331 // musttail bypasses the remaining checks: the checks either reject cases
10332 // we handle specially (indirect args are forwarded via incoming pointers,
10333 // stack-passed args reuse the matching incoming layout, sret is forwarded
10334 // like any other pointer arg) or are optimizations not applicable to
10335 // mandatory tail calls.
10336 if (IsMustTail)
10337 return true;
10338
10339 // Do not tail call opt if the stack is used to pass parameters.
10340 if (CCInfo.getStackSize() != 0)
10341 return false;
10342
10343 // Do not tail call opt if any parameters need to be passed indirectly.
10344 for (auto &VA : ArgLocs)
10345 if (VA.getLocInfo() == CCValAssign::Indirect)
10346 return false;
10347
10348 // Do not tail call opt if either caller or callee uses struct return
10349 // semantics.
10350 auto IsCallerStructRet = Caller.hasStructRetAttr();
10351 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
10352 if (IsCallerStructRet || IsCalleeStructRet)
10353 return false;
10354
10355 // The callee has to preserve all registers the caller needs to preserve.
10356 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
10357 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
10358 if (CalleeCC != CallerCC) {
10359 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
10360 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
10361 return false;
10362 }
10363 return true;
10364}
10365
10366static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
10367 return DAG.getDataLayout().getPrefTypeAlign(
10368 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
10369}
10370
10371// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
10372// and output parameter nodes.
10373SDValue
10374LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
10375 SmallVectorImpl<SDValue> &InVals) const {
10376 SelectionDAG &DAG = CLI.DAG;
10377 SDLoc &DL = CLI.DL;
10378 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
10379 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
10380 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
10381 SDValue Chain = CLI.Chain;
10382 SDValue Callee = CLI.Callee;
10383 CallingConv::ID CallConv = CLI.CallConv;
10384 bool IsVarArg = CLI.IsVarArg;
10385 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10386 MVT GRLenVT = Subtarget.getGRLenVT();
10387 bool &IsTailCall = CLI.IsTailCall;
10388
10389 MachineFunction &MF = DAG.getMachineFunction();
10390
10391 // Analyze the operands of the call, assigning locations to each operand.
10392 SmallVector<CCValAssign> ArgLocs;
10393 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10394
10395 if (CallConv == CallingConv::GHC)
10396 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_LoongArch_GHC);
10397 else
10398 analyzeOutputArgs(MF, CCInfo&: ArgCCInfo, Outs, /*IsRet=*/false, CLI: &CLI, Fn: CC_LoongArch);
10399
10400 // Check if it's really possible to do a tail call.
10401 if (IsTailCall)
10402 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
10403
10404 if (IsTailCall)
10405 ++NumTailCalls;
10406 else if (CLI.CB && CLI.CB->isMustTailCall())
10407 report_fatal_error(reason: "failed to perform tail call elimination on a call "
10408 "site marked musttail");
10409
10410 // Get a count of how many bytes are to be pushed on the stack.
10411 unsigned NumBytes = ArgCCInfo.getStackSize();
10412
10413 // Create local copies for byval args.
10414 SmallVector<SDValue> ByValArgs;
10415 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10416 ISD::ArgFlagsTy Flags = Outs[i].Flags;
10417 if (!Flags.isByVal())
10418 continue;
10419
10420 SDValue Arg = OutVals[i];
10421 unsigned Size = Flags.getByValSize();
10422 Align Alignment = Flags.getNonZeroByValAlign();
10423
10424 int FI =
10425 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
10426 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
10427 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: GRLenVT);
10428
10429 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
10430 /*IsVolatile=*/isVol: false,
10431 /*AlwaysInline=*/false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
10432 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
10433 ByValArgs.push_back(Elt: FIPtr);
10434 }
10435
10436 if (!IsTailCall)
10437 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
10438
10439 // Copy argument values to their designated locations.
10440 SmallVector<std::pair<Register, SDValue>> RegsToPass;
10441 SmallVector<SDValue> MemOpChains;
10442 SDValue StackPtr;
10443 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
10444 ++i, ++OutIdx) {
10445 CCValAssign &VA = ArgLocs[i];
10446 SDValue ArgValue = OutVals[OutIdx];
10447 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
10448
10449 // Handle passing f64 on LA32D with a soft float ABI as a special case.
10450 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10451 assert(VA.isRegLoc() && "Expected register VA assignment");
10452 assert(VA.needsCustom());
10453 SDValue SplitF64 =
10454 DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10455 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
10456 SDValue Lo = SplitF64.getValue(R: 0);
10457 SDValue Hi = SplitF64.getValue(R: 1);
10458
10459 Register RegLo = VA.getLocReg();
10460 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
10461
10462 // Get the CCValAssign for the Hi part.
10463 CCValAssign &HiVA = ArgLocs[++i];
10464
10465 if (HiVA.isMemLoc()) {
10466 // Second half of f64 is passed on the stack.
10467 if (!StackPtr.getNode())
10468 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10469 SDValue Address =
10470 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10471 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
10472 // Emit the store.
10473 MemOpChains.push_back(Elt: DAG.getStore(
10474 Chain, dl: DL, Val: Hi, Ptr: Address,
10475 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
10476 } else {
10477 // Second half of f64 is passed in another GPR.
10478 Register RegHigh = HiVA.getLocReg();
10479 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
10480 }
10481 continue;
10482 }
10483
10484 // Promote the value if needed.
10485 // For now, only handle fully promoted and indirect arguments.
10486 if (VA.getLocInfo() == CCValAssign::Indirect) {
10487 // For musttail calls, reuse incoming indirect pointers instead of
10488 // creating new stack temporaries. The incoming pointers point to the
10489 // caller's caller's frame, which remains valid after a tail call.
10490 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
10491 LoongArchMachineFunctionInfo *LAFI =
10492 MF.getInfo<LoongArchMachineFunctionInfo>();
10493 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
10494
10495 // Resolve which formal parameter is being passed at this call
10496 // position.
10497 //
10498 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
10499 // but Outs[].OrigArgIndex is an index into a filtered arg list
10500 // (empty types removed, via CallLoweringInfo in the target-
10501 // independent layer). IncomingIndirectArgs is keyed by the
10502 // caller's unfiltered Argument::getArgNo(), so we have to walk
10503 // the caller's formals (same filter) to translate the index.
10504 // This target-independent asymmetry should be normalized so
10505 // backends do not need to re-derive the mapping.
10506 //
10507 // Steps:
10508 // 1. Find the call operand at filtered position CallArgIdx.
10509 // 2. If it is an Argument, use getArgNo() directly (same filter
10510 // for caller formals and call operands).
10511 // 3. Otherwise (computed value), walk the caller's formals and
10512 // skip empty types to map the filtered index to getArgNo().
10513 const Argument *FormalArg = nullptr;
10514 unsigned FilteredIdx = 0;
10515 for (const auto &CallArg : CLI.CB->args()) {
10516 if (CallArg->getType()->isEmptyTy())
10517 continue;
10518 if (FilteredIdx == CallArgIdx) {
10519 FormalArg = dyn_cast<Argument>(Val: CallArg);
10520 break;
10521 }
10522 ++FilteredIdx;
10523 }
10524
10525 // For forwarded args, getArgNo() gives the unfiltered index directly.
10526 // For computed args, walk the caller's formals to resolve it.
10527 unsigned FormalArgIdx = CallArgIdx;
10528 if (FormalArg) {
10529 FormalArgIdx = FormalArg->getArgNo();
10530 } else {
10531 FilteredIdx = 0;
10532 for (const auto &Arg : MF.getFunction().args()) {
10533 if (Arg.getType()->isEmptyTy())
10534 continue;
10535 if (FilteredIdx == CallArgIdx) {
10536 FormalArgIdx = Arg.getArgNo();
10537 break;
10538 }
10539 ++FilteredIdx;
10540 }
10541 }
10542
10543 Register VReg = LAFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
10544 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
10545 // Thread the CopyFromReg output chain through MemOpChains so the
10546 // TokenFactor below sequences the copy with any stores we emit
10547 // for this argument.
10548 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
10549 SDValue IncomingPtr = CopyOp;
10550
10551 if (!FormalArg) {
10552 // Computed value: store into the incoming indirect pointer for the
10553 // same-position formal parameter (musttail guarantees matching
10554 // prototypes, so types match). The pointer survives the tail call
10555 // since it points to the caller's caller's frame.
10556 //
10557 // The data-flow edge through IncomingPtr already prevents the
10558 // store from being scheduled before the CopyFromReg. Threading
10559 // CopyOp.getValue(1) (the copy's output chain) into the store
10560 // makes that ordering explicit on the chain edge as well, which
10561 // is the convention for memory ops chaining off their producers.
10562 MemOpChains.push_back(
10563 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
10564 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10565 // Store any split parts at their respective offsets.
10566 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10567 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10568 SDValue PartValue = OutVals[OutIdx + 1];
10569 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10570 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10571 SDValue Addr =
10572 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
10573 MemOpChains.push_back(
10574 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
10575 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
10576 ++i;
10577 ++OutIdx;
10578 }
10579 }
10580 ArgValue = IncomingPtr;
10581
10582 // Skip any remaining split parts (for forwarded args, they are
10583 // covered by the forwarded pointer).
10584 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10585 ++i;
10586 ++OutIdx;
10587 }
10588 } else {
10589 // Store the argument in a stack slot and pass its address.
10590 Align StackAlign =
10591 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
10592 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
10593 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
10594 // If the original argument was split and passed by reference, we need
10595 // to store the required parts of it here (and pass just one address).
10596 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
10597 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10598 assert(ArgPartOffset == 0);
10599 // Calculate the total size to store. We don't have access to what we're
10600 // actually storing other than performing the loop and collecting the
10601 // info.
10602 SmallVector<std::pair<SDValue, SDValue>> Parts;
10603 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
10604 SDValue PartValue = OutVals[OutIdx + 1];
10605 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10606 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
10607 EVT PartVT = PartValue.getValueType();
10608 StoredSize += PartVT.getStoreSize();
10609 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
10610 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
10611 ++i;
10612 ++OutIdx;
10613 }
10614 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
10615 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
10616 MemOpChains.push_back(
10617 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
10618 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10619 for (const auto &Part : Parts) {
10620 SDValue PartValue = Part.first;
10621 SDValue PartOffset = Part.second;
10622 SDValue Address =
10623 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
10624 MemOpChains.push_back(
10625 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
10626 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
10627 }
10628 ArgValue = SpillSlot;
10629 }
10630 } else {
10631 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL);
10632 }
10633
10634 // Use local copy if it is a byval arg.
10635 if (Flags.isByVal())
10636 ArgValue = ByValArgs[j++];
10637
10638 if (VA.isRegLoc()) {
10639 // Queue up the argument copies and emit them at the end.
10640 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
10641 } else {
10642 assert(VA.isMemLoc() && "Argument not register or memory");
10643 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
10644 "Tail call not allowed if stack is used for passing parameters");
10645
10646 // Work out the address of the stack slot.
10647 if (!StackPtr.getNode())
10648 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
10649 SDValue Address =
10650 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
10651 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
10652
10653 // Emit the store.
10654 MemOpChains.push_back(
10655 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
10656 }
10657 }
10658
10659 // Join the stores, which are independent of one another.
10660 if (!MemOpChains.empty())
10661 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
10662
10663 SDValue Glue;
10664
10665 // Build a sequence of copy-to-reg nodes, chained and glued together.
10666 for (auto &Reg : RegsToPass) {
10667 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
10668 Glue = Chain.getValue(R: 1);
10669 }
10670
10671 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
10672 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
10673 // split it and then direct call can be matched by PseudoCALL_SMALL.
10674 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
10675 const GlobalValue *GV = S->getGlobal();
10676 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV)
10677 ? LoongArchII::MO_CALL
10678 : LoongArchII::MO_CALL_PLT;
10679 Callee = DAG.getTargetGlobalAddress(GV: S->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: OpFlags);
10680 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
10681 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV: nullptr)
10682 ? LoongArchII::MO_CALL
10683 : LoongArchII::MO_CALL_PLT;
10684 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: OpFlags);
10685 }
10686
10687 // The first call operand is the chain and the second is the target address.
10688 SmallVector<SDValue> Ops;
10689 Ops.push_back(Elt: Chain);
10690 Ops.push_back(Elt: Callee);
10691
10692 // Add argument registers to the end of the list so that they are
10693 // known live into the call.
10694 for (auto &Reg : RegsToPass)
10695 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
10696
10697 if (!IsTailCall) {
10698 // Add a register mask operand representing the call-preserved registers.
10699 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
10700 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
10701 assert(Mask && "Missing call preserved mask for calling convention");
10702 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
10703 }
10704
10705 // Glue the call to the argument copies, if any.
10706 if (Glue.getNode())
10707 Ops.push_back(Elt: Glue);
10708
10709 // Emit the call.
10710 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
10711 unsigned Op;
10712 switch (DAG.getTarget().getCodeModel()) {
10713 default:
10714 report_fatal_error(reason: "Unsupported code model");
10715 case CodeModel::Small:
10716 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
10717 break;
10718 case CodeModel::Medium:
10719 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
10720 break;
10721 case CodeModel::Large:
10722 assert(Subtarget.is64Bit() && "Large code model requires LA64");
10723 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
10724 break;
10725 }
10726
10727 if (IsTailCall) {
10728 MF.getFrameInfo().setHasTailCall();
10729 SDValue Ret = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10730 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
10731 return Ret;
10732 }
10733
10734 Chain = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
10735 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
10736 Glue = Chain.getValue(R: 1);
10737
10738 // Mark the end of the call, which is glued to the call itself.
10739 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
10740 Glue = Chain.getValue(R: 1);
10741
10742 // Assign locations to each value returned by this call.
10743 SmallVector<CCValAssign> RVLocs;
10744 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
10745 analyzeInputArgs(MF, CCInfo&: RetCCInfo, Ins, /*IsRet=*/true, Fn: CC_LoongArch);
10746
10747 // Copy all of the result registers out of their specified physreg.
10748 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
10749 auto &VA = RVLocs[i];
10750 // Copy the value out.
10751 SDValue RetValue =
10752 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
10753 // Glue the RetValue to the end of the call sequence.
10754 Chain = RetValue.getValue(R: 1);
10755 Glue = RetValue.getValue(R: 2);
10756
10757 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10758 assert(VA.needsCustom());
10759 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
10760 VT: MVT::i32, Glue);
10761 Chain = RetValue2.getValue(R: 1);
10762 Glue = RetValue2.getValue(R: 2);
10763 RetValue = DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64,
10764 N1: RetValue, N2: RetValue2);
10765 } else
10766 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL);
10767
10768 InVals.push_back(Elt: RetValue);
10769 }
10770
10771 return Chain;
10772}
10773
10774bool LoongArchTargetLowering::CanLowerReturn(
10775 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
10776 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
10777 const Type *RetTy) const {
10778 SmallVector<CCValAssign> RVLocs;
10779 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
10780
10781 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10782 LoongArchABI::ABI ABI =
10783 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10784 if (CC_LoongArch(DL: MF.getDataLayout(), ABI, ValNo: i, ValVT: Outs[i].VT, LocInfo: CCValAssign::Full,
10785 ArgFlags: Outs[i].Flags, State&: CCInfo, /*IsRet=*/true, OrigTy: nullptr))
10786 return false;
10787 }
10788 return true;
10789}
10790
10791SDValue LoongArchTargetLowering::LowerReturn(
10792 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10793 const SmallVectorImpl<ISD::OutputArg> &Outs,
10794 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
10795 SelectionDAG &DAG) const {
10796 // Stores the assignment of the return value to a location.
10797 SmallVector<CCValAssign> RVLocs;
10798
10799 // Info about the registers and stack slot.
10800 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
10801 *DAG.getContext());
10802
10803 analyzeOutputArgs(MF&: DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
10804 CLI: nullptr, Fn: CC_LoongArch);
10805 if (CallConv == CallingConv::GHC && !RVLocs.empty())
10806 report_fatal_error(reason: "GHC functions return void only");
10807 SDValue Glue;
10808 SmallVector<SDValue, 4> RetOps(1, Chain);
10809
10810 // Copy the result values into the output registers.
10811 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
10812 SDValue Val = OutVals[OutIdx];
10813 CCValAssign &VA = RVLocs[i];
10814 assert(VA.isRegLoc() && "Can only return in registers!");
10815
10816 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10817 // Handle returning f64 on LA32D with a soft float ABI.
10818 assert(VA.isRegLoc() && "Expected return via registers");
10819 assert(VA.needsCustom());
10820 SDValue SplitF64 = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
10821 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
10822 SDValue Lo = SplitF64.getValue(R: 0);
10823 SDValue Hi = SplitF64.getValue(R: 1);
10824 Register RegLo = VA.getLocReg();
10825 Register RegHi = RVLocs[++i].getLocReg();
10826
10827 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
10828 Glue = Chain.getValue(R: 1);
10829 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
10830 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
10831 Glue = Chain.getValue(R: 1);
10832 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
10833 } else {
10834 // Handle a 'normal' return.
10835 Val = convertValVTToLocVT(DAG, Val, VA, DL);
10836 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
10837
10838 // Guarantee that all emitted copies are stuck together.
10839 Glue = Chain.getValue(R: 1);
10840 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
10841 }
10842 }
10843
10844 RetOps[0] = Chain; // Update chain.
10845
10846 // Add the glue node if we have it.
10847 if (Glue.getNode())
10848 RetOps.push_back(Elt: Glue);
10849
10850 return DAG.getNode(Opcode: LoongArchISD::RET, DL, VT: MVT::Other, Ops: RetOps);
10851}
10852
10853// Check if a constant splat can be generated using [x]vldi, where imm[12] == 1.
10854// Note: The following prefixes are excluded:
10855// imm[11:8] == 4'b0000, 4'b0100, 4'b1000
10856// as they can be represented using [x]vrepli.[whb]
10857std::pair<bool, uint64_t> LoongArchTargetLowering::isImmVLDILegalForMode1(
10858 const APInt &SplatValue, const unsigned SplatBitSize) const {
10859 uint64_t RequiredImm = 0;
10860 uint64_t V = SplatValue.getZExtValue();
10861 if (SplatBitSize == 16 && !(V & 0x00FF)) {
10862 // 4'b0101
10863 RequiredImm = (0b10101 << 8) | (V >> 8);
10864 return {true, RequiredImm};
10865 } else if (SplatBitSize == 32) {
10866 // 4'b0001
10867 if (!(V & 0xFFFF00FF)) {
10868 RequiredImm = (0b10001 << 8) | (V >> 8);
10869 return {true, RequiredImm};
10870 }
10871 // 4'b0010
10872 if (!(V & 0xFF00FFFF)) {
10873 RequiredImm = (0b10010 << 8) | (V >> 16);
10874 return {true, RequiredImm};
10875 }
10876 // 4'b0011
10877 if (!(V & 0x00FFFFFF)) {
10878 RequiredImm = (0b10011 << 8) | (V >> 24);
10879 return {true, RequiredImm};
10880 }
10881 // 4'b0110
10882 if ((V & 0xFFFF00FF) == 0xFF) {
10883 RequiredImm = (0b10110 << 8) | (V >> 8);
10884 return {true, RequiredImm};
10885 }
10886 // 4'b0111
10887 if ((V & 0xFF00FFFF) == 0xFFFF) {
10888 RequiredImm = (0b10111 << 8) | (V >> 16);
10889 return {true, RequiredImm};
10890 }
10891 // 4'b1010
10892 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
10893 RequiredImm =
10894 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10895 return {true, RequiredImm};
10896 }
10897 } else if (SplatBitSize == 64) {
10898 // 4'b1011
10899 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
10900 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
10901 RequiredImm =
10902 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10903 return {true, RequiredImm};
10904 }
10905 // 4'b1100
10906 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
10907 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
10908 RequiredImm =
10909 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
10910 return {true, RequiredImm};
10911 }
10912 // 4'b1001
10913 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
10914 uint8_t res = 0;
10915 for (int i = 0; i < 8; ++i) {
10916 uint8_t byte = x & 0xFF;
10917 if (byte == 0 || byte == 0xFF)
10918 res |= ((byte & 1) << i);
10919 else
10920 return {false, 0};
10921 x >>= 8;
10922 }
10923 return {true, res};
10924 };
10925 auto [IsSame, Suffix] = sameBitsPreByte(V);
10926 if (IsSame) {
10927 RequiredImm = (0b11001 << 8) | Suffix;
10928 return {true, RequiredImm};
10929 }
10930 }
10931 return {false, RequiredImm};
10932}
10933
10934bool LoongArchTargetLowering::isFPImmVLDILegal(const APFloat &Imm,
10935 EVT VT) const {
10936 if (!Subtarget.hasExtLSX())
10937 return false;
10938
10939 if (VT == MVT::f32) {
10940 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
10941 return (masked == 0x3e000000 || masked == 0x40000000);
10942 }
10943
10944 if (VT == MVT::f64) {
10945 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
10946 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
10947 }
10948
10949 return false;
10950}
10951
10952bool LoongArchTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
10953 bool ForCodeSize) const {
10954 // TODO: Maybe need more checks here after vector extension is supported.
10955 if (VT == MVT::f32 && !Subtarget.hasBasicF())
10956 return false;
10957 if (VT == MVT::f64 && !Subtarget.hasBasicD())
10958 return false;
10959 return (Imm.isZero() || Imm.isOne() || isFPImmVLDILegal(Imm, VT));
10960}
10961
10962bool LoongArchTargetLowering::isCheapToSpeculateCttz(Type *) const {
10963 return true;
10964}
10965
10966bool LoongArchTargetLowering::isCheapToSpeculateCtlz(Type *) const {
10967 return true;
10968}
10969
10970bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
10971 const Instruction *I) const {
10972 if (!Subtarget.is64Bit())
10973 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
10974
10975 if (isa<LoadInst>(Val: I))
10976 return true;
10977
10978 // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
10979 // require fences beacuse we can use amswap_db.[w/d].
10980 Type *Ty = I->getOperand(i: 0)->getType();
10981 if (isa<StoreInst>(Val: I) && Ty->isIntegerTy()) {
10982 unsigned Size = Ty->getIntegerBitWidth();
10983 return (Size == 8 || Size == 16);
10984 }
10985
10986 return false;
10987}
10988
10989EVT LoongArchTargetLowering::getSetCCResultType(const DataLayout &DL,
10990 LLVMContext &Context,
10991 EVT VT) const {
10992 if (!VT.isVector())
10993 return getPointerTy(DL);
10994 return VT.changeVectorElementTypeToInteger();
10995}
10996
10997bool LoongArchTargetLowering::canMergeStoresTo(
10998 unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const {
10999 // Do not merge to float value size (128 or 256 bits) if no implicit
11000 // float attribute is set.
11001 bool NoFloat = MF.getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat);
11002 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
11003 if (NoFloat)
11004 return MemVT.getSizeInBits() <= MaxIntSize;
11005
11006 // Make sure we don't merge greater than our maximum supported vector width.
11007 if (Subtarget.hasExtLASX())
11008 MaxIntSize = 256;
11009 else if (Subtarget.hasExtLSX())
11010 MaxIntSize = 128;
11011
11012 return MemVT.getSizeInBits() <= MaxIntSize;
11013}
11014
11015bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
11016 EVT VT = Y.getValueType();
11017
11018 if (VT.isVector())
11019 return Subtarget.hasExtLSX() && VT.isInteger();
11020
11021 return VT.isScalarInteger() && !isa<ConstantSDNode>(Val: Y);
11022}
11023
11024void LoongArchTargetLowering::getTgtMemIntrinsic(
11025 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
11026 MachineFunction &MF, unsigned Intrinsic) const {
11027 switch (Intrinsic) {
11028 default:
11029 return;
11030 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
11031 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
11032 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
11033 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
11034 IntrinsicInfo Info;
11035 Info.opc = ISD::INTRINSIC_W_CHAIN;
11036 Info.memVT = MVT::i32;
11037 Info.ptrVal = I.getArgOperand(i: 0);
11038 Info.offset = 0;
11039 Info.align = Align(4);
11040 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
11041 MachineMemOperand::MOVolatile;
11042 Infos.push_back(Elt: Info);
11043 return;
11044 // TODO: Add more Intrinsics later.
11045 }
11046 }
11047}
11048
11049// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
11050// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
11051// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
11052// regression, we need to implement it manually.
11053void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
11054 AtomicRMWInst::BinOp Op = AI->getOperation();
11055
11056 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
11057 Op == AtomicRMWInst::And) &&
11058 "Unable to expand");
11059 unsigned MinWordSize = 4;
11060
11061 IRBuilder<> Builder(AI);
11062 LLVMContext &Ctx = Builder.getContext();
11063 const DataLayout &DL = AI->getDataLayout();
11064 Type *ValueType = AI->getType();
11065 Type *WordType = Type::getIntNTy(C&: Ctx, N: MinWordSize * 8);
11066
11067 Value *Addr = AI->getPointerOperand();
11068 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
11069 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
11070
11071 Value *AlignedAddr = Builder.CreateIntrinsic(
11072 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
11073 Args: {Addr, ConstantInt::get(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))}, FMFSource: nullptr,
11074 Name: "AlignedAddr");
11075
11076 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
11077 Value *PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
11078 Value *ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
11079 ShiftAmt = Builder.CreateTrunc(V: ShiftAmt, DestTy: WordType, Name: "ShiftAmt");
11080 Value *Mask = Builder.CreateShl(
11081 LHS: ConstantInt::get(Ty: WordType,
11082 V: (1 << (DL.getTypeStoreSize(Ty: ValueType) * 8)) - 1),
11083 RHS: ShiftAmt, Name: "Mask");
11084 Value *Inv_Mask = Builder.CreateNot(V: Mask, Name: "Inv_Mask");
11085 Value *ValOperand_Shifted =
11086 Builder.CreateShl(LHS: Builder.CreateZExt(V: AI->getValOperand(), DestTy: WordType),
11087 RHS: ShiftAmt, Name: "ValOperand_Shifted");
11088 Value *NewOperand;
11089 if (Op == AtomicRMWInst::And)
11090 NewOperand = Builder.CreateOr(LHS: ValOperand_Shifted, RHS: Inv_Mask, Name: "AndOperand");
11091 else
11092 NewOperand = ValOperand_Shifted;
11093
11094 AtomicRMWInst *NewAI =
11095 Builder.CreateAtomicRMW(Op, Ptr: AlignedAddr, Val: NewOperand, Align: Align(MinWordSize),
11096 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
11097
11098 Value *Shift = Builder.CreateLShr(LHS: NewAI, RHS: ShiftAmt, Name: "shifted");
11099 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: ValueType, Name: "extracted");
11100 Value *FinalOldResult = Builder.CreateBitCast(V: Trunc, DestTy: ValueType);
11101 AI->replaceAllUsesWith(V: FinalOldResult);
11102 AI->eraseFromParent();
11103}
11104
11105TargetLowering::AtomicExpansionKind
11106LoongArchTargetLowering::shouldExpandAtomicRMWInIR(
11107 const AtomicRMWInst *AI) const {
11108 // TODO: Add more AtomicRMWInst that needs to be extended.
11109
11110 // Since floating-point operation requires a non-trivial set of data
11111 // operations, use CmpXChg to expand.
11112 if (AI->isFloatingPointOperation() ||
11113 AI->getOperation() == AtomicRMWInst::UIncWrap ||
11114 AI->getOperation() == AtomicRMWInst::UDecWrap ||
11115 AI->getOperation() == AtomicRMWInst::USubCond ||
11116 AI->getOperation() == AtomicRMWInst::USubSat)
11117 return AtomicExpansionKind::CmpXChg;
11118
11119 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
11120 (AI->getOperation() == AtomicRMWInst::Xchg ||
11121 AI->getOperation() == AtomicRMWInst::Add ||
11122 AI->getOperation() == AtomicRMWInst::Sub)) {
11123 return AtomicExpansionKind::None;
11124 }
11125
11126 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11127 if (Subtarget.hasLAMCAS()) {
11128 if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
11129 AI->getOperation() == AtomicRMWInst::Or ||
11130 AI->getOperation() == AtomicRMWInst::Xor))
11131 return AtomicExpansionKind::CustomExpand;
11132 if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
11133 return AtomicExpansionKind::CmpXChg;
11134 }
11135
11136 if (Size == 8 || Size == 16)
11137 return AtomicExpansionKind::MaskedIntrinsic;
11138 return AtomicExpansionKind::None;
11139}
11140
11141static Intrinsic::ID
11142getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
11143 AtomicRMWInst::BinOp BinOp) {
11144 if (GRLen == 64) {
11145 switch (BinOp) {
11146 default:
11147 llvm_unreachable("Unexpected AtomicRMW BinOp");
11148 case AtomicRMWInst::Xchg:
11149 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
11150 case AtomicRMWInst::Add:
11151 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
11152 case AtomicRMWInst::Sub:
11153 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
11154 case AtomicRMWInst::Nand:
11155 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
11156 case AtomicRMWInst::UMax:
11157 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
11158 case AtomicRMWInst::UMin:
11159 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
11160 case AtomicRMWInst::Max:
11161 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
11162 case AtomicRMWInst::Min:
11163 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
11164 // TODO: support other AtomicRMWInst.
11165 }
11166 }
11167
11168 if (GRLen == 32) {
11169 switch (BinOp) {
11170 default:
11171 llvm_unreachable("Unexpected AtomicRMW BinOp");
11172 case AtomicRMWInst::Xchg:
11173 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
11174 case AtomicRMWInst::Add:
11175 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
11176 case AtomicRMWInst::Sub:
11177 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
11178 case AtomicRMWInst::Nand:
11179 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
11180 case AtomicRMWInst::UMax:
11181 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
11182 case AtomicRMWInst::UMin:
11183 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
11184 case AtomicRMWInst::Max:
11185 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
11186 case AtomicRMWInst::Min:
11187 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
11188 // TODO: support other AtomicRMWInst.
11189 }
11190 }
11191
11192 llvm_unreachable("Unexpected GRLen\n");
11193}
11194
11195TargetLowering::AtomicExpansionKind
11196LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
11197 const AtomicCmpXchgInst *CI) const {
11198
11199 if (Subtarget.hasLAMCAS())
11200 return AtomicExpansionKind::None;
11201
11202 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
11203 if (Size == 8 || Size == 16)
11204 return AtomicExpansionKind::MaskedIntrinsic;
11205 return AtomicExpansionKind::None;
11206}
11207
11208Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
11209 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
11210 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
11211 unsigned GRLen = Subtarget.getGRLen();
11212 AtomicOrdering FailOrd = CI->getFailureOrdering();
11213 Value *FailureOrdering =
11214 Builder.getIntN(N: Subtarget.getGRLen(), C: static_cast<uint64_t>(FailOrd));
11215 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
11216 if (GRLen == 64) {
11217 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
11218 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
11219 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
11220 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11221 }
11222 Type *Tys[] = {AlignedAddr->getType()};
11223 Value *Result = Builder.CreateIntrinsic(
11224 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
11225 if (GRLen == 64)
11226 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11227 return Result;
11228}
11229
11230Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic(
11231 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
11232 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
11233 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
11234 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
11235 // mask, as this produces better code than the LL/SC loop emitted by
11236 // int_loongarch_masked_atomicrmw_xchg.
11237 if (AI->getOperation() == AtomicRMWInst::Xchg &&
11238 isa<ConstantInt>(Val: AI->getValOperand())) {
11239 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
11240 if (CVal->isZero())
11241 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
11242 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
11243 Align: AI->getAlign(), Ordering: Ord);
11244 if (CVal->isMinusOne())
11245 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
11246 Align: AI->getAlign(), Ordering: Ord);
11247 }
11248
11249 unsigned GRLen = Subtarget.getGRLen();
11250 Value *Ordering =
11251 Builder.getIntN(N: GRLen, C: static_cast<uint64_t>(AI->getOrdering()));
11252 Type *Tys[] = {AlignedAddr->getType()};
11253 Function *LlwOpScwLoop = Intrinsic::getOrInsertDeclaration(
11254 M: AI->getModule(),
11255 id: getIntrinsicForMaskedAtomicRMWBinOp(GRLen, BinOp: AI->getOperation()), OverloadTys: Tys);
11256
11257 if (GRLen == 64) {
11258 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
11259 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
11260 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
11261 }
11262
11263 Value *Result;
11264
11265 // Must pass the shift amount needed to sign extend the loaded value prior
11266 // to performing a signed comparison for min/max. ShiftAmt is the number of
11267 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which
11268 // is the number of bits to left+right shift the value in order to
11269 // sign-extend.
11270 if (AI->getOperation() == AtomicRMWInst::Min ||
11271 AI->getOperation() == AtomicRMWInst::Max) {
11272 const DataLayout &DL = AI->getDataLayout();
11273 unsigned ValWidth =
11274 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
11275 Value *SextShamt =
11276 Builder.CreateSub(LHS: Builder.getIntN(N: GRLen, C: GRLen - ValWidth), RHS: ShiftAmt);
11277 Result = Builder.CreateCall(Callee: LlwOpScwLoop,
11278 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
11279 } else {
11280 Result =
11281 Builder.CreateCall(Callee: LlwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
11282 }
11283
11284 if (GRLen == 64)
11285 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
11286 return Result;
11287}
11288
11289bool LoongArchTargetLowering::isFMAFasterThanFMulAndFAdd(
11290 const MachineFunction &MF, EVT VT) const {
11291 VT = VT.getScalarType();
11292
11293 if (!VT.isSimple())
11294 return false;
11295
11296 switch (VT.getSimpleVT().SimpleTy) {
11297 case MVT::f32:
11298 case MVT::f64:
11299 return true;
11300 default:
11301 break;
11302 }
11303
11304 return false;
11305}
11306
11307Register LoongArchTargetLowering::getExceptionPointerRegister(
11308 ExceptionHandling EH, const Constant *PersonalityFn) const {
11309 return LoongArch::R4;
11310}
11311
11312Register LoongArchTargetLowering::getExceptionSelectorRegister(
11313 ExceptionHandling EH, const Constant *PersonalityFn) const {
11314 return LoongArch::R5;
11315}
11316
11317//===----------------------------------------------------------------------===//
11318// Target Optimization Hooks
11319//===----------------------------------------------------------------------===//
11320
11321static int getEstimateRefinementSteps(EVT VT,
11322 const LoongArchSubtarget &Subtarget) {
11323 // Feature FRECIPE instrucions relative accuracy is 2^-14.
11324 // IEEE float has 23 digits and double has 52 digits.
11325 int RefinementSteps = VT.getScalarType() == MVT::f64 ? 2 : 1;
11326 return RefinementSteps;
11327}
11328
11329static bool
11330isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget) {
11331 assert(Subtarget.hasFrecipe() &&
11332 "Reciprocal estimate queried on unsupported target");
11333
11334 if (!VT.isSimple())
11335 return false;
11336
11337 switch (VT.getSimpleVT().SimpleTy) {
11338 case MVT::f32:
11339 // f32 is the base type for reciprocal estimate instructions.
11340 return true;
11341
11342 case MVT::f64:
11343 return Subtarget.hasBasicD();
11344
11345 case MVT::v4f32:
11346 case MVT::v2f64:
11347 return Subtarget.hasExtLSX();
11348
11349 case MVT::v8f32:
11350 case MVT::v4f64:
11351 return Subtarget.hasExtLASX();
11352
11353 default:
11354 return false;
11355 }
11356}
11357
11358SDValue LoongArchTargetLowering::getSqrtEstimate(SDValue Operand,
11359 SelectionDAG &DAG, int Enabled,
11360 int &RefinementSteps,
11361 bool &UseOneConstNR,
11362 bool Reciprocal) const {
11363 assert(Enabled != ReciprocalEstimate::Disabled &&
11364 "Enabled should never be Disabled here");
11365
11366 if (!Subtarget.hasFrecipe())
11367 return SDValue();
11368
11369 SDLoc DL(Operand);
11370 EVT VT = Operand.getValueType();
11371
11372 // Check supported types.
11373 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11374 return SDValue();
11375
11376 // Handle refinement steps.
11377 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11378 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11379
11380 // LoongArch only has FRSQRTE which is 1.0 / sqrt(x).
11381 UseOneConstNR = false;
11382 SDValue Rsqrt = DAG.getNode(Opcode: LoongArchISD::FRSQRTE, DL, VT, Operand);
11383
11384 // If the caller wants 1.0 / sqrt(x), or if further refinement steps
11385 // are needed (which rely on the reciprocal form), return the raw reciprocal
11386 // estimate.
11387 if (Reciprocal || RefinementSteps > 0)
11388 return Rsqrt;
11389
11390 // Otherwise, return sqrt(x) by multiplying with the operand.
11391 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Operand, N2: Rsqrt);
11392}
11393
11394SDValue LoongArchTargetLowering::getRecipEstimate(SDValue Operand,
11395 SelectionDAG &DAG,
11396 int Enabled,
11397 int &RefinementSteps) const {
11398 assert(Enabled != ReciprocalEstimate::Disabled &&
11399 "Enabled should never be Disabled here");
11400
11401 if (!Subtarget.hasFrecipe())
11402 return SDValue();
11403
11404 SDLoc DL(Operand);
11405 EVT VT = Operand.getValueType();
11406
11407 // Check supported types.
11408 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
11409 return SDValue();
11410
11411 if (RefinementSteps == ReciprocalEstimate::Unspecified)
11412 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
11413
11414 // FRECIPE computes 1.0 / x.
11415 return DAG.getNode(Opcode: LoongArchISD::FRECIPE, DL, VT, Operand);
11416}
11417
11418//===----------------------------------------------------------------------===//
11419// LoongArch Inline Assembly Support
11420//===----------------------------------------------------------------------===//
11421
11422LoongArchTargetLowering::ConstraintType
11423LoongArchTargetLowering::getConstraintType(StringRef Constraint) const {
11424 // LoongArch specific constraints in GCC: config/loongarch/constraints.md
11425 //
11426 // 'f': A floating-point register (if available).
11427 // 'k': A memory operand whose address is formed by a base register and
11428 // (optionally scaled) index register.
11429 // 'l': A signed 16-bit constant.
11430 // 'm': A memory operand whose address is formed by a base register and
11431 // offset that is suitable for use in instructions with the same
11432 // addressing mode as st.w and ld.w.
11433 // 'q': A general-purpose register except for $r0 and $r1 (for the csrxchg
11434 // instruction)
11435 // 'I': A signed 12-bit constant (for arithmetic instructions).
11436 // 'J': Integer zero.
11437 // 'K': An unsigned 12-bit constant (for logic instructions).
11438 // "ZB": An address that is held in a general-purpose register. The offset is
11439 // zero.
11440 // "ZC": A memory operand whose address is formed by a base register and
11441 // offset that is suitable for use in instructions with the same
11442 // addressing mode as ll.w and sc.w.
11443 if (Constraint.size() == 1) {
11444 switch (Constraint[0]) {
11445 default:
11446 break;
11447 case 'f':
11448 case 'q':
11449 return C_RegisterClass;
11450 case 'l':
11451 case 'I':
11452 case 'J':
11453 case 'K':
11454 return C_Immediate;
11455 case 'k':
11456 return C_Memory;
11457 }
11458 }
11459
11460 if (Constraint == "ZC" || Constraint == "ZB")
11461 return C_Memory;
11462
11463 // 'm' is handled here.
11464 return TargetLowering::getConstraintType(Constraint);
11465}
11466
11467InlineAsm::ConstraintCode LoongArchTargetLowering::getInlineAsmMemConstraint(
11468 StringRef ConstraintCode) const {
11469 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
11470 .Case(S: "k", Value: InlineAsm::ConstraintCode::k)
11471 .Case(S: "ZB", Value: InlineAsm::ConstraintCode::ZB)
11472 .Case(S: "ZC", Value: InlineAsm::ConstraintCode::ZC)
11473 .Default(Value: TargetLowering::getInlineAsmMemConstraint(ConstraintCode));
11474}
11475
11476std::pair<unsigned, const TargetRegisterClass *>
11477LoongArchTargetLowering::getRegForInlineAsmConstraint(
11478 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
11479 // First, see if this is a constraint that directly corresponds to a LoongArch
11480 // register class.
11481 if (Constraint.size() == 1) {
11482 switch (Constraint[0]) {
11483 case 'r':
11484 // TODO: Support fixed vectors up to GRLen?
11485 if (VT.isVector())
11486 break;
11487 return std::make_pair(x: 0U, y: &LoongArch::GPRRegClass);
11488 case 'q':
11489 return std::make_pair(x: 0U, y: &LoongArch::GPRNoR0R1RegClass);
11490 case 'f':
11491 if (Subtarget.hasBasicF() && VT == MVT::f32)
11492 return std::make_pair(x: 0U, y: &LoongArch::FPR32RegClass);
11493 if (Subtarget.hasBasicD() && VT == MVT::f64)
11494 return std::make_pair(x: 0U, y: &LoongArch::FPR64RegClass);
11495 if (Subtarget.hasExtLSX() &&
11496 TRI->isTypeLegalForClass(RC: LoongArch::LSX128RegClass, T: VT))
11497 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11498 if (Subtarget.hasExtLSX() && VT == MVT::i128)
11499 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
11500 if (Subtarget.hasExtLASX() &&
11501 TRI->isTypeLegalForClass(RC: LoongArch::LASX256RegClass, T: VT))
11502 return std::make_pair(x: 0U, y: &LoongArch::LASX256RegClass);
11503 break;
11504 default:
11505 break;
11506 }
11507 }
11508
11509 // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen
11510 // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm
11511 // constraints while the official register name is prefixed with a '$'. So we
11512 // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.)
11513 // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is
11514 // case insensitive, so no need to convert the constraint to upper case here.
11515 //
11516 // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly
11517 // decode the usage of register name aliases into their official names. And
11518 // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use
11519 // official register names.
11520 if (Constraint.starts_with(Prefix: "{$r") || Constraint.starts_with(Prefix: "{$f") ||
11521 Constraint.starts_with(Prefix: "{$vr") || Constraint.starts_with(Prefix: "{$xr")) {
11522 bool IsFP = Constraint[2] == 'f';
11523 std::pair<StringRef, StringRef> Temp = Constraint.split(Separator: '$');
11524 std::pair<unsigned, const TargetRegisterClass *> R;
11525 R = TargetLowering::getRegForInlineAsmConstraint(
11526 TRI, Constraint: join_items(Separator: "", Items&: Temp.first, Items&: Temp.second), VT);
11527 // Match those names to the widest floating point register type available.
11528 if (IsFP) {
11529 unsigned RegNo = R.first;
11530 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
11531 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
11532 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
11533 return std::make_pair(x&: DReg, y: &LoongArch::FPR64RegClass);
11534 }
11535 }
11536 }
11537 return R;
11538 }
11539
11540 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
11541}
11542
11543void LoongArchTargetLowering::LowerAsmOperandForConstraint(
11544 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
11545 SelectionDAG &DAG) const {
11546 // Currently only support length 1 constraints.
11547 if (Constraint.size() == 1) {
11548 switch (Constraint[0]) {
11549 case 'l':
11550 // Validate & create a 16-bit signed immediate operand.
11551 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11552 uint64_t CVal = C->getSExtValue();
11553 if (isInt<16>(x: CVal))
11554 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11555 VT: Subtarget.getGRLenVT()));
11556 }
11557 return;
11558 case 'I':
11559 // Validate & create a 12-bit signed immediate operand.
11560 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11561 uint64_t CVal = C->getSExtValue();
11562 if (isInt<12>(x: CVal))
11563 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
11564 VT: Subtarget.getGRLenVT()));
11565 }
11566 return;
11567 case 'J':
11568 // Validate & create an integer zero operand.
11569 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
11570 if (C->getZExtValue() == 0)
11571 Ops.push_back(
11572 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11573 return;
11574 case 'K':
11575 // Validate & create a 12-bit unsigned immediate operand.
11576 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
11577 uint64_t CVal = C->getZExtValue();
11578 if (isUInt<12>(x: CVal))
11579 Ops.push_back(
11580 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
11581 }
11582 return;
11583 default:
11584 break;
11585 }
11586 }
11587 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
11588}
11589
11590#define GET_REGISTER_MATCHER
11591#include "LoongArchGenAsmMatcher.inc"
11592
11593Register
11594LoongArchTargetLowering::getRegisterByName(const char *RegName, LLT VT,
11595 const MachineFunction &MF) const {
11596 std::pair<StringRef, StringRef> Name = StringRef(RegName).split(Separator: '$');
11597 std::string NewRegName = Name.second.str();
11598 Register Reg = MatchRegisterAltName(Name: NewRegName);
11599 if (!Reg)
11600 Reg = MatchRegisterName(Name: NewRegName);
11601 if (!Reg)
11602 return Reg;
11603 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11604 if (!ReservedRegs.test(Idx: Reg))
11605 report_fatal_error(reason: Twine("Trying to obtain non-reserved register \"" +
11606 StringRef(RegName) + "\"."));
11607 return Reg;
11608}
11609
11610bool LoongArchTargetLowering::decomposeMulByConstant(LLVMContext &Context,
11611 EVT VT, SDValue C) const {
11612 // TODO: Support vectors.
11613 if (!VT.isScalarInteger())
11614 return false;
11615
11616 // Omit the optimization if the data size exceeds GRLen.
11617 if (VT.getSizeInBits() > Subtarget.getGRLen())
11618 return false;
11619
11620 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
11621 const APInt &Imm = ConstNode->getAPIntValue();
11622 // Break MUL into (SLLI + ADD/SUB) or ALSL.
11623 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
11624 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
11625 return true;
11626 // Break MUL into (ALSL x, (SLLI x, imm0), imm1).
11627 if (ConstNode->hasOneUse() &&
11628 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
11629 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
11630 return true;
11631 // Break (MUL x, imm) into (ADD (SLLI x, s0), (SLLI x, s1)),
11632 // in which the immediate has two set bits. Or Break (MUL x, imm)
11633 // into (SUB (SLLI x, s0), (SLLI x, s1)), in which the immediate
11634 // equals to (1 << s0) - (1 << s1).
11635 if (ConstNode->hasOneUse() && !(Imm.sge(RHS: -2048) && Imm.sle(RHS: 4095))) {
11636 unsigned Shifts = Imm.countr_zero();
11637 // Reject immediates which can be composed via a single LUI.
11638 if (Shifts >= 12)
11639 return false;
11640 // Reject multiplications can be optimized to
11641 // (SLLI (ALSL x, x, 1/2/3/4), s).
11642 APInt ImmPop = Imm.ashr(ShiftAmt: Shifts);
11643 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
11644 return false;
11645 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`,
11646 // since it needs one more instruction than other 3 cases.
11647 APInt ImmSmall = APInt(Imm.getBitWidth(), 1ULL << Shifts, true);
11648 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
11649 (ImmSmall - Imm).isPowerOf2())
11650 return true;
11651 }
11652 }
11653
11654 return false;
11655}
11656
11657bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
11658 const AddrMode &AM,
11659 Type *Ty, unsigned AS,
11660 Instruction *I) const {
11661 // LoongArch has four basic addressing modes:
11662 // 1. reg
11663 // 2. reg + 12-bit signed offset
11664 // 3. reg + 14-bit signed offset left-shifted by 2
11665 // 4. reg1 + reg2
11666 // TODO: Add more checks after support vector extension.
11667
11668 // No global is ever allowed as a base.
11669 if (AM.BaseGV)
11670 return false;
11671
11672 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
11673 // with `UAL` feature.
11674 if (!isInt<12>(x: AM.BaseOffs) &&
11675 !(isShiftedInt<14, 2>(x: AM.BaseOffs) && Subtarget.hasUAL()))
11676 return false;
11677
11678 switch (AM.Scale) {
11679 case 0:
11680 // "r+i" or just "i", depending on HasBaseReg.
11681 break;
11682 case 1:
11683 // "r+r+i" is not allowed.
11684 if (AM.HasBaseReg && AM.BaseOffs)
11685 return false;
11686 // Otherwise we have "r+r" or "r+i".
11687 break;
11688 case 2:
11689 // "2*r+r" or "2*r+i" is not allowed.
11690 if (AM.HasBaseReg || AM.BaseOffs)
11691 return false;
11692 // Allow "2*r" as "r+r".
11693 break;
11694 default:
11695 return false;
11696 }
11697
11698 return true;
11699}
11700
11701bool LoongArchTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
11702 return isInt<12>(x: Imm);
11703}
11704
11705bool LoongArchTargetLowering::isLegalAddImmediate(int64_t Imm) const {
11706 return isInt<12>(x: Imm);
11707}
11708
11709bool LoongArchTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11710 // Zexts are free if they can be combined with a load.
11711 // Don't advertise i32->i64 zextload as being free for LA64. It interacts
11712 // poorly with type legalization of compares preferring sext.
11713 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
11714 EVT MemVT = LD->getMemoryVT();
11715 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11716 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
11717 LD->getExtensionType() == ISD::ZEXTLOAD))
11718 return true;
11719 }
11720
11721 return TargetLowering::isZExtFree(Val, VT2);
11722}
11723
11724bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT,
11725 EVT DstVT) const {
11726 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
11727}
11728
11729bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const {
11730 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
11731}
11732
11733bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
11734 // TODO: Support vectors.
11735 if (Y.getValueType().isVector())
11736 return false;
11737
11738 return !isa<ConstantSDNode>(Val: Y);
11739}
11740
11741ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
11742 // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
11743 return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
11744}
11745
11746bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
11747 Type *Ty, bool IsSigned) const {
11748 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
11749 return true;
11750
11751 return IsSigned;
11752}
11753
11754bool LoongArchTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
11755 // Return false to suppress the unnecessary extensions if the LibCall
11756 // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
11757 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
11758 Type.getSizeInBits() < Subtarget.getGRLen()))
11759 return false;
11760 return true;
11761}
11762
11763// memcpy, and other memory intrinsics, typically tries to use wider load/store
11764// if the source/dest is aligned and the copy size is large enough. We therefore
11765// want to align such objects passed to memory intrinsics.
11766bool LoongArchTargetLowering::shouldAlignPointerArgs(CallInst *CI,
11767 unsigned &MinSize,
11768 Align &PrefAlign) const {
11769 if (!isa<MemIntrinsic>(Val: CI))
11770 return false;
11771
11772 if (Subtarget.is64Bit()) {
11773 MinSize = 8;
11774 PrefAlign = Align(8);
11775 } else {
11776 MinSize = 4;
11777 PrefAlign = Align(4);
11778 }
11779
11780 return true;
11781}
11782
11783TargetLoweringBase::LegalizeTypeAction
11784LoongArchTargetLowering::getPreferredVectorAction(MVT VT) const {
11785 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
11786 VT.getVectorElementType() != MVT::i1)
11787 return TypeWidenVector;
11788
11789 return TargetLoweringBase::getPreferredVectorAction(VT);
11790}
11791
11792bool LoongArchTargetLowering::splitValueIntoRegisterParts(
11793 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
11794 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
11795 bool IsABIRegCopy = CC.has_value();
11796 EVT ValueVT = Val.getValueType();
11797
11798 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11799 PartVT == MVT::f32) {
11800 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
11801 // nan, and cast to f32.
11802 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
11803 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
11804 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
11805 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
11806 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Val);
11807 Parts[0] = Val;
11808 return true;
11809 }
11810
11811 return false;
11812}
11813
11814SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
11815 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
11816 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
11817 bool IsABIRegCopy = CC.has_value();
11818
11819 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11820 PartVT == MVT::f32) {
11821 SDValue Val = Parts[0];
11822
11823 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
11824 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
11825 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
11826 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
11827 return Val;
11828 }
11829
11830 return SDValue();
11831}
11832
11833MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
11834 CallingConv::ID CC,
11835 EVT VT) const {
11836 // Use f32 to pass f16.
11837 if (VT == MVT::f16 && Subtarget.hasBasicF())
11838 return MVT::f32;
11839
11840 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
11841}
11842
11843unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
11844 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
11845 // Use f32 to pass f16.
11846 if (VT == MVT::f16 && Subtarget.hasBasicF())
11847 return 1;
11848
11849 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
11850}
11851
11852void LoongArchTargetLowering::computeKnownBitsForTargetNode(
11853 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
11854 const SelectionDAG &DAG, unsigned Depth) const {
11855 unsigned Opc = Op.getOpcode();
11856 Known.resetAll();
11857 switch (Opc) {
11858 default:
11859 break;
11860 case LoongArchISD::VPICK_ZEXT_ELT: {
11861 assert(isa<VTSDNode>(Op->getOperand(2)) && "Unexpected operand!");
11862 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 2))->getVT();
11863 unsigned VTBits = VT.getScalarSizeInBits();
11864 assert(Known.getBitWidth() >= VTBits && "Unexpected width!");
11865 Known.Zero.setBitsFrom(VTBits);
11866 break;
11867 }
11868 }
11869}
11870
11871bool LoongArchTargetLowering::SimplifyDemandedBitsForTargetNode(
11872 SDValue Op, const APInt &OriginalDemandedBits,
11873 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
11874 unsigned Depth) const {
11875 EVT VT = Op.getValueType();
11876 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
11877 unsigned Opc = Op.getOpcode();
11878 switch (Opc) {
11879 default:
11880 break;
11881 case LoongArchISD::CRC_W_B_W:
11882 case LoongArchISD::CRC_W_H_W:
11883 case LoongArchISD::CRCC_W_B_W:
11884 case LoongArchISD::CRCC_W_H_W: {
11885 KnownBits KnownSrc;
11886 APInt DemandedSrcBits =
11887 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: (Opc == LoongArchISD::CRC_W_B_W ||
11888 Opc == LoongArchISD::CRCC_W_B_W)
11889 ? 8
11890 : 16);
11891 return SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: DemandedSrcBits,
11892 DemandedElts: OriginalDemandedElts, Known&: KnownSrc, TLO, Depth: Depth + 1);
11893 }
11894 case LoongArchISD::VMSKLTZ:
11895 case LoongArchISD::XVMSKLTZ: {
11896 SDValue Src = Op.getOperand(i: 0);
11897 MVT SrcVT = Src.getSimpleValueType();
11898 unsigned SrcBits = SrcVT.getScalarSizeInBits();
11899 unsigned NumElts = SrcVT.getVectorNumElements();
11900
11901 // If we don't need the sign bits at all just return zero.
11902 if (OriginalDemandedBits.countr_zero() >= NumElts)
11903 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
11904
11905 // Only demand the vector elements of the sign bits we need.
11906 APInt KnownUndef, KnownZero;
11907 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(width: NumElts);
11908 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedElts, KnownUndef, KnownZero,
11909 TLO, Depth: Depth + 1))
11910 return true;
11911
11912 Known.Zero = KnownZero.zext(width: BitWidth);
11913 Known.Zero.setHighBits(BitWidth - NumElts);
11914
11915 // [X]VMSKLTZ only uses the MSB from each vector element.
11916 KnownBits KnownSrc;
11917 APInt DemandedSrcBits = APInt::getSignMask(BitWidth: SrcBits);
11918 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, Known&: KnownSrc, TLO,
11919 Depth: Depth + 1))
11920 return true;
11921
11922 if (KnownSrc.One[SrcBits - 1])
11923 Known.One.setLowBits(NumElts);
11924 else if (KnownSrc.Zero[SrcBits - 1])
11925 Known.Zero.setLowBits(NumElts);
11926
11927 // Attempt to avoid multi-use ops if we don't need anything from it.
11928 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
11929 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
11930 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT, Operand: NewSrc));
11931 return false;
11932 }
11933 }
11934
11935 return TargetLowering::SimplifyDemandedBitsForTargetNode(
11936 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
11937}
11938
11939bool LoongArchTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
11940 unsigned Opc = VecOp.getOpcode();
11941
11942 // Assume target opcodes can't be scalarized.
11943 // TODO - do we have any exceptions?
11944 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
11945 return false;
11946
11947 // If the vector op is not supported, try to convert to scalar.
11948 EVT VecVT = VecOp.getValueType();
11949 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
11950 return true;
11951
11952 // If the vector op is supported, but the scalar op is not, the transform may
11953 // not be worthwhile.
11954 EVT ScalarVT = VecVT.getScalarType();
11955 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
11956}
11957
11958TargetLowering::ExtractSubvectorCost
11959LoongArchTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
11960 unsigned Index) const {
11961 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
11962 return ExtractSubvectorCost::Expensive;
11963
11964 // Extract a 128-bit subvector from index 0 of a 256-bit vector is free.
11965 if (Index == 0)
11966 return ExtractSubvectorCost::Free;
11967 return ExtractSubvectorCost::Expensive;
11968}
11969
11970bool LoongArchTargetLowering::isExtractVecEltCheap(EVT VT,
11971 unsigned Index) const {
11972 EVT EltVT = VT.getScalarType();
11973
11974 // Extract a scalar FP value from index 0 of a vector is free.
11975 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
11976}
11977
11978bool LoongArchTargetLowering::hasInlineStackProbe(
11979 const MachineFunction &MF) const {
11980
11981 // If the function specifically requests inline stack probes, emit them.
11982 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
11983 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
11984 "inline-asm";
11985
11986 return false;
11987}
11988
11989unsigned LoongArchTargetLowering::getStackProbeSize(const MachineFunction &MF,
11990 Align StackAlign) const {
11991 // The default stack probe size is 4096 if the function has no
11992 // stack-probe-size attribute.
11993 const Function &Fn = MF.getFunction();
11994 unsigned StackProbeSize =
11995 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
11996 // Round down to the stack alignment.
11997 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
11998 return StackProbeSize ? StackProbeSize : StackAlign.value();
11999}
12000
12001SDValue
12002LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
12003 SelectionDAG &DAG) const {
12004 MachineFunction &MF = DAG.getMachineFunction();
12005 if (!hasInlineStackProbe(MF))
12006 return SDValue();
12007
12008 const MVT GRLenVT = Subtarget.getGRLenVT();
12009 // Get the inputs.
12010 SDValue Chain = Op.getOperand(i: 0);
12011 SDValue Size = Op.getOperand(i: 1);
12012
12013 const MaybeAlign Align =
12014 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
12015 const SDLoc dl(Op);
12016 const EVT VT = Op.getValueType();
12017
12018 // Construct the new SP value in a GPR.
12019 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: LoongArch::R3, VT: GRLenVT);
12020 Chain = SP.getValue(R: 1);
12021 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: GRLenVT, N1: SP, N2: Size);
12022 if (Align)
12023 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
12024 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
12025
12026 // Set the real SP to the new value with a probing loop.
12027 Chain = DAG.getNode(Opcode: LoongArchISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
12028 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
12029}
12030
12031MachineBasicBlock *
12032LoongArchTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
12033 MachineBasicBlock *MBB) const {
12034 MachineFunction &MF = *MBB->getParent();
12035 MachineBasicBlock::iterator MBBI = MI.getIterator();
12036 DebugLoc DL = MBB->findDebugLoc(MBBI);
12037 const Register TargetReg = MI.getOperand(i: 0).getReg();
12038
12039 const LoongArchInstrInfo *TII = Subtarget.getInstrInfo();
12040 const bool IsLA64 = Subtarget.is64Bit();
12041 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
12042 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
12043 const uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
12044
12045 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
12046 MachineBasicBlock *const LoopTestMBB =
12047 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12048 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
12049 MachineBasicBlock *const ExitMBB =
12050 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12051 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
12052 const Register SPReg = LoongArch::R3;
12053 const Register ScratchReg =
12054 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
12055
12056 // ScratchReg = ProbeSize
12057 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
12058
12059 // LoopTest:
12060 // sub.{w/d} $sp, $sp, ScratchReg
12061 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12062 MCID: TII->get(Opcode: IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W), DestReg: SPReg)
12063 .addReg(RegNo: SPReg)
12064 .addReg(RegNo: ScratchReg);
12065
12066 // st.{w/d} $zero, $sp, 0
12067 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12068 MCID: TII->get(Opcode: IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
12069 .addReg(RegNo: LoongArch::R0)
12070 .addReg(RegNo: SPReg)
12071 .addImm(Val: 0);
12072
12073 // bltu TargetReg, $sp, LoopTest
12074 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::BLTU))
12075 .addReg(RegNo: TargetReg)
12076 .addReg(RegNo: SPReg)
12077 .addMBB(MBB: LoopTestMBB);
12078
12079 // move $sp, TargetReg
12080 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::OR), DestReg: SPReg)
12081 .addReg(RegNo: TargetReg)
12082 .addReg(RegNo: LoongArch::R0);
12083
12084 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
12085 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
12086
12087 LoopTestMBB->addSuccessor(Succ: ExitMBB);
12088 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
12089 MBB->addSuccessor(Succ: LoopTestMBB);
12090
12091 MI.eraseFromParent();
12092 MF.getInfo<LoongArchMachineFunctionInfo>()->setDynamicAllocation();
12093 return ExitMBB->begin()->getParent();
12094}
12095