1//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that LoongArch uses to lower LLVM code into
10// a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LoongArchISelLowering.h"
15#include "LoongArch.h"
16#include "LoongArchMachineFunctionInfo.h"
17#include "LoongArchRegisterInfo.h"
18#include "LoongArchSelectionDAGInfo.h"
19#include "LoongArchSubtarget.h"
20#include "MCTargetDesc/LoongArchBaseInfo.h"
21#include "MCTargetDesc/LoongArchMCTargetDesc.h"
22#include "MCTargetDesc/LoongArchMatInt.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/RuntimeLibcallUtil.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/IntrinsicsLoongArch.h"
33#include "llvm/Support/CodeGen.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include <llvm/Analysis/VectorUtils.h>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "loongarch-isel-lowering"
43
44STATISTIC(NumTailCalls, "Number of tail calls");
45
46enum MaterializeFPImm {
47 NoMaterializeFPImm = 0,
48 MaterializeFPImm2Ins = 2,
49 MaterializeFPImm3Ins = 3,
50 MaterializeFPImm4Ins = 4,
51 MaterializeFPImm5Ins = 5,
52 MaterializeFPImm6Ins = 6
53};
54
55static cl::opt<MaterializeFPImm> MaterializeFPImmInsNum(
56 "loongarch-materialize-float-imm", cl::Hidden,
57 cl::desc("Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
60 cl::init(Val: MaterializeFPImm3Ins),
61 cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"),
62 clEnumValN(MaterializeFPImm2Ins, "2",
63 "Materialize FP immediate within 2 instructions"),
64 clEnumValN(MaterializeFPImm3Ins, "3",
65 "Materialize FP immediate within 3 instructions"),
66 clEnumValN(MaterializeFPImm4Ins, "4",
67 "Materialize FP immediate within 4 instructions"),
68 clEnumValN(MaterializeFPImm5Ins, "5",
69 "Materialize FP immediate within 5 instructions"),
70 clEnumValN(MaterializeFPImm6Ins, "6",
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
73
74static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
75 cl::desc("Trap on integer division by zero."),
76 cl::init(Val: false));
77
78LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
79 const LoongArchSubtarget &STI)
80 : TargetLowering(TM, STI), Subtarget(STI) {
81
82 MVT GRLenVT = Subtarget.getGRLenVT();
83
84 // Set up the register classes.
85
86 addRegisterClass(VT: GRLenVT, RC: &LoongArch::GPRRegClass);
87 if (Subtarget.hasBasicF())
88 addRegisterClass(VT: MVT::f32, RC: &LoongArch::FPR32RegClass);
89 if (Subtarget.hasBasicD())
90 addRegisterClass(VT: MVT::f64, RC: &LoongArch::FPR64RegClass);
91
92 static const MVT::SimpleValueType LSXVTs[] = {
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
94 static const MVT::SimpleValueType LASXVTs[] = {
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
96
97 if (Subtarget.hasExtLSX())
98 for (MVT VT : LSXVTs)
99 addRegisterClass(VT, RC: &LoongArch::LSX128RegClass);
100
101 if (Subtarget.hasExtLASX())
102 for (MVT VT : LASXVTs)
103 addRegisterClass(VT, RC: &LoongArch::LASX256RegClass);
104
105 // Set operations for LA32 and LA64.
106
107 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: GRLenVT,
108 MemVT: MVT::i1, Action: Promote);
109
110 setOperationAction(Op: ISD::SHL_PARTS, VT: GRLenVT, Action: Custom);
111 setOperationAction(Op: ISD::SRA_PARTS, VT: GRLenVT, Action: Custom);
112 setOperationAction(Op: ISD::SRL_PARTS, VT: GRLenVT, Action: Custom);
113 setOperationAction(Op: ISD::FP_TO_SINT, VT: GRLenVT, Action: Custom);
114 if (Subtarget.hasBasicF())
115 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT: GRLenVT, Action: Custom);
116 setOperationAction(Op: ISD::ROTL, VT: GRLenVT, Action: Expand);
117 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Expand);
118
119 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
120 ISD::JumpTable, ISD::GlobalTLSAddress},
121 VT: GRLenVT, Action: Custom);
122
123 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: GRLenVT, Action: Custom);
124
125 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: GRLenVT, Action: Custom);
126 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
127 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
128 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
129
130 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
131 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
132
133 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
134 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
135 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
136
137 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
138
139 // BITREV/REVB requires the 32S feature.
140 if (STI.has32S()) {
141 // Expand bitreverse.i16 with native-width bitrev and shift for now, before
142 // we get to know which of sll and revb.2h is faster.
143 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
144 setOperationAction(Op: ISD::BITREVERSE, VT: GRLenVT, Action: Legal);
145
146 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
147 // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
148 // and i32 could still be byte-swapped relatively cheaply.
149 setOperationAction(Op: ISD::BSWAP, VT: MVT::i16, Action: Custom);
150 } else {
151 setOperationAction(Op: ISD::BSWAP, VT: GRLenVT, Action: Expand);
152 setOperationAction(Op: ISD::CTTZ, VT: GRLenVT, Action: Expand);
153 setOperationAction(Op: ISD::CTLZ, VT: GRLenVT, Action: Expand);
154 setOperationAction(Op: ISD::ROTR, VT: GRLenVT, Action: Expand);
155 setOperationAction(Op: ISD::SELECT, VT: GRLenVT, Action: Custom);
156 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
157 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
158 }
159
160 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
161 setOperationAction(Op: ISD::BR_CC, VT: GRLenVT, Action: Expand);
162 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
163 setOperationAction(Op: ISD::SELECT_CC, VT: GRLenVT, Action: Expand);
164 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
165 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: GRLenVT, Action: Expand);
166
167 setOperationAction(Op: ISD::FP_TO_UINT, VT: GRLenVT, Action: Custom);
168 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Expand);
169
170 // Set operations for LA64 only.
171
172 if (Subtarget.is64Bit()) {
173 setOperationAction(Op: ISD::ADD, VT: MVT::i32, Action: Custom);
174 setOperationAction(Op: ISD::SUB, VT: MVT::i32, Action: Custom);
175 setOperationAction(Op: ISD::SHL, VT: MVT::i32, Action: Custom);
176 setOperationAction(Op: ISD::SRA, VT: MVT::i32, Action: Custom);
177 setOperationAction(Op: ISD::SRL, VT: MVT::i32, Action: Custom);
178 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
179 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
180 setOperationAction(Op: ISD::ROTR, VT: MVT::i32, Action: Custom);
181 setOperationAction(Op: ISD::ROTL, VT: MVT::i32, Action: Custom);
182 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Custom);
183 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Custom);
184 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
185 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i32, Action: Custom);
186 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i32, Action: Custom);
187 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
188 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
189 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i32, Action: Custom);
190
191 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Custom);
192 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Custom);
193 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: MVT::i32,
194 Action: Custom);
195 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
196 }
197
198 // Set operations for LA32 only.
199
200 if (!Subtarget.is64Bit()) {
201 setOperationAction(Op: ISD::READ_REGISTER, VT: MVT::i64, Action: Custom);
202 setOperationAction(Op: ISD::WRITE_REGISTER, VT: MVT::i64, Action: Custom);
203 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i64, Action: Custom);
204 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
205 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i64, Action: Custom);
206 if (Subtarget.hasBasicD())
207 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
208 }
209
210 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
211
212 static const ISD::CondCode FPCCToExpand[] = {
213 ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
214 ISD::SETGE, ISD::SETNE, ISD::SETGT};
215
216 // Set operations for 'F' feature.
217
218 if (Subtarget.hasBasicF()) {
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
220 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
222 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
223 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
224
225 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
226 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
227 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
228 setOperationAction(Op: ISD::FMA, VT: MVT::f32, Action: Legal);
229 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
230 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
231 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
232 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
233 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
234 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
235 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
236 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Legal);
237 setOperationAction(Op: ISD::FSIN, VT: MVT::f32, Action: Expand);
238 setOperationAction(Op: ISD::FCOS, VT: MVT::f32, Action: Expand);
239 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
240 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Expand);
241 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
242 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32,
243 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
244 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32,
245 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
246 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
247 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
248 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
249 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
250 setOperationAction(Op: ISD::GET_ROUNDING, VT: GRLenVT, Action: Custom);
251
252 if (Subtarget.is64Bit())
253 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
254
255 if (!Subtarget.hasBasicD()) {
256 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
257 if (Subtarget.is64Bit()) {
258 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
259 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
260 }
261 }
262 }
263
264 // Set operations for 'D' feature.
265
266 if (Subtarget.hasBasicD()) {
267 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
268 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
269 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
271 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
272 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
273 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
274
275 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
276 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
277 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
278 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
279 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
280 setOperationAction(Op: ISD::FMA, VT: MVT::f64, Action: Legal);
281 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
282 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
283 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
284 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
285 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
286 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Legal);
287 setOperationAction(Op: ISD::FSIN, VT: MVT::f64, Action: Expand);
288 setOperationAction(Op: ISD::FCOS, VT: MVT::f64, Action: Expand);
289 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
290 setOperationAction(Op: ISD::FPOW, VT: MVT::f64, Action: Expand);
291 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
292 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
293 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64,
294 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
295 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
296 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
297 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
298
299 if (Subtarget.is64Bit())
300 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
301 }
302
303 // Set operations for 'LSX' feature.
304
305 if (Subtarget.hasExtLSX()) {
306 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
307 // Expand all truncating stores and extending loads.
308 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
309 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
310 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
311 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
312 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
313 }
314 // By default everything must be expanded. Then we will selectively turn
315 // on ones that can be effectively codegen'd.
316 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
317 setOperationAction(Op, VT, Action: Expand);
318 }
319
320 for (MVT VT : LSXVTs) {
321 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
322 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
323 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
324
325 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
326 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
327 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
328
329 setOperationAction(Op: ISD::SETCC, VT, Action: Legal);
330 setOperationAction(Op: ISD::VSELECT, VT, Action: Custom);
331 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
332 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT, Action: Legal);
333 }
334 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
335 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
336 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
337 Action: Legal);
338 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
339 VT, Action: Legal);
340 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
341 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
342 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
343 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
344 setCondCodeAction(
345 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
346 Action: Expand);
347 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
348 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
349 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
350 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
351 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
352 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
353 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
354 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
355 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
356 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
357 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
358 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
359 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
360 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
361 }
362 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
363 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
364 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
365 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
366 for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
367 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Action: Legal);
368 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
369 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT, Action: Custom);
370 }
371 setOperationAction(Op: ISD::UINT_TO_FP, VT: GRLenVT, Action: Custom);
372 for (MVT VT : {MVT::v4f32, MVT::v2f64}) {
373 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
374 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
375 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
376 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
377 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
378 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
379 ISD::SETUGE, ISD::SETUGT},
380 VT, Action: Expand);
381 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
382 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
383 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
384 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
385 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
386 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
387 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
388 }
389 setOperationAction(Op: ISD::CTPOP, VT: GRLenVT, Action: Legal);
390 setOperationAction(Ops: ISD::FCEIL, VTs: {MVT::f32, MVT::f64}, Action: Legal);
391 setOperationAction(Ops: ISD::FFLOOR, VTs: {MVT::f32, MVT::f64}, Action: Legal);
392 setOperationAction(Ops: ISD::FTRUNC, VTs: {MVT::f32, MVT::f64}, Action: Legal);
393 setOperationAction(Ops: ISD::FROUNDEVEN, VTs: {MVT::f32, MVT::f64}, Action: Legal);
394
395 for (MVT VT :
396 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
397 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
398 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Custom);
399 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
400 setOperationAction(Op: ISD::VECREDUCE_AND, VT, Action: Custom);
401 setOperationAction(Op: ISD::VECREDUCE_OR, VT, Action: Custom);
402 setOperationAction(Op: ISD::VECREDUCE_XOR, VT, Action: Custom);
403 setOperationAction(Op: ISD::VECREDUCE_SMAX, VT, Action: Custom);
404 setOperationAction(Op: ISD::VECREDUCE_SMIN, VT, Action: Custom);
405 setOperationAction(Op: ISD::VECREDUCE_UMAX, VT, Action: Custom);
406 setOperationAction(Op: ISD::VECREDUCE_UMIN, VT, Action: Custom);
407 }
408 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v2f32, Action: Custom);
409 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
410 // We want to legalize this to an f64 load rather than an i64 load.
411 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Custom);
412 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
413 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
414 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
415 MVT::v16i64})
416 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Custom);
417 }
418
419 // Set operations for 'LASX' feature.
420
421 if (Subtarget.hasExtLASX()) {
422 for (MVT VT : LASXVTs) {
423 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Legal);
424 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
425 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
426
427 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Custom);
428 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Custom);
429 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
430 setOperationAction(Op: ISD::CONCAT_VECTORS, VT, Action: Custom);
431 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT, Action: Legal);
432
433 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
434 setOperationAction(Op: ISD::VSELECT, VT, Action: Custom);
435 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
436 }
437 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
438 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT, Action: Legal);
439 setOperationAction(Ops: {ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,
440 Action: Legal);
441 setOperationAction(Ops: {ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},
442 VT, Action: Legal);
443 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Legal);
444 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL}, VT, Action: Legal);
445 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT, Action: Legal);
446 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Legal);
447 setCondCodeAction(
448 CCs: {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,
449 Action: Expand);
450 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
451 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
452 setOperationAction(Op: ISD::ABDS, VT, Action: Legal);
453 setOperationAction(Op: ISD::ABDU, VT, Action: Legal);
454 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
455 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
456 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
457 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
458 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
459 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
460 setOperationAction(Op: ISD::ROTR, VT, Action: Custom);
461 setOperationAction(Op: ISD::AVGFLOORS, VT, Action: Legal);
462 setOperationAction(Op: ISD::AVGFLOORU, VT, Action: Legal);
463 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
464 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
465 }
466 for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
467 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Custom);
468 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
469 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
470 for (MVT VT : {MVT::v8i32, MVT::v4i64})
471 setOperationAction(Op: ISD::FP_TO_SINT_SAT, VT, Action: Custom);
472 for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
473 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Action: Legal);
474 setOperationAction(Op: ISD::SINT_TO_FP, VT, Action: Legal);
475 setOperationAction(Op: ISD::UINT_TO_FP, VT, Action: Custom);
476 }
477 for (MVT VT : {MVT::v8f32, MVT::v4f64}) {
478 setOperationAction(Ops: {ISD::FADD, ISD::FSUB}, VT, Action: Legal);
479 setOperationAction(Ops: {ISD::FMUL, ISD::FDIV}, VT, Action: Legal);
480 setOperationAction(Op: ISD::FMA, VT, Action: Legal);
481 setOperationAction(Op: ISD::FSQRT, VT, Action: Legal);
482 setOperationAction(Op: ISD::FNEG, VT, Action: Legal);
483 setCondCodeAction(CCs: {ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,
484 ISD::SETUGE, ISD::SETUGT},
485 VT, Action: Expand);
486 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Legal);
487 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
488 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
489 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
490 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
491 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
492 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
493 }
494 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v4f32, Action: Custom);
495 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f64, Action: Custom);
496 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
497 setOperationAction(Op: ISD::SIGN_EXTEND, VT, Action: Legal);
498 setOperationAction(Op: ISD::ZERO_EXTEND, VT, Action: Legal);
499 setOperationAction(Op: ISD::ANY_EXTEND, VT, Action: Custom);
500 }
501 for (MVT VT :
502 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
503 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Legal);
504 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Legal);
505 }
506 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
507 setOperationAction(Op: ISD::TRUNCATE, VT, Action: Legal);
508 }
509
510 // Set DAG combine for LA32 and LA64.
511 if (Subtarget.hasBasicF()) {
512 setTargetDAGCombine(ISD::SINT_TO_FP);
513 }
514
515 setTargetDAGCombine(ISD::AND);
516 setTargetDAGCombine(ISD::OR);
517 setTargetDAGCombine(ISD::XOR);
518 setTargetDAGCombine(ISD::SRL);
519 setTargetDAGCombine(ISD::SETCC);
520 setTargetDAGCombine(ISD::SUB);
521 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
522
523 // On targets with the 32S feature, `select` is expanded into
524 // maskeqz + masknez + or (3 instructions), which is more expensive than
525 // on most other architectures where a single cmov-like instruction
526 // suffices. Enable a combine that can turn
527 // select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
528 // select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
529 // for binop in {add, or, xor, sub}, replacing the 3-insn select (plus
530 // the original binop) with a single mask instruction plus the binop.
531 if (Subtarget.has32S())
532 setTargetDAGCombine(ISD::SELECT);
533
534 // Set DAG combine for 'LSX' feature.
535
536 if (Subtarget.hasExtLSX()) {
537 setTargetDAGCombine(ISD::ADD);
538 setTargetDAGCombine(ISD::SUB);
539 setTargetDAGCombine(ISD::SHL);
540 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
541 setTargetDAGCombine(ISD::BITCAST);
542 setTargetDAGCombine(ISD::VSELECT);
543 setTargetDAGCombine(ISD::FP_TO_SINT);
544 setTargetDAGCombine(ISD::FP_TO_UINT);
545 setTargetDAGCombine(ISD::UINT_TO_FP);
546 setTargetDAGCombine(ISD::ZERO_EXTEND);
547 setTargetDAGCombine(ISD::SIGN_EXTEND);
548 setTargetDAGCombine(ISD::MUL);
549 }
550
551 // Set DAG combine for 'LASX' feature.
552 if (Subtarget.hasExtLASX()) {
553 setTargetDAGCombine(ISD::ANY_EXTEND);
554 setTargetDAGCombine(ISD::CONCAT_VECTORS);
555 }
556
557 // Compute derived properties from the register classes.
558 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
559
560 setStackPointerRegisterToSaveRestore(LoongArch::R3);
561
562 setBooleanContents(ZeroOrOneBooleanContent);
563 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
564
565 setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());
566
567 setMinCmpXchgSizeInBits(32);
568
569 // Function alignments.
570 setMinFunctionAlignment(Align(4));
571 // Set preferred alignments.
572 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
573 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
574 setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
575
576 // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
577 if (Subtarget.hasLAMCAS())
578 setMinCmpXchgSizeInBits(8);
579
580 if (Subtarget.hasSCQ()) {
581 setMaxAtomicSizeInBitsSupported(128);
582 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i128, Action: Custom);
583 }
584
585 // Disable strict node mutation.
586 IsStrictFPEnabled = true;
587}
588
589bool LoongArchTargetLowering::isOffsetFoldingLegal(
590 const GlobalAddressSDNode *GA) const {
591 // In order to maximise the opportunity for common subexpression elimination,
592 // keep a separate ADD node for the global address offset instead of folding
593 // it in the global address node. Later peephole optimisations may choose to
594 // fold it back in when profitable.
595 return false;
596}
597
598SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
599 SelectionDAG &DAG) const {
600 switch (Op.getOpcode()) {
601 case ISD::ATOMIC_FENCE:
602 return lowerATOMIC_FENCE(Op, DAG);
603 case ISD::EH_DWARF_CFA:
604 return lowerEH_DWARF_CFA(Op, DAG);
605 case ISD::GlobalAddress:
606 return lowerGlobalAddress(Op, DAG);
607 case ISD::GlobalTLSAddress:
608 return lowerGlobalTLSAddress(Op, DAG);
609 case ISD::INTRINSIC_WO_CHAIN:
610 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
611 case ISD::INTRINSIC_W_CHAIN:
612 return lowerINTRINSIC_W_CHAIN(Op, DAG);
613 case ISD::INTRINSIC_VOID:
614 return lowerINTRINSIC_VOID(Op, DAG);
615 case ISD::BlockAddress:
616 return lowerBlockAddress(Op, DAG);
617 case ISD::JumpTable:
618 return lowerJumpTable(Op, DAG);
619 case ISD::SHL_PARTS:
620 return lowerShiftLeftParts(Op, DAG);
621 case ISD::SRA_PARTS:
622 return lowerShiftRightParts(Op, DAG, IsSRA: true);
623 case ISD::SRL_PARTS:
624 return lowerShiftRightParts(Op, DAG, IsSRA: false);
625 case ISD::ConstantPool:
626 return lowerConstantPool(Op, DAG);
627 case ISD::FP_TO_SINT:
628 return lowerFP_TO_SINT(Op, DAG);
629 case ISD::FP_TO_UINT:
630 return lowerFP_TO_UINT(Op, DAG);
631 case ISD::FP_TO_SINT_SAT:
632 return lowerFP_TO_SINT_SAT(Op, DAG);
633 case ISD::BITCAST:
634 return lowerBITCAST(Op, DAG);
635 case ISD::UINT_TO_FP:
636 return lowerUINT_TO_FP(Op, DAG);
637 case ISD::SINT_TO_FP:
638 return lowerSINT_TO_FP(Op, DAG);
639 case ISD::VASTART:
640 return lowerVASTART(Op, DAG);
641 case ISD::FRAMEADDR:
642 return lowerFRAMEADDR(Op, DAG);
643 case ISD::RETURNADDR:
644 return lowerRETURNADDR(Op, DAG);
645 case ISD::SET_ROUNDING:
646 return lowerSET_ROUNDING(Op, DAG);
647 case ISD::GET_ROUNDING:
648 return lowerGET_ROUNDING(Op, DAG);
649 case ISD::WRITE_REGISTER:
650 return lowerWRITE_REGISTER(Op, DAG);
651 case ISD::INSERT_VECTOR_ELT:
652 return lowerINSERT_VECTOR_ELT(Op, DAG);
653 case ISD::EXTRACT_VECTOR_ELT:
654 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
655 case ISD::BUILD_VECTOR:
656 return lowerBUILD_VECTOR(Op, DAG);
657 case ISD::CONCAT_VECTORS:
658 return lowerCONCAT_VECTORS(Op, DAG);
659 case ISD::VECTOR_SHUFFLE:
660 return lowerVECTOR_SHUFFLE(Op, DAG);
661 case ISD::BITREVERSE:
662 return lowerBITREVERSE(Op, DAG);
663 case ISD::SCALAR_TO_VECTOR:
664 return lowerSCALAR_TO_VECTOR(Op, DAG);
665 case ISD::PREFETCH:
666 return lowerPREFETCH(Op, DAG);
667 case ISD::SELECT:
668 return lowerSELECT(Op, DAG);
669 case ISD::BRCOND:
670 return lowerBRCOND(Op, DAG);
671 case ISD::FP_TO_FP16:
672 return lowerFP_TO_FP16(Op, DAG);
673 case ISD::FP16_TO_FP:
674 return lowerFP16_TO_FP(Op, DAG);
675 case ISD::FP_TO_BF16:
676 return lowerFP_TO_BF16(Op, DAG);
677 case ISD::BF16_TO_FP:
678 return lowerBF16_TO_FP(Op, DAG);
679 case ISD::VECREDUCE_ADD:
680 return lowerVECREDUCE_ADD(Op, DAG);
681 case ISD::ROTL:
682 case ISD::ROTR:
683 return lowerRotate(Op, DAG);
684 case ISD::VECREDUCE_AND:
685 case ISD::VECREDUCE_OR:
686 case ISD::VECREDUCE_XOR:
687 case ISD::VECREDUCE_SMAX:
688 case ISD::VECREDUCE_SMIN:
689 case ISD::VECREDUCE_UMAX:
690 case ISD::VECREDUCE_UMIN:
691 return lowerVECREDUCE(Op, DAG);
692 case ISD::ConstantFP:
693 return lowerConstantFP(Op, DAG);
694 case ISD::SETCC:
695 return lowerSETCC(Op, DAG);
696 case ISD::FP_ROUND:
697 return lowerFP_ROUND(Op, DAG);
698 case ISD::FP_EXTEND:
699 return lowerFP_EXTEND(Op, DAG);
700 case ISD::SIGN_EXTEND_VECTOR_INREG:
701 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
702 case ISD::DYNAMIC_STACKALLOC:
703 return lowerDYNAMIC_STACKALLOC(Op, DAG);
704 case ISD::ANY_EXTEND:
705 return lowerANY_EXTEND(Op, DAG);
706 case ISD::VSELECT:
707 return lowerVSELECT(Op, DAG);
708 }
709 return SDValue();
710}
711
712// Helper to attempt to return a cheaper, bit-inverted version of \p V.
713static SDValue isNOT(SDValue V, SelectionDAG &DAG) {
714 // TODO: don't always ignore oneuse constraints.
715 V = peekThroughBitcasts(V);
716 EVT VT = V.getValueType();
717
718 // Match not(xor X, -1) -> X.
719 if (V.getOpcode() == ISD::XOR &&
720 (ISD::isBuildVectorAllOnes(V: V.getOperand(i: 1)) ||
721 isAllOnesConstant(V: V.getOperand(i: 1))))
722 return V.getOperand(i: 0);
723
724 // Match not(extract_subvector(not(X)) -> extract_subvector(X).
725 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
726 (isNullConstant(V: V.getOperand(i: 1)) || V.getOperand(i: 0).hasOneUse())) {
727 if (SDValue Not = isNOT(V: V.getOperand(i: 0), DAG)) {
728 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
729 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Not), VT, N1: Not,
730 N2: V.getOperand(i: 1));
731 }
732 }
733
734 // Match not(SplatVector(not(X)) -> SplatVector(X).
735 if (V.getOpcode() == ISD::BUILD_VECTOR) {
736 if (SDValue SplatValue =
737 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
738 if (!V->isOnlyUserOf(N: SplatValue.getNode()))
739 return SDValue();
740
741 if (SDValue Not = isNOT(V: SplatValue, DAG)) {
742 Not = DAG.getBitcast(VT: V.getOperand(i: 0).getValueType(), V: Not);
743 return DAG.getSplat(VT, DL: SDLoc(Not), Op: Not);
744 }
745 }
746 }
747
748 // Match not(or(not(X),not(Y))) -> and(X, Y).
749 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
750 V.getOperand(i: 0).hasOneUse() && V.getOperand(i: 1).hasOneUse()) {
751 // TODO: Handle cases with single NOT operand -> VANDN
752 if (SDValue Op1 = isNOT(V: V.getOperand(i: 1), DAG))
753 if (SDValue Op0 = isNOT(V: V.getOperand(i: 0), DAG))
754 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(V), VT, N1: DAG.getBitcast(VT, V: Op0),
755 N2: DAG.getBitcast(VT, V: Op1));
756 }
757
758 // TODO: Add more matching patterns. Such as,
759 // not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
760 // not(slt(C, X)) -> slt(X - 1, C)
761 return SDValue();
762}
763
764// Combine two ISD::FP_ROUND / LoongArchISD::VFCVT nodes with same type to
765// LoongArchISD::VFCVT. For example:
766// x1 = fp_round x, 0
767// y1 = fp_round y, 0
768// z = concat_vectors x1, y1
769// Or
770// x1 = LoongArch::VFCVT undef, x
771// y1 = LoongArch::VFCVT undef, y
772// z = LoongArchISD::VPACKEV y1, x1; or LoongArchISD::VPERMI y1, x1, 68
773// can be combined to:
774// z = LoongArch::VFCVT y, x
775static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG,
776 const LoongArchSubtarget &Subtarget) {
777 assert(((N->getOpcode() == ISD::CONCAT_VECTORS && N->getNumOperands() == 2) ||
778 (N->getOpcode() == LoongArchISD::VPACKEV) ||
779 (N->getOpcode() == LoongArchISD::VPERMI)) &&
780 "Invalid Node");
781
782 SDValue Op0 = peekThroughBitcasts(V: N->getOperand(Num: 0));
783 SDValue Op1 = peekThroughBitcasts(V: N->getOperand(Num: 1));
784 unsigned Opcode0 = Op0.getOpcode();
785 unsigned Opcode1 = Op1.getOpcode();
786 if (Opcode0 != Opcode1)
787 return SDValue();
788
789 if (Opcode0 != ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
790 return SDValue();
791
792 // Check if two nodes have only one use.
793 if (!Op0.hasOneUse() || !Op1.hasOneUse())
794 return SDValue();
795
796 EVT VT = N.getValueType();
797 EVT SVT0 = Op0.getValueType();
798 EVT SVT1 = Op1.getValueType();
799 // Check if two nodes have the same result type.
800 if (SVT0 != SVT1)
801 return SDValue();
802
803 // Check if two nodes have the same operand type.
804 EVT SSVT0 = Op0.getOperand(i: 0).getValueType();
805 EVT SSVT1 = Op1.getOperand(i: 0).getValueType();
806 if (SSVT0 != SSVT1)
807 return SDValue();
808
809 if (N->getOpcode() == ISD::CONCAT_VECTORS && Opcode0 == ISD::FP_ROUND) {
810 if (Subtarget.hasExtLASX() && VT.is256BitVector() && SVT0 == MVT::v4f32 &&
811 SSVT0 == MVT::v4f64) {
812 // A vector_shuffle is required in the final step, as xvfcvt instruction
813 // operates on each 128-bit segament as a lane.
814 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v8f32,
815 N1: Op1.getOperand(i: 0), N2: Op0.getOperand(i: 0));
816 SDValue Undef = DAG.getUNDEF(VT: Res.getValueType());
817 // After VFCVT, the high part of Res comes from the high parts of Op0 and
818 // Op1, and the low part comes from the low parts of Op0 and Op1. However,
819 // the desired order requires Op0 to fully occupy the lower half and Op1
820 // the upper half of Res. The Mask reorders the elements of Res to achieve
821 // this:
822 // - The first four elements (0, 1, 4, 5) come from Op0.
823 // - The next four elements (2, 3, 6, 7) come from Op1.
824 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
825 Res = DAG.getVectorShuffle(VT: Res.getValueType(), dl: DL, N1: Res, N2: Undef, Mask);
826 return DAG.getBitcast(VT, V: Res);
827 }
828 }
829
830 if ((N->getOpcode() == LoongArchISD::VPACKEV ||
831 N->getOpcode() == LoongArchISD::VPERMI) &&
832 Opcode0 == LoongArchISD::VFCVT) {
833 // For VPACKEV or VPERMI, check if the first operation of VFCVT is undef.
834 if (!Op0.getOperand(i: 0).isUndef() || !Op1.getOperand(i: 0).isUndef())
835 return SDValue();
836
837 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
838 return SDValue();
839
840 if (N->getOpcode() == LoongArchISD::VPACKEV &&
841 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
842 SDValue Res = DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32,
843 N1: Op0.getOperand(i: 1), N2: Op1.getOperand(i: 1));
844 return DAG.getBitcast(VT, V: Res);
845 }
846
847 if (N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
848 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
849 if (Imm != 68)
850 return SDValue();
851 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Op0.getOperand(i: 1),
852 N2: Op1.getOperand(i: 1));
853 }
854 }
855
856 return SDValue();
857}
858
859SDValue LoongArchTargetLowering::lowerFP_ROUND(SDValue Op,
860 SelectionDAG &DAG) const {
861 SDLoc DL(Op);
862 SDValue In = Op.getOperand(i: 0);
863 MVT VT = Op.getSimpleValueType();
864 MVT SVT = In.getSimpleValueType();
865
866 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
867 SDValue Lo, Hi;
868 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: In, DL);
869 return DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT, N1: Hi, N2: Lo);
870 }
871
872 return SDValue();
873}
874
875SDValue LoongArchTargetLowering::lowerFP_EXTEND(SDValue Op,
876 SelectionDAG &DAG) const {
877
878 SDLoc DL(Op);
879 EVT VT = Op.getValueType();
880 SDValue Src = Op->getOperand(Num: 0);
881 EVT SVT = Src.getValueType();
882
883 bool V2F32ToV2F64 =
884 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
885 bool V4F32ToV4F64 =
886 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
887 if (!V2F32ToV2F64 && !V4F32ToV4F64)
888 return SDValue();
889
890 // Check if Op is the high part of vector.
891 auto CheckVecHighPart = [](SDValue Op) {
892 Op = peekThroughBitcasts(V: Op);
893 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
894 SDValue SOp = Op.getOperand(i: 0);
895 EVT SVT = SOp.getValueType();
896 if (!SVT.isVector() || (SVT.getVectorNumElements() % 2 != 0))
897 return SDValue();
898
899 const uint64_t Imm = Op.getConstantOperandVal(i: 1);
900 if (Imm == SVT.getVectorNumElements() / 2)
901 return SOp;
902 return SDValue();
903 }
904 return SDValue();
905 };
906
907 unsigned Opcode;
908 SDValue VFCVTOp;
909 EVT WideOpVT = SVT.getSimpleVT().getDoubleNumVectorElementsVT();
910 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
911
912 // If the operand of ISD::FP_EXTEND comes from the high part of vector,
913 // generate LoongArchISD::VFCVTH, otherwise LoongArchISD::VFCVTL.
914 if (SDValue V = CheckVecHighPart(Src)) {
915 assert(V.getValueSizeInBits() == WideOpVT.getSizeInBits() &&
916 "Unexpected wide vector");
917 Opcode = LoongArchISD::VFCVTH;
918 VFCVTOp = DAG.getBitcast(VT: WideOpVT, V);
919 } else {
920 Opcode = LoongArchISD::VFCVTL;
921 VFCVTOp = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideOpVT,
922 N1: DAG.getUNDEF(VT: WideOpVT), N2: Src, N3: ZeroIdx);
923 }
924
925 // v2f64 = fp_extend v2f32
926 if (V2F32ToV2F64)
927 return DAG.getNode(Opcode, DL, VT, Operand: VFCVTOp);
928
929 // v4f64 = fp_extend v4f32
930 if (V4F32ToV4F64) {
931 // XVFCVT instruction operates on each 128-bit segment as a lane, so a
932 // vector_shuffle is required firstly.
933 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
934 SDValue Res = DAG.getVectorShuffle(VT: WideOpVT, dl: DL, N1: VFCVTOp,
935 N2: DAG.getUNDEF(VT: WideOpVT), Mask);
936 Res = DAG.getNode(Opcode, DL, VT, Operand: Res);
937 return Res;
938 }
939
940 return SDValue();
941}
942
943SDValue LoongArchTargetLowering::lowerVSELECT(SDValue Op,
944 SelectionDAG &DAG) const {
945 SDValue Cond = Op.getOperand(i: 0);
946 SDValue LHS = Op.getOperand(i: 1);
947 SDValue RHS = Op.getOperand(i: 2);
948 MVT VT = Op.getSimpleValueType();
949 SDLoc DL(Op);
950
951 // Try to lower vselect to vector_shuffle. All cases with constant
952 // build_vector condition will be handled. NOTE: On 32-bit platform, vselect
953 // with v2i64/v4i64 type condition will never enter this because of the extra
954 // bitcast. It can be considered separately later.
955 if (ISD::isBuildVectorOfConstantSDNodes(V: Cond)) {
956 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val: Cond.getNode());
957 APInt SplatValue, SplatUndef;
958 unsigned SplatBitSize;
959 bool HasAnyUndefs;
960
961 // If the Cond is a BUILD_VECTOR with splat constants, using a vldi or
962 // constant broadcast instruction and a vbitsel is better.
963 if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
964 /*MinSplatBits=*/8) &&
965 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
966 SplatBitSize == 64))
967 return Op;
968
969 SmallVector<int, 32> Mask;
970 EVT CondVT = Cond.getValueType();
971 int NumElts = CondVT.getVectorNumElements();
972 Mask.resize(N: NumElts, NV: -1);
973 for (int i = 0; i < NumElts; ++i) {
974 SDValue CondOp = Cond.getOperand(i);
975 if (CondOp.isUndef())
976 continue;
977 auto *CondElt = dyn_cast<ConstantSDNode>(Val&: CondOp);
978 if (CondElt->getZExtValue() == 0)
979 Mask[i] = i + NumElts;
980 else
981 Mask[i] = i;
982 }
983
984 return DAG.getVectorShuffle(VT, dl: DL, N1: LHS, N2: RHS, Mask);
985 }
986
987 // Fallback to match patterns in tablegen.
988 return Op;
989}
990
991SDValue LoongArchTargetLowering::lowerConstantFP(SDValue Op,
992 SelectionDAG &DAG) const {
993 EVT VT = Op.getValueType();
994 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Op);
995 const APFloat &FPVal = CFP->getValueAPF();
996 SDLoc DL(CFP);
997
998 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
999 (VT == MVT::f64 && Subtarget.hasBasicD()));
1000
1001 // If value is 0.0 or -0.0, just ignore it.
1002 if (FPVal.isZero())
1003 return SDValue();
1004
1005 // If lsx enabled, use cheaper 'vldi' instruction if possible.
1006 if (isFPImmVLDILegal(Imm: FPVal, VT))
1007 return SDValue();
1008
1009 // The legalizer builds immediates the size of a GPR from their bits. Fall
1010 // back to a load from the constant pool for expensive immediates.
1011 if (VT.getSizeInBits() == Subtarget.getGRLen() ||
1012 !isFPImmCheapInGPR(Imm: FPVal, VT))
1013 return SDValue();
1014
1015 // Construct as integer, and move to float register.
1016 APInt INTVal = FPVal.bitcastToAPInt();
1017 if (VT == MVT::f32) {
1018 assert(Subtarget.is64Bit());
1019 SDValue NewVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64,
1020 Operand: DAG.getConstant(Val: INTVal, DL, VT: MVT::i32));
1021 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT, Operand: NewVal);
1022 }
1023
1024 assert(VT == MVT::f64 && !Subtarget.is64Bit());
1025 SDValue Lo = DAG.getConstant(Val: INTVal.trunc(width: 32), DL, VT: MVT::i32);
1026 SDValue Hi = DAG.getConstant(Val: INTVal.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
1027 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_D_LO_HI, DL, VT, N1: Lo, N2: Hi);
1028}
1029
1030// Ensure SETCC result and operand have the same bit width; isel does not
1031// support mismatched widths.
1032SDValue LoongArchTargetLowering::lowerSETCC(SDValue Op,
1033 SelectionDAG &DAG) const {
1034 SDLoc DL(Op);
1035 EVT ResultVT = Op.getValueType();
1036 EVT OperandVT = Op.getOperand(i: 0).getValueType();
1037
1038 EVT SetCCResultVT =
1039 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT);
1040
1041 if (ResultVT == SetCCResultVT)
1042 return Op;
1043
1044 assert(Op.getOperand(0).getValueType() == Op.getOperand(1).getValueType() &&
1045 "SETCC operands must have the same type!");
1046
1047 SDValue SetCCNode =
1048 DAG.getNode(Opcode: ISD::SETCC, DL, VT: SetCCResultVT, N1: Op.getOperand(i: 0),
1049 N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2));
1050
1051 if (ResultVT.bitsGT(VT: SetCCResultVT))
1052 SetCCNode = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: ResultVT, Operand: SetCCNode);
1053 else if (ResultVT.bitsLT(VT: SetCCResultVT))
1054 SetCCNode = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: SetCCNode);
1055
1056 return SetCCNode;
1057}
1058
1059// Lower SIGN_EXTEND_VECTOR_INREG with VSLLWIL, which sign-extends the low half
1060// of the source elements (and shifts them left by 0).
1061// For example:
1062// %b = sext <8 x i8> %a to <8 x i16>
1063// can be lowered to:
1064// VSLLWIL.H.B vr1, vr1, 0
1065//
1066// VSLLWIL is slower than VILVL and VSRAI, so a 4x extension instead
1067// replicates each source element across its destination element by
1068// interleaving the source with itself twice, then shifts the copy in the top
1069// bits down arithmetically:
1070// %b = sext <4 x i8> %a to <4 x i32>
1071// can be lowered to:
1072// VILVL.B vr1, vr1, vr1
1073// VILVL.H vr1, vr1, vr1
1074// VSRAI.W vr1, vr1, 24
1075//
1076// If every extended source element is already all-ones or all-zeros, e.g. a
1077// comparison result, the interleaves alone produce the sign extension, so
1078// neither VSLLWIL nor the VSRAI is needed:
1079// %b = sext <8 x i8> %mask to <8 x i16>
1080// can be lowered to:
1081// VILVL.B vr1, vr1, vr1
1082SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1083 SDValue Op, SelectionDAG &DAG) const {
1084 SDLoc DL(Op);
1085 SDValue Src = Op.getOperand(i: 0);
1086
1087 // Only whole 128-bit vectors of simple element types can be lowered this
1088 // way; fall back to the default lowering for other sources.
1089 if (!Src.getValueType().isSimple())
1090 return SDValue();
1091 MVT SrcVT = Src.getSimpleValueType();
1092 MVT DstVT = Op.getSimpleValueType();
1093 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
1094 unsigned DstEltBits = DstVT.getScalarSizeInBits();
1095
1096 // Leave larger ratios to the generic shuffle + shift expansion, which only
1097 // needs a single VEXTRINS to move the second element into place.
1098 unsigned Ratio = DstEltBits / SrcEltBits;
1099 if (!SrcVT.is128BitVector() || Ratio > 4)
1100 return SDValue();
1101
1102 // Only the low elements of the source are extended.
1103 APInt DemandedElts = APInt::getLowBitsSet(numBits: SrcVT.getVectorNumElements(),
1104 loBitsSet: DstVT.getVectorNumElements());
1105 bool IsSignSplat = DAG.ComputeNumSignBits(Op: Src, DemandedElts) == SrcEltBits;
1106
1107 if (Ratio == 2 && !IsSignSplat)
1108 return DAG.getNode(Opcode: LoongArchISD::VSLLWIL, DL, VT: DstVT, N1: Src,
1109 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
1110
1111 SDValue Res = Src;
1112 for (unsigned EltBits = SrcEltBits; EltBits < DstEltBits; EltBits *= 2) {
1113 MVT VT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits), NumElements: 128 / EltBits);
1114 Res = DAG.getBitcast(VT, V: Res);
1115 Res = DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: Res, N2: Res);
1116 }
1117 Res = DAG.getBitcast(VT: DstVT, V: Res);
1118 if (IsSignSplat)
1119 return Res;
1120 return DAG.getNode(Opcode: ISD::SRA, DL, VT: DstVT, N1: Res,
1121 N2: DAG.getConstant(Val: DstEltBits - SrcEltBits, DL, VT: DstVT));
1122}
1123
1124// ANY_EXTEND can be replaced by ZERO_EXTEND when LASX is enabled.
1125SDValue LoongArchTargetLowering::lowerANY_EXTEND(SDValue Op,
1126 SelectionDAG &DAG) const {
1127 assert(Subtarget.hasExtLASX());
1128 // We don't have corresponding instrunction for ANY_EXTEND, lowering it to
1129 // ZERO_EXTEND won't break its semantics, while avoid scalar extract/insert.
1130 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: Op.getValueType(),
1131 Operand: Op.getOperand(i: 0));
1132}
1133
1134// Lower vecreduce_add using vhaddw instructions.
1135// For Example:
1136// call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
1137// can be lowered to:
1138// VHADDW_D_W vr0, vr0, vr0
1139// VHADDW_Q_D vr0, vr0, vr0
1140// VPICKVE2GR_D a0, vr0, 0
1141// ADDI_W a0, a0, 0
1142SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
1143 SelectionDAG &DAG) const {
1144
1145 SDLoc DL(Op);
1146 MVT OpVT = Op.getSimpleValueType();
1147 SDValue Val = Op.getOperand(i: 0);
1148
1149 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1150 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1151 unsigned ResBits = OpVT.getScalarSizeInBits();
1152
1153 unsigned LegalVecSize = 128;
1154 bool isLASX256Vector =
1155 Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
1156
1157 // Ensure operand type legal or enable it legal.
1158 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1159 Val = DAG.WidenVector(N: Val, DL);
1160 }
1161
1162 // NumEles is designed for iterations count, v4i32 for LSX
1163 // and v8i32 for LASX should have the same count.
1164 if (isLASX256Vector) {
1165 NumEles /= 2;
1166 LegalVecSize = 256;
1167 }
1168
1169 EleBits *= 2;
1170 for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1171 EleBits = std::min(a: EleBits, b: 64u);
1172 MVT IntTy = MVT::getIntegerVT(BitWidth: EleBits);
1173 MVT VecTy = MVT::getVectorVT(VT: IntTy, NumElements: LegalVecSize / EleBits);
1174 Val = DAG.getNode(Opcode: LoongArchISD::VHADDW, DL, VT: VecTy, N1: Val, N2: Val);
1175 }
1176
1177 if (isLASX256Vector) {
1178 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: Val,
1179 N2: DAG.getConstant(Val: 2, DL, VT: Subtarget.getGRLenVT()));
1180 Val = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::v4i64, N1: Tmp, N2: Val);
1181 }
1182
1183 Val = DAG.getBitcast(VT: MVT::getVectorVT(VT: OpVT, NumElements: LegalVecSize / ResBits), V: Val);
1184 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1185 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
1186}
1187
1188// Lower vecreduce_and/or/xor/[s/u]max/[s/u]min.
1189// For Example:
1190// call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
1191// can be lowered to:
1192// VBSRL_V vr1, vr0, 8
1193// VMAX_W vr0, vr1, vr0
1194// VBSRL_V vr1, vr0, 4
1195// VMAX_W vr0, vr1, vr0
1196// VPICKVE2GR_W a0, vr0, 0
1197// For 256 bit vector, it is illegal and will be spilt into
1198// two 128 bit vector by default then processed by this.
1199SDValue LoongArchTargetLowering::lowerVECREDUCE(SDValue Op,
1200 SelectionDAG &DAG) const {
1201 SDLoc DL(Op);
1202
1203 MVT OpVT = Op.getSimpleValueType();
1204 SDValue Val = Op.getOperand(i: 0);
1205
1206 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1207 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1208
1209 // Ensure operand type legal or enable it legal.
1210 while (!isTypeLegal(VT: Val.getSimpleValueType())) {
1211 Val = DAG.WidenVector(N: Val, DL);
1212 }
1213
1214 unsigned Opcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
1215 MVT VecTy = Val.getSimpleValueType();
1216 MVT GRLenVT = Subtarget.getGRLenVT();
1217
1218 for (int i = NumEles; i > 1; i /= 2) {
1219 SDValue ShiftAmt = DAG.getConstant(Val: i * EleBits / 16, DL, VT: GRLenVT);
1220 SDValue Tmp = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: VecTy, N1: Val, N2: ShiftAmt);
1221 Val = DAG.getNode(Opcode, DL, VT: VecTy, N1: Tmp, N2: Val);
1222 }
1223
1224 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Val,
1225 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
1226}
1227
1228SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
1229 SelectionDAG &DAG) const {
1230 unsigned IsData = Op.getConstantOperandVal(i: 4);
1231
1232 // We don't support non-data prefetch.
1233 // Just preserve the chain.
1234 if (!IsData)
1235 return Op.getOperand(i: 0);
1236
1237 return Op;
1238}
1239
1240SDValue LoongArchTargetLowering::lowerRotate(SDValue Op,
1241 SelectionDAG &DAG) const {
1242 MVT VT = Op.getSimpleValueType();
1243 assert(VT.isVector() && "Unexpected type");
1244
1245 SDLoc DL(Op);
1246 SDValue R = Op.getOperand(i: 0);
1247 SDValue Amt = Op.getOperand(i: 1);
1248 unsigned Opcode = Op.getOpcode();
1249 unsigned EltSizeInBits = VT.getScalarSizeInBits();
1250
1251 auto checkCstSplat = [](SDValue V, APInt &CstSplatValue) {
1252 if (V.getOpcode() != ISD::BUILD_VECTOR)
1253 return false;
1254 if (SDValue SplatValue =
1255 cast<BuildVectorSDNode>(Val: V.getNode())->getSplatValue()) {
1256 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
1257 CstSplatValue = C->getAPIntValue();
1258 return true;
1259 }
1260 }
1261 return false;
1262 };
1263
1264 // Check for constant splat rotation amount.
1265 APInt CstSplatValue;
1266 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1267 bool isROTL = Opcode == ISD::ROTL;
1268
1269 // Check for splat rotate by zero.
1270 if (IsCstSplat && CstSplatValue.urem(RHS: EltSizeInBits) == 0)
1271 return R;
1272
1273 // LoongArch targets always prefer ISD::ROTR.
1274 if (isROTL) {
1275 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
1276 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: R,
1277 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: Amt));
1278 }
1279
1280 // Rotate by a immediate.
1281 if (IsCstSplat) {
1282 // ISD::ROTR: Attemp to rotate by a positive immediate.
1283 SDValue Bits = DAG.getConstant(Val: EltSizeInBits, DL, VT);
1284 if (SDValue Urem =
1285 DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL, VT, Ops: {Amt, Bits}))
1286 return DAG.getNode(Opcode, DL, VT, N1: R, N2: Urem);
1287 }
1288
1289 return Op;
1290}
1291
1292// Return true if Val is equal to (setcc LHS, RHS, CC).
1293// Return false if Val is the inverse of (setcc LHS, RHS, CC).
1294// Otherwise, return std::nullopt.
1295static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
1296 ISD::CondCode CC, SDValue Val) {
1297 assert(Val->getOpcode() == ISD::SETCC);
1298 SDValue LHS2 = Val.getOperand(i: 0);
1299 SDValue RHS2 = Val.getOperand(i: 1);
1300 ISD::CondCode CC2 = Val.getOperandCondCode(i: 2);
1301
1302 if (LHS == LHS2 && RHS == RHS2) {
1303 if (CC == CC2)
1304 return true;
1305 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1306 return false;
1307 } else if (LHS == RHS2 && RHS == LHS2) {
1308 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
1309 if (CC == CC2)
1310 return true;
1311 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
1312 return false;
1313 }
1314
1315 return std::nullopt;
1316}
1317
1318static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
1319 const LoongArchSubtarget &Subtarget) {
1320 SDValue CondV = N->getOperand(Num: 0);
1321 SDValue TrueV = N->getOperand(Num: 1);
1322 SDValue FalseV = N->getOperand(Num: 2);
1323 MVT VT = N->getSimpleValueType(ResNo: 0);
1324 SDLoc DL(N);
1325
1326 // (select c, -1, y) -> -c | y
1327 if (isAllOnesConstant(V: TrueV)) {
1328 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1329 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1330 }
1331 // (select c, y, -1) -> (c-1) | y
1332 if (isAllOnesConstant(V: FalseV)) {
1333 SDValue Neg =
1334 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1335 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1336 }
1337
1338 // (select c, 0, y) -> (c-1) & y
1339 if (isNullConstant(V: TrueV)) {
1340 SDValue Neg =
1341 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
1342 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
1343 }
1344 // (select c, y, 0) -> -c & y
1345 if (isNullConstant(V: FalseV)) {
1346 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1347 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
1348 }
1349
1350 // select c, ~x, x --> xor -c, x
1351 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
1352 const APInt &TrueVal = TrueV->getAsAPIntVal();
1353 const APInt &FalseVal = FalseV->getAsAPIntVal();
1354 if (~TrueVal == FalseVal) {
1355 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
1356 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
1357 }
1358 }
1359
1360 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
1361 // when both truev and falsev are also setcc.
1362 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
1363 FalseV.getOpcode() == ISD::SETCC) {
1364 SDValue LHS = CondV.getOperand(i: 0);
1365 SDValue RHS = CondV.getOperand(i: 1);
1366 ISD::CondCode CC = CondV.getOperandCondCode(i: 2);
1367
1368 // (select x, x, y) -> x | y
1369 // (select !x, x, y) -> x & y
1370 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
1371 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
1372 N2: DAG.getFreeze(V: FalseV));
1373 }
1374 // (select x, y, x) -> x & y
1375 // (select !x, y, x) -> x | y
1376 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
1377 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
1378 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
1379 }
1380 }
1381
1382 return SDValue();
1383}
1384
1385// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
1386// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
1387// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
1388// being `0` or `-1`. In such cases we can replace `select` with `and`.
1389// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
1390// than `c0`?
1391static SDValue
1392foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
1393 const LoongArchSubtarget &Subtarget) {
1394 unsigned SelOpNo = 0;
1395 SDValue Sel = BO->getOperand(Num: 0);
1396 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1397 SelOpNo = 1;
1398 Sel = BO->getOperand(Num: 1);
1399 }
1400
1401 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1402 return SDValue();
1403
1404 unsigned ConstSelOpNo = 1;
1405 unsigned OtherSelOpNo = 2;
1406 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
1407 ConstSelOpNo = 2;
1408 OtherSelOpNo = 1;
1409 }
1410 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
1411 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
1412 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
1413 return SDValue();
1414
1415 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
1416 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
1417 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
1418 return SDValue();
1419
1420 SDLoc DL(Sel);
1421 EVT VT = BO->getValueType(ResNo: 0);
1422
1423 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1424 if (SelOpNo == 1)
1425 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
1426
1427 SDValue NewConstOp =
1428 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
1429 if (!NewConstOp)
1430 return SDValue();
1431
1432 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
1433 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
1434 return SDValue();
1435
1436 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
1437 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1438 if (SelOpNo == 1)
1439 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
1440 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
1441
1442 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1443 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1444 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
1445}
1446
1447// Changes the condition code and swaps operands if necessary, so the SetCC
1448// operation matches one of the comparisons supported directly by branches
1449// in the LoongArch ISA. May adjust compares to favor compare with 0 over
1450// compare with 1/-1.
1451static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1452 ISD::CondCode &CC, SelectionDAG &DAG) {
1453 // If this is a single bit test that can't be handled by ANDI, shift the
1454 // bit to be tested to the MSB and perform a signed compare with 0.
1455 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
1456 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1457 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
1458 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
1459 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
1460 unsigned ShAmt = 0;
1461 if (isPowerOf2_64(Value: Mask)) {
1462 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
1463 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
1464 } else {
1465 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
1466 }
1467
1468 LHS = LHS.getOperand(i: 0);
1469 if (ShAmt != 0)
1470 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
1471 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
1472 return;
1473 }
1474 }
1475
1476 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
1477 int64_t C = RHSC->getSExtValue();
1478 switch (CC) {
1479 default:
1480 break;
1481 case ISD::SETGT:
1482 // Convert X > -1 to X >= 0.
1483 if (C == -1) {
1484 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1485 CC = ISD::SETGE;
1486 return;
1487 }
1488 break;
1489 case ISD::SETLT:
1490 // Convert X < 1 to 0 >= X.
1491 if (C == 1) {
1492 RHS = LHS;
1493 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
1494 CC = ISD::SETGE;
1495 return;
1496 }
1497 break;
1498 }
1499 }
1500
1501 switch (CC) {
1502 default:
1503 break;
1504 case ISD::SETGT:
1505 case ISD::SETLE:
1506 case ISD::SETUGT:
1507 case ISD::SETULE:
1508 CC = ISD::getSetCCSwappedOperands(Operation: CC);
1509 std::swap(a&: LHS, b&: RHS);
1510 break;
1511 }
1512}
1513
1514SDValue LoongArchTargetLowering::lowerSELECT(SDValue Op,
1515 SelectionDAG &DAG) const {
1516 SDValue CondV = Op.getOperand(i: 0);
1517 SDValue TrueV = Op.getOperand(i: 1);
1518 SDValue FalseV = Op.getOperand(i: 2);
1519 SDLoc DL(Op);
1520 MVT VT = Op.getSimpleValueType();
1521 MVT GRLenVT = Subtarget.getGRLenVT();
1522
1523 if (SDValue V = combineSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
1524 return V;
1525
1526 if (Op.hasOneUse()) {
1527 unsigned UseOpc = Op->user_begin()->getOpcode();
1528 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
1529 SDNode *BinOp = *Op->user_begin();
1530 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
1531 DAG, Subtarget)) {
1532 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
1533 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
1534 // may return a constant node and cause crash in lowerSELECT.
1535 if (NewSel.getOpcode() == ISD::SELECT)
1536 return lowerSELECT(Op: NewSel, DAG);
1537 return NewSel;
1538 }
1539 }
1540 }
1541
1542 // If the condition is not an integer SETCC which operates on GRLenVT, we need
1543 // to emit a LoongArchISD::SELECT_CC comparing the condition to zero. i.e.:
1544 // (select condv, truev, falsev)
1545 // -> (loongarchisd::select_cc condv, zero, setne, truev, falsev)
1546 if (CondV.getOpcode() != ISD::SETCC ||
1547 CondV.getOperand(i: 0).getSimpleValueType() != GRLenVT) {
1548 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: GRLenVT);
1549 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
1550
1551 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
1552
1553 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1554 }
1555
1556 // If the CondV is the output of a SETCC node which operates on GRLenVT
1557 // inputs, then merge the SETCC node into the lowered LoongArchISD::SELECT_CC
1558 // to take advantage of the integer compare+branch instructions. i.e.: (select
1559 // (setcc lhs, rhs, cc), truev, falsev)
1560 // -> (loongarchisd::select_cc lhs, rhs, cc, truev, falsev)
1561 SDValue LHS = CondV.getOperand(i: 0);
1562 SDValue RHS = CondV.getOperand(i: 1);
1563 ISD::CondCode CCVal = CondV.getOperandCondCode(i: 2);
1564
1565 // Special case for a select of 2 constants that have a difference of 1.
1566 // Normally this is done by DAGCombine, but if the select is introduced by
1567 // type legalization or op legalization, we miss it. Restricting to SETLT
1568 // case for now because that is what signed saturating add/sub need.
1569 // FIXME: We don't need the condition to be SETLT or even a SETCC,
1570 // but we would probably want to swap the true/false values if the condition
1571 // is SETGE/SETLE to avoid an XORI.
1572 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
1573 CCVal == ISD::SETLT) {
1574 const APInt &TrueVal = TrueV->getAsAPIntVal();
1575 const APInt &FalseVal = FalseV->getAsAPIntVal();
1576 if (TrueVal - 1 == FalseVal)
1577 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
1578 if (TrueVal + 1 == FalseVal)
1579 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
1580 }
1581
1582 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1583 // 1 < x ? x : 1 -> 0 < x ? x : 1
1584 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
1585 RHS == TrueV && LHS == FalseV) {
1586 LHS = DAG.getConstant(Val: 0, DL, VT);
1587 // 0 <u x is the same as x != 0.
1588 if (CCVal == ISD::SETULT) {
1589 std::swap(a&: LHS, b&: RHS);
1590 CCVal = ISD::SETNE;
1591 }
1592 }
1593
1594 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
1595 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
1596 RHS == FalseV) {
1597 RHS = DAG.getConstant(Val: 0, DL, VT);
1598 }
1599
1600 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1601
1602 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
1603 // (select (setcc lhs, rhs, CC), constant, falsev)
1604 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
1605 std::swap(a&: TrueV, b&: FalseV);
1606 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
1607 }
1608
1609 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
1610 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT, Ops);
1611}
1612
1613SDValue LoongArchTargetLowering::lowerBRCOND(SDValue Op,
1614 SelectionDAG &DAG) const {
1615 SDValue CondV = Op.getOperand(i: 1);
1616 SDLoc DL(Op);
1617 MVT GRLenVT = Subtarget.getGRLenVT();
1618
1619 if (CondV.getOpcode() == ISD::SETCC) {
1620 if (CondV.getOperand(i: 0).getValueType() == GRLenVT) {
1621 SDValue LHS = CondV.getOperand(i: 0);
1622 SDValue RHS = CondV.getOperand(i: 1);
1623 ISD::CondCode CCVal = CondV.getOperandCondCode(i: 2);
1624
1625 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
1626
1627 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
1628 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1629 N1: Op.getOperand(i: 0), N2: LHS, N3: RHS, N4: TargetCC,
1630 N5: Op.getOperand(i: 2));
1631 } else if (CondV.getOperand(i: 0).getValueType().isFloatingPoint()) {
1632 return DAG.getNode(Opcode: LoongArchISD::BRCOND, DL, VT: Op.getValueType(),
1633 N1: Op.getOperand(i: 0), N2: CondV, N3: Op.getOperand(i: 2));
1634 }
1635 }
1636
1637 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: Op.getValueType(),
1638 N1: Op.getOperand(i: 0), N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT),
1639 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
1640}
1641
1642SDValue
1643LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
1644 SelectionDAG &DAG) const {
1645 SDLoc DL(Op);
1646 MVT OpVT = Op.getSimpleValueType();
1647
1648 SDValue Vector = DAG.getUNDEF(VT: OpVT);
1649 SDValue Val = Op.getOperand(i: 0);
1650 SDValue Idx = DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT());
1651
1652 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: OpVT, N1: Vector, N2: Val, N3: Idx);
1653}
1654
1655SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
1656 SelectionDAG &DAG) const {
1657 EVT ResTy = Op->getValueType(ResNo: 0);
1658 SDValue Src = Op->getOperand(Num: 0);
1659 SDLoc DL(Op);
1660
1661 // LoongArchISD::BITREV_8B is not supported on LA32.
1662 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1663 return SDValue();
1664
1665 EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
1666 unsigned int OrigEltNum = ResTy.getVectorNumElements();
1667 unsigned int NewEltNum = NewVT.getVectorNumElements();
1668
1669 SDValue NewSrc = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: NewVT, Operand: Src);
1670
1671 SmallVector<SDValue, 8> Ops;
1672 for (unsigned int i = 0; i < NewEltNum; i++) {
1673 SDValue Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: NewSrc,
1674 N2: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
1675 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1676 ? (unsigned)LoongArchISD::BITREV_8B
1677 : (unsigned)ISD::BITREVERSE;
1678 Ops.push_back(Elt: DAG.getNode(Opcode: RevOp, DL, VT: MVT::i64, Operand: Op));
1679 }
1680 SDValue Res =
1681 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResTy, Operand: DAG.getBuildVector(VT: NewVT, DL, Ops));
1682
1683 switch (ResTy.getSimpleVT().SimpleTy) {
1684 default:
1685 return SDValue();
1686 case MVT::v16i8:
1687 case MVT::v32i8:
1688 return Res;
1689 case MVT::v8i16:
1690 case MVT::v16i16:
1691 case MVT::v4i32:
1692 case MVT::v8i32: {
1693 SmallVector<int, 32> Mask;
1694 for (unsigned int i = 0; i < NewEltNum; i++)
1695 for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
1696 Mask.push_back(Elt: j + (OrigEltNum / NewEltNum) * i);
1697 return DAG.getVectorShuffle(VT: ResTy, dl: DL, N1: Res, N2: DAG.getUNDEF(VT: ResTy), Mask);
1698 }
1699 }
1700}
1701
1702// Widen element type to get a new mask value (if possible).
1703// For example:
1704// shufflevector <4 x i32> %a, <4 x i32> %b,
1705// <4 x i32> <i32 6, i32 7, i32 2, i32 3>
1706// is equivalent to:
1707// shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 1>
1708// can be lowered to:
1709// VPACKOD_D vr0, vr0, vr1
1710static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
1711 SDValue V1, SDValue V2, SelectionDAG &DAG) {
1712 unsigned EltBits = VT.getScalarSizeInBits();
1713
1714 if (EltBits > 32 || EltBits == 1)
1715 return SDValue();
1716
1717 SmallVector<int, 8> NewMask;
1718 if (widenShuffleMaskElts(M: Mask, NewMask)) {
1719 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: EltBits * 2)
1720 : MVT::getIntegerVT(BitWidth: EltBits * 2);
1721 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
1722 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT)) {
1723 SDValue NewV1 = DAG.getBitcast(VT: NewVT, V: V1);
1724 SDValue NewV2 = DAG.getBitcast(VT: NewVT, V: V2);
1725 return DAG.getBitcast(
1726 VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: NewV1, N2: NewV2, Mask: NewMask));
1727 }
1728 }
1729
1730 return SDValue();
1731}
1732
1733/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
1734/// instruction.
1735// The funciton matches elements from one of the input vector shuffled to the
1736// left or right with zeroable elements 'shifted in'. It handles both the
1737// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
1738// lane.
1739// Mostly copied from X86.
1740static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
1741 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
1742 int MaskOffset, const APInt &Zeroable) {
1743 int Size = Mask.size();
1744 unsigned SizeInBits = Size * ScalarSizeInBits;
1745
1746 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
1747 for (int i = 0; i < Size; i += Scale)
1748 for (int j = 0; j < Shift; ++j)
1749 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
1750 return false;
1751
1752 return true;
1753 };
1754
1755 auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
1756 int Step = 1) {
1757 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
1758 if (!(Mask[i] == -1 || Mask[i] == Low))
1759 return false;
1760 return true;
1761 };
1762
1763 auto MatchShift = [&](int Shift, int Scale, bool Left) {
1764 for (int i = 0; i != Size; i += Scale) {
1765 unsigned Pos = Left ? i + Shift : i;
1766 unsigned Low = Left ? i : i + Shift;
1767 unsigned Len = Scale - Shift;
1768 if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
1769 return -1;
1770 }
1771
1772 int ShiftEltBits = ScalarSizeInBits * Scale;
1773 bool ByteShift = ShiftEltBits > 64;
1774 Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1775 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1776 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1777
1778 // Normalize the scale for byte shifts to still produce an i64 element
1779 // type.
1780 Scale = ByteShift ? Scale / 2 : Scale;
1781
1782 // We need to round trip through the appropriate type for the shift.
1783 MVT ShiftSVT = MVT::getIntegerVT(BitWidth: ScalarSizeInBits * Scale);
1784 ShiftVT = ByteShift ? MVT::getVectorVT(VT: MVT::i8, NumElements: SizeInBits / 8)
1785 : MVT::getVectorVT(VT: ShiftSVT, NumElements: Size / Scale);
1786 return (int)ShiftAmt;
1787 };
1788
1789 unsigned MaxWidth = 128;
1790 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1791 for (int Shift = 1; Shift != Scale; ++Shift)
1792 for (bool Left : {true, false})
1793 if (CheckZeros(Shift, Scale, Left)) {
1794 int ShiftAmt = MatchShift(Shift, Scale, Left);
1795 if (0 < ShiftAmt)
1796 return ShiftAmt;
1797 }
1798
1799 // no match
1800 return -1;
1801}
1802
1803/// Lower VECTOR_SHUFFLE as shift (if possible).
1804///
1805/// For example:
1806/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1807/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
1808/// is lowered to:
1809/// (VBSLL_V $v0, $v0, 4)
1810///
1811/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1812/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
1813/// is lowered to:
1814/// (VSLLI_D $v0, $v0, 32)
1815static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef<int> Mask,
1816 MVT VT, SDValue V1, SDValue V2,
1817 SelectionDAG &DAG,
1818 const LoongArchSubtarget &Subtarget,
1819 const APInt &Zeroable) {
1820 int Size = Mask.size();
1821 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
1822
1823 MVT ShiftVT;
1824 SDValue V = V1;
1825 unsigned Opcode;
1826
1827 // Try to match shuffle against V1 shift.
1828 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1829 Mask, MaskOffset: 0, Zeroable);
1830
1831 // If V1 failed, try to match shuffle against V2 shift.
1832 if (ShiftAmt < 0) {
1833 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, ScalarSizeInBits: VT.getScalarSizeInBits(),
1834 Mask, MaskOffset: Size, Zeroable);
1835 V = V2;
1836 }
1837
1838 if (ShiftAmt < 0)
1839 return SDValue();
1840
1841 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
1842 "Illegal integer vector type");
1843 V = DAG.getBitcast(VT: ShiftVT, V);
1844 V = DAG.getNode(Opcode, DL, VT: ShiftVT, N1: V,
1845 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: Subtarget.getGRLenVT()));
1846 return DAG.getBitcast(VT, V);
1847}
1848
1849/// Determine whether a range fits a regular pattern of values.
1850/// This function accounts for the possibility of jumping over the End iterator.
1851template <typename ValType>
1852static bool
1853fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,
1854 unsigned CheckStride,
1855 typename SmallVectorImpl<ValType>::const_iterator End,
1856 ValType ExpectedIndex, unsigned ExpectedIndexStride) {
1857 auto &I = Begin;
1858
1859 while (I != End) {
1860 if (*I != -1 && *I != ExpectedIndex)
1861 return false;
1862 ExpectedIndex += ExpectedIndexStride;
1863
1864 // Incrementing past End is undefined behaviour so we must increment one
1865 // step at a time and check for End at each step.
1866 for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
1867 ; // Empty loop body.
1868 }
1869 return true;
1870}
1871
1872/// Compute whether each element of a shuffle is zeroable.
1873///
1874/// A "zeroable" vector shuffle element is one which can be lowered to zero.
1875static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
1876 SDValue V2, APInt &KnownUndef,
1877 APInt &KnownZero) {
1878 int Size = Mask.size();
1879 KnownUndef = KnownZero = APInt::getZero(numBits: Size);
1880
1881 V1 = peekThroughBitcasts(V: V1);
1882 V2 = peekThroughBitcasts(V: V2);
1883
1884 bool V1IsZero = ISD::isBuildVectorAllZeros(V: V1);
1885 bool V2IsZero = ISD::isBuildVectorAllZeros(V: V2);
1886
1887 int VectorSizeInBits = V1.getValueSizeInBits();
1888 int ScalarSizeInBits = VectorSizeInBits / Size;
1889 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
1890 (void)ScalarSizeInBits;
1891
1892 for (int i = 0; i < Size; ++i) {
1893 int M = Mask[i];
1894 if (M < 0) {
1895 KnownUndef.setBit(i);
1896 continue;
1897 }
1898 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
1899 KnownZero.setBit(i);
1900 continue;
1901 }
1902 }
1903}
1904
1905/// Test whether a shuffle mask is equivalent within each sub-lane.
1906///
1907/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
1908/// non-trivial to compute in the face of undef lanes. The representation is
1909/// suitable for use with existing 128-bit shuffles as entries from the second
1910/// vector have been remapped to [LaneSize, 2*LaneSize).
1911static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
1912 ArrayRef<int> Mask,
1913 SmallVectorImpl<int> &RepeatedMask) {
1914 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
1915 RepeatedMask.assign(NumElts: LaneSize, Elt: -1);
1916 int Size = Mask.size();
1917 for (int i = 0; i < Size; ++i) {
1918 assert(Mask[i] == -1 || Mask[i] >= 0);
1919 if (Mask[i] < 0)
1920 continue;
1921 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
1922 // This entry crosses lanes, so there is no way to model this shuffle.
1923 return false;
1924
1925 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
1926 // Adjust second vector indices to start at LaneSize instead of Size.
1927 int LocalM =
1928 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1929 if (RepeatedMask[i % LaneSize] < 0)
1930 // This is the first non-undef entry in this slot of a 128-bit lane.
1931 RepeatedMask[i % LaneSize] = LocalM;
1932 else if (RepeatedMask[i % LaneSize] != LocalM)
1933 // Found a mismatch with the repeated mask.
1934 return false;
1935 }
1936 return true;
1937}
1938
1939/// Attempts to match vector shuffle as byte rotation.
1940static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
1941 ArrayRef<int> Mask) {
1942
1943 SDValue Lo, Hi;
1944 SmallVector<int, 16> RepeatedMask;
1945
1946 if (!isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask, RepeatedMask))
1947 return -1;
1948
1949 int NumElts = RepeatedMask.size();
1950 int Rotation = 0;
1951 int Scale = 16 / NumElts;
1952
1953 for (int i = 0; i < NumElts; ++i) {
1954 int M = RepeatedMask[i];
1955 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1956 "Unexpected mask index.");
1957 if (M < 0)
1958 continue;
1959
1960 // Determine where a rotated vector would have started.
1961 int StartIdx = i - (M % NumElts);
1962 if (StartIdx == 0)
1963 return -1;
1964
1965 // If we found the tail of a vector the rotation must be the missing
1966 // front. If we found the head of a vector, it must be how much of the
1967 // head.
1968 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1969
1970 if (Rotation == 0)
1971 Rotation = CandidateRotation;
1972 else if (Rotation != CandidateRotation)
1973 return -1;
1974
1975 // Compute which value this mask is pointing at.
1976 SDValue MaskV = M < NumElts ? V1 : V2;
1977
1978 // Compute which of the two target values this index should be assigned
1979 // to. This reflects whether the high elements are remaining or the low
1980 // elements are remaining.
1981 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
1982
1983 // Either set up this value if we've not encountered it before, or check
1984 // that it remains consistent.
1985 if (!TargetV)
1986 TargetV = MaskV;
1987 else if (TargetV != MaskV)
1988 return -1;
1989 }
1990
1991 // Check that we successfully analyzed the mask, and normalize the results.
1992 assert(Rotation != 0 && "Failed to locate a viable rotation!");
1993 assert((Lo || Hi) && "Failed to find a rotated input vector!");
1994 if (!Lo)
1995 Lo = Hi;
1996 else if (!Hi)
1997 Hi = Lo;
1998
1999 V1 = Lo;
2000 V2 = Hi;
2001
2002 return Rotation * Scale;
2003}
2004
2005/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
2006///
2007/// For example:
2008/// %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
2009/// <2 x i32> <i32 3, i32 0>
2010/// is lowered to:
2011/// (VBSRL_V $v1, $v1, 8)
2012/// (VBSLL_V $v0, $v0, 8)
2013/// (VOR_V $v0, $V0, $v1)
2014static SDValue
2015lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2016 SDValue V1, SDValue V2, SelectionDAG &DAG,
2017 const LoongArchSubtarget &Subtarget) {
2018
2019 SDValue Lo = V1, Hi = V2;
2020 int ByteRotation = matchShuffleAsByteRotate(VT, V1&: Lo, V2&: Hi, Mask);
2021 if (ByteRotation <= 0)
2022 return SDValue();
2023
2024 MVT ByteVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VT.getSizeInBits() / 8);
2025 Lo = DAG.getBitcast(VT: ByteVT, V: Lo);
2026 Hi = DAG.getBitcast(VT: ByteVT, V: Hi);
2027
2028 int LoByteShift = 16 - ByteRotation;
2029 int HiByteShift = ByteRotation;
2030 MVT GRLenVT = Subtarget.getGRLenVT();
2031
2032 SDValue LoShift = DAG.getNode(Opcode: LoongArchISD::VBSLL, DL, VT: ByteVT, N1: Lo,
2033 N2: DAG.getConstant(Val: LoByteShift, DL, VT: GRLenVT));
2034 SDValue HiShift = DAG.getNode(Opcode: LoongArchISD::VBSRL, DL, VT: ByteVT, N1: Hi,
2035 N2: DAG.getConstant(Val: HiByteShift, DL, VT: GRLenVT));
2036 return DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::OR, DL, VT: ByteVT, N1: LoShift, N2: HiShift));
2037}
2038
2039/// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
2040///
2041/// For example:
2042/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
2043/// <4 x i32> <i32 0, i32 4, i32 1, i32 4>
2044/// %3 = bitcast <4 x i32> %2 to <2 x i64>
2045/// is lowered to:
2046/// (VREPLI $v1, 0)
2047/// (VILVL $v0, $v1, $v0)
2048static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL,
2049 ArrayRef<int> Mask, MVT VT,
2050 SDValue V1, SDValue V2,
2051 SelectionDAG &DAG,
2052 const APInt &Zeroable) {
2053 int Bits = VT.getSizeInBits();
2054 int EltBits = VT.getScalarSizeInBits();
2055 int NumElements = VT.getVectorNumElements();
2056
2057 if (Zeroable.isAllOnes())
2058 return DAG.getConstant(Val: 0, DL, VT);
2059
2060 // Define a helper function to check a particular ext-scale and lower to it if
2061 // valid.
2062 auto Lower = [&](int Scale) -> SDValue {
2063 SDValue InputV;
2064 bool AnyExt = true;
2065 int Offset = 0;
2066 for (int i = 0; i < NumElements; i++) {
2067 int M = Mask[i];
2068 if (M < 0)
2069 continue;
2070 if (i % Scale != 0) {
2071 // Each of the extended elements need to be zeroable.
2072 if (!Zeroable[i])
2073 return SDValue();
2074
2075 AnyExt = false;
2076 continue;
2077 }
2078
2079 // Each of the base elements needs to be consecutive indices into the
2080 // same input vector.
2081 SDValue V = M < NumElements ? V1 : V2;
2082 M = M % NumElements;
2083 if (!InputV) {
2084 InputV = V;
2085 Offset = M - (i / Scale);
2086
2087 // These offset can't be handled
2088 if (Offset % (NumElements / Scale))
2089 return SDValue();
2090 } else if (InputV != V)
2091 return SDValue();
2092
2093 if (M != (Offset + (i / Scale)))
2094 return SDValue(); // Non-consecutive strided elements.
2095 }
2096
2097 // If we fail to find an input, we have a zero-shuffle which should always
2098 // have already been handled.
2099 if (!InputV)
2100 return SDValue();
2101
2102 do {
2103 unsigned VilVLoHi = LoongArchISD::VILVL;
2104 if (Offset >= (NumElements / 2)) {
2105 VilVLoHi = LoongArchISD::VILVH;
2106 Offset -= (NumElements / 2);
2107 }
2108
2109 MVT InputVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits), NumElements);
2110 SDValue Ext =
2111 AnyExt ? DAG.getFreeze(V: InputV) : DAG.getConstant(Val: 0, DL, VT: InputVT);
2112 InputV = DAG.getBitcast(VT: InputVT, V: InputV);
2113 Ext = DAG.getBitcast(VT: InputVT, V: Ext);
2114 InputV = DAG.getNode(Opcode: VilVLoHi, DL, VT: InputVT, N1: Ext, N2: InputV);
2115 Scale /= 2;
2116 EltBits *= 2;
2117 NumElements /= 2;
2118 } while (Scale > 1);
2119 return DAG.getBitcast(VT, V: InputV);
2120 };
2121
2122 // Each iteration, try extending the elements half as much, but into twice as
2123 // many elements.
2124 for (int NumExtElements = Bits / 64; NumExtElements < NumElements;
2125 NumExtElements *= 2) {
2126 if (SDValue V = Lower(NumElements / NumExtElements))
2127 return V;
2128 }
2129 return SDValue();
2130}
2131
2132/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
2133///
2134/// VREPLVEI performs vector broadcast based on an element specified by an
2135/// integer immediate, with its mask being similar to:
2136/// <x, x, x, ...>
2137/// where x is any valid index.
2138///
2139/// When undef's appear in the mask they are treated as if they were whatever
2140/// value is necessary in order to fit the above form.
2141static SDValue
2142lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2143 SDValue V1, SelectionDAG &DAG,
2144 const LoongArchSubtarget &Subtarget) {
2145 int SplatIndex = -1;
2146 for (const auto &M : Mask) {
2147 if (M != -1) {
2148 SplatIndex = M;
2149 break;
2150 }
2151 }
2152
2153 if (SplatIndex == -1)
2154 return DAG.getUNDEF(VT);
2155
2156 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2157 if (fitsRegularPattern<int>(Begin: Mask.begin(), CheckStride: 1, End: Mask.end(), ExpectedIndex: SplatIndex, ExpectedIndexStride: 0)) {
2158 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2159 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2160 }
2161
2162 return SDValue();
2163}
2164
2165/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
2166///
2167/// VSHUF4I splits the vector into blocks of four elements, then shuffles these
2168/// elements according to a <4 x i2> constant (encoded as an integer immediate).
2169///
2170/// It is therefore possible to lower into VSHUF4I when the mask takes the form:
2171/// <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2172/// When undef's appear they are treated as if they were whatever value is
2173/// necessary in order to fit the above forms.
2174///
2175/// For example:
2176/// %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2177/// <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2178/// i32 7, i32 6, i32 5, i32 4>
2179/// is lowered to:
2180/// (VSHUF4I_H $v0, $v1, 27)
2181/// where the 27 comes from:
2182/// 3 + (2 << 2) + (1 << 4) + (0 << 6)
2183static SDValue
2184lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2185 SDValue V1, SDValue V2, SelectionDAG &DAG,
2186 const LoongArchSubtarget &Subtarget) {
2187
2188 unsigned SubVecSize = 4;
2189 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2190 SubVecSize = 2;
2191
2192 int SubMask[4] = {-1, -1, -1, -1};
2193 for (unsigned i = 0; i < SubVecSize; ++i) {
2194 for (unsigned j = i; j < Mask.size(); j += SubVecSize) {
2195 int M = Mask[j];
2196
2197 // Convert from vector index to 4-element subvector index
2198 // If an index refers to an element outside of the subvector then give up
2199 if (M != -1) {
2200 M -= 4 * (j / SubVecSize);
2201 if (M < 0 || M >= 4)
2202 return SDValue();
2203 }
2204
2205 // If the mask has an undef, replace it with the current index.
2206 // Note that it might still be undef if the current index is also undef
2207 if (SubMask[i] == -1)
2208 SubMask[i] = M;
2209 // Check that non-undef values are the same as in the mask. If they
2210 // aren't then give up
2211 else if (M != -1 && M != SubMask[i])
2212 return SDValue();
2213 }
2214 }
2215
2216 // Calculate the immediate. Replace any remaining undefs with zero
2217 int Imm = 0;
2218 for (int i = SubVecSize - 1; i >= 0; --i) {
2219 int M = SubMask[i];
2220
2221 if (M == -1)
2222 M = 0;
2223
2224 Imm <<= 2;
2225 Imm |= M & 0x3;
2226 }
2227
2228 MVT GRLenVT = Subtarget.getGRLenVT();
2229
2230 // Return vshuf4i.d
2231 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2232 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2233 N3: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2234
2235 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT, N1: V1,
2236 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
2237}
2238
2239/// Lower VECTOR_SHUFFLE whose result is the reversed source vector.
2240///
2241/// It is possible to do optimization for VECTOR_SHUFFLE performing vector
2242/// reverse whose mask likes:
2243/// <7, 6, 5, 4, 3, 2, 1, 0>
2244///
2245/// When undef's appear in the mask they are treated as if they were whatever
2246/// value is necessary in order to fit the above forms.
2247static SDValue
2248lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2249 SDValue V1, SelectionDAG &DAG,
2250 const LoongArchSubtarget &Subtarget) {
2251 // Only vectors with i8/i16 elements which cannot match other patterns
2252 // directly needs to do this.
2253 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2254 VT != MVT::v16i16)
2255 return SDValue();
2256
2257 if (!ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: Mask.size()))
2258 return SDValue();
2259
2260 int WidenNumElts = VT.getVectorNumElements() / 4;
2261 SmallVector<int, 16> WidenMask(WidenNumElts, -1);
2262 for (int i = 0; i < WidenNumElts; ++i)
2263 WidenMask[i] = WidenNumElts - 1 - i;
2264
2265 MVT WidenVT = MVT::getVectorVT(
2266 VT: VT.getVectorElementType() == MVT::i8 ? MVT::i32 : MVT::i64, NumElements: WidenNumElts);
2267 SDValue NewV1 = DAG.getBitcast(VT: WidenVT, V: V1);
2268 SDValue WidenRev = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: NewV1,
2269 N2: DAG.getUNDEF(VT: WidenVT), Mask: WidenMask);
2270
2271 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I, DL, VT,
2272 N1: DAG.getBitcast(VT, V: WidenRev),
2273 N2: DAG.getConstant(Val: 27, DL, VT: Subtarget.getGRLenVT()));
2274}
2275
2276/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).
2277///
2278/// VPACKEV interleaves the even elements from each vector.
2279///
2280/// It is possible to lower into VPACKEV when the mask consists of two of the
2281/// following forms interleaved:
2282/// <0, 2, 4, ...>
2283/// <n, n+2, n+4, ...>
2284/// where n is the number of elements in the vector.
2285/// For example:
2286/// <0, 0, 2, 2, 4, 4, ...>
2287/// <0, n, 2, n+2, 4, n+4, ...>
2288///
2289/// When undef's appear in the mask they are treated as if they were whatever
2290/// value is necessary in order to fit the above forms.
2291static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2292 MVT VT, SDValue V1, SDValue V2,
2293 SelectionDAG &DAG) {
2294
2295 const auto &Begin = Mask.begin();
2296 const auto &End = Mask.end();
2297 SDValue OriV1 = V1, OriV2 = V2;
2298
2299 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2300 V1 = OriV1;
2301 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2302 V1 = OriV2;
2303 else
2304 return SDValue();
2305
2306 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2307 V2 = OriV1;
2308 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2309 V2 = OriV2;
2310 else
2311 return SDValue();
2312
2313 return DAG.getNode(Opcode: LoongArchISD::VPACKEV, DL, VT, N1: V2, N2: V1);
2314}
2315
2316/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).
2317///
2318/// VPACKOD interleaves the odd elements from each vector.
2319///
2320/// It is possible to lower into VPACKOD when the mask consists of two of the
2321/// following forms interleaved:
2322/// <1, 3, 5, ...>
2323/// <n+1, n+3, n+5, ...>
2324/// where n is the number of elements in the vector.
2325/// For example:
2326/// <1, 1, 3, 3, 5, 5, ...>
2327/// <1, n+1, 3, n+3, 5, n+5, ...>
2328///
2329/// When undef's appear in the mask they are treated as if they were whatever
2330/// value is necessary in order to fit the above forms.
2331static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2332 MVT VT, SDValue V1, SDValue V2,
2333 SelectionDAG &DAG) {
2334
2335 const auto &Begin = Mask.begin();
2336 const auto &End = Mask.end();
2337 SDValue OriV1 = V1, OriV2 = V2;
2338
2339 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2340 V1 = OriV1;
2341 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2342 V1 = OriV2;
2343 else
2344 return SDValue();
2345
2346 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2347 V2 = OriV1;
2348 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2349 V2 = OriV2;
2350 else
2351 return SDValue();
2352
2353 return DAG.getNode(Opcode: LoongArchISD::VPACKOD, DL, VT, N1: V2, N2: V1);
2354}
2355
2356/// Lower VECTOR_SHUFFLE into VILVH (if possible).
2357///
2358/// VILVH interleaves consecutive elements from the left (highest-indexed) half
2359/// of each vector.
2360///
2361/// It is possible to lower into VILVH when the mask consists of two of the
2362/// following forms interleaved:
2363/// <x, x+1, x+2, ...>
2364/// <n+x, n+x+1, n+x+2, ...>
2365/// where n is the number of elements in the vector and x is half n.
2366/// For example:
2367/// <x, x, x+1, x+1, x+2, x+2, ...>
2368/// <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2369///
2370/// When undef's appear in the mask they are treated as if they were whatever
2371/// value is necessary in order to fit the above forms.
2372static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef<int> Mask,
2373 MVT VT, SDValue V1, SDValue V2,
2374 SelectionDAG &DAG) {
2375
2376 const auto &Begin = Mask.begin();
2377 const auto &End = Mask.end();
2378 unsigned HalfSize = Mask.size() / 2;
2379 SDValue OriV1 = V1, OriV2 = V2;
2380
2381 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2382 V1 = OriV1;
2383 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
2384 V1 = OriV2;
2385 else
2386 return SDValue();
2387
2388 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
2389 V2 = OriV1;
2390 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size() + HalfSize,
2391 ExpectedIndexStride: 1))
2392 V2 = OriV2;
2393 else
2394 return SDValue();
2395
2396 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
2397}
2398
2399/// Lower VECTOR_SHUFFLE into VILVL (if possible).
2400///
2401/// VILVL interleaves consecutive elements from the right (lowest-indexed) half
2402/// of each vector.
2403///
2404/// It is possible to lower into VILVL when the mask consists of two of the
2405/// following forms interleaved:
2406/// <0, 1, 2, ...>
2407/// <n, n+1, n+2, ...>
2408/// where n is the number of elements in the vector.
2409/// For example:
2410/// <0, 0, 1, 1, 2, 2, ...>
2411/// <0, n, 1, n+1, 2, n+2, ...>
2412///
2413/// When undef's appear in the mask they are treated as if they were whatever
2414/// value is necessary in order to fit the above forms.
2415static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef<int> Mask,
2416 MVT VT, SDValue V1, SDValue V2,
2417 SelectionDAG &DAG) {
2418
2419 const auto &Begin = Mask.begin();
2420 const auto &End = Mask.end();
2421 SDValue OriV1 = V1, OriV2 = V2;
2422
2423 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2424 V1 = OriV1;
2425 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2426 V1 = OriV2;
2427 else
2428 return SDValue();
2429
2430 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: 0, ExpectedIndexStride: 1))
2431 V2 = OriV1;
2432 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1))
2433 V2 = OriV2;
2434 else
2435 return SDValue();
2436
2437 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
2438}
2439
2440/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).
2441///
2442/// VPICKEV copies the even elements of each vector into the result vector.
2443///
2444/// It is possible to lower into VPICKEV when the mask consists of two of the
2445/// following forms concatenated:
2446/// <0, 2, 4, ...>
2447/// <n, n+2, n+4, ...>
2448/// where n is the number of elements in the vector.
2449/// For example:
2450/// <0, 2, 4, ..., 0, 2, 4, ...>
2451/// <0, 2, 4, ..., n, n+2, n+4, ...>
2452///
2453/// When undef's appear in the mask they are treated as if they were whatever
2454/// value is necessary in order to fit the above forms.
2455static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
2456 MVT VT, SDValue V1, SDValue V2,
2457 SelectionDAG &DAG) {
2458
2459 const auto &Begin = Mask.begin();
2460 const auto &Mid = Mask.begin() + Mask.size() / 2;
2461 const auto &End = Mask.end();
2462 SDValue OriV1 = V1, OriV2 = V2;
2463
2464 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2))
2465 V1 = OriV1;
2466 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2467 V1 = OriV2;
2468 else
2469 return SDValue();
2470
2471 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 0, ExpectedIndexStride: 2))
2472 V2 = OriV1;
2473 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2))
2474 V2 = OriV2;
2475
2476 else
2477 return SDValue();
2478
2479 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
2480}
2481
2482/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).
2483///
2484/// VPICKOD copies the odd elements of each vector into the result vector.
2485///
2486/// It is possible to lower into VPICKOD when the mask consists of two of the
2487/// following forms concatenated:
2488/// <1, 3, 5, ...>
2489/// <n+1, n+3, n+5, ...>
2490/// where n is the number of elements in the vector.
2491/// For example:
2492/// <1, 3, 5, ..., 1, 3, 5, ...>
2493/// <1, 3, 5, ..., n+1, n+3, n+5, ...>
2494///
2495/// When undef's appear in the mask they are treated as if they were whatever
2496/// value is necessary in order to fit the above forms.
2497static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
2498 MVT VT, SDValue V1, SDValue V2,
2499 SelectionDAG &DAG) {
2500
2501 const auto &Begin = Mask.begin();
2502 const auto &Mid = Mask.begin() + Mask.size() / 2;
2503 const auto &End = Mask.end();
2504 SDValue OriV1 = V1, OriV2 = V2;
2505
2506 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2))
2507 V1 = OriV1;
2508 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2509 V1 = OriV2;
2510 else
2511 return SDValue();
2512
2513 if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: 1, ExpectedIndexStride: 2))
2514 V2 = OriV1;
2515 else if (fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2))
2516 V2 = OriV2;
2517 else
2518 return SDValue();
2519
2520 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
2521}
2522
2523/// Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
2524///
2525/// VEXTRINS copies one element of a vector into any place of the result
2526/// vector and makes no change to the rest elements of the result vector.
2527///
2528/// It is possible to lower into VEXTRINS when the mask takes the form:
2529/// <0, 1, 2, ..., n+i, ..., n-1> or <n, n+1, n+2, ..., i, ..., 2n-1> or
2530/// <0, 1, 2, ..., i, ..., n-1> or <n, n+1, n+2, ..., n+i, ..., 2n-1>
2531/// where n is the number of elements in the vector and i is in [0, n).
2532/// For example:
2533/// <0, 1, 2, 3, 4, 5, 6, 8> , <2, 9, 10, 11, 12, 13, 14, 15> ,
2534/// <0, 1, 2, 6, 4, 5, 6, 7> , <8, 9, 10, 11, 12, 9, 14, 15>
2535///
2536/// When undef's appear in the mask they are treated as if they were whatever
2537/// value is necessary in order to fit the above forms.
2538static SDValue
2539lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2540 SDValue V1, SDValue V2, SelectionDAG &DAG,
2541 const LoongArchSubtarget &Subtarget) {
2542 unsigned NumElts = VT.getVectorNumElements();
2543 MVT EltVT = VT.getVectorElementType();
2544 MVT GRLenVT = Subtarget.getGRLenVT();
2545
2546 if (Mask.size() != NumElts)
2547 return SDValue();
2548
2549 auto tryLowerToExtrAndIns = [&](unsigned Base) -> SDValue {
2550 int DiffCount = 0;
2551 int DiffPos = -1;
2552 for (unsigned i = 0; i < NumElts; ++i) {
2553 if (Mask[i] == -1)
2554 continue;
2555 if (Mask[i] != int(Base + i)) {
2556 ++DiffCount;
2557 DiffPos = int(i);
2558 if (DiffCount > 1)
2559 return SDValue();
2560 }
2561 }
2562
2563 // Need exactly one differing element to lower into VEXTRINS.
2564 if (DiffCount != 1)
2565 return SDValue();
2566
2567 // DiffMask must be in [0, 2N).
2568 int DiffMask = Mask[DiffPos];
2569 if (DiffMask < 0 || DiffMask >= int(2 * NumElts))
2570 return SDValue();
2571
2572 // Determine source vector and source index.
2573 SDValue SrcVec;
2574 unsigned SrcIdx;
2575 if (unsigned(DiffMask) < NumElts) {
2576 SrcVec = V1;
2577 SrcIdx = unsigned(DiffMask);
2578 } else {
2579 SrcVec = V2;
2580 SrcIdx = unsigned(DiffMask) - NumElts;
2581 }
2582
2583 // Replace with EXTRACT_VECTOR_ELT + INSERT_VECTOR_ELT, it will match the
2584 // patterns of VEXTRINS in tablegen.
2585 SDValue Extracted = DAG.getNode(
2586 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT.isFloatingPoint() ? EltVT : GRLenVT,
2587 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdx, DL, VT: GRLenVT));
2588 SDValue Result =
2589 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: (Base == 0) ? V1 : V2,
2590 N2: Extracted, N3: DAG.getConstant(Val: DiffPos, DL, VT: GRLenVT));
2591
2592 return Result;
2593 };
2594
2595 // Try [0, n-1) insertion then [n, 2n-1) insertion.
2596 if (SDValue Result = tryLowerToExtrAndIns(0))
2597 return Result;
2598 return tryLowerToExtrAndIns(NumElts);
2599}
2600
2601// Check the Mask and then build SrcVec and MaskImm infos which will
2602// be used to build LoongArchISD nodes for VPERMI_W or XVPERMI_W.
2603// On success, return true. Otherwise, return false.
2604static bool buildVPERMIInfo(ArrayRef<int> Mask, SDValue V1, SDValue V2,
2605 SmallVectorImpl<SDValue> &SrcVec,
2606 unsigned &MaskImm) {
2607 unsigned MaskSize = Mask.size();
2608
2609 auto isValid = [&](int M, int Off) {
2610 return (M == -1) || (M >= Off && M < Off + 4);
2611 };
2612
2613 auto buildImm = [&](int MLo, int MHi, unsigned Off, unsigned I) {
2614 auto immPart = [&](int M, unsigned Off) {
2615 return (M == -1 ? 0 : (M - Off)) & 0x3;
2616 };
2617 MaskImm |= immPart(MLo, Off) << (I * 2);
2618 MaskImm |= immPart(MHi, Off) << ((I + 1) * 2);
2619 };
2620
2621 for (unsigned i = 0; i < 4; i += 2) {
2622 int MLo = Mask[i];
2623 int MHi = Mask[i + 1];
2624
2625 if (MaskSize == 8) { // Only v8i32/v8f32 need this check.
2626 auto isValid2 = [&](int &M, int M2) {
2627 // If high half index is undef, it's always valid.
2628 if (M2 == -1)
2629 return true;
2630 if (M == -1) {
2631 // If low half index is undef, use index from high half,
2632 // remapped to low half.
2633 if ((M2 % MaskSize) < 4)
2634 return false;
2635 M = M2 - 4;
2636 return true;
2637 }
2638 // Index in low half must be same as index in high half.
2639 return M2 == M + 4;
2640 };
2641 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2642 return false;
2643 }
2644
2645 if (isValid(MLo, 0) && isValid(MHi, 0)) {
2646 SrcVec.push_back(Elt: V1);
2647 buildImm(MLo, MHi, 0, i);
2648 } else if (isValid(MLo, MaskSize) && isValid(MHi, MaskSize)) {
2649 SrcVec.push_back(Elt: V2);
2650 buildImm(MLo, MHi, MaskSize, i);
2651 } else {
2652 return false;
2653 }
2654 }
2655
2656 return true;
2657}
2658
2659/// Lower VECTOR_SHUFFLE into VPERMI (if possible).
2660///
2661/// VPERMI selects two elements from each of the two vectors based on the
2662/// mask and places them in the corresponding positions of the result vector
2663/// in order. Only v4i32 and v4f32 types are allowed.
2664///
2665/// It is possible to lower into VPERMI when the mask consists of two of the
2666/// following forms concatenated:
2667/// <i, j, u, v>
2668/// <u, v, i, j>
2669/// where i,j are in [0,4) and u,v are in [4, 8).
2670/// For example:
2671/// <2, 3, 4, 5>
2672/// <5, 7, 0, 2>
2673///
2674/// When undef's appear in the mask they are treated as if they were whatever
2675/// value is necessary in order to fit the above forms.
2676static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef<int> Mask,
2677 MVT VT, SDValue V1, SDValue V2,
2678 SelectionDAG &DAG,
2679 const LoongArchSubtarget &Subtarget) {
2680 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2681 Mask.size() != VT.getVectorNumElements())
2682 return SDValue();
2683
2684 SmallVector<SDValue, 2> SrcVec;
2685 unsigned MaskImm = 0;
2686 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2687 return SDValue();
2688
2689 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2690 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2691}
2692
2693/// Lower VECTOR_SHUFFLE into VSHUF.
2694///
2695/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and
2696/// adding it as an operand to the resulting VSHUF.
2697static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef<int> Mask,
2698 MVT VT, SDValue V1, SDValue V2,
2699 SelectionDAG &DAG,
2700 const LoongArchSubtarget &Subtarget) {
2701
2702 SmallVector<SDValue, 16> Ops;
2703 for (auto M : Mask)
2704 Ops.push_back(Elt: DAG.getSignedConstant(Val: M, DL, VT: Subtarget.getGRLenVT()));
2705
2706 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
2707 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops);
2708
2709 // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.
2710 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>
2711 // VSHF concatenates the vectors in a bitwise fashion:
2712 // <0b00, 0b01> + <0b10, 0b11> ->
2713 // 0b0100 + 0b1110 -> 0b01001110
2714 // <0b10, 0b11, 0b00, 0b01>
2715 // We must therefore swap the operands to get the correct result.
2716 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
2717}
2718
2719/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
2720///
2721/// This routine breaks down the specific type of 128-bit shuffle and
2722/// dispatches to the lowering routines accordingly.
2723static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2724 SDValue V1, SDValue V2, SelectionDAG &DAG,
2725 const LoongArchSubtarget &Subtarget) {
2726 assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||
2727 VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||
2728 VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&
2729 "Vector type is unsupported for lsx!");
2730 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
2731 "Two operands have different types!");
2732 assert(VT.getVectorNumElements() == Mask.size() &&
2733 "Unexpected mask size for shuffle!");
2734 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
2735
2736 APInt KnownUndef, KnownZero;
2737 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
2738 APInt Zeroable = KnownUndef | KnownZero;
2739
2740 SDValue Result;
2741 // TODO: Add more comparison patterns.
2742 if (V2.isUndef()) {
2743 if ((Result =
2744 lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
2745 return Result;
2746 if ((Result =
2747 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2748 return Result;
2749 if ((Result =
2750 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
2751 return Result;
2752
2753 // TODO: This comment may be enabled in the future to better match the
2754 // pattern for instruction selection.
2755 /* V2 = V1; */
2756 }
2757
2758 // It is recommended not to change the pattern comparison order for better
2759 // performance.
2760 if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))
2761 return Result;
2762 if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))
2763 return Result;
2764 if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))
2765 return Result;
2766 if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))
2767 return Result;
2768 if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))
2769 return Result;
2770 if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
2771 return Result;
2772 if ((VT.SimpleTy == MVT::v2i64 || VT.SimpleTy == MVT::v2f64) &&
2773 (Result =
2774 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2775 return Result;
2776 if ((Result =
2777 lowerVECTOR_SHUFFLE_VEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2778 return Result;
2779 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
2780 Zeroable)))
2781 return Result;
2782 if ((Result =
2783 lowerVECTOR_SHUFFLE_VPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2784 return Result;
2785 if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
2786 Zeroable)))
2787 return Result;
2788 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
2789 Subtarget)))
2790 return Result;
2791 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
2792 return NewShuffle;
2793 if ((Result =
2794 lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2795 return Result;
2796 return SDValue();
2797}
2798
2799/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
2800///
2801/// It is a XVREPLVEI when the mask is:
2802/// <x, x, x, ..., x+n, x+n, x+n, ...>
2803/// where the number of x is equal to n and n is half the length of vector.
2804///
2805/// When undef's appear in the mask they are treated as if they were whatever
2806/// value is necessary in order to fit the above form.
2807static SDValue
2808lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2809 SDValue V1, SelectionDAG &DAG,
2810 const LoongArchSubtarget &Subtarget) {
2811 int SplatIndex = -1;
2812 for (const auto &M : Mask) {
2813 if (M != -1) {
2814 SplatIndex = M;
2815 break;
2816 }
2817 }
2818
2819 if (SplatIndex == -1)
2820 return DAG.getUNDEF(VT);
2821
2822 const auto &Begin = Mask.begin();
2823 const auto &End = Mask.end();
2824 int HalfSize = Mask.size() / 2;
2825
2826 if (SplatIndex >= HalfSize)
2827 return SDValue();
2828
2829 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2830 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: End - HalfSize, ExpectedIndex: SplatIndex, ExpectedIndexStride: 0) &&
2831 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 1, End, ExpectedIndex: SplatIndex + HalfSize,
2832 ExpectedIndexStride: 0)) {
2833 return DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT, N1: V1,
2834 N2: DAG.getConstant(Val: SplatIndex, DL, VT: Subtarget.getGRLenVT()));
2835 }
2836
2837 return SDValue();
2838}
2839
2840/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
2841static SDValue
2842lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2843 SDValue V1, SDValue V2, SelectionDAG &DAG,
2844 const LoongArchSubtarget &Subtarget) {
2845 // XVSHUF4I_D must be handled separately because it is different from other
2846 // types of [X]VSHUF4I instructions.
2847 if (Mask.size() == 4) {
2848 unsigned MaskImm = 0;
2849 for (int i = 1; i >= 0; --i) {
2850 int MLo = Mask[i];
2851 int MHi = Mask[i + 2];
2852 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2853 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2854 return SDValue();
2855 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2856 return SDValue();
2857
2858 MaskImm <<= 2;
2859 if (MLo != -1)
2860 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2861 else if (MHi != -1)
2862 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2863 }
2864
2865 return DAG.getNode(Opcode: LoongArchISD::VSHUF4I_D, DL, VT, N1: V1, N2: V2,
2866 N3: DAG.getConstant(Val: MaskImm, DL, VT: Subtarget.getGRLenVT()));
2867 }
2868
2869 return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget);
2870}
2871
2872/// Lower VECTOR_SHUFFLE into XVREPLVE0 (if possible).
2873///
2874/// It is a XVREPLVE0 when the VECTOR_SHUFFLE is a broadcast of element 0,
2875/// i.e. the mask is:
2876/// <0, 0, 0, ...>
2877///
2878/// When undef's appear in the mask they are treated as if they were whatever
2879/// value is necessary in order to fit the above form.
2880static SDValue lowerVECTOR_SHUFFLE_XVREPLVE0(const SDLoc &DL,
2881 ArrayRef<int> Mask, MVT VT,
2882 SDValue V1, SelectionDAG &DAG) {
2883 if (!ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts: Mask.size()))
2884 return SDValue();
2885
2886 return DAG.getNode(Opcode: LoongArchISD::XVREPLVE0, DL, VT, Operand: V1);
2887}
2888
2889/// Lower VECTOR_SHUFFLE into XVPERMI (if possible).
2890static SDValue
2891lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
2892 SDValue V1, SDValue V2, SelectionDAG &DAG,
2893 const LoongArchSubtarget &Subtarget) {
2894 MVT GRLenVT = Subtarget.getGRLenVT();
2895 unsigned MaskSize = Mask.size();
2896 if (MaskSize != VT.getVectorNumElements())
2897 return SDValue();
2898
2899 // Consider XVPERMI_W.
2900 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2901 SmallVector<SDValue, 2> SrcVec;
2902 unsigned MaskImm = 0;
2903 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2904 return SDValue();
2905
2906 return DAG.getNode(Opcode: LoongArchISD::VPERMI, DL, VT, N1: SrcVec[1], N2: SrcVec[0],
2907 N3: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2908 }
2909
2910 // Consider XVPERMI_D.
2911 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2912 unsigned MaskImm = 0;
2913 for (unsigned i = 0; i < MaskSize; ++i) {
2914 if (Mask[i] == -1)
2915 continue;
2916 if (Mask[i] >= (int)MaskSize)
2917 return SDValue();
2918 MaskImm |= Mask[i] << (i * 2);
2919 }
2920
2921 return DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT, N1: V1,
2922 N2: DAG.getConstant(Val: MaskImm, DL, VT: GRLenVT));
2923 }
2924
2925 return SDValue();
2926}
2927
2928/// Lower VECTOR_SHUFFLE into XVPERM (if possible).
2929static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef<int> Mask,
2930 MVT VT, SDValue V1, SelectionDAG &DAG,
2931 const LoongArchSubtarget &Subtarget) {
2932 // LoongArch LASX only have XVPERM_W.
2933 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2934 return SDValue();
2935
2936 unsigned NumElts = VT.getVectorNumElements();
2937 unsigned HalfSize = NumElts / 2;
2938 bool FrontLo = true, FrontHi = true;
2939 bool BackLo = true, BackHi = true;
2940
2941 auto inRange = [](int val, int low, int high) {
2942 return (val == -1) || (val >= low && val < high);
2943 };
2944
2945 for (unsigned i = 0; i < HalfSize; ++i) {
2946 int Fronti = Mask[i];
2947 int Backi = Mask[i + HalfSize];
2948
2949 FrontLo &= inRange(Fronti, 0, HalfSize);
2950 FrontHi &= inRange(Fronti, HalfSize, NumElts);
2951 BackLo &= inRange(Backi, 0, HalfSize);
2952 BackHi &= inRange(Backi, HalfSize, NumElts);
2953 }
2954
2955 // If both the lower and upper 128-bit parts access only one half of the
2956 // vector (either lower or upper), avoid using xvperm.w. The latency of
2957 // xvperm.w(3) is higher than using xvshuf(1) and xvori(1).
2958 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2959 return SDValue();
2960
2961 SmallVector<SDValue, 8> Masks;
2962 MVT GRLenVT = Subtarget.getGRLenVT();
2963 for (unsigned i = 0; i < NumElts; ++i)
2964 Masks.push_back(Elt: Mask[i] == -1 ? DAG.getUNDEF(VT: GRLenVT)
2965 : DAG.getConstant(Val: Mask[i], DL, VT: GRLenVT));
2966 SDValue MaskVec = DAG.getBuildVector(VT: MVT::v8i32, DL, Ops: Masks);
2967
2968 return DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT, N1: V1, N2: MaskVec);
2969}
2970
2971/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
2972static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef<int> Mask,
2973 MVT VT, SDValue V1, SDValue V2,
2974 SelectionDAG &DAG) {
2975 return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);
2976}
2977
2978/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
2979static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef<int> Mask,
2980 MVT VT, SDValue V1, SDValue V2,
2981 SelectionDAG &DAG) {
2982 return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);
2983}
2984
2985/// Lower VECTOR_SHUFFLE into XVILVH (if possible).
2986static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef<int> Mask,
2987 MVT VT, SDValue V1, SDValue V2,
2988 SelectionDAG &DAG) {
2989
2990 const auto &Begin = Mask.begin();
2991 const auto &End = Mask.end();
2992 unsigned HalfSize = Mask.size() / 2;
2993 unsigned LeftSize = HalfSize / 2;
2994 SDValue OriV1 = V1, OriV2 = V2;
2995
2996 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
2997 ExpectedIndexStride: 1) &&
2998 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize, ExpectedIndexStride: 1))
2999 V1 = OriV1;
3000 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize,
3001 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
3002 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
3003 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
3004 V1 = OriV2;
3005 else
3006 return SDValue();
3007
3008 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: HalfSize - LeftSize,
3009 ExpectedIndexStride: 1) &&
3010 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize + LeftSize,
3011 ExpectedIndexStride: 1))
3012 V2 = OriV1;
3013 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize,
3014 ExpectedIndex: Mask.size() + HalfSize - LeftSize, ExpectedIndexStride: 1) &&
3015 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
3016 ExpectedIndex: Mask.size() + HalfSize + LeftSize, ExpectedIndexStride: 1))
3017 V2 = OriV2;
3018 else
3019 return SDValue();
3020
3021 return DAG.getNode(Opcode: LoongArchISD::VILVH, DL, VT, N1: V2, N2: V1);
3022}
3023
3024/// Lower VECTOR_SHUFFLE into XVILVL (if possible).
3025static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef<int> Mask,
3026 MVT VT, SDValue V1, SDValue V2,
3027 SelectionDAG &DAG) {
3028
3029 const auto &Begin = Mask.begin();
3030 const auto &End = Mask.end();
3031 unsigned HalfSize = Mask.size() / 2;
3032 SDValue OriV1 = V1, OriV2 = V2;
3033
3034 if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
3035 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
3036 V1 = OriV1;
3037 else if (fitsRegularPattern<int>(Begin, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(), ExpectedIndexStride: 1) &&
3038 fitsRegularPattern<int>(Begin: Begin + HalfSize, CheckStride: 2, End,
3039 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
3040 V1 = OriV2;
3041 else
3042 return SDValue();
3043
3044 if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: 0, ExpectedIndexStride: 1) &&
3045 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 1))
3046 V2 = OriV1;
3047 else if (fitsRegularPattern<int>(Begin: Begin + 1, CheckStride: 2, End: End - HalfSize, ExpectedIndex: Mask.size(),
3048 ExpectedIndexStride: 1) &&
3049 fitsRegularPattern<int>(Begin: Begin + 1 + HalfSize, CheckStride: 2, End,
3050 ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 1))
3051 V2 = OriV2;
3052 else
3053 return SDValue();
3054
3055 return DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT, N1: V2, N2: V1);
3056}
3057
3058/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
3059static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef<int> Mask,
3060 MVT VT, SDValue V1, SDValue V2,
3061 SelectionDAG &DAG) {
3062
3063 const auto &Begin = Mask.begin();
3064 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3065 const auto &Mid = Mask.begin() + Mask.size() / 2;
3066 const auto &RightMid = Mask.end() - Mask.size() / 4;
3067 const auto &End = Mask.end();
3068 unsigned HalfSize = Mask.size() / 2;
3069 SDValue OriV1 = V1, OriV2 = V2;
3070
3071 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
3072 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
3073 V1 = OriV1;
3074 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
3075 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
3076 V1 = OriV2;
3077 else
3078 return SDValue();
3079
3080 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 0, ExpectedIndexStride: 2) &&
3081 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize, ExpectedIndexStride: 2))
3082 V2 = OriV1;
3083 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size(), ExpectedIndexStride: 2) &&
3084 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize, ExpectedIndexStride: 2))
3085 V2 = OriV2;
3086
3087 else
3088 return SDValue();
3089
3090 return DAG.getNode(Opcode: LoongArchISD::VPICKEV, DL, VT, N1: V2, N2: V1);
3091}
3092
3093/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
3094static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef<int> Mask,
3095 MVT VT, SDValue V1, SDValue V2,
3096 SelectionDAG &DAG) {
3097
3098 const auto &Begin = Mask.begin();
3099 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3100 const auto &Mid = Mask.begin() + Mask.size() / 2;
3101 const auto &RightMid = Mask.end() - Mask.size() / 4;
3102 const auto &End = Mask.end();
3103 unsigned HalfSize = Mask.size() / 2;
3104 SDValue OriV1 = V1, OriV2 = V2;
3105
3106 if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3107 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3108 V1 = OriV1;
3109 else if (fitsRegularPattern<int>(Begin, CheckStride: 1, End: LeftMid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3110 fitsRegularPattern<int>(Begin: Mid, CheckStride: 1, End: RightMid, ExpectedIndex: Mask.size() + HalfSize + 1,
3111 ExpectedIndexStride: 2))
3112 V1 = OriV2;
3113 else
3114 return SDValue();
3115
3116 if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: 1, ExpectedIndexStride: 2) &&
3117 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: HalfSize + 1, ExpectedIndexStride: 2))
3118 V2 = OriV1;
3119 else if (fitsRegularPattern<int>(Begin: LeftMid, CheckStride: 1, End: Mid, ExpectedIndex: Mask.size() + 1, ExpectedIndexStride: 2) &&
3120 fitsRegularPattern<int>(Begin: RightMid, CheckStride: 1, End, ExpectedIndex: Mask.size() + HalfSize + 1,
3121 ExpectedIndexStride: 2))
3122 V2 = OriV2;
3123 else
3124 return SDValue();
3125
3126 return DAG.getNode(Opcode: LoongArchISD::VPICKOD, DL, VT, N1: V2, N2: V1);
3127}
3128
3129/// Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
3130static SDValue
3131lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3132 SDValue V1, SDValue V2, SelectionDAG &DAG,
3133 const LoongArchSubtarget &Subtarget) {
3134 int NumElts = VT.getVectorNumElements();
3135 int HalfSize = NumElts / 2;
3136 MVT EltVT = VT.getVectorElementType();
3137 MVT GRLenVT = Subtarget.getGRLenVT();
3138
3139 if ((int)Mask.size() != NumElts)
3140 return SDValue();
3141
3142 auto tryLowerToExtrAndIns = [&](int Base) -> SDValue {
3143 SmallVector<int> DiffPos;
3144 for (int i = 0; i < NumElts; ++i) {
3145 if (Mask[i] == -1)
3146 continue;
3147 if (Mask[i] != Base + i) {
3148 DiffPos.push_back(Elt: i);
3149 if (DiffPos.size() > 2)
3150 return SDValue();
3151 }
3152 }
3153
3154 // Need exactly two differing element to lower into XVEXTRINS.
3155 // If only one differing element, the element at a distance of
3156 // HalfSize from it must be undef.
3157 if (DiffPos.size() == 1) {
3158 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3159 DiffPos.push_back(Elt: DiffPos[0] + HalfSize);
3160 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3161 DiffPos.insert(I: DiffPos.begin(), Elt: DiffPos[0] - HalfSize);
3162 else
3163 return SDValue();
3164 }
3165 if (DiffPos.size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3166 return SDValue();
3167
3168 // DiffMask must be in its low or high part.
3169 int DiffMaskLo = Mask[DiffPos[0]];
3170 int DiffMaskHi = Mask[DiffPos[1]];
3171 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3172 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3173 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3174 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3175 return SDValue();
3176 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3177 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3178 return SDValue();
3179 if (DiffMaskHi != DiffMaskLo + HalfSize)
3180 return SDValue();
3181
3182 // Determine source vector and source index.
3183 SDValue SrcVec = (DiffMaskLo < HalfSize) ? V1 : V2;
3184 int SrcIdxLo =
3185 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3186 bool IsEltFP = EltVT.isFloatingPoint();
3187
3188 // Replace with 2*EXTRACT_VECTOR_ELT + 2*INSERT_VECTOR_ELT, it will match
3189 // the patterns of XVEXTRINS in tablegen.
3190 SDValue BaseVec = (Base == 0) ? V1 : V2;
3191 SDValue EltLo =
3192 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3193 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo, DL, VT: GRLenVT));
3194 SDValue InsLo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: BaseVec, N2: EltLo,
3195 N3: DAG.getConstant(Val: DiffPos[0], DL, VT: GRLenVT));
3196 SDValue EltHi =
3197 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IsEltFP ? EltVT : GRLenVT,
3198 N1: SrcVec, N2: DAG.getConstant(Val: SrcIdxLo + HalfSize, DL, VT: GRLenVT));
3199 SDValue Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: InsLo, N2: EltHi,
3200 N3: DAG.getConstant(Val: DiffPos[1], DL, VT: GRLenVT));
3201
3202 return Result;
3203 };
3204
3205 // Try [0, n-1) insertion then [n, 2n-1) insertion.
3206 if (SDValue Result = tryLowerToExtrAndIns(0))
3207 return Result;
3208 return tryLowerToExtrAndIns(NumElts);
3209}
3210
3211/// Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
3212static SDValue
3213lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3214 SDValue V1, SDValue V2, SelectionDAG &DAG,
3215 const LoongArchSubtarget &Subtarget) {
3216 // LoongArch LASX only supports xvinsve0.{w/d}.
3217 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3218 VT != MVT::v4f64)
3219 return SDValue();
3220
3221 MVT GRLenVT = Subtarget.getGRLenVT();
3222 int MaskSize = Mask.size();
3223 assert(MaskSize == (int)VT.getVectorNumElements() && "Unexpected mask size");
3224
3225 // Check if exactly one element of the Mask is replaced by 'Replaced', while
3226 // all other elements are either 'Base + i' or undef (-1). On success, return
3227 // the index of the replaced element. Otherwise, just return -1.
3228 auto checkReplaceOne = [&](int Base, int Replaced) -> int {
3229 int Idx = -1;
3230 for (int i = 0; i < MaskSize; ++i) {
3231 if (Mask[i] == Base + i || Mask[i] == -1)
3232 continue;
3233 if (Mask[i] != Replaced)
3234 return -1;
3235 if (Idx == -1)
3236 Idx = i;
3237 else
3238 return -1;
3239 }
3240 return Idx;
3241 };
3242
3243 // Case 1: the lowest element of V2 replaces one element in V1.
3244 int Idx = checkReplaceOne(0, MaskSize);
3245 if (Idx != -1)
3246 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V1, N2: V2,
3247 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3248
3249 // Case 2: the lowest element of V1 replaces one element in V2.
3250 Idx = checkReplaceOne(MaskSize, 0);
3251 if (Idx != -1)
3252 return DAG.getNode(Opcode: LoongArchISD::XVINSVE0, DL, VT, N1: V2, N2: V1,
3253 N3: DAG.getConstant(Val: Idx, DL, VT: GRLenVT));
3254
3255 return SDValue();
3256}
3257
3258/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).
3259static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,
3260 MVT VT, SDValue V1, SDValue V2,
3261 SelectionDAG &DAG) {
3262
3263 int MaskSize = Mask.size();
3264 int HalfSize = Mask.size() / 2;
3265 const auto &Begin = Mask.begin();
3266 const auto &Mid = Mask.begin() + HalfSize;
3267 const auto &End = Mask.end();
3268
3269 // VECTOR_SHUFFLE concatenates the vectors:
3270 // <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>
3271 // shuffling ->
3272 // <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>
3273 //
3274 // XVSHUF concatenates the vectors:
3275 // <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>
3276 // shuffling ->
3277 // <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>
3278 SmallVector<SDValue, 8> MaskAlloc;
3279 for (auto it = Begin; it < Mid; it++) {
3280 if (*it < 0) // UNDEF
3281 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3282 else if ((*it >= 0 && *it < HalfSize) ||
3283 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3284 int M = *it < HalfSize ? *it : *it - HalfSize;
3285 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3286 } else
3287 return SDValue();
3288 }
3289 assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");
3290
3291 for (auto it = Mid; it < End; it++) {
3292 if (*it < 0) // UNDEF
3293 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
3294 else if ((*it >= HalfSize && *it < MaskSize) ||
3295 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3296 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3297 MaskAlloc.push_back(Elt: DAG.getTargetConstant(Val: M, DL, VT: MVT::i64));
3298 } else
3299 return SDValue();
3300 }
3301 assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");
3302
3303 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
3304 SDValue MaskVec = DAG.getBuildVector(VT: MaskVecTy, DL, Ops: MaskAlloc);
3305 return DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT, N1: MaskVec, N2: V2, N3: V1);
3306}
3307
3308/// Shuffle vectors by lane to generate more optimized instructions.
3309/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
3310///
3311/// Therefore, except for the following four cases, other cases are regarded
3312/// as cross-lane shuffles, where optimization is relatively limited.
3313///
3314/// - Shuffle high, low lanes of two inputs vector
3315/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
3316/// - Shuffle low, high lanes of two inputs vector
3317/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
3318/// - Shuffle low, low lanes of two inputs vector
3319/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
3320/// - Shuffle high, high lanes of two inputs vector
3321/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
3322///
3323/// The first case is the closest to LoongArch instructions and the other
3324/// cases need to be converted to it for processing.
3325///
3326/// This function will return true for the last three cases above and will
3327/// modify V1, V2 and Mask. Otherwise, return false for the first case and
3328/// cross-lane shuffle cases.
3329static bool canonicalizeShuffleVectorByLane(
3330 const SDLoc &DL, MutableArrayRef<int> Mask, MVT VT, SDValue &V1,
3331 SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
3332
3333 enum HalfMaskType { HighLaneTy, LowLaneTy, None };
3334
3335 int MaskSize = Mask.size();
3336 int HalfSize = Mask.size() / 2;
3337 MVT GRLenVT = Subtarget.getGRLenVT();
3338
3339 HalfMaskType preMask = None, postMask = None;
3340
3341 if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3342 return M < 0 || (M >= 0 && M < HalfSize) ||
3343 (M >= MaskSize && M < MaskSize + HalfSize);
3344 }))
3345 preMask = HighLaneTy;
3346 else if (std::all_of(first: Mask.begin(), last: Mask.begin() + HalfSize, pred: [&](int M) {
3347 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3348 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3349 }))
3350 preMask = LowLaneTy;
3351
3352 if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3353 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3354 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3355 }))
3356 postMask = LowLaneTy;
3357 else if (std::all_of(first: Mask.begin() + HalfSize, last: Mask.end(), pred: [&](int M) {
3358 return M < 0 || (M >= 0 && M < HalfSize) ||
3359 (M >= MaskSize && M < MaskSize + HalfSize);
3360 }))
3361 postMask = HighLaneTy;
3362
3363 // The pre-half of mask is high lane type, and the post-half of mask
3364 // is low lane type, which is closest to the LoongArch instructions.
3365 //
3366 // Note: In the LoongArch architecture, the high lane of mask corresponds
3367 // to the lower 128-bit of vector register, and the low lane of mask
3368 // corresponds the higher 128-bit of vector register.
3369 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3370 return false;
3371 }
3372 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3373 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3374 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3375 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3376 V1 = DAG.getBitcast(VT, V: V1);
3377
3378 if (!V2.isUndef()) {
3379 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3380 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3381 N2: DAG.getConstant(Val: 0b01001110, DL, VT: GRLenVT));
3382 V2 = DAG.getBitcast(VT, V: V2);
3383 }
3384
3385 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3386 *it = *it < 0 ? *it : *it - HalfSize;
3387 }
3388 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3389 *it = *it < 0 ? *it : *it + HalfSize;
3390 }
3391 } else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3392 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3393 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3394 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3395 V1 = DAG.getBitcast(VT, V: V1);
3396
3397 if (!V2.isUndef()) {
3398 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3399 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3400 N2: DAG.getConstant(Val: 0b11101110, DL, VT: GRLenVT));
3401 V2 = DAG.getBitcast(VT, V: V2);
3402 }
3403
3404 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3405 *it = *it < 0 ? *it : *it - HalfSize;
3406 }
3407 } else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3408 V1 = DAG.getBitcast(VT: MVT::v4i64, V: V1);
3409 V1 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V1,
3410 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3411 V1 = DAG.getBitcast(VT, V: V1);
3412
3413 if (!V2.isUndef()) {
3414 V2 = DAG.getBitcast(VT: MVT::v4i64, V: V2);
3415 V2 = DAG.getNode(Opcode: LoongArchISD::XVPERMI, DL, VT: MVT::v4i64, N1: V2,
3416 N2: DAG.getConstant(Val: 0b01000100, DL, VT: GRLenVT));
3417 V2 = DAG.getBitcast(VT, V: V2);
3418 }
3419
3420 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3421 *it = *it < 0 ? *it : *it + HalfSize;
3422 }
3423 } else { // cross-lane
3424 return false;
3425 }
3426
3427 return true;
3428}
3429
3430/// Lower VECTOR_SHUFFLE into XVPERMI_Q (if possible).
3431///
3432/// For example:
3433/// %2 = shufflevector <4 x i64> %0, <4 x i64> %1,
3434/// <4 x i32> <i32 2, i32 3, i32 4, i32 5>
3435/// is lowered to:
3436/// (XVPERMI_Q $xr1, $xr0, 33)
3437static SDValue
3438lowerVECTOR_SHUFFLE_XVPERMIQ(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3439 SDValue V1, SDValue V2, SelectionDAG &DAG,
3440 const LoongArchSubtarget &Subtarget) {
3441 // Each 128-bit lane of the result must be a whole lane of V1 or V2.
3442 SmallVector<int, 2> LaneMask;
3443 if (!widenShuffleMaskElts(Scale: Mask.size() / 2, Mask, ScaledMask&: LaneMask))
3444 return SDValue();
3445
3446 // Lanes 0-3 are V1.lo, V1.hi, V2.lo and V2.hi, the same as the XVPERMI_Q
3447 // selector with xj = V1 and xd = V2.
3448 int Lo = LaneMask[0] < 0 ? 0 : LaneMask[0];
3449 int Hi = LaneMask[1] < 0 ? 1 : LaneMask[1];
3450 return DAG.getNode(
3451 Opcode: LoongArchISD::XVPERMIQ, DL, VT, N1: V2, N2: V1,
3452 N3: DAG.getConstant(Val: Lo | (Hi << 4), DL, VT: Subtarget.getGRLenVT()));
3453}
3454
3455/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
3456/// Only for 256-bit vector.
3457///
3458/// Each 128-bit lane of the result may use at most two lanes of the inputs.
3459/// These lanes are first moved into place, then shuffled within the lane.
3460///
3461/// For example:
3462/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
3463/// <4 x i64> <i32 0, i32 3, i32 2, i32 0>
3464/// is lowerded to:
3465/// (XVPERMI $xr2, $xr0, 78)
3466/// (XVSHUF $xr1, $xr2, $xr0)
3467/// (XVORI $xr0, $xr1, 0)
3468///
3469/// and the splice:
3470/// %2 = shufflevector <8 x i32> %0, <8 x i32> %1,
3471/// <8 x i32> <i32 7, i32 8, i32 9, i32 10,
3472/// i32 11, i32 12, i32 13, i32 14>
3473/// is lowered to:
3474/// (XVBSLL_V $xr2, $xr1, 4)
3475/// (XVPERMI_Q $xr1, $xr0, 33)
3476/// (XVBSRL_V $xr0, $xr1, 12)
3477/// (XVOR_V $xr0, $xr2, $xr0)
3478static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL,
3479 ArrayRef<int> Mask,
3480 MVT VT, SDValue V1,
3481 SDValue V2,
3482 SelectionDAG &DAG) {
3483 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
3484 int Size = Mask.size();
3485 int LaneSize = Size / 2;
3486
3487 // For each result lane, the input lanes (0/1: V1, 2/3: V2) used by the two
3488 // operands of the in-lane shuffle.
3489 int LaneSrcs[2][2] = {{-1, -1}, {-1, -1}};
3490 SmallVector<int, 32> InLaneMask(Size, -1);
3491 bool LaneCrossing = false;
3492 for (int i = 0; i < Size; ++i) {
3493 int M = Mask[i];
3494 if (M < 0)
3495 continue;
3496 int Lane = i / LaneSize;
3497 int SrcLane = M / LaneSize;
3498
3499 // Put lanes that stay in place in the first operand and the others in the
3500 // second one, so a single input becomes V1 and V1 with lanes swapped.
3501 int Op = (SrcLane % 2) != Lane;
3502 LaneCrossing |= Op;
3503 if (LaneSrcs[Lane][Op] >= 0 && LaneSrcs[Lane][Op] != SrcLane)
3504 Op ^= 1;
3505 if (LaneSrcs[Lane][Op] >= 0 && LaneSrcs[Lane][Op] != SrcLane)
3506 return SDValue();
3507 LaneSrcs[Lane][Op] = SrcLane;
3508 InLaneMask[i] = Op * Size + Lane * LaneSize + M % LaneSize;
3509 }
3510
3511 if (!LaneCrossing)
3512 return SDValue();
3513
3514 // Swap the operands of the high lane if that makes both lanes use the same
3515 // in-lane mask, which more patterns (e.g. XVPICKEV) can match.
3516 SmallVector<int, 16> RepeatedMask;
3517 if (!isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask: InLaneMask, RepeatedMask)) {
3518 SmallVector<int, 32> SwappedMask(InLaneMask);
3519 for (int i = LaneSize; i < Size; ++i)
3520 if (SwappedMask[i] >= 0)
3521 SwappedMask[i] = (SwappedMask[i] + Size) % (2 * Size);
3522 if (isRepeatedShuffleMask(LaneSizeInBits: 128, VT, Mask: SwappedMask, RepeatedMask)) {
3523 std::swap(a&: LaneSrcs[1][0], b&: LaneSrcs[1][1]);
3524 InLaneMask = SwappedMask;
3525 }
3526 }
3527
3528 // Move the input lanes of each operand into place.
3529 SDValue Ops[2];
3530 for (int Op = 0; Op < 2; ++Op) {
3531 SmallVector<int, 32> LaneMask(Size, -1);
3532 for (int Lane = 0; Lane < 2; ++Lane)
3533 if (LaneSrcs[Lane][Op] >= 0)
3534 for (int i = 0; i < LaneSize; ++i)
3535 LaneMask[Lane * LaneSize + i] = LaneSrcs[Lane][Op] * LaneSize + i;
3536 Ops[Op] = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask: LaneMask);
3537
3538 // Exit if we got back the shuffle we started with, to avoid infinite
3539 // recursion.
3540 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val&: Ops[Op]))
3541 if (SVN->getMask() == Mask)
3542 return SDValue();
3543 }
3544 return DAG.getVectorShuffle(VT, dl: DL, N1: Ops[0], N2: Ops[1], Mask: InLaneMask);
3545}
3546
3547/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
3548///
3549/// This routine breaks down the specific type of 256-bit shuffle and
3550/// dispatches to the lowering routines accordingly.
3551static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
3552 SDValue V1, SDValue V2, SelectionDAG &DAG,
3553 const LoongArchSubtarget &Subtarget) {
3554 assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||
3555 VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||
3556 VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&
3557 "Vector type is unsupported for lasx!");
3558 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
3559 "Two operands have different types!");
3560 assert(VT.getVectorNumElements() == Mask.size() &&
3561 "Unexpected mask size for shuffle!");
3562 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
3563 assert(Mask.size() >= 4 && "Mask size is less than 4.");
3564
3565 APInt KnownUndef, KnownZero;
3566 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
3567 APInt Zeroable = KnownUndef | KnownZero;
3568
3569 SDValue Result;
3570 // TODO: Add more comparison patterns.
3571 if (V2.isUndef()) {
3572 if ((Result =
3573 lowerVECTOR_SHUFFLE_XVREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
3574 return Result;
3575 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG,
3576 Subtarget)))
3577 return Result;
3578 if ((Result = lowerVECTOR_SHUFFLE_XVREPLVE0(DL, Mask, VT, V1, DAG)))
3579 return Result;
3580 // Try to widen vectors to gain more optimization opportunities.
3581 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
3582 return NewShuffle;
3583 if ((Result =
3584 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3585 return Result;
3586 if ((Result = lowerVECTOR_SHUFFLE_XVPERM(DL, Mask, VT, V1, DAG, Subtarget)))
3587 return Result;
3588 if ((Result =
3589 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
3590 return Result;
3591
3592 // TODO: This comment may be enabled in the future to better match the
3593 // pattern for instruction selection.
3594 /* V2 = V1; */
3595 }
3596
3597 // It is recommended not to change the pattern comparison order for better
3598 // performance.
3599 if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, Mask, VT, V1, V2, DAG)))
3600 return Result;
3601 if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, Mask, VT, V1, V2, DAG)))
3602 return Result;
3603 if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, Mask, VT, V1, V2, DAG)))
3604 return Result;
3605 if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, Mask, VT, V1, V2, DAG)))
3606 return Result;
3607 if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, Mask, VT, V1, V2, DAG)))
3608 return Result;
3609 if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, Mask, VT, V1, V2, DAG)))
3610 return Result;
3611 if ((VT.SimpleTy == MVT::v4i64 || VT.SimpleTy == MVT::v4f64) &&
3612 (Result =
3613 lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3614 return Result;
3615 if ((Result =
3616 lowerVECTOR_SHUFFLE_XVEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3617 return Result;
3618 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
3619 Zeroable)))
3620 return Result;
3621 if ((Result =
3622 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3623 return Result;
3624 if ((Result =
3625 lowerVECTOR_SHUFFLE_XVINSVE0(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3626 return Result;
3627 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
3628 Subtarget)))
3629 return Result;
3630
3631 if ((Result =
3632 lowerVECTOR_SHUFFLE_XVPERMIQ(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3633 return Result;
3634
3635 // canonicalize non cross-lane shuffle vector
3636 SmallVector<int> NewMask(Mask);
3637 if (canonicalizeShuffleVectorByLane(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget))
3638 return lower256BitShuffle(DL, Mask: NewMask, VT, V1, V2, DAG, Subtarget);
3639
3640 // FIXME: Handling the remaining cases earlier can degrade performance
3641 // in some situations. Further analysis is required to enable more
3642 // effective optimizations.
3643 if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, Mask: NewMask, VT, V1,
3644 V2, DAG)))
3645 return Result;
3646
3647 if (SDValue NewShuffle = widenShuffleMask(DL, Mask: NewMask, VT, V1, V2, DAG))
3648 return NewShuffle;
3649 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, Mask: NewMask, VT, V1, V2, DAG)))
3650 return Result;
3651
3652 return SDValue();
3653}
3654
3655SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3656 SelectionDAG &DAG) const {
3657 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
3658 ArrayRef<int> OrigMask = SVOp->getMask();
3659 SDValue V1 = Op.getOperand(i: 0);
3660 SDValue V2 = Op.getOperand(i: 1);
3661 MVT VT = Op.getSimpleValueType();
3662 int NumElements = VT.getVectorNumElements();
3663 SDLoc DL(Op);
3664
3665 bool V1IsUndef = V1.isUndef();
3666 bool V2IsUndef = V2.isUndef();
3667 if (V1IsUndef && V2IsUndef)
3668 return DAG.getUNDEF(VT);
3669
3670 // When we create a shuffle node we put the UNDEF node to second operand,
3671 // but in some cases the first operand may be transformed to UNDEF.
3672 // In this case we should just commute the node.
3673 if (V1IsUndef)
3674 return DAG.getCommutedVectorShuffle(SV: *SVOp);
3675
3676 // Check for non-undef masks pointing at an undef vector and make the masks
3677 // undef as well. This makes it easier to match the shuffle based solely on
3678 // the mask.
3679 if (V2IsUndef &&
3680 any_of(Range&: OrigMask, P: [NumElements](int M) { return M >= NumElements; })) {
3681 SmallVector<int, 8> NewMask(OrigMask);
3682 for (int &M : NewMask)
3683 if (M >= NumElements)
3684 M = -1;
3685 return DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask: NewMask);
3686 }
3687
3688 // Check for illegal shuffle mask element index values.
3689 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
3690 (void)MaskUpperLimit;
3691 assert(llvm::all_of(OrigMask,
3692 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
3693 "Out of bounds shuffle index");
3694
3695 // For each vector width, delegate to a specialized lowering routine.
3696 if (VT.is128BitVector())
3697 return lower128BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3698
3699 if (VT.is256BitVector())
3700 return lower256BitShuffle(DL, Mask: OrigMask, VT, V1, V2, DAG, Subtarget);
3701
3702 return SDValue();
3703}
3704
3705SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
3706 SelectionDAG &DAG) const {
3707 // Custom lower to ensure the libcall return is passed in an FPR on hard
3708 // float ABIs.
3709 SDLoc DL(Op);
3710 MakeLibCallOptions CallOptions;
3711 SDValue Op0 = Op.getOperand(i: 0);
3712 SDValue Chain = SDValue();
3713 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
3714 SDValue Res;
3715 std::tie(args&: Res, args&: Chain) =
3716 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
3717 if (Subtarget.is64Bit())
3718 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3719 return DAG.getBitcast(VT: MVT::i32, V: Res);
3720}
3721
3722SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
3723 SelectionDAG &DAG) const {
3724 // Custom lower to ensure the libcall argument is passed in an FPR on hard
3725 // float ABIs.
3726 SDLoc DL(Op);
3727 MakeLibCallOptions CallOptions;
3728 SDValue Op0 = Op.getOperand(i: 0);
3729 SDValue Chain = SDValue();
3730 SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3731 DL, VT: MVT::f32, Operand: Op0)
3732 : DAG.getBitcast(VT: MVT::f32, V: Op0);
3733 SDValue Res;
3734 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
3735 CallOptions, dl: DL, Chain);
3736 return Res;
3737}
3738
3739SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
3740 SelectionDAG &DAG) const {
3741 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3742 SDLoc DL(Op);
3743 MakeLibCallOptions CallOptions;
3744 RTLIB::Libcall LC =
3745 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
3746 SDValue Res =
3747 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
3748 if (Subtarget.is64Bit())
3749 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Res);
3750 return DAG.getBitcast(VT: MVT::i32, V: Res);
3751}
3752
3753SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
3754 SelectionDAG &DAG) const {
3755 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3756 MVT VT = Op.getSimpleValueType();
3757 SDLoc DL(Op);
3758 Op = DAG.getNode(
3759 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
3760 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
3761 SDValue Res = Subtarget.is64Bit() ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64,
3762 DL, VT: MVT::f32, Operand: Op)
3763 : DAG.getBitcast(VT: MVT::f32, V: Op);
3764 if (VT != MVT::f32)
3765 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
3766 return Res;
3767}
3768
3769// Lower BUILD_VECTOR as broadcast load (if possible).
3770// For example:
3771// %a = load i8, ptr %ptr
3772// %b = build_vector %a, %a, %a, %a
3773// is lowered to :
3774// (VLDREPL_B $a0, 0)
3775static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp,
3776 const SDLoc &DL,
3777 SelectionDAG &DAG) {
3778 MVT VT = BVOp->getSimpleValueType(ResNo: 0);
3779 int NumOps = BVOp->getNumOperands();
3780
3781 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3782 "Unsupported vector type for broadcast.");
3783
3784 SDValue IdentitySrc;
3785 bool IsIdeneity = true;
3786
3787 for (int i = 0; i != NumOps; i++) {
3788 SDValue Op = BVOp->getOperand(Num: i);
3789 if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
3790 IsIdeneity = false;
3791 break;
3792 }
3793 IdentitySrc = BVOp->getOperand(Num: 0);
3794 }
3795
3796 // make sure that this load is valid and only has one user.
3797 if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(N: IdentitySrc.getNode()))
3798 return SDValue();
3799
3800 auto *LN = cast<LoadSDNode>(Val&: IdentitySrc);
3801 auto ExtType = LN->getExtensionType();
3802
3803 if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
3804 VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) {
3805 // Indexed loads and stores are not supported on LoongArch.
3806 assert(LN->isUnindexed() && "Unexpected indexed load.");
3807
3808 SDVTList Tys = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3809 // The offset operand of unindexed load is always undefined, so there is
3810 // no need to pass it to VLDREPL.
3811 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3812 SDValue BCast = DAG.getNode(Opcode: LoongArchISD::VLDREPL, DL, VTList: Tys, Ops);
3813 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: BCast.getValue(R: 1));
3814 return BCast;
3815 }
3816 return SDValue();
3817}
3818
3819// Sequentially insert elements from Ops into Vector, from low to high indices.
3820// Note: Ops can have fewer elements than Vector.
3821static void fillVector(ArrayRef<SDValue> Ops, SelectionDAG &DAG, SDLoc DL,
3822 const LoongArchSubtarget &Subtarget, SDValue &Vector,
3823 EVT ResTy) {
3824 assert(Ops.size() <= ResTy.getVectorNumElements());
3825
3826 SDValue Op0 = Ops[0];
3827 if (!Op0.isUndef())
3828 Vector = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: ResTy, Operand: Op0);
3829 for (unsigned i = 1; i < Ops.size(); ++i) {
3830 SDValue Opi = Ops[i];
3831 if (Opi.isUndef())
3832 continue;
3833 Vector = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Vector, N2: Opi,
3834 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3835 }
3836}
3837
3838// Build a ResTy subvector from Node, taking NumElts elements starting at index
3839// 'first'.
3840static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node,
3841 SelectionDAG &DAG, SDLoc DL,
3842 const LoongArchSubtarget &Subtarget,
3843 EVT ResTy, unsigned first) {
3844 unsigned NumElts = ResTy.getVectorNumElements();
3845
3846 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3847
3848 SmallVector<SDValue, 16> Ops(Node->op_begin() + first,
3849 Node->op_begin() + first + NumElts);
3850 SDValue Vector = DAG.getUNDEF(VT: ResTy);
3851 fillVector(Ops, DAG, DL, Subtarget, Vector, ResTy);
3852 return Vector;
3853}
3854
3855SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
3856 SelectionDAG &DAG) const {
3857 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Val&: Op);
3858 MVT VT = Node->getSimpleValueType(ResNo: 0);
3859 EVT ResTy = Op->getValueType(ResNo: 0);
3860 unsigned NumElts = ResTy.getVectorNumElements();
3861 SDLoc DL(Op);
3862 APInt SplatValue, SplatUndef;
3863 unsigned SplatBitSize;
3864 bool HasAnyUndefs;
3865 bool IsConstant = false;
3866 bool UseSameConstant = true;
3867 SDValue ConstantValue;
3868 bool Is128Vec = ResTy.is128BitVector();
3869 bool Is256Vec = ResTy.is256BitVector();
3870
3871 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3872 (!Subtarget.hasExtLASX() || !Is256Vec))
3873 return SDValue();
3874
3875 if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(BVOp: Node, DL, DAG))
3876 return Result;
3877
3878 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3879 /*MinSplatBits=*/8) &&
3880 SplatBitSize <= 64) {
3881 // We can only cope with 8, 16, 32, or 64-bit elements.
3882 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3883 SplatBitSize != 64)
3884 return SDValue();
3885
3886 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3887 // We can only handle 64-bit elements that are within
3888 // the signed 10-bit range or match vldi patterns on 32-bit targets.
3889 // See the BUILD_VECTOR case in LoongArchDAGToDAGISel::Select().
3890 if (!SplatValue.isSignedIntN(N: 10) &&
3891 !isImmVLDILegalForMode1(SplatValue, SplatBitSize).first)
3892 return SDValue();
3893 if ((Is128Vec && ResTy == MVT::v4i32) ||
3894 (Is256Vec && ResTy == MVT::v8i32))
3895 return Op;
3896 }
3897
3898 EVT ViaVecTy;
3899
3900 switch (SplatBitSize) {
3901 default:
3902 return SDValue();
3903 case 8:
3904 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3905 break;
3906 case 16:
3907 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3908 break;
3909 case 32:
3910 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3911 break;
3912 case 64:
3913 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3914 break;
3915 }
3916
3917 // SelectionDAG::getConstant will promote SplatValue appropriately.
3918 SDValue Result = DAG.getConstant(Val: SplatValue, DL, VT: ViaVecTy);
3919
3920 // Bitcast to the type we originally wanted.
3921 if (ViaVecTy != ResTy)
3922 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Node), VT: ResTy, Operand: Result);
3923
3924 return Result;
3925 }
3926
3927 if (DAG.isSplatValue(V: Op, /*AllowUndefs=*/false))
3928 return Op;
3929
3930 for (unsigned i = 0; i < NumElts; ++i) {
3931 SDValue Opi = Node->getOperand(Num: i);
3932 if (isIntOrFPConstant(V: Opi)) {
3933 IsConstant = true;
3934 if (!ConstantValue.getNode())
3935 ConstantValue = Opi;
3936 else if (ConstantValue != Opi)
3937 UseSameConstant = false;
3938 }
3939 }
3940
3941 // If the type of BUILD_VECTOR is v2f64, custom legalizing it has no benefits.
3942 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3943 SDValue Result = DAG.getSplatBuildVector(VT: ResTy, DL, Op: ConstantValue);
3944 for (unsigned i = 0; i < NumElts; ++i) {
3945 SDValue Opi = Node->getOperand(Num: i);
3946 if (!isIntOrFPConstant(V: Opi))
3947 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: ResTy, N1: Result, N2: Opi,
3948 N3: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
3949 }
3950 return Result;
3951 }
3952
3953 if (!IsConstant) {
3954 SmallVector<SDValue> Sequence;
3955 BitVector UndefElements;
3956 bool IsRepeated = Node->getRepeatedSequence(Sequence, UndefElements: &UndefElements);
3957 unsigned SeqLen = Sequence.size();
3958 unsigned NumUndefElts = UndefElements.count();
3959
3960 // When a BUILD_VECTOR consists of the same element (ignoring undefs),
3961 // prefer emitting a broadcast instead of multiple insertions.
3962 if (IsRepeated && SeqLen == 1) {
3963 bool IsSplatBetter = true;
3964
3965 // Integer vectors benefit from splat except in the case where the only
3966 // non-undef element is at index 0 and is a load with a base+offset addr.
3967 if (ResTy.isInteger()) {
3968 SDValue Op0 = Node->getOperand(Num: 0);
3969 if (NumUndefElts == NumElts - 1 && Op0.getOpcode() == ISD::LOAD &&
3970 DAG.isBaseWithConstantOffset(Op: Op0.getOperand(i: 1)))
3971 IsSplatBetter = false;
3972 }
3973
3974 // Several floating-point cases suffer performance regressions or have
3975 // no improvement when
3976 // only one element is non-undef,
3977 // one of the only two non-undefs at index 0,
3978 // the two non-undefs of v8f32 buildvector at index (1,2)/(1,3)/(2,3).
3979 if (ResTy.isFloatingPoint()) {
3980 if (NumUndefElts == NumElts - 1)
3981 IsSplatBetter = false;
3982 if (NumUndefElts == NumElts - 2 && !UndefElements[0])
3983 IsSplatBetter = false;
3984 if (ResTy == MVT::v8f32 && NumUndefElts == NumElts - 2 &&
3985 ((!UndefElements[1] && !UndefElements[2]) ||
3986 (!UndefElements[1] && !UndefElements[3]) ||
3987 (!UndefElements[2] && !UndefElements[3])))
3988 IsSplatBetter = false;
3989 }
3990
3991 if (IsSplatBetter)
3992 return DAG.getSplatBuildVector(VT: ResTy, DL, Op: Sequence[0]);
3993 }
3994
3995 // If the BUILD_VECTOR has a repeated pattern, use INSERT_VECTOR_ELT to fill
3996 // the sub-sequence of the vector and then broadcast the sub-sequence.
3997 //
3998 // TODO: If the BUILD_VECTOR contains undef elements, consider falling
3999 // back to use INSERT_VECTOR_ELT to materialize the vector, because it
4000 // generates worse code in some cases. This could be further optimized
4001 // with more consideration.
4002 if (IsRepeated && NumUndefElts == 0) {
4003 // Using LSX instructions to fill the sub-sequence of 256-bits vector,
4004 // because the high part can be simply treated as undef.
4005 SDValue Vector = DAG.getUNDEF(VT: ResTy);
4006 EVT FillTy = Is256Vec
4007 ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext())
4008 : ResTy;
4009 SDValue FillVec =
4010 Is256Vec ? DAG.getExtractSubvector(DL, VT: FillTy, Vec: Vector, Idx: 0) : Vector;
4011
4012 fillVector(Ops: Sequence, DAG, DL, Subtarget, Vector&: FillVec, ResTy: FillTy);
4013
4014 unsigned SplatLen = NumElts / SeqLen;
4015 MVT SplatEltTy = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits() * SeqLen);
4016 MVT SplatTy = MVT::getVectorVT(VT: SplatEltTy, NumElements: SplatLen);
4017
4018 // If size of the sub-sequence is half of a 256-bits vector, bitcast the
4019 // vector to v4i64 type in order to match the pattern of XVREPLVE0Q.
4020 if (SplatEltTy == MVT::i128)
4021 SplatTy = MVT::v4i64;
4022
4023 SDValue SplatVec;
4024 SDValue SrcVec = DAG.getBitcast(
4025 VT: SplatTy,
4026 V: Is256Vec ? DAG.getInsertSubvector(DL, Vec: Vector, SubVec: FillVec, Idx: 0) : FillVec);
4027 if (Is256Vec) {
4028 SplatVec =
4029 DAG.getNode(Opcode: (SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
4030 : LoongArchISD::XVREPLVE0,
4031 DL, VT: SplatTy, Operand: SrcVec);
4032 } else {
4033 SplatVec = DAG.getNode(Opcode: LoongArchISD::VREPLVEI, DL, VT: SplatTy, N1: SrcVec,
4034 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
4035 }
4036
4037 return DAG.getBitcast(VT: ResTy, V: SplatVec);
4038 }
4039
4040 // Use INSERT_VECTOR_ELT operations rather than expand to stores, because
4041 // using memory operations is much lower.
4042 //
4043 // For 256-bit vectors, normally split into two halves and concatenate.
4044 // Special case: for v8i32/v8f32/v4i64/v4f64, if the upper half has only
4045 // one non-undef element, skip spliting to avoid a worse result.
4046 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
4047 ResTy == MVT::v4f64) {
4048 unsigned NonUndefCount = 0;
4049 for (unsigned i = NumElts / 2; i < NumElts; ++i) {
4050 if (!Node->getOperand(Num: i).isUndef()) {
4051 ++NonUndefCount;
4052 if (NonUndefCount > 1)
4053 break;
4054 }
4055 }
4056 if (NonUndefCount == 1)
4057 return fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy, first: 0);
4058 }
4059
4060 EVT VecTy =
4061 Is256Vec ? ResTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext()) : ResTy;
4062 SDValue Vector =
4063 fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy: VecTy, first: 0);
4064
4065 if (Is128Vec)
4066 return Vector;
4067
4068 SDValue VectorHi = fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget,
4069 ResTy: VecTy, first: NumElts / 2);
4070
4071 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResTy, N1: Vector, N2: VectorHi);
4072 }
4073
4074 return SDValue();
4075}
4076
4077SDValue LoongArchTargetLowering::lowerCONCAT_VECTORS(SDValue Op,
4078 SelectionDAG &DAG) const {
4079 SDLoc DL(Op);
4080 MVT ResVT = Op.getSimpleValueType();
4081 assert(ResVT.is256BitVector() && Op.getNumOperands() == 2);
4082
4083 if (Op.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
4084 Op.getOperand(i: 1).getOpcode() == ISD::TRUNCATE)
4085 return Op;
4086
4087 unsigned NumOperands = Op.getNumOperands();
4088 unsigned NumFreezeUndef = 0;
4089 unsigned NumZero = 0;
4090 unsigned NumNonZero = 0;
4091 unsigned NonZeros = 0;
4092 SmallSet<SDValue, 4> Undefs;
4093 for (unsigned i = 0; i != NumOperands; ++i) {
4094 SDValue SubVec = Op.getOperand(i);
4095 if (SubVec.isUndef())
4096 continue;
4097 if (ISD::isFreezeUndef(V: SubVec)) {
4098 // If the freeze(undef) has multiple uses then we must fold to zero.
4099 if (SubVec.hasOneUse()) {
4100 ++NumFreezeUndef;
4101 } else {
4102 ++NumZero;
4103 Undefs.insert(V: SubVec);
4104 }
4105 } else if (ISD::isBuildVectorAllZeros(V: SubVec))
4106 ++NumZero;
4107 else {
4108 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
4109 NonZeros |= 1 << i;
4110 ++NumNonZero;
4111 }
4112 }
4113
4114 // If we have more than 2 non-zeros, build each half separately.
4115 if (NumNonZero > 2) {
4116 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
4117 ArrayRef<SDUse> Ops = Op->ops();
4118 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
4119 Ops: Ops.slice(N: 0, M: NumOperands / 2));
4120 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
4121 Ops: Ops.slice(N: NumOperands / 2));
4122 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
4123 }
4124
4125 // Otherwise, build it up through insert_subvectors.
4126 SDValue Vec = NumZero ? DAG.getConstant(Val: 0, DL, VT: ResVT)
4127 : (NumFreezeUndef ? DAG.getFreeze(V: DAG.getUNDEF(VT: ResVT))
4128 : DAG.getUNDEF(VT: ResVT));
4129
4130 // Replace Undef operands with ZeroVector.
4131 for (SDValue U : Undefs)
4132 DAG.ReplaceAllUsesWith(From: U, To: DAG.getConstant(Val: 0, DL, VT: U.getSimpleValueType()));
4133
4134 MVT SubVT = Op.getOperand(i: 0).getSimpleValueType();
4135 unsigned NumSubElems = SubVT.getVectorNumElements();
4136 for (unsigned i = 0; i != NumOperands; ++i) {
4137 if ((NonZeros & (1 << i)) == 0)
4138 continue;
4139
4140 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ResVT, N1: Vec, N2: Op.getOperand(i),
4141 N3: DAG.getVectorIdxConstant(Val: i * NumSubElems, DL));
4142 }
4143
4144 return Vec;
4145}
4146
4147SDValue
4148LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4149 SelectionDAG &DAG) const {
4150 MVT EltVT = Op.getSimpleValueType();
4151 SDValue Vec = Op->getOperand(Num: 0);
4152 EVT VecTy = Vec->getValueType(ResNo: 0);
4153 SDValue Idx = Op->getOperand(Num: 1);
4154 SDLoc DL(Op);
4155 MVT GRLenVT = Subtarget.getGRLenVT();
4156
4157 assert(VecTy.is256BitVector() && "Unexpected EXTRACT_VECTOR_ELT vector type");
4158
4159 if (isa<ConstantSDNode>(Val: Idx))
4160 return Op;
4161
4162 switch (VecTy.getSimpleVT().SimpleTy) {
4163 default:
4164 llvm_unreachable("Unexpected type");
4165 case MVT::v32i8:
4166 case MVT::v16i16:
4167 case MVT::v4i64:
4168 case MVT::v4f64: {
4169 // Extract the high half subvector and place it to the low half of a new
4170 // vector. It doesn't matter what the high half of the new vector is.
4171 EVT HalfTy = VecTy.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4172 SDValue VecHi =
4173 DAG.getExtractSubvector(DL, VT: HalfTy, Vec, Idx: HalfTy.getVectorNumElements());
4174 SDValue TmpVec =
4175 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: VecTy, N1: DAG.getUNDEF(VT: VecTy),
4176 N2: VecHi, N3: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
4177
4178 // Shuffle the origin Vec and the TmpVec using MaskVec, the lowest element
4179 // of MaskVec is Idx, the rest do not matter. ResVec[0] will hold the
4180 // desired element.
4181 SDValue IdxCp =
4182 Subtarget.is64Bit()
4183 ? DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Idx)
4184 : DAG.getBitcast(VT: MVT::f32, V: Idx);
4185 SDValue IdxVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v8f32, Operand: IdxCp);
4186 SDValue MaskVec =
4187 DAG.getBitcast(VT: (VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, V: IdxVec);
4188 SDValue ResVec =
4189 DAG.getNode(Opcode: LoongArchISD::VSHUF, DL, VT: VecTy, N1: MaskVec, N2: TmpVec, N3: Vec);
4190
4191 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: ResVec,
4192 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
4193 }
4194 case MVT::v8i32:
4195 case MVT::v8f32: {
4196 SDValue SplatIdx = DAG.getSplatBuildVector(VT: MVT::v8i32, DL, Op: Idx);
4197 SDValue SplatValue =
4198 DAG.getNode(Opcode: LoongArchISD::XVPERM, DL, VT: VecTy, N1: Vec, N2: SplatIdx);
4199
4200 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: SplatValue,
4201 N2: DAG.getConstant(Val: 0, DL, VT: GRLenVT));
4202 }
4203 }
4204}
4205
4206SDValue
4207LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4208 SelectionDAG &DAG) const {
4209 MVT VT = Op.getSimpleValueType();
4210 MVT EltVT = VT.getVectorElementType();
4211 unsigned NumElts = VT.getVectorNumElements();
4212 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
4213 SDLoc DL(Op);
4214 SDValue Op0 = Op.getOperand(i: 0);
4215 SDValue Op1 = Op.getOperand(i: 1);
4216 SDValue Op2 = Op.getOperand(i: 2);
4217
4218 // Lower inserting a loaded integer into lane 0 of an undefined vector as a
4219 // vector load that writes the whole register: FLD.{S/D} for 32/64-bit
4220 // elements and [X]VLDREPL.{B/H} for 8/16-bit ones. Unlike a scalar load +
4221 // [X]VINSGR2VR, it has no false dependency on the previous value of the
4222 // destination and doesn't need to move the value from a GPR.
4223 // For example:
4224 // %a = load i64, ptr %ptr
4225 // %b = insert_vector_elt undef, %a, 0
4226 // is lowered to:
4227 // (FLD_D $a0, 0)
4228 if (Op0.isUndef() && isNullConstant(V: Op2) && EltVT.isInteger() &&
4229 Op1.getOpcode() == ISD::LOAD && Op1.hasOneUse()) {
4230 auto *LN = cast<LoadSDNode>(Val&: Op1);
4231 auto ExtType = LN->getExtensionType();
4232 if (LN->isSimple() &&
4233 (ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
4234 LN->getMemoryVT().getSizeInBits() == EltSizeInBits) {
4235 // Indexed loads and stores are not supported on LoongArch.
4236 assert(LN->isUnindexed() && "Unexpected indexed load.");
4237 SDValue Res;
4238 if (EltSizeInBits >= 32) {
4239 MVT FltVT = MVT::getFloatingPointVT(BitWidth: EltSizeInBits);
4240 // Don't reuse the memory operand, its range metadata (if any) only
4241 // applies to integer loads.
4242 SDValue Ld = DAG.getLoad(
4243 VT: FltVT, dl: DL, Chain: LN->getChain(), Ptr: LN->getBasePtr(), PtrInfo: LN->getPointerInfo(),
4244 Alignment: LN->getAlign(), MMOFlags: LN->getMemOperand()->getFlags(), Metadata: LN->getAAInfo());
4245 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL,
4246 VT: MVT::getVectorVT(VT: FltVT, NumElements: NumElts), Operand: Ld);
4247 Res = DAG.getBitcast(VT, V: Res);
4248 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: Ld.getValue(R: 1));
4249 } else {
4250 SDVTList Tys = DAG.getVTList(VT1: VT, VT2: MVT::Other);
4251 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
4252 Res = DAG.getMemIntrinsicNode(Opcode: LoongArchISD::VLDREPL, dl: DL, VTList: Tys, Ops,
4253 MemVT: LN->getMemoryVT(), MMO: LN->getMemOperand());
4254 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN, 1), To: Res.getValue(R: 1));
4255 }
4256 return Res;
4257 }
4258 }
4259
4260 if (isa<ConstantSDNode>(Val: Op2))
4261 return Op;
4262
4263 MVT IdxTy = MVT::getIntegerVT(BitWidth: EltSizeInBits);
4264 MVT IdxVTy = MVT::getVectorVT(VT: IdxTy, NumElements: NumElts);
4265
4266 if (!isTypeLegal(VT) || !isTypeLegal(VT: IdxVTy))
4267 return SDValue();
4268
4269 SDValue SplatElt = DAG.getSplatBuildVector(VT, DL, Op: Op1);
4270 SmallVector<SDValue, 32> RawIndices;
4271 SDValue SplatIdx;
4272 SDValue Indices;
4273
4274 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4275 MVT PairVTy = MVT::getVectorVT(VT: MVT::i32, NumElements: NumElts * 2);
4276 for (unsigned i = 0; i < NumElts; ++i) {
4277 RawIndices.push_back(Elt: Op2);
4278 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4279 }
4280 SplatIdx = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4281 SplatIdx = DAG.getBitcast(VT: IdxVTy, V: SplatIdx);
4282
4283 RawIndices.clear();
4284 for (unsigned i = 0; i < NumElts; ++i) {
4285 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: MVT::i32));
4286 RawIndices.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4287 }
4288 Indices = DAG.getBuildVector(VT: PairVTy, DL, Ops: RawIndices);
4289 Indices = DAG.getBitcast(VT: IdxVTy, V: Indices);
4290 } else {
4291 SplatIdx = DAG.getSplatBuildVector(VT: IdxVTy, DL, Op: Op2);
4292
4293 for (unsigned i = 0; i < NumElts; ++i)
4294 RawIndices.push_back(Elt: DAG.getConstant(Val: i, DL, VT: Subtarget.getGRLenVT()));
4295 Indices = DAG.getBuildVector(VT: IdxVTy, DL, Ops: RawIndices);
4296 }
4297
4298 // insert vec, elt, idx
4299 // =>
4300 // select (splatidx == {0,1,2...}) ? splatelt : vec
4301 SDValue SelectCC =
4302 DAG.getSetCC(DL, VT: IdxVTy, LHS: SplatIdx, RHS: Indices, Cond: ISD::CondCode::SETEQ);
4303 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectCC, N2: SplatElt, N3: Op0);
4304}
4305
4306SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4307 SelectionDAG &DAG) const {
4308 SDLoc DL(Op);
4309 SyncScope::ID FenceSSID =
4310 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
4311
4312 // singlethread fences only synchronize with signal handlers on the same
4313 // thread and thus only need to preserve instruction order, not actually
4314 // enforce memory ordering.
4315 if (FenceSSID == SyncScope::SingleThread)
4316 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4317 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
4318
4319 return Op;
4320}
4321
4322static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL,
4323 MVT GRLenVT, SDValue RMValue) {
4324 // LLVM rounding mode encoding differs from LoongArch FCSR encoding:
4325 // LLVM: 0=RTZ, 1=RNE, 2=RUP, 3=RDN
4326 // FCSR: 0=RNE, 1=RZ, 2=RP, 3=RN
4327 //
4328 // The conversion swaps encodings 0 and 1 while preserving 2 and 3.
4329 // Since the transformation is self-inverse, it applies in both directions:
4330 // LLVM RM <-> LoongArch FCSR RM
4331 //
4332 // Transformation: RM ^ (~(RM >> 1) & 1)
4333 SDValue ShiftRight1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4334 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4335
4336 SDValue SwapMask = DAG.getNode(Opcode: ISD::AND, DL, VT: GRLenVT,
4337 N1: DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: ShiftRight1,
4338 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT)),
4339 N2: DAG.getConstant(Val: 1, DL, VT: GRLenVT));
4340
4341 return DAG.getNode(Opcode: ISD::XOR, DL, VT: GRLenVT, N1: RMValue, N2: SwapMask);
4342}
4343
4344SDValue LoongArchTargetLowering::lowerSET_ROUNDING(SDValue Op,
4345 SelectionDAG &DAG) const {
4346 MVT GRLenVT = Subtarget.getGRLenVT();
4347 SDLoc DL(Op);
4348 SDValue Chain = Op.getOperand(i: 0);
4349 SDValue RMValue = Op.getOperand(i: 1);
4350
4351 if (auto *CVal = dyn_cast<ConstantSDNode>(Val&: RMValue)) {
4352 uint64_t RM = CVal->getZExtValue();
4353 if (RM > 3) {
4354 MachineFunction &MF = DAG.getMachineFunction();
4355 LLVMContext &C = MF.getFunction().getContext();
4356 C.diagnose(DI: DiagnosticInfoUnsupported(
4357 MF.getFunction(),
4358 "rounding mode is not supported by LoongArch hardware",
4359 DiagnosticLocation(DL.getDebugLoc()), DS_Error));
4360 return Chain;
4361 }
4362 }
4363
4364 RMValue = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: RMValue);
4365 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4366
4367 // The RM field in FCSR is at bits [9:8]. Shift the rounding mode value
4368 // into position before writing via WRFCSR.
4369 RMValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: RMValue,
4370 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4371
4372 // FCSR3 is an alias of the RM field; writing it avoids clobbering
4373 // unrelated fields in FCSR0.
4374 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4375 MachineSDNode *RN = DAG.getMachineNode(Opcode: LoongArch::WRFCSR, dl: DL, VT: MVT::Other,
4376 Op1: FCSRNo, Op2: RMValue, Op3: Chain);
4377 return SDValue(RN, 0);
4378}
4379
4380SDValue LoongArchTargetLowering::lowerGET_ROUNDING(SDValue Op,
4381 SelectionDAG &DAG) const {
4382 MVT GRLenVT = Subtarget.getGRLenVT();
4383 SDLoc DL(Op);
4384 SDValue Chain = Op->getOperand(Num: 0);
4385
4386 // FCSR3 is an alias of the RM field.
4387 SDValue FCSRNo = DAG.getTargetConstant(Val: 3, DL, VT: GRLenVT);
4388 MachineSDNode *FCSR = DAG.getMachineNode(Opcode: LoongArch::RDFCSR, dl: DL, VT1: GRLenVT,
4389 VT2: MVT::Other, Op1: FCSRNo, Op2: Chain);
4390 SDValue RMValue = SDValue(FCSR, 0);
4391 Chain = SDValue(FCSR, 1);
4392
4393 // The RM field in FCSR is at bits [9:8].
4394 RMValue = DAG.getNode(Opcode: ISD::SRL, DL, VT: GRLenVT, N1: RMValue,
4395 N2: DAG.getConstant(Val: 8, DL, VT: GRLenVT));
4396 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4397
4398 SDValue RetVal = DAG.getZExtOrTrunc(Op: RMValue, DL, VT: Op.getValueType());
4399 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl: DL);
4400}
4401
4402SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
4403 SelectionDAG &DAG) const {
4404
4405 if (Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i32) {
4406 DAG.getContext()->emitError(
4407 ErrorStr: "On LA64, only 64-bit registers can be written.");
4408 return Op.getOperand(i: 0);
4409 }
4410
4411 if (!Subtarget.is64Bit() && Op.getOperand(i: 2).getValueType() == MVT::i64) {
4412 DAG.getContext()->emitError(
4413 ErrorStr: "On LA32, only 32-bit registers can be written.");
4414 return Op.getOperand(i: 0);
4415 }
4416
4417 return Op;
4418}
4419
4420SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
4421 SelectionDAG &DAG) const {
4422 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 0))) {
4423 DAG.getContext()->emitError(ErrorStr: "argument to '__builtin_frame_address' must "
4424 "be a constant integer");
4425 return SDValue();
4426 }
4427
4428 MachineFunction &MF = DAG.getMachineFunction();
4429 MF.getFrameInfo().setFrameAddressIsTaken(true);
4430 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4431 EVT VT = Op.getValueType();
4432 SDLoc DL(Op);
4433 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
4434 unsigned Depth = Op.getConstantOperandVal(i: 0);
4435 int GRLenInBytes = Subtarget.getGRLen() / 8;
4436
4437 while (Depth--) {
4438 int Offset = -(GRLenInBytes * 2);
4439 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
4440 N2: DAG.getSignedConstant(Val: Offset, DL, VT));
4441 FrameAddr =
4442 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
4443 }
4444 return FrameAddr;
4445}
4446
4447SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
4448 SelectionDAG &DAG) const {
4449 // Currently only support lowering return address for current frame.
4450 if (Op.getConstantOperandVal(i: 0) != 0) {
4451 DAG.getContext()->emitError(
4452 ErrorStr: "return address can only be determined for the current frame");
4453 return SDValue();
4454 }
4455
4456 MachineFunction &MF = DAG.getMachineFunction();
4457 MF.getFrameInfo().setReturnAddressIsTaken(true);
4458 MVT GRLenVT = Subtarget.getGRLenVT();
4459
4460 // Return the value of the return address register, marking it an implicit
4461 // live-in.
4462 Register Reg = MF.addLiveIn(PReg: Subtarget.getRegisterInfo()->getRARegister(),
4463 RC: getRegClassFor(VT: GRLenVT));
4464 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg, VT: GRLenVT);
4465}
4466
4467SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
4468 SelectionDAG &DAG) const {
4469 MachineFunction &MF = DAG.getMachineFunction();
4470 auto Size = Subtarget.getGRLen() / 8;
4471 auto FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: 0, IsImmutable: false);
4472 return DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
4473}
4474
4475SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,
4476 SelectionDAG &DAG) const {
4477 MachineFunction &MF = DAG.getMachineFunction();
4478 auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();
4479
4480 SDLoc DL(Op);
4481 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4482 VT: getPointerTy(DL: MF.getDataLayout()));
4483
4484 // vastart just stores the address of the VarArgsFrameIndex slot into the
4485 // memory location argument.
4486 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4487 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
4488 PtrInfo: MachinePointerInfo(SV));
4489}
4490
4491// Lower an unsigned vXi64 -> vXf32 conversion to the same round-to-odd
4492// sequence that the generic legalizer uses for the scalar case:
4493//
4494// Odd = (x >> 1) | (x & 1)
4495// Doubled = (float)(int64_t)Odd + (float)(int64_t)Odd // exact, no rounding
4496// Signed = (float)(int64_t)x // valid for x < 2^63
4497// Result = x < 0 ? Doubled : Signed
4498//
4499// The generic half-word expansion is not usable here: it converts the high and
4500// low 32-bit halves separately and adds them, rounding three times, which is
4501// not enough to emulate the single rounding required by uitofp.
4502SDValue LoongArchTargetLowering::lowerUINT_TO_FPToF32(SDValue Op,
4503 SelectionDAG &DAG) const {
4504 SDLoc DL(Op);
4505 SDValue Src = Op.getOperand(i: 0);
4506 EVT SrcVT = Src.getValueType();
4507 EVT DstVT = Op.getValueType();
4508
4509 SDValue Shift = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL);
4510 SDValue Half = DAG.getNode(Opcode: ISD::SRL, DL, VT: SrcVT, N1: Src, N2: Shift);
4511 SDValue Sticky =
4512 DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src, N2: DAG.getConstant(Val: 1, DL, VT: SrcVT));
4513 SDValue Odd = DAG.getNode(Opcode: ISD::OR, DL, VT: SrcVT, N1: Half, N2: Sticky);
4514
4515 // Doubling a value with at most 24 significant bits is exact.
4516 SDValue RoundToOdd = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: Odd);
4517 SDValue Doubled = DAG.getNode(Opcode: ISD::FADD, DL, VT: DstVT, N1: RoundToOdd, N2: RoundToOdd);
4518
4519 // Values below 2^63 convert directly; above that the signed conversion is
4520 // negative, so the round-to-odd path is used instead.
4521 SDValue Signed = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DstVT, Operand: Src);
4522
4523 // The signed conversion is negative exactly when the source did not fit, so
4524 // its sign bit is the mask: broadcasting it with an arithmetic shift avoids
4525 // a comparison followed by a truncate of the 64-bit result to the width of
4526 // the result vector.
4527 EVT MaskVT = DstVT.changeVectorElementTypeToInteger();
4528 SDValue Mask =
4529 DAG.getNode(Opcode: ISD::SRA, DL, VT: MaskVT, N1: DAG.getBitcast(VT: MaskVT, V: Signed),
4530 N2: DAG.getShiftAmountConstant(Val: 31, VT: MaskVT, DL));
4531 return DAG.getSelect(DL, VT: DstVT, Cond: Mask, LHS: Doubled, RHS: Signed);
4532}
4533
4534SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
4535 SelectionDAG &DAG) const {
4536 SDLoc DL(Op);
4537 SDValue Op0 = Op.getOperand(i: 0);
4538 EVT VT = Op.getValueType();
4539 EVT Op0VT = Op0.getValueType();
4540
4541 if (VT.isVector()) {
4542 if (VT.getScalarSizeInBits() == Op0VT.getScalarSizeInBits())
4543 return Op;
4544
4545 // There is no unsigned 64-bit to 32-bit conversion instruction, so
4546 // synthesize one with a single rounding. A value known to be
4547 // non-negative converts exactly as signed, and that form is native.
4548 if (Subtarget.hasExtLASX() && Op0VT.getScalarType() == MVT::i64 &&
4549 VT.getScalarType() == MVT::f32) {
4550 if (DAG.SignBitIsZero(Op: Op0) || Op->getFlags().hasNonNeg())
4551 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: Op0);
4552 return lowerUINT_TO_FPToF32(Op, DAG);
4553 }
4554
4555 return SDValue();
4556 }
4557
4558 if ((DAG.SignBitIsZero(Op: Op0) || Op->getFlags().hasNonNeg()) &&
4559 !isOperationLegal(Op: ISD::UINT_TO_FP, VT: Op0VT) &&
4560 isOperationLegal(Op: ISD::SINT_TO_FP, VT: Op0VT))
4561 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: Op0);
4562
4563 // We can't do uint64 -> double -> float because of double-rounding issue.
4564 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4565 Op0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i64, Operand: Op0);
4566 SDValue Conv = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::v2f64, Operand: Op0);
4567 Conv = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f64, N1: Conv,
4568 N2: DAG.getIntPtrConstant(Val: 0, DL));
4569 return Conv;
4570 }
4571
4572 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4573 return SDValue();
4574
4575 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4576 !Subtarget.hasBasicD() && "unexpected target features");
4577
4578 if (Op0->getOpcode() == ISD::AND) {
4579 auto *C = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
4580 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4581 return Op;
4582 }
4583
4584 if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&
4585 Op0.getConstantOperandVal(i: 1) < UINT64_C(0X1F) &&
4586 Op0.getConstantOperandVal(i: 2) == UINT64_C(0))
4587 return Op;
4588
4589 if (Op0.getOpcode() == ISD::AssertZext &&
4590 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLT(VT: MVT::i32))
4591 return Op;
4592
4593 EVT OpVT = Op0.getValueType();
4594 EVT RetVT = Op.getValueType();
4595 RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
4596 MakeLibCallOptions CallOptions;
4597 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4598 SDValue Chain = SDValue();
4599 SDValue Result;
4600 std::tie(args&: Result, args&: Chain) =
4601 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4602 return Result;
4603}
4604
4605SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
4606 SelectionDAG &DAG) const {
4607 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4608 !Subtarget.hasBasicD() && "unexpected target features");
4609
4610 SDLoc DL(Op);
4611 SDValue Op0 = Op.getOperand(i: 0);
4612
4613 if ((Op0.getOpcode() == ISD::AssertSext ||
4614 Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&
4615 dyn_cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT().bitsLE(VT: MVT::i32))
4616 return Op;
4617
4618 EVT OpVT = Op0.getValueType();
4619 EVT RetVT = Op.getValueType();
4620 RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
4621 MakeLibCallOptions CallOptions;
4622 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT);
4623 SDValue Chain = SDValue();
4624 SDValue Result;
4625 std::tie(args&: Result, args&: Chain) =
4626 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op0, CallOptions, dl: DL, Chain);
4627 return Result;
4628}
4629
4630SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
4631 SelectionDAG &DAG) const {
4632
4633 SDLoc DL(Op);
4634 EVT VT = Op.getValueType();
4635 SDValue Op0 = Op.getOperand(i: 0);
4636 EVT Op0VT = Op0.getValueType();
4637
4638 if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4639 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4640 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4641 return DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: NewOp0);
4642 }
4643 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4644 SDValue Lo, Hi;
4645 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
4646 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
4647 }
4648 return Op;
4649}
4650
4651SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
4652 SelectionDAG &DAG) const {
4653
4654 SDLoc DL(Op);
4655 SDValue Op0 = Op.getOperand(i: 0);
4656
4657 if (Op0.getValueType() == MVT::f16)
4658 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
4659
4660 if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4661 !Subtarget.hasBasicD()) {
4662 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f32, Operand: Op0);
4663 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Dst);
4664 }
4665
4666 EVT FPTy = EVT::getFloatingPointVT(BitWidth: Op.getValueSizeInBits());
4667 SDValue Trunc = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FPTy, Operand: Op0);
4668 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op.getValueType(), Operand: Trunc);
4669}
4670
4671// [X]VFTINTRZ and FTINTRZ saturate out-of-range values and convert NaN to 0,
4672// which is exactly the semantics of FP_TO_SINT_SAT with a native saturation
4673// width.
4674SDValue LoongArchTargetLowering::lowerFP_TO_SINT_SAT(SDValue Op,
4675 SelectionDAG &DAG) const {
4676 SDLoc DL(Op);
4677 SDValue Src = Op.getOperand(i: 0);
4678 EVT VT = Op.getValueType();
4679 EVT SrcVT = Src.getValueType();
4680 unsigned SatWidth = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4681
4682 // Vectors are selected by patterns.
4683 if (VT.isVector()) {
4684 if (SatWidth == VT.getScalarSizeInBits() &&
4685 SatWidth == SrcVT.getScalarSizeInBits())
4686 return Op;
4687 return SDValue();
4688 }
4689
4690 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4691 return SDValue();
4692
4693 if (SatWidth == 32) {
4694 SDValue Ftint = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f32, Operand: Src);
4695 if (Subtarget.is64Bit())
4696 return DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT, Operand: Ftint);
4697 return DAG.getBitcast(VT, V: Ftint);
4698 }
4699
4700 if (SatWidth == 64 && Subtarget.is64Bit() && Subtarget.hasBasicD()) {
4701 SDValue Ftint = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: MVT::f64, Operand: Src);
4702 return DAG.getBitcast(VT, V: Ftint);
4703 }
4704
4705 return SDValue();
4706}
4707
4708SDValue LoongArchTargetLowering::lowerFP_TO_UINT(SDValue Op,
4709 SelectionDAG &DAG) const {
4710 if (!Subtarget.hasExtLSX())
4711 return SDValue();
4712
4713 SDLoc DL(Op);
4714 SDValue Src = Op.getOperand(i: 0);
4715 EVT VT = Op.getValueType();
4716 EVT SrcVT = Src.getValueType();
4717
4718 if (VT != MVT::i64)
4719 return SDValue();
4720
4721 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4722 return SDValue();
4723
4724 if (SrcVT == MVT::f32)
4725 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Src);
4726 Src = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2f64, Operand: Src);
4727 SDValue Conv = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: MVT::v2i64, Operand: Src);
4728 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: Conv,
4729 N2: DAG.getIntPtrConstant(Val: 0, DL));
4730}
4731
4732static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
4733 SelectionDAG &DAG, unsigned Flags) {
4734 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
4735}
4736
4737static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
4738 SelectionDAG &DAG, unsigned Flags) {
4739 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
4740 TargetFlags: Flags);
4741}
4742
4743static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
4744 SelectionDAG &DAG, unsigned Flags) {
4745 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
4746 Offset: N->getOffset(), TargetFlags: Flags);
4747}
4748
4749static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
4750 SelectionDAG &DAG, unsigned Flags) {
4751 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
4752}
4753
4754template <class NodeTy>
4755SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4756 CodeModel::Model M,
4757 bool IsLocal) const {
4758 SDLoc DL(N);
4759 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4760 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
4761 SDValue Load;
4762
4763 switch (M) {
4764 default:
4765 report_fatal_error(reason: "Unsupported code model");
4766
4767 case CodeModel::Large: {
4768 assert(Subtarget.is64Bit() && "Large code model requires LA64");
4769
4770 // This is not actually used, but is necessary for successfully matching
4771 // the PseudoLA_*_LARGE nodes.
4772 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4773 if (IsLocal) {
4774 // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that
4775 // eventually becomes the desired 5-insn code sequence.
4776 Load = SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL_LARGE, dl: DL, VT: Ty,
4777 Op1: Tmp, Op2: Addr),
4778 0);
4779 } else {
4780 // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that
4781 // eventually becomes the desired 5-insn code sequence.
4782 Load = SDValue(
4783 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT_LARGE, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr),
4784 0);
4785 }
4786 break;
4787 }
4788
4789 case CodeModel::Small:
4790 case CodeModel::Medium:
4791 if (IsLocal) {
4792 // This generates the pattern (PseudoLA_PCREL sym), which
4793 //
4794 // for la32r expands to:
4795 // (addi.w (pcaddu12i %pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4796 //
4797 // for la32s and la64 expands to:
4798 // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).
4799 Load = SDValue(
4800 DAG.getMachineNode(Opcode: LoongArch::PseudoLA_PCREL, dl: DL, VT: Ty, Op1: Addr), 0);
4801 } else {
4802 // This generates the pattern (PseudoLA_GOT sym), which
4803 //
4804 // for la32r expands to:
4805 // (ld.w (pcaddu12i %got_pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4806 //
4807 // for la32s and la64 expands to:
4808 // (ld.w/d (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).
4809 Load =
4810 SDValue(DAG.getMachineNode(Opcode: LoongArch::PseudoLA_GOT, dl: DL, VT: Ty, Op1: Addr), 0);
4811 }
4812 }
4813
4814 if (!IsLocal) {
4815 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4816 MachineFunction &MF = DAG.getMachineFunction();
4817 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4818 PtrInfo: MachinePointerInfo::getGOT(MF),
4819 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4820 MachineMemOperand::MOInvariant,
4821 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4822 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
4823 }
4824
4825 return Load;
4826}
4827
4828SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,
4829 SelectionDAG &DAG) const {
4830 return getAddr(N: cast<BlockAddressSDNode>(Val&: Op), DAG,
4831 M: DAG.getTarget().getCodeModel());
4832}
4833
4834SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,
4835 SelectionDAG &DAG) const {
4836 return getAddr(N: cast<JumpTableSDNode>(Val&: Op), DAG,
4837 M: DAG.getTarget().getCodeModel());
4838}
4839
4840SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,
4841 SelectionDAG &DAG) const {
4842 return getAddr(N: cast<ConstantPoolSDNode>(Val&: Op), DAG,
4843 M: DAG.getTarget().getCodeModel());
4844}
4845
4846SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,
4847 SelectionDAG &DAG) const {
4848 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4849 assert(N->getOffset() == 0 && "unexpected offset in global node");
4850 auto CM = DAG.getTarget().getCodeModel();
4851 const GlobalValue *GV = N->getGlobal();
4852
4853 if (GV->isDSOLocal() && isa<GlobalVariable>(Val: GV)) {
4854 if (auto GCM = dyn_cast<GlobalVariable>(Val: GV)->getCodeModel())
4855 CM = *GCM;
4856 }
4857
4858 return getAddr(N, DAG, M: CM, IsLocal: GV->isDSOLocal());
4859}
4860
4861SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
4862 SelectionDAG &DAG,
4863 unsigned Opc, bool UseGOT,
4864 bool Large) const {
4865 SDLoc DL(N);
4866 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4867 MVT GRLenVT = Subtarget.getGRLenVT();
4868
4869 // This is not actually used, but is necessary for successfully matching the
4870 // PseudoLA_*_LARGE nodes.
4871 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4872 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4873
4874 // Only IE needs an extra argument for large code model.
4875 SDValue Offset = Opc == LoongArch::PseudoLA_TLS_IE_LARGE
4876 ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4877 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4878
4879 // If it is LE for normal/medium code model, the add tp operation will occur
4880 // during the pseudo-instruction expansion.
4881 if (Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4882 return Offset;
4883
4884 if (UseGOT) {
4885 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4886 MachineFunction &MF = DAG.getMachineFunction();
4887 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4888 PtrInfo: MachinePointerInfo::getGOT(MF),
4889 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4890 MachineMemOperand::MOInvariant,
4891 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
4892 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Offset.getNode()), NewMemRefs: {MemOp});
4893 }
4894
4895 // Add the thread pointer.
4896 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Offset,
4897 N2: DAG.getRegister(Reg: LoongArch::R2, VT: GRLenVT));
4898}
4899
4900SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
4901 SelectionDAG &DAG,
4902 unsigned Opc,
4903 bool Large) const {
4904 SDLoc DL(N);
4905 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4906 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
4907
4908 // This is not actually used, but is necessary for successfully matching the
4909 // PseudoLA_*_LARGE nodes.
4910 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4911
4912 // Use a PC-relative addressing mode to access the dynamic GOT address.
4913 SDValue Addr = DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: 0);
4914 SDValue Load = Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4915 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4916
4917 // Prepare argument list to generate call.
4918 ArgListTy Args;
4919 Args.emplace_back(args&: Load, args&: CallTy);
4920
4921 // Setup call to __tls_get_addr.
4922 TargetLowering::CallLoweringInfo CLI(DAG);
4923 CLI.setDebugLoc(DL)
4924 .setChain(DAG.getEntryNode())
4925 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
4926 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
4927 ArgsList: std::move(Args));
4928
4929 return LowerCallTo(CLI).first;
4930}
4931
4932SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
4933 SelectionDAG &DAG, unsigned Opc,
4934 bool Large) const {
4935 SDLoc DL(N);
4936 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
4937 const GlobalValue *GV = N->getGlobal();
4938
4939 // This is not actually used, but is necessary for successfully matching the
4940 // PseudoLA_*_LARGE nodes.
4941 SDValue Tmp = DAG.getConstant(Val: 0, DL, VT: Ty);
4942
4943 // Use a PC-relative addressing mode to access the global dynamic GOT address.
4944 // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).
4945 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
4946 return Large ? SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Tmp, Op2: Addr), 0)
4947 : SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VT: Ty, Op1: Addr), 0);
4948}
4949
4950SDValue
4951LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,
4952 SelectionDAG &DAG) const {
4953 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4954 CallingConv::GHC)
4955 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4956
4957 bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;
4958 assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");
4959
4960 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
4961 assert(N->getOffset() == 0 && "unexpected offset in global node");
4962
4963 if (DAG.getTarget().useEmulatedTLS())
4964 reportFatalUsageError(reason: "the emulated TLS is prohibited");
4965
4966 bool IsDesc = DAG.getTarget().useTLSDESC();
4967
4968 switch (getTargetMachine().getTLSModel(GV: N->getGlobal())) {
4969 case TLSModel::GeneralDynamic:
4970 // In this model, application code calls the dynamic linker function
4971 // __tls_get_addr to locate TLS offsets into the dynamic thread vector at
4972 // runtime.
4973 if (!IsDesc)
4974 return getDynamicTLSAddr(N, DAG,
4975 Opc: Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4976 : LoongArch::PseudoLA_TLS_GD,
4977 Large);
4978 break;
4979 case TLSModel::LocalDynamic:
4980 // Same as GeneralDynamic, except for assembly modifiers and relocation
4981 // records.
4982 if (!IsDesc)
4983 return getDynamicTLSAddr(N, DAG,
4984 Opc: Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4985 : LoongArch::PseudoLA_TLS_LD,
4986 Large);
4987 break;
4988 case TLSModel::InitialExec:
4989 // This model uses the GOT to resolve TLS offsets.
4990 return getStaticTLSAddr(N, DAG,
4991 Opc: Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4992 : LoongArch::PseudoLA_TLS_IE,
4993 /*UseGOT=*/true, Large);
4994 case TLSModel::LocalExec:
4995 // This model is used when static linking as the TLS offsets are resolved
4996 // during program linking.
4997 //
4998 // This node doesn't need an extra argument for the large code model.
4999 return getStaticTLSAddr(N, DAG, Opc: LoongArch::PseudoLA_TLS_LE,
5000 /*UseGOT=*/false, Large);
5001 }
5002
5003 return getTLSDescAddr(N, DAG,
5004 Opc: Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
5005 : LoongArch::PseudoLA_TLS_DESC,
5006 Large);
5007}
5008
5009template <unsigned N>
5010static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp,
5011 SelectionDAG &DAG, bool IsSigned = false) {
5012 auto *CImm = cast<ConstantSDNode>(Val: Op->getOperand(Num: ImmOp));
5013 // Check the ImmArg.
5014 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
5015 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
5016 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) +
5017 ": argument out of range.");
5018 return DAG.getNode(Opcode: ISD::UNDEF, DL: SDLoc(Op), VT: Op.getValueType());
5019 }
5020 return SDValue();
5021}
5022
5023SDValue
5024LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5025 SelectionDAG &DAG) const {
5026 switch (Op.getConstantOperandVal(i: 0)) {
5027 default:
5028 return SDValue(); // Don't custom lower most intrinsics.
5029 case Intrinsic::thread_pointer: {
5030 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
5031 return DAG.getRegister(Reg: LoongArch::R2, VT: PtrVT);
5032 }
5033 case Intrinsic::loongarch_lsx_vpickve2gr_d:
5034 case Intrinsic::loongarch_lsx_vpickve2gr_du:
5035 case Intrinsic::loongarch_lsx_vreplvei_d:
5036 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
5037 return checkIntrinsicImmArg<1>(Op, ImmOp: 2, DAG);
5038 case Intrinsic::loongarch_lsx_vreplvei_w:
5039 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
5040 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
5041 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
5042 case Intrinsic::loongarch_lasx_xvpickve_d:
5043 case Intrinsic::loongarch_lasx_xvpickve_d_f:
5044 return checkIntrinsicImmArg<2>(Op, ImmOp: 2, DAG);
5045 case Intrinsic::loongarch_lasx_xvinsve0_d:
5046 return checkIntrinsicImmArg<2>(Op, ImmOp: 3, DAG);
5047 case Intrinsic::loongarch_lsx_vsat_b:
5048 case Intrinsic::loongarch_lsx_vsat_bu:
5049 case Intrinsic::loongarch_lsx_vrotri_b:
5050 case Intrinsic::loongarch_lsx_vsllwil_h_b:
5051 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
5052 case Intrinsic::loongarch_lsx_vsrlri_b:
5053 case Intrinsic::loongarch_lsx_vsrari_b:
5054 case Intrinsic::loongarch_lsx_vreplvei_h:
5055 case Intrinsic::loongarch_lasx_xvsat_b:
5056 case Intrinsic::loongarch_lasx_xvsat_bu:
5057 case Intrinsic::loongarch_lasx_xvrotri_b:
5058 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
5059 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
5060 case Intrinsic::loongarch_lasx_xvsrlri_b:
5061 case Intrinsic::loongarch_lasx_xvsrari_b:
5062 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
5063 case Intrinsic::loongarch_lasx_xvpickve_w:
5064 case Intrinsic::loongarch_lasx_xvpickve_w_f:
5065 return checkIntrinsicImmArg<3>(Op, ImmOp: 2, DAG);
5066 case Intrinsic::loongarch_lasx_xvinsve0_w:
5067 return checkIntrinsicImmArg<3>(Op, ImmOp: 3, DAG);
5068 case Intrinsic::loongarch_lsx_vsat_h:
5069 case Intrinsic::loongarch_lsx_vsat_hu:
5070 case Intrinsic::loongarch_lsx_vrotri_h:
5071 case Intrinsic::loongarch_lsx_vsllwil_w_h:
5072 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
5073 case Intrinsic::loongarch_lsx_vsrlri_h:
5074 case Intrinsic::loongarch_lsx_vsrari_h:
5075 case Intrinsic::loongarch_lsx_vreplvei_b:
5076 case Intrinsic::loongarch_lasx_xvsat_h:
5077 case Intrinsic::loongarch_lasx_xvsat_hu:
5078 case Intrinsic::loongarch_lasx_xvrotri_h:
5079 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
5080 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
5081 case Intrinsic::loongarch_lasx_xvsrlri_h:
5082 case Intrinsic::loongarch_lasx_xvsrari_h:
5083 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
5084 return checkIntrinsicImmArg<4>(Op, ImmOp: 2, DAG);
5085 case Intrinsic::loongarch_lsx_vsrlni_b_h:
5086 case Intrinsic::loongarch_lsx_vsrani_b_h:
5087 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
5088 case Intrinsic::loongarch_lsx_vsrarni_b_h:
5089 case Intrinsic::loongarch_lsx_vssrlni_b_h:
5090 case Intrinsic::loongarch_lsx_vssrani_b_h:
5091 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
5092 case Intrinsic::loongarch_lsx_vssrani_bu_h:
5093 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
5094 case Intrinsic::loongarch_lsx_vssrarni_b_h:
5095 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
5096 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
5097 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
5098 case Intrinsic::loongarch_lasx_xvsrani_b_h:
5099 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
5100 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
5101 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
5102 case Intrinsic::loongarch_lasx_xvssrani_b_h:
5103 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
5104 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
5105 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
5106 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
5107 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
5108 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
5109 return checkIntrinsicImmArg<4>(Op, ImmOp: 3, DAG);
5110 case Intrinsic::loongarch_lsx_vsat_w:
5111 case Intrinsic::loongarch_lsx_vsat_wu:
5112 case Intrinsic::loongarch_lsx_vrotri_w:
5113 case Intrinsic::loongarch_lsx_vsllwil_d_w:
5114 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
5115 case Intrinsic::loongarch_lsx_vsrlri_w:
5116 case Intrinsic::loongarch_lsx_vsrari_w:
5117 case Intrinsic::loongarch_lsx_vslei_bu:
5118 case Intrinsic::loongarch_lsx_vslei_hu:
5119 case Intrinsic::loongarch_lsx_vslei_wu:
5120 case Intrinsic::loongarch_lsx_vslei_du:
5121 case Intrinsic::loongarch_lsx_vslti_bu:
5122 case Intrinsic::loongarch_lsx_vslti_hu:
5123 case Intrinsic::loongarch_lsx_vslti_wu:
5124 case Intrinsic::loongarch_lsx_vslti_du:
5125 case Intrinsic::loongarch_lsx_vbsll_v:
5126 case Intrinsic::loongarch_lsx_vbsrl_v:
5127 case Intrinsic::loongarch_lasx_xvsat_w:
5128 case Intrinsic::loongarch_lasx_xvsat_wu:
5129 case Intrinsic::loongarch_lasx_xvrotri_w:
5130 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
5131 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
5132 case Intrinsic::loongarch_lasx_xvsrlri_w:
5133 case Intrinsic::loongarch_lasx_xvsrari_w:
5134 case Intrinsic::loongarch_lasx_xvslei_bu:
5135 case Intrinsic::loongarch_lasx_xvslei_hu:
5136 case Intrinsic::loongarch_lasx_xvslei_wu:
5137 case Intrinsic::loongarch_lasx_xvslei_du:
5138 case Intrinsic::loongarch_lasx_xvslti_bu:
5139 case Intrinsic::loongarch_lasx_xvslti_hu:
5140 case Intrinsic::loongarch_lasx_xvslti_wu:
5141 case Intrinsic::loongarch_lasx_xvslti_du:
5142 case Intrinsic::loongarch_lasx_xvbsll_v:
5143 case Intrinsic::loongarch_lasx_xvbsrl_v:
5144 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG);
5145 case Intrinsic::loongarch_lsx_vseqi_b:
5146 case Intrinsic::loongarch_lsx_vseqi_h:
5147 case Intrinsic::loongarch_lsx_vseqi_w:
5148 case Intrinsic::loongarch_lsx_vseqi_d:
5149 case Intrinsic::loongarch_lsx_vslei_b:
5150 case Intrinsic::loongarch_lsx_vslei_h:
5151 case Intrinsic::loongarch_lsx_vslei_w:
5152 case Intrinsic::loongarch_lsx_vslei_d:
5153 case Intrinsic::loongarch_lsx_vslti_b:
5154 case Intrinsic::loongarch_lsx_vslti_h:
5155 case Intrinsic::loongarch_lsx_vslti_w:
5156 case Intrinsic::loongarch_lsx_vslti_d:
5157 case Intrinsic::loongarch_lasx_xvseqi_b:
5158 case Intrinsic::loongarch_lasx_xvseqi_h:
5159 case Intrinsic::loongarch_lasx_xvseqi_w:
5160 case Intrinsic::loongarch_lasx_xvseqi_d:
5161 case Intrinsic::loongarch_lasx_xvslei_b:
5162 case Intrinsic::loongarch_lasx_xvslei_h:
5163 case Intrinsic::loongarch_lasx_xvslei_w:
5164 case Intrinsic::loongarch_lasx_xvslei_d:
5165 case Intrinsic::loongarch_lasx_xvslti_b:
5166 case Intrinsic::loongarch_lasx_xvslti_h:
5167 case Intrinsic::loongarch_lasx_xvslti_w:
5168 case Intrinsic::loongarch_lasx_xvslti_d:
5169 return checkIntrinsicImmArg<5>(Op, ImmOp: 2, DAG, /*IsSigned=*/true);
5170 case Intrinsic::loongarch_lsx_vsrlni_h_w:
5171 case Intrinsic::loongarch_lsx_vsrani_h_w:
5172 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
5173 case Intrinsic::loongarch_lsx_vsrarni_h_w:
5174 case Intrinsic::loongarch_lsx_vssrlni_h_w:
5175 case Intrinsic::loongarch_lsx_vssrani_h_w:
5176 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
5177 case Intrinsic::loongarch_lsx_vssrani_hu_w:
5178 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
5179 case Intrinsic::loongarch_lsx_vssrarni_h_w:
5180 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
5181 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
5182 case Intrinsic::loongarch_lsx_vfrstpi_b:
5183 case Intrinsic::loongarch_lsx_vfrstpi_h:
5184 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
5185 case Intrinsic::loongarch_lasx_xvsrani_h_w:
5186 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
5187 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
5188 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
5189 case Intrinsic::loongarch_lasx_xvssrani_h_w:
5190 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
5191 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
5192 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
5193 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
5194 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
5195 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
5196 case Intrinsic::loongarch_lasx_xvfrstpi_b:
5197 case Intrinsic::loongarch_lasx_xvfrstpi_h:
5198 return checkIntrinsicImmArg<5>(Op, ImmOp: 3, DAG);
5199 case Intrinsic::loongarch_lsx_vsat_d:
5200 case Intrinsic::loongarch_lsx_vsat_du:
5201 case Intrinsic::loongarch_lsx_vrotri_d:
5202 case Intrinsic::loongarch_lsx_vsrlri_d:
5203 case Intrinsic::loongarch_lsx_vsrari_d:
5204 case Intrinsic::loongarch_lasx_xvsat_d:
5205 case Intrinsic::loongarch_lasx_xvsat_du:
5206 case Intrinsic::loongarch_lasx_xvrotri_d:
5207 case Intrinsic::loongarch_lasx_xvsrlri_d:
5208 case Intrinsic::loongarch_lasx_xvsrari_d:
5209 return checkIntrinsicImmArg<6>(Op, ImmOp: 2, DAG);
5210 case Intrinsic::loongarch_lsx_vsrlni_w_d:
5211 case Intrinsic::loongarch_lsx_vsrani_w_d:
5212 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
5213 case Intrinsic::loongarch_lsx_vsrarni_w_d:
5214 case Intrinsic::loongarch_lsx_vssrlni_w_d:
5215 case Intrinsic::loongarch_lsx_vssrani_w_d:
5216 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
5217 case Intrinsic::loongarch_lsx_vssrani_wu_d:
5218 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
5219 case Intrinsic::loongarch_lsx_vssrarni_w_d:
5220 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
5221 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
5222 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
5223 case Intrinsic::loongarch_lasx_xvsrani_w_d:
5224 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
5225 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
5226 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
5227 case Intrinsic::loongarch_lasx_xvssrani_w_d:
5228 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
5229 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
5230 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
5231 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
5232 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
5233 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
5234 return checkIntrinsicImmArg<6>(Op, ImmOp: 3, DAG);
5235 case Intrinsic::loongarch_lsx_vsrlni_d_q:
5236 case Intrinsic::loongarch_lsx_vsrani_d_q:
5237 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
5238 case Intrinsic::loongarch_lsx_vsrarni_d_q:
5239 case Intrinsic::loongarch_lsx_vssrlni_d_q:
5240 case Intrinsic::loongarch_lsx_vssrani_d_q:
5241 case Intrinsic::loongarch_lsx_vssrlni_du_q:
5242 case Intrinsic::loongarch_lsx_vssrani_du_q:
5243 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
5244 case Intrinsic::loongarch_lsx_vssrarni_d_q:
5245 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
5246 case Intrinsic::loongarch_lsx_vssrarni_du_q:
5247 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
5248 case Intrinsic::loongarch_lasx_xvsrani_d_q:
5249 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
5250 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
5251 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
5252 case Intrinsic::loongarch_lasx_xvssrani_d_q:
5253 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
5254 case Intrinsic::loongarch_lasx_xvssrani_du_q:
5255 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
5256 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
5257 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
5258 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
5259 return checkIntrinsicImmArg<7>(Op, ImmOp: 3, DAG);
5260 case Intrinsic::loongarch_lsx_vnori_b:
5261 case Intrinsic::loongarch_lsx_vshuf4i_b:
5262 case Intrinsic::loongarch_lsx_vshuf4i_h:
5263 case Intrinsic::loongarch_lsx_vshuf4i_w:
5264 case Intrinsic::loongarch_lasx_xvnori_b:
5265 case Intrinsic::loongarch_lasx_xvshuf4i_b:
5266 case Intrinsic::loongarch_lasx_xvshuf4i_h:
5267 case Intrinsic::loongarch_lasx_xvshuf4i_w:
5268 case Intrinsic::loongarch_lasx_xvpermi_d:
5269 return checkIntrinsicImmArg<8>(Op, ImmOp: 2, DAG);
5270 case Intrinsic::loongarch_lsx_vshuf4i_d:
5271 case Intrinsic::loongarch_lsx_vpermi_w:
5272 case Intrinsic::loongarch_lsx_vbitseli_b:
5273 case Intrinsic::loongarch_lsx_vextrins_b:
5274 case Intrinsic::loongarch_lsx_vextrins_h:
5275 case Intrinsic::loongarch_lsx_vextrins_w:
5276 case Intrinsic::loongarch_lsx_vextrins_d:
5277 case Intrinsic::loongarch_lasx_xvshuf4i_d:
5278 case Intrinsic::loongarch_lasx_xvpermi_w:
5279 case Intrinsic::loongarch_lasx_xvpermi_q:
5280 case Intrinsic::loongarch_lasx_xvbitseli_b:
5281 case Intrinsic::loongarch_lasx_xvextrins_b:
5282 case Intrinsic::loongarch_lasx_xvextrins_h:
5283 case Intrinsic::loongarch_lasx_xvextrins_w:
5284 case Intrinsic::loongarch_lasx_xvextrins_d:
5285 return checkIntrinsicImmArg<8>(Op, ImmOp: 3, DAG);
5286 case Intrinsic::loongarch_lsx_vrepli_b:
5287 case Intrinsic::loongarch_lsx_vrepli_h:
5288 case Intrinsic::loongarch_lsx_vrepli_w:
5289 case Intrinsic::loongarch_lsx_vrepli_d:
5290 case Intrinsic::loongarch_lasx_xvrepli_b:
5291 case Intrinsic::loongarch_lasx_xvrepli_h:
5292 case Intrinsic::loongarch_lasx_xvrepli_w:
5293 case Intrinsic::loongarch_lasx_xvrepli_d:
5294 return checkIntrinsicImmArg<10>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
5295 case Intrinsic::loongarch_lsx_vldi:
5296 case Intrinsic::loongarch_lasx_xvldi:
5297 return checkIntrinsicImmArg<13>(Op, ImmOp: 1, DAG, /*IsSigned=*/true);
5298 }
5299}
5300
5301// Helper function that emits error message for intrinsics with chain and return
5302// merge values of a UNDEF and the chain.
5303static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op,
5304 StringRef ErrorMsg,
5305 SelectionDAG &DAG) {
5306 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5307 return DAG.getMergeValues(Ops: {DAG.getUNDEF(VT: Op.getValueType()), Op.getOperand(i: 0)},
5308 dl: SDLoc(Op));
5309}
5310
5311SDValue
5312LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5313 SelectionDAG &DAG) const {
5314 SDLoc DL(Op);
5315 MVT GRLenVT = Subtarget.getGRLenVT();
5316 EVT VT = Op.getValueType();
5317 SDValue Chain = Op.getOperand(i: 0);
5318 const StringRef ErrorMsgOOR = "argument out of range";
5319 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5320 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5321
5322 switch (Op.getConstantOperandVal(i: 1)) {
5323 default:
5324 return Op;
5325 case Intrinsic::loongarch_crc_w_b_w:
5326 case Intrinsic::loongarch_crc_w_h_w:
5327 case Intrinsic::loongarch_crc_w_w_w:
5328 case Intrinsic::loongarch_crc_w_d_w:
5329 case Intrinsic::loongarch_crcc_w_b_w:
5330 case Intrinsic::loongarch_crcc_w_h_w:
5331 case Intrinsic::loongarch_crcc_w_w_w:
5332 case Intrinsic::loongarch_crcc_w_d_w:
5333 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5334 case Intrinsic::loongarch_csrrd_w:
5335 case Intrinsic::loongarch_csrrd_d: {
5336 unsigned Imm = Op.getConstantOperandVal(i: 2);
5337 return !isUInt<14>(x: Imm)
5338 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5339 : DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
5340 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5341 }
5342 case Intrinsic::loongarch_csrwr_w:
5343 case Intrinsic::loongarch_csrwr_d: {
5344 unsigned Imm = Op.getConstantOperandVal(i: 3);
5345 return !isUInt<14>(x: Imm)
5346 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5347 : DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
5348 Ops: {Chain, Op.getOperand(i: 2),
5349 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5350 }
5351 case Intrinsic::loongarch_csrxchg_w:
5352 case Intrinsic::loongarch_csrxchg_d: {
5353 unsigned Imm = Op.getConstantOperandVal(i: 4);
5354 return !isUInt<14>(x: Imm)
5355 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5356 : DAG.getNode(Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
5357 Ops: {Chain, Op.getOperand(i: 2), Op.getOperand(i: 3),
5358 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5359 }
5360 case Intrinsic::loongarch_iocsrrd_d: {
5361 return DAG.getNode(
5362 Opcode: LoongArchISD::IOCSRRD_D, DL, ResultTys: {GRLenVT, MVT::Other},
5363 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op.getOperand(i: 2))});
5364 }
5365#define IOCSRRD_CASE(NAME, NODE) \
5366 case Intrinsic::loongarch_##NAME: { \
5367 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5368 {Chain, Op.getOperand(2)}); \
5369 }
5370 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5371 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5372 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5373#undef IOCSRRD_CASE
5374 case Intrinsic::loongarch_cpucfg: {
5375 return DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
5376 Ops: {Chain, Op.getOperand(i: 2)});
5377 }
5378 case Intrinsic::loongarch_lddir_d: {
5379 unsigned Imm = Op.getConstantOperandVal(i: 3);
5380 return !isUInt<8>(x: Imm)
5381 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5382 : Op;
5383 }
5384 case Intrinsic::loongarch_movfcsr2gr: {
5385 if (!Subtarget.hasBasicF())
5386 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5387 unsigned Imm = Op.getConstantOperandVal(i: 2);
5388 return !isUInt<2>(x: Imm)
5389 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5390 : DAG.getNode(Opcode: LoongArchISD::MOVFCSR2GR, DL, ResultTys: {VT, MVT::Other},
5391 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
5392 }
5393 case Intrinsic::loongarch_lsx_vld:
5394 case Intrinsic::loongarch_lsx_vldrepl_b:
5395 case Intrinsic::loongarch_lasx_xvld:
5396 case Intrinsic::loongarch_lasx_xvldrepl_b:
5397 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5398 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5399 : SDValue();
5400 case Intrinsic::loongarch_lsx_vldrepl_h:
5401 case Intrinsic::loongarch_lasx_xvldrepl_h:
5402 return !isShiftedInt<11, 1>(
5403 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5404 ? emitIntrinsicWithChainErrorMessage(
5405 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5406 : SDValue();
5407 case Intrinsic::loongarch_lsx_vldrepl_w:
5408 case Intrinsic::loongarch_lasx_xvldrepl_w:
5409 return !isShiftedInt<10, 2>(
5410 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5411 ? emitIntrinsicWithChainErrorMessage(
5412 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5413 : SDValue();
5414 case Intrinsic::loongarch_lsx_vldrepl_d:
5415 case Intrinsic::loongarch_lasx_xvldrepl_d:
5416 return !isShiftedInt<9, 3>(
5417 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 3))->getSExtValue())
5418 ? emitIntrinsicWithChainErrorMessage(
5419 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5420 : SDValue();
5421 }
5422}
5423
5424// Helper function that emits error message for intrinsics with void return
5425// value and return the chain.
5426static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg,
5427 SelectionDAG &DAG) {
5428
5429 DAG.getContext()->emitError(ErrorStr: Op->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5430 return Op.getOperand(i: 0);
5431}
5432
5433SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
5434 SelectionDAG &DAG) const {
5435 SDLoc DL(Op);
5436 MVT GRLenVT = Subtarget.getGRLenVT();
5437 SDValue Chain = Op.getOperand(i: 0);
5438 uint64_t IntrinsicEnum = Op.getConstantOperandVal(i: 1);
5439 SDValue Op2 = Op.getOperand(i: 2);
5440 const StringRef ErrorMsgOOR = "argument out of range";
5441 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5442 const StringRef ErrorMsgReqLA32 = "requires loongarch32";
5443 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5444
5445 switch (IntrinsicEnum) {
5446 default:
5447 // TODO: Add more Intrinsics.
5448 return SDValue();
5449 case Intrinsic::loongarch_cacop_d:
5450 case Intrinsic::loongarch_cacop_w: {
5451 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5452 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG);
5453 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5454 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA32, DAG);
5455 // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)
5456 unsigned Imm1 = Op2->getAsZExtVal();
5457 int Imm2 = cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue();
5458 if (!isUInt<5>(x: Imm1) || !isInt<12>(x: Imm2))
5459 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG);
5460 return Op;
5461 }
5462 case Intrinsic::loongarch_dbar: {
5463 unsigned Imm = Op2->getAsZExtVal();
5464 return !isUInt<15>(x: Imm)
5465 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5466 : DAG.getNode(Opcode: LoongArchISD::DBAR, DL, VT: MVT::Other, N1: Chain,
5467 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5468 }
5469 case Intrinsic::loongarch_ibar: {
5470 unsigned Imm = Op2->getAsZExtVal();
5471 return !isUInt<15>(x: Imm)
5472 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5473 : DAG.getNode(Opcode: LoongArchISD::IBAR, DL, VT: MVT::Other, N1: Chain,
5474 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5475 }
5476 case Intrinsic::loongarch_break: {
5477 unsigned Imm = Op2->getAsZExtVal();
5478 return !isUInt<15>(x: Imm)
5479 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5480 : DAG.getNode(Opcode: LoongArchISD::BREAK, DL, VT: MVT::Other, N1: Chain,
5481 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5482 }
5483 case Intrinsic::loongarch_movgr2fcsr: {
5484 if (!Subtarget.hasBasicF())
5485 return emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqF, DAG);
5486 unsigned Imm = Op2->getAsZExtVal();
5487 return !isUInt<2>(x: Imm)
5488 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5489 : DAG.getNode(Opcode: LoongArchISD::MOVGR2FCSR, DL, VT: MVT::Other, N1: Chain,
5490 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT),
5491 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT,
5492 Operand: Op.getOperand(i: 3)));
5493 }
5494 case Intrinsic::loongarch_syscall: {
5495 unsigned Imm = Op2->getAsZExtVal();
5496 return !isUInt<15>(x: Imm)
5497 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5498 : DAG.getNode(Opcode: LoongArchISD::SYSCALL, DL, VT: MVT::Other, N1: Chain,
5499 N2: DAG.getConstant(Val: Imm, DL, VT: GRLenVT));
5500 }
5501#define IOCSRWR_CASE(NAME, NODE) \
5502 case Intrinsic::loongarch_##NAME: { \
5503 SDValue Op3 = Op.getOperand(3); \
5504 return Subtarget.is64Bit() \
5505 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5506 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5507 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5508 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5509 Op3); \
5510 }
5511 IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);
5512 IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);
5513 IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);
5514#undef IOCSRWR_CASE
5515 case Intrinsic::loongarch_iocsrwr_d: {
5516 return !Subtarget.is64Bit()
5517 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5518 : DAG.getNode(Opcode: LoongArchISD::IOCSRWR_D, DL, VT: MVT::Other, N1: Chain,
5519 N2: Op2,
5520 N3: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64,
5521 Operand: Op.getOperand(i: 3)));
5522 }
5523#define ASRT_LE_GT_CASE(NAME) \
5524 case Intrinsic::loongarch_##NAME: { \
5525 return !Subtarget.is64Bit() \
5526 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5527 : Op; \
5528 }
5529 ASRT_LE_GT_CASE(asrtle_d)
5530 ASRT_LE_GT_CASE(asrtgt_d)
5531#undef ASRT_LE_GT_CASE
5532 case Intrinsic::loongarch_ldpte_d: {
5533 unsigned Imm = Op.getConstantOperandVal(i: 3);
5534 return !Subtarget.is64Bit()
5535 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgReqLA64, DAG)
5536 : !isUInt<8>(x: Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5537 : Op;
5538 }
5539 case Intrinsic::loongarch_lsx_vst:
5540 case Intrinsic::loongarch_lasx_xvst:
5541 return !isInt<12>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue())
5542 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5543 : SDValue();
5544 case Intrinsic::loongarch_lasx_xvstelm_b:
5545 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5546 !isUInt<5>(x: Op.getConstantOperandVal(i: 5)))
5547 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5548 : SDValue();
5549 case Intrinsic::loongarch_lsx_vstelm_b:
5550 return (!isInt<8>(x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5551 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5552 ? emitIntrinsicErrorMessage(Op, ErrorMsg: ErrorMsgOOR, DAG)
5553 : SDValue();
5554 case Intrinsic::loongarch_lasx_xvstelm_h:
5555 return (!isShiftedInt<8, 1>(
5556 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5557 !isUInt<4>(x: Op.getConstantOperandVal(i: 5)))
5558 ? emitIntrinsicErrorMessage(
5559 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5560 : SDValue();
5561 case Intrinsic::loongarch_lsx_vstelm_h:
5562 return (!isShiftedInt<8, 1>(
5563 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5564 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5565 ? emitIntrinsicErrorMessage(
5566 Op, ErrorMsg: "argument out of range or not a multiple of 2", DAG)
5567 : SDValue();
5568 case Intrinsic::loongarch_lasx_xvstelm_w:
5569 return (!isShiftedInt<8, 2>(
5570 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5571 !isUInt<3>(x: Op.getConstantOperandVal(i: 5)))
5572 ? emitIntrinsicErrorMessage(
5573 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5574 : SDValue();
5575 case Intrinsic::loongarch_lsx_vstelm_w:
5576 return (!isShiftedInt<8, 2>(
5577 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5578 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5579 ? emitIntrinsicErrorMessage(
5580 Op, ErrorMsg: "argument out of range or not a multiple of 4", DAG)
5581 : SDValue();
5582 case Intrinsic::loongarch_lasx_xvstelm_d:
5583 return (!isShiftedInt<8, 3>(
5584 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5585 !isUInt<2>(x: Op.getConstantOperandVal(i: 5)))
5586 ? emitIntrinsicErrorMessage(
5587 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5588 : SDValue();
5589 case Intrinsic::loongarch_lsx_vstelm_d:
5590 return (!isShiftedInt<8, 3>(
5591 x: cast<ConstantSDNode>(Val: Op.getOperand(i: 4))->getSExtValue()) ||
5592 !isUInt<1>(x: Op.getConstantOperandVal(i: 5)))
5593 ? emitIntrinsicErrorMessage(
5594 Op, ErrorMsg: "argument out of range or not a multiple of 8", DAG)
5595 : SDValue();
5596 }
5597}
5598
5599SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,
5600 SelectionDAG &DAG) const {
5601 SDLoc DL(Op);
5602 SDValue Lo = Op.getOperand(i: 0);
5603 SDValue Hi = Op.getOperand(i: 1);
5604 SDValue Shamt = Op.getOperand(i: 2);
5605 EVT VT = Lo.getValueType();
5606
5607 // if Shamt-GRLen < 0: // Shamt < GRLen
5608 // Lo = Lo << Shamt
5609 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))
5610 // else:
5611 // Lo = 0
5612 // Hi = Lo << (Shamt-GRLen)
5613
5614 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5615 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5616 SDValue MinusGRLen =
5617 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5618 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5619 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5620 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5621
5622 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
5623 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
5624 SDValue ShiftRightLo =
5625 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: GRLenMinus1Shamt);
5626 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
5627 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
5628 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusGRLen);
5629
5630 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5631
5632 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
5633 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5634
5635 SDValue Parts[2] = {Lo, Hi};
5636 return DAG.getMergeValues(Ops: Parts, dl: DL);
5637}
5638
5639SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,
5640 SelectionDAG &DAG,
5641 bool IsSRA) const {
5642 SDLoc DL(Op);
5643 SDValue Lo = Op.getOperand(i: 0);
5644 SDValue Hi = Op.getOperand(i: 1);
5645 SDValue Shamt = Op.getOperand(i: 2);
5646 EVT VT = Lo.getValueType();
5647
5648 // SRA expansion:
5649 // if Shamt-GRLen < 0: // Shamt < GRLen
5650 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5651 // Hi = Hi >>s Shamt
5652 // else:
5653 // Lo = Hi >>s (Shamt-GRLen);
5654 // Hi = Hi >>s (GRLen-1)
5655 //
5656 // SRL expansion:
5657 // if Shamt-GRLen < 0: // Shamt < GRLen
5658 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5659 // Hi = Hi >>u Shamt
5660 // else:
5661 // Lo = Hi >>u (Shamt-GRLen);
5662 // Hi = 0;
5663
5664 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
5665
5666 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5667 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5668 SDValue MinusGRLen =
5669 DAG.getSignedConstant(Val: -(int)Subtarget.getGRLen(), DL, VT);
5670 SDValue GRLenMinus1 = DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT);
5671 SDValue ShamtMinusGRLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusGRLen);
5672 SDValue GRLenMinus1Shamt = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Shamt, N2: GRLenMinus1);
5673
5674 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
5675 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
5676 SDValue ShiftLeftHi =
5677 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: GRLenMinus1Shamt);
5678 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
5679 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
5680 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusGRLen);
5681 SDValue HiFalse =
5682 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: GRLenMinus1) : Zero;
5683
5684 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusGRLen, RHS: Zero, Cond: ISD::SETLT);
5685
5686 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
5687 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
5688
5689 SDValue Parts[2] = {Lo, Hi};
5690 return DAG.getMergeValues(Ops: Parts, dl: DL);
5691}
5692
5693// Returns the opcode of the target-specific SDNode that implements the 32-bit
5694// form of the given Opcode.
5695static unsigned getLoongArchWOpcode(unsigned Opcode) {
5696 switch (Opcode) {
5697 default:
5698 llvm_unreachable("Unexpected opcode");
5699 case ISD::SDIV:
5700 return LoongArchISD::DIV_W;
5701 case ISD::UDIV:
5702 return LoongArchISD::DIV_WU;
5703 case ISD::SREM:
5704 return LoongArchISD::MOD_W;
5705 case ISD::UREM:
5706 return LoongArchISD::MOD_WU;
5707 case ISD::SHL:
5708 return LoongArchISD::SLL_W;
5709 case ISD::SRA:
5710 return LoongArchISD::SRA_W;
5711 case ISD::SRL:
5712 return LoongArchISD::SRL_W;
5713 case ISD::ROTL:
5714 case ISD::ROTR:
5715 return LoongArchISD::ROTR_W;
5716 case ISD::CTTZ:
5717 return LoongArchISD::CTZ_W;
5718 case ISD::CTLZ:
5719 return LoongArchISD::CLZ_W;
5720 }
5721}
5722
5723// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5724// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
5725// otherwise be promoted to i64, making it difficult to select the
5726// SLL_W/.../*W later one because the fact the operation was originally of
5727// type i8/i16/i32 is lost.
5728static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
5729 unsigned ExtOpc = ISD::ANY_EXTEND) {
5730 SDLoc DL(N);
5731 unsigned WOpcode = getLoongArchWOpcode(Opcode: N->getOpcode());
5732 SDValue NewOp0, NewRes;
5733
5734 switch (NumOp) {
5735 default:
5736 llvm_unreachable("Unexpected NumOp");
5737 case 1: {
5738 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5739 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, Operand: NewOp0);
5740 break;
5741 }
5742 case 2: {
5743 NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5744 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5745 if (N->getOpcode() == ISD::ROTL) {
5746 SDValue TmpOp = DAG.getConstant(Val: 32, DL, VT: MVT::i64);
5747 NewOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: TmpOp, N2: NewOp1);
5748 }
5749 NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5750 break;
5751 }
5752 // TODO:Handle more NumOp.
5753 }
5754
5755 // ReplaceNodeResults requires we maintain the same type for the return
5756 // value.
5757 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
5758}
5759
5760// Converts the given 32-bit operation to a i64 operation with signed extension
5761// semantic to reduce the signed extension instructions.
5762static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
5763 SDLoc DL(N);
5764 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
5765 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
5766 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
5767 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
5768 N2: DAG.getValueType(MVT::i32));
5769 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
5770}
5771
5772// Helper function that emits error message for intrinsics with/without chain
5773// and return a UNDEF or and the chain as the results.
5774static void emitErrorAndReplaceIntrinsicResults(
5775 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
5776 StringRef ErrorMsg, bool WithChain = true) {
5777 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
5778 Results.push_back(Elt: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)));
5779 if (!WithChain)
5780 return;
5781 Results.push_back(Elt: N->getOperand(Num: 0));
5782}
5783
5784template <unsigned N>
5785static void
5786replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl<SDValue> &Results,
5787 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget,
5788 unsigned ResOp) {
5789 const StringRef ErrorMsgOOR = "argument out of range";
5790 unsigned Imm = Node->getConstantOperandVal(Num: 2);
5791 if (!isUInt<N>(Imm)) {
5792 emitErrorAndReplaceIntrinsicResults(N: Node, Results, DAG, ErrorMsg: ErrorMsgOOR,
5793 /*WithChain=*/false);
5794 return;
5795 }
5796 SDLoc DL(Node);
5797 SDValue Vec = Node->getOperand(Num: 1);
5798
5799 SDValue PickElt =
5800 DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), N1: Vec,
5801 N2: DAG.getConstant(Val: Imm, DL, VT: Subtarget.getGRLenVT()),
5802 N3: DAG.getValueType(Vec.getValueType().getVectorElementType()));
5803 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Node->getValueType(ResNo: 0),
5804 Operand: PickElt.getValue(R: 0)));
5805}
5806
5807static void replaceVecCondBranchResults(SDNode *N,
5808 SmallVectorImpl<SDValue> &Results,
5809 SelectionDAG &DAG,
5810 const LoongArchSubtarget &Subtarget,
5811 unsigned ResOp) {
5812 SDLoc DL(N);
5813 SDValue Vec = N->getOperand(Num: 1);
5814
5815 SDValue CB = DAG.getNode(Opcode: ResOp, DL, VT: Subtarget.getGRLenVT(), Operand: Vec);
5816 Results.push_back(
5817 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: CB.getValue(R: 0)));
5818}
5819
5820static void
5821replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
5822 SelectionDAG &DAG,
5823 const LoongArchSubtarget &Subtarget) {
5824 switch (N->getConstantOperandVal(Num: 0)) {
5825 default:
5826 llvm_unreachable("Unexpected Intrinsic.");
5827 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5828 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5829 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5830 break;
5831 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5832 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5833 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5834 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5835 break;
5836 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5837 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5838 ResOp: LoongArchISD::VPICK_SEXT_ELT);
5839 break;
5840 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5841 replaceVPICKVE2GRResults<4>(Node: N, Results, DAG, Subtarget,
5842 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5843 break;
5844 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5845 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5846 replaceVPICKVE2GRResults<3>(Node: N, Results, DAG, Subtarget,
5847 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5848 break;
5849 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5850 replaceVPICKVE2GRResults<2>(Node: N, Results, DAG, Subtarget,
5851 ResOp: LoongArchISD::VPICK_ZEXT_ELT);
5852 break;
5853 case Intrinsic::loongarch_lsx_bz_b:
5854 case Intrinsic::loongarch_lsx_bz_h:
5855 case Intrinsic::loongarch_lsx_bz_w:
5856 case Intrinsic::loongarch_lsx_bz_d:
5857 case Intrinsic::loongarch_lasx_xbz_b:
5858 case Intrinsic::loongarch_lasx_xbz_h:
5859 case Intrinsic::loongarch_lasx_xbz_w:
5860 case Intrinsic::loongarch_lasx_xbz_d:
5861 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5862 ResOp: LoongArchISD::VALL_ZERO);
5863 break;
5864 case Intrinsic::loongarch_lsx_bz_v:
5865 case Intrinsic::loongarch_lasx_xbz_v:
5866 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5867 ResOp: LoongArchISD::VANY_ZERO);
5868 break;
5869 case Intrinsic::loongarch_lsx_bnz_b:
5870 case Intrinsic::loongarch_lsx_bnz_h:
5871 case Intrinsic::loongarch_lsx_bnz_w:
5872 case Intrinsic::loongarch_lsx_bnz_d:
5873 case Intrinsic::loongarch_lasx_xbnz_b:
5874 case Intrinsic::loongarch_lasx_xbnz_h:
5875 case Intrinsic::loongarch_lasx_xbnz_w:
5876 case Intrinsic::loongarch_lasx_xbnz_d:
5877 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5878 ResOp: LoongArchISD::VALL_NONZERO);
5879 break;
5880 case Intrinsic::loongarch_lsx_bnz_v:
5881 case Intrinsic::loongarch_lasx_xbnz_v:
5882 replaceVecCondBranchResults(N, Results, DAG, Subtarget,
5883 ResOp: LoongArchISD::VANY_NONZERO);
5884 break;
5885 }
5886}
5887
5888static void replaceCMP_XCHG_128Results(SDNode *N,
5889 SmallVectorImpl<SDValue> &Results,
5890 SelectionDAG &DAG) {
5891 assert(N->getValueType(0) == MVT::i128 &&
5892 "AtomicCmpSwap on types less than 128 should be legal");
5893 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
5894
5895 unsigned Opcode;
5896 switch (MemOp->getMergedOrdering()) {
5897 case AtomicOrdering::Acquire:
5898 case AtomicOrdering::AcquireRelease:
5899 case AtomicOrdering::SequentiallyConsistent:
5900 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5901 break;
5902 case AtomicOrdering::Monotonic:
5903 case AtomicOrdering::Release:
5904 Opcode = LoongArch::PseudoCmpXchg128;
5905 break;
5906 default:
5907 llvm_unreachable("Unexpected ordering!");
5908 }
5909
5910 SDLoc DL(N);
5911 auto CmpVal = DAG.SplitScalar(N: N->getOperand(Num: 2), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5912 auto NewVal = DAG.SplitScalar(N: N->getOperand(Num: 3), DL, LoVT: MVT::i64, HiVT: MVT::i64);
5913 SDValue Ops[] = {N->getOperand(Num: 1), CmpVal.first, CmpVal.second,
5914 NewVal.first, NewVal.second, N->getOperand(Num: 0)};
5915
5916 SDNode *CmpSwap = DAG.getMachineNode(
5917 Opcode, dl: SDLoc(N), VTs: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::i64, VT4: MVT::Other),
5918 Ops);
5919 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
5920 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
5921 N1: SDValue(CmpSwap, 0), N2: SDValue(CmpSwap, 1)));
5922 Results.push_back(Elt: SDValue(CmpSwap, 3));
5923}
5924
5925void LoongArchTargetLowering::ReplaceNodeResults(
5926 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
5927 SDLoc DL(N);
5928 EVT VT = N->getValueType(ResNo: 0);
5929 switch (N->getOpcode()) {
5930 default:
5931 llvm_unreachable("Don't know how to legalize this operation");
5932 case ISD::ADD:
5933 case ISD::SUB:
5934 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5935 "Unexpected custom legalisation");
5936 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
5937 break;
5938 case ISD::SDIV:
5939 case ISD::UDIV:
5940 case ISD::SREM:
5941 case ISD::UREM:
5942 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5943 "Unexpected custom legalisation");
5944 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2,
5945 ExtOpc: Subtarget.hasDiv32() && VT == MVT::i32
5946 ? ISD::ANY_EXTEND
5947 : ISD::SIGN_EXTEND));
5948 break;
5949 case ISD::SHL:
5950 case ISD::SRA:
5951 case ISD::SRL:
5952 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5953 "Unexpected custom legalisation");
5954 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
5955 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5956 break;
5957 }
5958 break;
5959 case ISD::ROTL:
5960 case ISD::ROTR:
5961 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5962 "Unexpected custom legalisation");
5963 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 2));
5964 break;
5965 case ISD::LOAD: {
5966 // Use an f64 load and a scalar_to_vector for v2f32 loads. This avoids
5967 // scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
5968 // cast since type legalization will try to use an i64 load.
5969 MVT VT = N->getSimpleValueType(ResNo: 0);
5970 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5971 "Unexpected custom legalisation");
5972 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
5973 "Unexpected type action!");
5974 if (!ISD::isNON_EXTLoad(N))
5975 return;
5976 auto *Ld = cast<LoadSDNode>(Val: N);
5977 SDValue Res = DAG.getLoad(VT: MVT::f64, dl: DL, Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
5978 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getBaseAlign(),
5979 MMOFlags: Ld->getMemOperand()->getFlags());
5980 SDValue Chain = Res.getValue(R: 1);
5981 MVT VecVT = MVT::getVectorVT(VT: MVT::f64, NumElements: 2);
5982 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecVT, Operand: Res);
5983 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5984 Res = DAG.getBitcast(VT: WideVT, V: Res);
5985 Results.push_back(Elt: Res);
5986 Results.push_back(Elt: Chain);
5987 break;
5988 }
5989 case ISD::FP_TO_SINT: {
5990 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5991 "Unexpected custom legalisation");
5992 SDValue Src = N->getOperand(Num: 0);
5993 EVT FVT = EVT::getFloatingPointVT(BitWidth: N->getValueSizeInBits(ResNo: 0));
5994 if (getTypeAction(Context&: *DAG.getContext(), VT: Src.getValueType()) !=
5995 TargetLowering::TypeSoftenFloat) {
5996 if (!isTypeLegal(VT: Src.getValueType()))
5997 return;
5998 if (Src.getValueType() == MVT::f16)
5999 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
6000 SDValue Dst = DAG.getNode(Opcode: LoongArchISD::FTINT, DL, VT: FVT, Operand: Src);
6001 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Dst));
6002 return;
6003 }
6004 // If the FP type needs to be softened, emit a library call using the 'si'
6005 // version. If we left it to default legalization we'd end up with 'di'.
6006 RTLIB::Libcall LC;
6007 LC = RTLIB::getFPTOSINT(OpVT: Src.getValueType(), RetVT: VT);
6008 MakeLibCallOptions CallOptions;
6009 EVT OpVT = Src.getValueType();
6010 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: VT);
6011 SDValue Chain = SDValue();
6012 SDValue Result;
6013 std::tie(args&: Result, args&: Chain) =
6014 makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl: DL, Chain);
6015 Results.push_back(Elt: Result);
6016 break;
6017 }
6018 case ISD::BITCAST: {
6019 SDValue Src = N->getOperand(Num: 0);
6020 EVT SrcVT = Src.getValueType();
6021 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
6022 Subtarget.hasBasicF()) {
6023 SDValue Dst =
6024 DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Src);
6025 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Dst));
6026 } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
6027 SDValue NewReg = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
6028 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Src);
6029 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64,
6030 N1: NewReg.getValue(R: 0), N2: NewReg.getValue(R: 1));
6031 Results.push_back(Elt: RetReg);
6032 }
6033 break;
6034 }
6035 case ISD::FP_TO_UINT: {
6036 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6037 "Unexpected custom legalisation");
6038 auto &TLI = DAG.getTargetLoweringInfo();
6039 SDValue Tmp1, Tmp2;
6040 TLI.expandFP_TO_UINT(N, Result&: Tmp1, Chain&: Tmp2, DAG);
6041 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Tmp1));
6042 break;
6043 }
6044 case ISD::FP_ROUND: {
6045 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
6046 "Unexpected custom legalisation");
6047 // On LSX platforms, rounding from v2f64 to v4f32 (after legalization from
6048 // v2f32) is scalarized. Add a customized v2f32 widening to convert it into
6049 // a target-specific LoongArchISD::VFCVT to optimize it.
6050 SDValue Op0 = N->getOperand(Num: 0);
6051 EVT OpVT = Op0.getValueType();
6052 if (OpVT == MVT::v2f64) {
6053 SDValue Undef = DAG.getUNDEF(VT: OpVT);
6054 SDValue Dst =
6055 DAG.getNode(Opcode: LoongArchISD::VFCVT, DL, VT: MVT::v4f32, N1: Undef, N2: Op0);
6056 Results.push_back(Elt: Dst);
6057 }
6058 break;
6059 }
6060 case ISD::BSWAP: {
6061 SDValue Src = N->getOperand(Num: 0);
6062 assert((VT == MVT::i16 || VT == MVT::i32) &&
6063 "Unexpected custom legalization");
6064 MVT GRLenVT = Subtarget.getGRLenVT();
6065 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
6066 SDValue Tmp;
6067 switch (VT.getSizeInBits()) {
6068 default:
6069 llvm_unreachable("Unexpected operand width");
6070 case 16:
6071 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2H, DL, VT: GRLenVT, Operand: NewSrc);
6072 break;
6073 case 32:
6074 // Only LA64 will get to here due to the size mismatch between VT and
6075 // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
6076 Tmp = DAG.getNode(Opcode: LoongArchISD::REVB_2W, DL, VT: GRLenVT, Operand: NewSrc);
6077 break;
6078 }
6079 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
6080 break;
6081 }
6082 case ISD::BITREVERSE: {
6083 SDValue Src = N->getOperand(Num: 0);
6084 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
6085 "Unexpected custom legalization");
6086 MVT GRLenVT = Subtarget.getGRLenVT();
6087 SDValue NewSrc = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: GRLenVT, Operand: Src);
6088 SDValue Tmp;
6089 switch (VT.getSizeInBits()) {
6090 default:
6091 llvm_unreachable("Unexpected operand width");
6092 case 8:
6093 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL, VT: GRLenVT, Operand: NewSrc);
6094 break;
6095 case 32:
6096 Tmp = DAG.getNode(Opcode: LoongArchISD::BITREV_W, DL, VT: GRLenVT, Operand: NewSrc);
6097 break;
6098 }
6099 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Tmp));
6100 break;
6101 }
6102 case ISD::CTLZ:
6103 case ISD::CTTZ: {
6104 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6105 "Unexpected custom legalisation");
6106 Results.push_back(Elt: customLegalizeToWOp(N, DAG, NumOp: 1));
6107 break;
6108 }
6109 case ISD::INTRINSIC_W_CHAIN: {
6110 SDValue Chain = N->getOperand(Num: 0);
6111 SDValue Op2 = N->getOperand(Num: 2);
6112 MVT GRLenVT = Subtarget.getGRLenVT();
6113 const StringRef ErrorMsgOOR = "argument out of range";
6114 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
6115 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
6116
6117 switch (N->getConstantOperandVal(Num: 1)) {
6118 default:
6119 llvm_unreachable("Unexpected Intrinsic.");
6120 case Intrinsic::loongarch_movfcsr2gr: {
6121 if (!Subtarget.hasBasicF()) {
6122 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqF);
6123 return;
6124 }
6125 unsigned Imm = Op2->getAsZExtVal();
6126 if (!isUInt<2>(x: Imm)) {
6127 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
6128 return;
6129 }
6130 SDValue MOVFCSR2GRResults = DAG.getNode(
6131 Opcode: LoongArchISD::MOVFCSR2GR, DL: SDLoc(N), ResultTys: {MVT::i64, MVT::Other},
6132 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
6133 Results.push_back(
6134 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: MOVFCSR2GRResults.getValue(R: 0)));
6135 Results.push_back(Elt: MOVFCSR2GRResults.getValue(R: 1));
6136 break;
6137 }
6138#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
6139 case Intrinsic::loongarch_##NAME: { \
6140 SDValue NODE = DAG.getNode( \
6141 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6142 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
6143 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6144 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6145 Results.push_back(NODE.getValue(1)); \
6146 break; \
6147 }
6148 CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
6149 CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)
6150 CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)
6151 CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)
6152 CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)
6153 CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)
6154#undef CRC_CASE_EXT_BINARYOP
6155
6156#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
6157 case Intrinsic::loongarch_##NAME: { \
6158 SDValue NODE = DAG.getNode( \
6159 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6160 {Chain, Op2, \
6161 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6162 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6163 Results.push_back(NODE.getValue(1)); \
6164 break; \
6165 }
6166 CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
6167 CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)
6168#undef CRC_CASE_EXT_UNARYOP
6169#define CSR_CASE(ID) \
6170 case Intrinsic::loongarch_##ID: { \
6171 if (!Subtarget.is64Bit()) \
6172 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
6173 break; \
6174 }
6175 CSR_CASE(csrrd_d);
6176 CSR_CASE(csrwr_d);
6177 CSR_CASE(csrxchg_d);
6178 CSR_CASE(iocsrrd_d);
6179#undef CSR_CASE
6180 case Intrinsic::loongarch_csrrd_w: {
6181 unsigned Imm = Op2->getAsZExtVal();
6182 if (!isUInt<14>(x: Imm)) {
6183 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
6184 return;
6185 }
6186 SDValue CSRRDResults =
6187 DAG.getNode(Opcode: LoongArchISD::CSRRD, DL, ResultTys: {GRLenVT, MVT::Other},
6188 Ops: {Chain, DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
6189 Results.push_back(
6190 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRRDResults.getValue(R: 0)));
6191 Results.push_back(Elt: CSRRDResults.getValue(R: 1));
6192 break;
6193 }
6194 case Intrinsic::loongarch_csrwr_w: {
6195 unsigned Imm = N->getConstantOperandVal(Num: 3);
6196 if (!isUInt<14>(x: Imm)) {
6197 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
6198 return;
6199 }
6200 SDValue CSRWRResults =
6201 DAG.getNode(Opcode: LoongArchISD::CSRWR, DL, ResultTys: {GRLenVT, MVT::Other},
6202 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
6203 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
6204 Results.push_back(
6205 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRWRResults.getValue(R: 0)));
6206 Results.push_back(Elt: CSRWRResults.getValue(R: 1));
6207 break;
6208 }
6209 case Intrinsic::loongarch_csrxchg_w: {
6210 unsigned Imm = N->getConstantOperandVal(Num: 4);
6211 if (!isUInt<14>(x: Imm)) {
6212 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgOOR);
6213 return;
6214 }
6215 SDValue CSRXCHGResults = DAG.getNode(
6216 Opcode: LoongArchISD::CSRXCHG, DL, ResultTys: {GRLenVT, MVT::Other},
6217 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2),
6218 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 3)),
6219 DAG.getConstant(Val: Imm, DL, VT: GRLenVT)});
6220 Results.push_back(
6221 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CSRXCHGResults.getValue(R: 0)));
6222 Results.push_back(Elt: CSRXCHGResults.getValue(R: 1));
6223 break;
6224 }
6225#define IOCSRRD_CASE(NAME, NODE) \
6226 case Intrinsic::loongarch_##NAME: { \
6227 SDValue IOCSRRDResults = \
6228 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6229 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
6230 Results.push_back( \
6231 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
6232 Results.push_back(IOCSRRDResults.getValue(1)); \
6233 break; \
6234 }
6235 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
6236 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
6237 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
6238#undef IOCSRRD_CASE
6239 case Intrinsic::loongarch_cpucfg: {
6240 SDValue CPUCFGResults =
6241 DAG.getNode(Opcode: LoongArchISD::CPUCFG, DL, ResultTys: {GRLenVT, MVT::Other},
6242 Ops: {Chain, DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op2)});
6243 Results.push_back(
6244 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CPUCFGResults.getValue(R: 0)));
6245 Results.push_back(Elt: CPUCFGResults.getValue(R: 1));
6246 break;
6247 }
6248 case Intrinsic::loongarch_lddir_d: {
6249 if (!Subtarget.is64Bit()) {
6250 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsg: ErrorMsgReqLA64);
6251 return;
6252 }
6253 break;
6254 }
6255 }
6256 break;
6257 }
6258 case ISD::READ_REGISTER: {
6259 if (Subtarget.is64Bit())
6260 DAG.getContext()->emitError(
6261 ErrorStr: "On LA64, only 64-bit registers can be read.");
6262 else
6263 DAG.getContext()->emitError(
6264 ErrorStr: "On LA32, only 32-bit registers can be read.");
6265 Results.push_back(Elt: DAG.getUNDEF(VT));
6266 Results.push_back(Elt: N->getOperand(Num: 0));
6267 break;
6268 }
6269 case ISD::INTRINSIC_WO_CHAIN: {
6270 replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);
6271 break;
6272 }
6273 case ISD::LROUND: {
6274 SDValue Op0 = N->getOperand(Num: 0);
6275 EVT OpVT = Op0.getValueType();
6276 RTLIB::Libcall LC =
6277 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
6278 MakeLibCallOptions CallOptions;
6279 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
6280 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
6281 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
6282 Results.push_back(Elt: Result);
6283 break;
6284 }
6285 case ISD::ATOMIC_CMP_SWAP: {
6286 replaceCMP_XCHG_128Results(N, Results, DAG);
6287 break;
6288 }
6289 case ISD::TRUNCATE: {
6290 MVT VT = N->getSimpleValueType(ResNo: 0);
6291 if (getTypeAction(Context&: *DAG.getContext(), VT) != TypeWidenVector)
6292 return;
6293
6294 MVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT).getSimpleVT();
6295 SDValue In = N->getOperand(Num: 0);
6296 EVT InVT = In.getValueType();
6297 EVT InEltVT = InVT.getVectorElementType();
6298 EVT EltVT = VT.getVectorElementType();
6299 unsigned MinElts = VT.getVectorNumElements();
6300 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6301 unsigned InBits = InVT.getSizeInBits();
6302
6303 // v8i64 -> (v8i32) -> v8i8
6304 if (InVT == MVT::v8i64 && WidenVT.is128BitVector()) {
6305 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 256 / MinElts), NumElements: MinElts);
6306 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
6307 InBits = 256;
6308 }
6309
6310 // v8i32 -> v8i8 / v4i64 -> v4i16 / v4i64 -> v4i8
6311 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
6312 WidenVT.is128BitVector()) {
6313 InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 128 / MinElts), NumElements: MinElts);
6314 In = DAG.getNode(Opcode: N->getOpcode(), DL, VT: InVT, Operand: In);
6315 InBits = 128;
6316 InEltVT = InVT.getVectorElementType();
6317 }
6318
6319 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
6320 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
6321 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
6322 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
6323 for (unsigned I = 0; I < MinElts; ++I)
6324 TruncMask[I] = Scale * I;
6325
6326 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
6327 MVT SVT = In.getSimpleValueType().getScalarType();
6328 MVT VT = MVT::getVectorVT(VT: SVT, NumElements: WidenNumElts);
6329 SDValue WidenIn =
6330 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT), N2: In,
6331 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6332 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
6333 "Illegal vector type in truncation");
6334 WidenIn = DAG.getBitcast(VT: WidenVT, V: WidenIn);
6335 Results.push_back(
6336 Elt: DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: WidenIn, N2: WidenIn, Mask: TruncMask));
6337 return;
6338 }
6339 }
6340
6341 break;
6342 }
6343 case ISD::SIGN_EXTEND: {
6344 // LASX has native VEXT2XV_* for sign extension.
6345 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
6346 return;
6347
6348 EVT DstVT = N->getValueType(ResNo: 0);
6349 SDValue Src = N->getOperand(Num: 0);
6350
6351 // The VSLTI + VILVL/VILVH expansion below only supports simple vector
6352 // sources no wider than 128 bits. Fall back to the default legalization
6353 // for anything else, e.g. when the mask of an i24 vector compare is
6354 // sign-extended, the promoted <4 x i24> source is not a simple type.
6355 EVT SrcEVT = Src.getValueType();
6356 if (!SrcEVT.isSimple() || SrcEVT.getSizeInBits() > 128)
6357 return;
6358 MVT SrcVT = SrcEVT.getSimpleVT();
6359
6360 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6361 unsigned DstEltBits = DstVT.getScalarSizeInBits();
6362 unsigned NumElts = DstVT.getVectorNumElements();
6363
6364 if (!DstVT.isVector() || DstVT.getSizeInBits() <= 128)
6365 return;
6366
6367 // If every source element is all-ones or all-zeros, so is every extended
6368 // element, and interleaving a block with itself extends its low half.
6369 bool IsSignSplat = DAG.ComputeNumSignBits(Op: Src) == SrcEltBits;
6370
6371 // Legalize and extend the src to 128-bit first.
6372 if (SrcVT.getSizeInBits() < 128) {
6373 unsigned WidenSrcElts = 128 / SrcEltBits;
6374 MVT WidenSrcVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: WidenSrcElts);
6375 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WidenSrcVT,
6376 N1: DAG.getUNDEF(VT: WidenSrcVT), N2: Src,
6377 N3: DAG.getVectorIdxConstant(Val: 0, DL));
6378 SrcVT = WidenSrcVT;
6379
6380 unsigned FirstStageEltBits = 128 / NumElts;
6381 MVT FirstStageEltVT = MVT::getIntegerVT(BitWidth: FirstStageEltBits);
6382 MVT FirstStageVT = MVT::getVectorVT(VT: FirstStageEltVT, NumElements: NumElts);
6383 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: FirstStageVT, Operand: Src);
6384 SrcVT = FirstStageVT;
6385 SrcEltBits = FirstStageEltBits;
6386 }
6387
6388 SmallVector<SDValue, 8> Blocks;
6389 Blocks.push_back(Elt: Src);
6390
6391 // Sign-extend the src by using VSLLWIL (or VILVL for a sign splat) + VEXTH
6392 // recursively.
6393 while (SrcEltBits < DstEltBits) {
6394 unsigned NextEltBits = SrcEltBits * 2;
6395 MVT NextEltVT = MVT::getIntegerVT(BitWidth: NextEltBits);
6396 unsigned CurEltsPerBlock = SrcVT.getVectorNumElements();
6397 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6398 MVT NextBlockVT = MVT::getVectorVT(VT: NextEltVT, NumElements: NextEltsPerBlock);
6399
6400 SmallVector<SDValue, 8> NextBlocks;
6401 NextBlocks.reserve(N: Blocks.size() * 2);
6402 for (SDValue Block : Blocks) {
6403 SDValue Lo;
6404 if (IsSignSplat) {
6405 Lo = DAG.getNode(Opcode: LoongArchISD::VILVL, DL, VT: SrcVT, N1: Block, N2: Block);
6406 Lo = DAG.getBitcast(VT: NextBlockVT, V: Lo);
6407 } else {
6408 Lo = DAG.getNode(Opcode: LoongArchISD::VSLLWIL, DL, VT: NextBlockVT, N1: Block,
6409 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()));
6410 }
6411 NextBlocks.push_back(Elt: Lo);
6412 NextBlocks.push_back(
6413 Elt: DAG.getNode(Opcode: LoongArchISD::VEXTH, DL, VT: NextBlockVT, Operand: Block));
6414 }
6415
6416 Blocks = std::move(NextBlocks);
6417 SrcVT = NextBlockVT;
6418 SrcEltBits = NextEltBits;
6419 }
6420
6421 Results.push_back(Elt: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Blocks));
6422 break;
6423 }
6424 case ISD::FP_EXTEND:
6425 // FP_EXTEND may reach here due to the Custom action for v2f32 results, but
6426 // no target-specific lowering is required. Leave it unchanged and rely on
6427 // the default type legalization.
6428 break;
6429 }
6430}
6431
6432/// Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
6433static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL,
6434 SelectionDAG &DAG) {
6435 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDN");
6436
6437 MVT VT = N->getSimpleValueType(ResNo: 0);
6438 if (!VT.is128BitVector() && !VT.is256BitVector())
6439 return SDValue();
6440
6441 SDValue X, Y;
6442 SDValue N0 = N->getOperand(Num: 0);
6443 SDValue N1 = N->getOperand(Num: 1);
6444
6445 if (SDValue Not = isNOT(V: N0, DAG)) {
6446 X = Not;
6447 Y = N1;
6448 } else if (SDValue Not = isNOT(V: N1, DAG)) {
6449 X = Not;
6450 Y = N0;
6451 } else
6452 return SDValue();
6453
6454 X = DAG.getBitcast(VT, V: X);
6455 Y = DAG.getBitcast(VT, V: Y);
6456 return DAG.getNode(Opcode: LoongArchISD::VANDN, DL, VT, N1: X, N2: Y);
6457}
6458
6459static bool isConstantSplatVector(SDValue N, APInt &SplatValue,
6460 unsigned MinSizeInBits) {
6461 N = peekThroughBitcasts(V: N);
6462 BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(Val&: N);
6463
6464 if (!Node)
6465 return false;
6466
6467 APInt SplatUndef;
6468 unsigned SplatBitSize;
6469 bool HasAnyUndefs;
6470
6471 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6472 HasAnyUndefs, MinSplatBits: MinSizeInBits,
6473 /*IsBigEndian=*/isBigEndian: false);
6474}
6475
6476static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex) {
6477 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6478 if (!BV)
6479 return SDValue();
6480
6481 SDValue Src;
6482 int Start = -1;
6483
6484 for (unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6485 SDValue Op = BV->getOperand(Num: i);
6486 if (Op.isUndef())
6487 continue;
6488 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6489 return SDValue();
6490
6491 auto *IdxC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6492 if (!IdxC)
6493 return SDValue();
6494
6495 unsigned EltIdx = IdxC->getZExtValue();
6496 if (Start < 0)
6497 Start = (int)EltIdx - (int)(i * 2);
6498 if (Start < 0 || Start > 1 || EltIdx != (unsigned)(Start + (int)(i * 2)))
6499 return SDValue();
6500
6501 SDValue CurSrc = Op.getOperand(i: 0);
6502 if (!Src)
6503 Src = CurSrc;
6504 else if (Src != CurSrc)
6505 return SDValue();
6506 }
6507
6508 if (!Src || Start < 0)
6509 return SDValue();
6510
6511 StartIndex = (unsigned)Start;
6512 return Src;
6513}
6514
6515// Return true if values of type VT are kept in vector registers, i.e. VT is
6516// a vector type whose elements are legal in LSX vectors.
6517static bool isVectorRegisterVT(EVT VT, SelectionDAG &DAG) {
6518 if (!VT.isVector())
6519 return false;
6520 EVT EltVT = VT.getVectorElementType();
6521 unsigned EltBits = EltVT.getSizeInBits();
6522 if (EltBits < 8 || EltBits > 64)
6523 return false;
6524 EVT LSXVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: 128 / EltBits);
6525 return DAG.getTargetLoweringInfo().isTypeLegal(VT: LSXVT);
6526}
6527
6528// Return true if all users of the value loaded by Ld can use it in vector
6529// registers.
6530static bool hasOnlyVectorUsers(LoadSDNode *Ld, SelectionDAG &DAG) {
6531 EVT VT = Ld->getValueType(ResNo: 0);
6532 for (SDUse &U : Ld->uses()) {
6533 if (U.getResNo() != 0)
6534 continue;
6535 SDNode *User = U.getUser();
6536 switch (User->getOpcode()) {
6537 case ISD::ADD:
6538 case ISD::SUB:
6539 if (User->getValueType(ResNo: 0) == VT)
6540 continue;
6541 break;
6542 case ISD::STORE:
6543 // The stored value, not the address.
6544 if (U.getOperandNo() == 1)
6545 continue;
6546 break;
6547 case ISD::BITCAST:
6548 if (isVectorRegisterVT(VT: User->getValueType(ResNo: 0), DAG))
6549 continue;
6550 break;
6551 }
6552 return false;
6553 }
6554 return true;
6555}
6556
6557// Return true if V, a vNi128 value, is cheap to obtain in vector registers
6558// of type PartVT.
6559static bool isCheapVNi128VectorOperand(SDValue V, MVT PartVT,
6560 SelectionDAG &DAG) {
6561 switch (V.getOpcode()) {
6562 case ISD::LOAD: {
6563 auto *Ld = cast<LoadSDNode>(Val&: V);
6564 unsigned Fast = 0;
6565 return ISD::isNormalLoad(N: Ld) && Ld->isSimple() &&
6566 DAG.getTargetLoweringInfo().allowsMemoryAccess(
6567 Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: PartVT,
6568 MMO: *Ld->getMemOperand(), Fast: &Fast) &&
6569 Fast && hasOnlyVectorUsers(Ld, DAG);
6570 }
6571 case ISD::BITCAST: {
6572 // Vectors built from GPRs are not in vector registers.
6573 SDValue Src = V.getOperand(i: 0);
6574 switch (Src.getOpcode()) {
6575 case ISD::BUILD_VECTOR:
6576 return ISD::isBuildVectorOfConstantSDNodes(V: Src);
6577 case ISD::INSERT_VECTOR_ELT:
6578 case ISD::SCALAR_TO_VECTOR:
6579 return false;
6580 }
6581 return isVectorRegisterVT(VT: Src.getValueType(), DAG);
6582 }
6583 default:
6584 return ISD::isBuildVectorOfConstantSDNodes(V);
6585 }
6586}
6587
6588// Combine add/sub on vNi128 to [X]VADD.Q/[X]VSUB.Q:
6589//
6590// (add vNi128:a, vNi128:b)
6591// -> (bitcast (VADD_Q (bitcast a to vMi64), (bitcast b to vMi64)))
6592//
6593// vNi128 types are not legal and would otherwise be expanded to scalar
6594// add/sub with carry on GPRs, so this has to happen before type legalization.
6595// Vectors wider than a vector register are split into register-sized parts.
6596// Only do it when both operands are already in, or cheap to get into, vector
6597// registers, e.g. loaded from memory. Values passed in GPRs (such as function
6598// arguments) are better handled in GPRs.
6599static SDValue performVNi128AddSubCombine(SDNode *N, SelectionDAG &DAG,
6600 TargetLowering::DAGCombinerInfo &DCI,
6601 const LoongArchSubtarget &Subtarget) {
6602 if (!Subtarget.hasExtLSX())
6603 return SDValue();
6604 if (!DCI.isBeforeLegalize())
6605 return SDValue();
6606
6607 EVT VT = N->getValueType(ResNo: 0);
6608 if (!VT.isFixedLengthVector() || VT.getVectorElementType() != MVT::i128)
6609 return SDValue();
6610
6611 // Use 256-bit parts with LASX and 128-bit parts for the rest.
6612 unsigned NumElts = VT.getVectorNumElements();
6613 bool UseLASX = Subtarget.hasExtLASX() && NumElts >= 2;
6614 MVT PartVT = UseLASX ? MVT::v4i64 : MVT::v2i64;
6615 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i64, NumElements: NumElts * 2);
6616
6617 SDValue LHS = N->getOperand(Num: 0);
6618 SDValue RHS = N->getOperand(Num: 1);
6619 if (!isCheapVNi128VectorOperand(V: LHS, PartVT, DAG) ||
6620 !isCheapVNi128VectorOperand(V: RHS, PartVT, DAG))
6621 return SDValue();
6622
6623 // Replace loads with loads of the i64 vector type, so that the other users
6624 // of the loaded value (e.g. another add/sub) can use it directly as well.
6625 // The range metadata, if any, is dropped as it doesn't match the new type.
6626 // The old loads are only replaced after N, as replacing them may CSE N into
6627 // another node and delete it.
6628 SmallVector<std::pair<LoadSDNode *, SDValue>, 2> LoadsToReplace;
6629 auto ConvertOperand = [&](SDValue V) {
6630 auto *Ld = dyn_cast<LoadSDNode>(Val&: V);
6631 if (!Ld)
6632 return DAG.getBitcast(VT: NewVT, V);
6633 SDValue NewLd =
6634 DAG.getLoad(VT: NewVT, dl: SDLoc(Ld), Chain: Ld->getChain(), Ptr: Ld->getBasePtr(),
6635 PtrInfo: Ld->getPointerInfo(), Alignment: Ld->getBaseAlign(),
6636 MMOFlags: Ld->getMemOperand()->getFlags(), Metadata: Ld->getAAInfo());
6637 LoadsToReplace.push_back(Elt: {Ld, NewLd});
6638 return NewLd;
6639 };
6640 SDValue NewLHS = ConvertOperand(LHS);
6641 SDValue NewRHS = LHS == RHS ? NewLHS : ConvertOperand(RHS);
6642
6643 SDLoc DL(N);
6644 unsigned Opc =
6645 N->getOpcode() == ISD::ADD ? LoongArchISD::VADD_Q : LoongArchISD::VSUB_Q;
6646 SDValue Res = DAG.getUNDEF(VT: NewVT);
6647 for (unsigned Idx = 0, End = NumElts * 2; Idx != End;) {
6648 MVT CurVT = UseLASX && End - Idx >= 4 ? MVT::v4i64 : MVT::v2i64;
6649 SDValue PartLHS = DAG.getExtractSubvector(DL, VT: CurVT, Vec: NewLHS, Idx);
6650 SDValue PartRHS = DAG.getExtractSubvector(DL, VT: CurVT, Vec: NewRHS, Idx);
6651 SDValue Part = DAG.getNode(Opcode: Opc, DL, VT: CurVT, N1: PartLHS, N2: PartRHS);
6652 Res = DAG.getInsertSubvector(DL, Vec: Res, SubVec: Part, Idx);
6653 Idx += CurVT.getVectorNumElements();
6654 }
6655 DCI.CombineTo(N, Res: DAG.getBitcast(VT, V: Res));
6656 for (auto [Ld, NewLd] : LoadsToReplace)
6657 DCI.CombineTo(N: Ld, Res0: DAG.getBitcast(VT, V: NewLd), Res1: NewLd.getValue(R: 1));
6658 return SDValue(N, 0);
6659}
6660
6661static SDValue
6662performHorizWideningCombine(SDNode *N, SelectionDAG &DAG,
6663 const LoongArchSubtarget &Subtarget) {
6664 if (!Subtarget.hasExtLSX())
6665 return SDValue();
6666
6667 unsigned Opc = N->getOpcode();
6668 assert((Opc == ISD::ADD || Opc == ISD::SUB) && "Unexpected opcode");
6669
6670 EVT VT = N->getValueType(ResNo: 0);
6671 SDLoc DL(N);
6672
6673 SDValue LHS = N->getOperand(Num: 0);
6674 SDValue RHS = N->getOperand(Num: 1);
6675
6676 bool isSigned;
6677 unsigned ExtOpc = LHS.getOpcode();
6678 if (ExtOpc == ISD::SIGN_EXTEND)
6679 isSigned = true;
6680 else if (ExtOpc == ISD::ZERO_EXTEND)
6681 isSigned = false;
6682 else
6683 return SDValue();
6684
6685 if (ExtOpc != RHS.getOpcode())
6686 return SDValue();
6687
6688 if (!LHS.hasOneUse() || !RHS.hasOneUse())
6689 return SDValue();
6690
6691 unsigned OddIdx, EvenIdx;
6692 SDValue LHSVec = matchDeinterleaveBuildVector(N: LHS.getOperand(i: 0), StartIndex&: OddIdx);
6693 SDValue RHSVec = matchDeinterleaveBuildVector(N: RHS.getOperand(i: 0), StartIndex&: EvenIdx);
6694
6695 if (!LHSVec || !RHSVec)
6696 return SDValue();
6697 if (OddIdx != 1 || EvenIdx != 0)
6698 return SDValue();
6699 if (LHSVec.getValueType() != RHSVec.getValueType())
6700 return SDValue();
6701
6702 EVT SrcVT = LHSVec.getValueType();
6703 EVT SrcEltVT = SrcVT.getVectorElementType();
6704 EVT DstEltVT = VT.getVectorElementType();
6705 auto &TLI = DAG.getTargetLoweringInfo();
6706
6707 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
6708 return SDValue();
6709 if (!SrcVT.isVector() || !VT.isVector())
6710 return SDValue();
6711 if (SrcVT.getSizeInBits() != VT.getSizeInBits())
6712 return SDValue();
6713 if (DstEltVT.getSizeInBits() != SrcEltVT.getSizeInBits() * 2)
6714 return SDValue();
6715 if (!SrcEltVT.isInteger() || SrcEltVT.getSizeInBits() > 32)
6716 return SDValue();
6717
6718 unsigned TargetOpc;
6719 if (Opc == ISD::ADD)
6720 TargetOpc = isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6721 else
6722 TargetOpc = isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6723
6724 return DAG.getNode(Opcode: TargetOpc, DL, VT, N1: LHSVec, N2: RHSVec);
6725}
6726
6727static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
6728 TargetLowering::DAGCombinerInfo &DCI,
6729 const LoongArchSubtarget &Subtarget) {
6730 if (SDValue V = performVNi128AddSubCombine(N, DAG, DCI, Subtarget))
6731 return V;
6732
6733 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
6734 return V;
6735
6736 if (DCI.isBeforeLegalizeOps())
6737 return SDValue();
6738
6739 EVT VT = N->getValueType(ResNo: 0);
6740 if (!VT.isVector())
6741 return SDValue();
6742
6743 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6744 return SDValue();
6745
6746 EVT EltVT = VT.getVectorElementType();
6747 if (!EltVT.isInteger())
6748 return SDValue();
6749
6750 // match:
6751 //
6752 // add
6753 // (and
6754 // (srl X, shift-1) / X
6755 // 1)
6756 // (srl/sra X, shift)
6757
6758 SDValue Add0 = N->getOperand(Num: 0);
6759 SDValue Add1 = N->getOperand(Num: 1);
6760 SDValue And;
6761 SDValue Shr;
6762
6763 if (Add0.getOpcode() == ISD::AND) {
6764 And = Add0;
6765 Shr = Add1;
6766 } else if (Add1.getOpcode() == ISD::AND) {
6767 And = Add1;
6768 Shr = Add0;
6769 } else {
6770 return SDValue();
6771 }
6772
6773 // match:
6774 //
6775 // srl/sra X, shift
6776
6777 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
6778 return SDValue();
6779
6780 SDValue X = Shr.getOperand(i: 0);
6781 SDValue Shift = Shr.getOperand(i: 1);
6782 APInt ShiftVal;
6783
6784 if (!isConstantSplatVector(N: Shift, SplatValue&: ShiftVal, MinSizeInBits: EltVT.getSizeInBits()))
6785 return SDValue();
6786
6787 if (ShiftVal == 0)
6788 return SDValue();
6789
6790 // match:
6791 //
6792 // and
6793 // (srl X, shift-1) / X
6794 // 1
6795
6796 SDValue One = And.getOperand(i: 1);
6797 APInt SplatVal;
6798
6799 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6800 return SDValue();
6801
6802 if (SplatVal != 1)
6803 return SDValue();
6804
6805 if (And.getOperand(i: 0) == X) {
6806 // match:
6807 //
6808 // shift == 1
6809
6810 if (ShiftVal != 1)
6811 return SDValue();
6812 } else {
6813 // match:
6814 //
6815 // srl X, shift-1
6816
6817 SDValue Srl = And.getOperand(i: 0);
6818
6819 if (Srl.getOpcode() != ISD::SRL)
6820 return SDValue();
6821
6822 if (Srl.getOperand(i: 0) != X)
6823 return SDValue();
6824
6825 // match:
6826 //
6827 // shift-1
6828
6829 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
6830
6831 if (!isConstantSplatVector(N: ShiftMinus1, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
6832 return SDValue();
6833
6834 if (ShiftVal != (SplatVal + 1))
6835 return SDValue();
6836 }
6837
6838 // We matched a rounded right shift pattern and can lower it
6839 // to a single vector rounded shift instruction.
6840
6841 SDLoc DL(N);
6842 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
6843 : LoongArchISD::VSRAR,
6844 DL, VT, N1: X, N2: Shift);
6845}
6846
6847static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
6848 TargetLowering::DAGCombinerInfo &DCI,
6849 const LoongArchSubtarget &Subtarget) {
6850 if (DCI.isBeforeLegalizeOps())
6851 return SDValue();
6852
6853 SDValue FirstOperand = N->getOperand(Num: 0);
6854 SDValue SecondOperand = N->getOperand(Num: 1);
6855 unsigned FirstOperandOpc = FirstOperand.getOpcode();
6856 EVT ValTy = N->getValueType(ResNo: 0);
6857 SDLoc DL(N);
6858 uint64_t lsb, msb;
6859 unsigned SMIdx, SMLen;
6860 ConstantSDNode *CN;
6861 SDValue NewOperand;
6862 MVT GRLenVT = Subtarget.getGRLenVT();
6863
6864 if (SDValue R = combineAndNotIntoVANDN(N, DL, DAG))
6865 return R;
6866
6867 // BSTRPICK requires the 32S feature.
6868 if (!Subtarget.has32S())
6869 return SDValue();
6870
6871 // Op's second operand must be a shifted mask.
6872 if (!(CN = dyn_cast<ConstantSDNode>(Val&: SecondOperand)) ||
6873 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx&: SMIdx, MaskLen&: SMLen))
6874 return SDValue();
6875
6876 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
6877 // Pattern match BSTRPICK.
6878 // $dst = and ((sra or srl) $src , lsb), (2**len - 1)
6879 // => BSTRPICK $dst, $src, msb, lsb
6880 // where msb = lsb + len - 1
6881
6882 // The second operand of the shift must be an immediate.
6883 if (!(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))))
6884 return SDValue();
6885
6886 lsb = CN->getZExtValue();
6887
6888 // Return if the shifted mask does not start at bit 0 or the sum of its
6889 // length and lsb exceeds the word's size.
6890 if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())
6891 return SDValue();
6892
6893 NewOperand = FirstOperand.getOperand(i: 0);
6894 } else {
6895 // Pattern match BSTRPICK.
6896 // $dst = and $src, (2**len- 1) , if len > 12
6897 // => BSTRPICK $dst, $src, msb, lsb
6898 // where lsb = 0 and msb = len - 1
6899
6900 // If the mask is <= 0xfff, andi can be used instead.
6901 if (CN->getZExtValue() <= 0xfff)
6902 return SDValue();
6903
6904 // Return if the MSB exceeds.
6905 if (SMIdx + SMLen > ValTy.getSizeInBits())
6906 return SDValue();
6907
6908 if (SMIdx > 0) {
6909 // Omit if the constant has more than 2 uses. This a conservative
6910 // decision. Whether it is a win depends on the HW microarchitecture.
6911 // However it should always be better for 1 and 2 uses.
6912 if (CN->use_size() > 2)
6913 return SDValue();
6914 // Return if the constant can be composed by a single LU12I.W.
6915 if ((CN->getZExtValue() & 0xfff) == 0)
6916 return SDValue();
6917 // Return if the constand can be composed by a single ADDI with
6918 // the zero register.
6919 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)
6920 return SDValue();
6921 }
6922
6923 lsb = SMIdx;
6924 NewOperand = FirstOperand;
6925 }
6926
6927 msb = lsb + SMLen - 1;
6928 SDValue NR0 = DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy, N1: NewOperand,
6929 N2: DAG.getConstant(Val: msb, DL, VT: GRLenVT),
6930 N3: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6931 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)
6932 return NR0;
6933 // Try to optimize to
6934 // bstrpick $Rd, $Rs, msb, lsb
6935 // slli $Rd, $Rd, lsb
6936 return DAG.getNode(Opcode: ISD::SHL, DL, VT: ValTy, N1: NR0,
6937 N2: DAG.getConstant(Val: lsb, DL, VT: GRLenVT));
6938}
6939
6940// Return the original source vector if N consists of the half
6941// of each 128-bit lane.
6942static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow) {
6943 N = peekThroughBitcasts(V: N);
6944
6945 EVT DstVT = N.getValueType();
6946 if (!DstVT.isVector())
6947 return SDValue();
6948
6949 unsigned NumElts = DstVT.getVectorNumElements();
6950
6951 // LSX canonical form:
6952 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6953 SDValue Src = N.getOperand(i: 0);
6954 EVT SrcVT = Src.getValueType();
6955
6956 if (!SrcVT.isVector() || !SrcVT.is128BitVector())
6957 return SDValue();
6958 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6959 return SDValue();
6960 if (SrcVT.getVectorNumElements() != NumElts * 2)
6961 return SDValue();
6962 if (N.getConstantOperandVal(i: 1) != (isLow ? 0 : NumElts))
6963 return SDValue();
6964
6965 return Src;
6966 }
6967
6968 // LASX canonical form:
6969 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
6970 if (!BV)
6971 return SDValue();
6972
6973 if (NumElts % 2 != 0)
6974 return SDValue();
6975
6976 SDValue Src;
6977 EVT SrcVT;
6978
6979 for (unsigned I = 0; I != NumElts; ++I) {
6980 SDValue Elt = BV->getOperand(Num: I);
6981 if (Elt.isUndef())
6982 continue;
6983 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6984 return SDValue();
6985
6986 SDValue ThisSrc = Elt.getOperand(i: 0);
6987 SDValue Idx = Elt.getOperand(i: 1);
6988 auto *CI = dyn_cast<ConstantSDNode>(Val&: Idx);
6989 if (!CI)
6990 return SDValue();
6991
6992 if (!Src) {
6993 Src = ThisSrc;
6994 SrcVT = Src.getValueType();
6995 if (!SrcVT.isVector())
6996 return SDValue();
6997
6998 if (!SrcVT.is256BitVector())
6999 return SDValue();
7000 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
7001 return SDValue();
7002 if (SrcVT.getVectorNumElements() != NumElts * 2)
7003 return SDValue();
7004 } else if (ThisSrc != Src) {
7005 return SDValue();
7006 }
7007
7008 unsigned Half = NumElts / 2;
7009 unsigned ExpectedIdx = (I < Half) ? I : (I + Half);
7010 ExpectedIdx += isLow ? 0 : Half;
7011
7012 if (CI->getZExtValue() != ExpectedIdx)
7013 return SDValue();
7014 }
7015
7016 return Src;
7017}
7018
7019static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
7020 TargetLowering::DAGCombinerInfo &DCI,
7021 const LoongArchSubtarget &Subtarget) {
7022 assert(N->getOpcode() == ISD::SHL && "Unexpected opcode");
7023
7024 EVT VT = N->getValueType(ResNo: 0);
7025 SDLoc DL(N);
7026
7027 SDValue LHS = N->getOperand(Num: 0);
7028 SDValue RHS = N->getOperand(Num: 1);
7029
7030 bool isSigned;
7031 unsigned ExtOpc = LHS.getOpcode();
7032 if (ExtOpc == ISD::SIGN_EXTEND)
7033 isSigned = true;
7034 else if (ExtOpc == ISD::ZERO_EXTEND)
7035 isSigned = false;
7036 else
7037 return SDValue();
7038
7039 if (!LHS.hasOneUse())
7040 return SDValue();
7041
7042 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
7043 N->getValueSizeInBits(ResNo: 0) != LHS->getOperand(Num: 0).getValueSizeInBits() * 2)
7044 return SDValue();
7045
7046 SDValue Vec = matchHalfOf128BitLanes(N: LHS.getOperand(i: 0), /*isLow=*/true);
7047 if (!Vec)
7048 return SDValue();
7049
7050 EVT SrcVT = Vec.getValueType();
7051 EVT SrcEltVT = SrcVT.getVectorElementType();
7052 EVT DstEltVT = VT.getVectorElementType();
7053 APInt Imm;
7054 if (!isConstantSplatVector(N: RHS, SplatValue&: Imm, MinSizeInBits: DstEltVT.getSizeInBits()))
7055 return SDValue();
7056 if (!Imm.ult(RHS: SrcEltVT.getSizeInBits()))
7057 return SDValue();
7058
7059 unsigned Opc = isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
7060 SDValue Sht = DAG.getConstant(Val: Imm.getZExtValue(), DL, VT: Subtarget.getGRLenVT());
7061 return DAG.getNode(Opcode: Opc, DL, VT, N1: Vec, N2: Sht);
7062}
7063
7064static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
7065 TargetLowering::DAGCombinerInfo &DCI,
7066 const LoongArchSubtarget &Subtarget) {
7067 // BSTRPICK requires the 32S feature.
7068 if (!Subtarget.has32S())
7069 return SDValue();
7070
7071 if (DCI.isBeforeLegalizeOps())
7072 return SDValue();
7073
7074 // $dst = srl (and $src, Mask), Shamt
7075 // =>
7076 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt
7077 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1
7078 //
7079
7080 SDValue FirstOperand = N->getOperand(Num: 0);
7081 ConstantSDNode *CN;
7082 EVT ValTy = N->getValueType(ResNo: 0);
7083 SDLoc DL(N);
7084 MVT GRLenVT = Subtarget.getGRLenVT();
7085 unsigned MaskIdx, MaskLen;
7086 uint64_t Shamt;
7087
7088 // The first operand must be an AND and the second operand of the AND must be
7089 // a shifted mask.
7090 if (FirstOperand.getOpcode() != ISD::AND ||
7091 !(CN = dyn_cast<ConstantSDNode>(Val: FirstOperand.getOperand(i: 1))) ||
7092 !isShiftedMask_64(Value: CN->getZExtValue(), MaskIdx, MaskLen))
7093 return SDValue();
7094
7095 // The second operand (shift amount) must be an immediate.
7096 if (!(CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))))
7097 return SDValue();
7098
7099 Shamt = CN->getZExtValue();
7100 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
7101 return DAG.getNode(Opcode: LoongArchISD::BSTRPICK, DL, VT: ValTy,
7102 N1: FirstOperand->getOperand(Num: 0),
7103 N2: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7104 N3: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
7105
7106 return SDValue();
7107}
7108
7109// Try to turn a sub boolean RHS and constant LHS into an addi. LoongArch's
7110// SUB has no immediate form, so (sub C, X) normally requires either
7111// materializing C or using sub with R0 plus an addi; rewriting it as
7112// (add X', C - 1) lets the immediate fold into a single ADDI.
7113static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
7114 SDValue N0 = N->getOperand(Num: 0);
7115 SDValue N1 = N->getOperand(Num: 1);
7116 EVT VT = N->getValueType(ResNo: 0);
7117 SDLoc DL(N);
7118
7119 // Require a constant LHS.
7120 auto *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
7121 if (!N0C)
7122 return SDValue();
7123
7124 // (sub 0, X) is a negation, not a constant materialization; rewriting it
7125 // as (add (setcc inv), -1) changes the condition code and can duplicate a
7126 // compare, without saving any instruction.
7127 if (N0C->isZero() && !N->hasOneUse())
7128 return SDValue();
7129
7130 // The result of this fold is only folded into an ADDI when it is used
7131 // directly. When it feeds an extend, the original (sub C, X) form usually
7132 // combines better with the surrounding zext/negate/xor.
7133 if (N->hasOneUse()) {
7134 unsigned UserOpc = N->use_begin()->getUser()->getOpcode();
7135 if (UserOpc == ISD::ZERO_EXTEND || UserOpc == ISD::SIGN_EXTEND ||
7136 UserOpc == ISD::ANY_EXTEND || UserOpc == ISD::SUB)
7137 return SDValue();
7138 }
7139
7140 // All our optimizations involve subtracting 1 from the immediate and
7141 // forming an ADDI. Make sure the new immediate is valid for an ADDI.
7142 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
7143 if (!ImmValMinus1.isSignedIntN(N: 12))
7144 return SDValue();
7145
7146 SDValue NewLHS;
7147 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
7148 // (sub constant, (setcc x, y, eq/neq)) ->
7149 // (add (setcc x, y, neq/eq), constant - 1)
7150 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
7151 EVT SetCCOpVT = N1.getOperand(i: 0).getValueType();
7152 if (!ISD::isIntEqualitySetCC(Code: CCVal) || !SetCCOpVT.isInteger())
7153 return SDValue();
7154 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
7155 NewLHS =
7156 DAG.getSetCC(DL: SDLoc(N1), VT, LHS: N1.getOperand(i: 0), RHS: N1.getOperand(i: 1), Cond: CCVal);
7157 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(V: N1.getOperand(i: 1)) &&
7158 N1.getOperand(i: 0).getOpcode() == ISD::SETCC) {
7159 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
7160 // Since setcc returns a bool the xor is equivalent to 1-setcc.
7161 NewLHS = N1.getOperand(i: 0);
7162 } else
7163 return SDValue();
7164
7165 SDValue NewRHS = DAG.getConstant(Val: ImmValMinus1, DL, VT);
7166 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewLHS, N2: NewRHS);
7167}
7168
7169static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
7170 TargetLowering::DAGCombinerInfo &DCI,
7171 const LoongArchSubtarget &Subtarget) {
7172 if (SDValue V = performVNi128AddSubCombine(N, DAG, DCI, Subtarget))
7173 return V;
7174
7175 if (SDValue V = performHorizWideningCombine(N, DAG, Subtarget))
7176 return V;
7177
7178 if (N->getValueType(ResNo: 0) != Subtarget.getGRLenVT())
7179 return SDValue();
7180
7181 SDValue N0 = N->getOperand(Num: 0);
7182 SDValue N1 = N->getOperand(Num: 1);
7183 EVT VT = N->getValueType(ResNo: 0);
7184
7185 if (SDValue V = combineSubOfBoolean(N, DAG))
7186 return V;
7187
7188 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
7189 if (isNullConstant(V: N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
7190 isNullConstant(V: N1.getOperand(i: 1)) &&
7191 N1.getValueType() == N1.getOperand(i: 0).getValueType()) {
7192 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
7193 if (CCVal == ISD::SETLT) {
7194 SDLoc DL(N);
7195 unsigned ShAmt = N0.getValueSizeInBits() - 1;
7196 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N1.getOperand(i: 0),
7197 N2: DAG.getConstant(Val: ShAmt, DL, VT));
7198 }
7199 }
7200
7201 return SDValue();
7202}
7203
7204// Helper to peek through bitops/trunc/setcc to determine size of source vector.
7205// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
7206static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
7207 unsigned Depth) {
7208 // Limit recursion.
7209 if (Depth >= SelectionDAG::MaxRecursionDepth)
7210 return false;
7211 switch (Src.getOpcode()) {
7212 case ISD::SETCC:
7213 case ISD::TRUNCATE:
7214 return Src.getOperand(i: 0).getValueSizeInBits() == Size;
7215 case ISD::FREEZE:
7216 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1);
7217 case ISD::AND:
7218 case ISD::XOR:
7219 case ISD::OR:
7220 return checkBitcastSrcVectorSize(Src: Src.getOperand(i: 0), Size, Depth: Depth + 1) &&
7221 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1);
7222 case ISD::SELECT:
7223 case ISD::VSELECT:
7224 return Src.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
7225 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 1), Size, Depth: Depth + 1) &&
7226 checkBitcastSrcVectorSize(Src: Src.getOperand(i: 2), Size, Depth: Depth + 1);
7227 case ISD::BUILD_VECTOR:
7228 return ISD::isBuildVectorAllZeros(V: Src) || ISD::isBuildVectorAllOnes(V: Src);
7229 }
7230 return false;
7231}
7232
7233// Helper to push sign extension of vXi1 SETCC result through bitops.
7234static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
7235 SDValue Src, const SDLoc &DL) {
7236 switch (Src.getOpcode()) {
7237 case ISD::SETCC:
7238 case ISD::FREEZE:
7239 case ISD::TRUNCATE:
7240 case ISD::BUILD_VECTOR:
7241 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
7242 case ISD::AND:
7243 case ISD::XOR:
7244 case ISD::OR:
7245 return DAG.getNode(
7246 Opcode: Src.getOpcode(), DL, VT: SExtVT,
7247 N1: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 0), DL),
7248 N2: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL));
7249 case ISD::SELECT:
7250 case ISD::VSELECT:
7251 return DAG.getSelect(
7252 DL, VT: SExtVT, Cond: Src.getOperand(i: 0),
7253 LHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 1), DL),
7254 RHS: signExtendBitcastSrcVector(DAG, SExtVT, Src: Src.getOperand(i: 2), DL));
7255 }
7256 llvm_unreachable("Unexpected node type for vXi1 sign extension");
7257}
7258
7259static SDValue
7260performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
7261 TargetLowering::DAGCombinerInfo &DCI,
7262 const LoongArchSubtarget &Subtarget) {
7263 SDLoc DL(N);
7264 EVT VT = N->getValueType(ResNo: 0);
7265 SDValue Src = N->getOperand(Num: 0);
7266 EVT SrcVT = Src.getValueType();
7267
7268 if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse())
7269 return SDValue();
7270
7271 bool UseLASX;
7272 unsigned Opc = ISD::DELETED_NODE;
7273 EVT CmpVT = Src.getOperand(i: 0).getValueType();
7274 EVT EltVT = CmpVT.getVectorElementType();
7275
7276 if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128)
7277 UseLASX = false;
7278 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
7279 CmpVT.getSizeInBits() == 256)
7280 UseLASX = true;
7281 else
7282 return SDValue();
7283
7284 SDValue SrcN1 = Src.getOperand(i: 1);
7285 switch (Src.getOperandCondCode(i: 2)) {
7286 default:
7287 break;
7288 case ISD::SETEQ:
7289 // x == 0 => not (vmsknez.b x)
7290 if (ISD::isBuildVectorAllZeros(V: SrcN1) && EltVT == MVT::i8)
7291 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
7292 break;
7293 case ISD::SETGT:
7294 // x > -1 => vmskgez.b x
7295 if (ISD::isBuildVectorAllOnes(V: SrcN1) && EltVT == MVT::i8)
7296 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7297 break;
7298 case ISD::SETGE:
7299 // x >= 0 => vmskgez.b x
7300 if (ISD::isBuildVectorAllZeros(V: SrcN1) && EltVT == MVT::i8)
7301 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7302 break;
7303 case ISD::SETLT:
7304 // x < 0 => vmskltz.{b,h,w,d} x
7305 if (ISD::isBuildVectorAllZeros(V: SrcN1) &&
7306 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7307 EltVT == MVT::i64))
7308 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7309 break;
7310 case ISD::SETLE:
7311 // x <= -1 => vmskltz.{b,h,w,d} x
7312 if (ISD::isBuildVectorAllOnes(V: SrcN1) &&
7313 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7314 EltVT == MVT::i64))
7315 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7316 break;
7317 case ISD::SETNE:
7318 // x != 0 => vmsknez.b x
7319 if (ISD::isBuildVectorAllZeros(V: SrcN1) && EltVT == MVT::i8)
7320 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
7321 break;
7322 }
7323
7324 if (Opc == ISD::DELETED_NODE)
7325 return SDValue();
7326
7327 SDValue V = DAG.getNode(Opcode: Opc, DL, VT: Subtarget.getGRLenVT(), Operand: Src.getOperand(i: 0));
7328 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
7329 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
7330 return DAG.getBitcast(VT, V);
7331}
7332
7333static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG,
7334 TargetLowering::DAGCombinerInfo &DCI,
7335 const LoongArchSubtarget &Subtarget) {
7336 SDLoc DL(N);
7337 EVT VT = N->getValueType(ResNo: 0);
7338 SDValue Src = N->getOperand(Num: 0);
7339 EVT SrcVT = Src.getValueType();
7340 MVT GRLenVT = Subtarget.getGRLenVT();
7341
7342 if (!DCI.isBeforeLegalizeOps())
7343 return SDValue();
7344
7345 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
7346 return SDValue();
7347
7348 // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
7349 SDValue Res = performSETCC_BITCASTCombine(N, DAG, DCI, Subtarget);
7350 if (Res)
7351 return Res;
7352
7353 // Generate vXi1 using [X]VMSKLTZ
7354 MVT SExtVT;
7355 unsigned Opc;
7356 bool UseLASX = false;
7357 bool PropagateSExt = false;
7358
7359 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
7360 EVT CmpVT = Src.getOperand(i: 0).getValueType();
7361 if (CmpVT.getSizeInBits() > 256)
7362 return SDValue();
7363 }
7364
7365 switch (SrcVT.getSimpleVT().SimpleTy) {
7366 default:
7367 return SDValue();
7368 case MVT::v2i1:
7369 SExtVT = MVT::v2i64;
7370 break;
7371 case MVT::v4i1:
7372 SExtVT = MVT::v4i32;
7373 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
7374 SExtVT = MVT::v4i64;
7375 UseLASX = true;
7376 PropagateSExt = true;
7377 }
7378 break;
7379 case MVT::v8i1:
7380 SExtVT = MVT::v8i16;
7381 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
7382 SExtVT = MVT::v8i32;
7383 UseLASX = true;
7384 PropagateSExt = true;
7385 }
7386 break;
7387 case MVT::v16i1:
7388 SExtVT = MVT::v16i8;
7389 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, Size: 256, Depth: 0)) {
7390 SExtVT = MVT::v16i16;
7391 UseLASX = true;
7392 PropagateSExt = true;
7393 }
7394 break;
7395 case MVT::v32i1:
7396 SExtVT = MVT::v32i8;
7397 UseLASX = true;
7398 break;
7399 };
7400 Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
7401 : DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: SExtVT, Operand: Src);
7402
7403 SDValue V;
7404 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
7405 if (Src.getSimpleValueType() == MVT::v32i8) {
7406 SDValue Lo, Hi;
7407 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Src, DL);
7408 Lo = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Lo);
7409 Hi = DAG.getNode(Opcode: LoongArchISD::VMSKLTZ, DL, VT: GRLenVT, Operand: Hi);
7410 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: GRLenVT, N1: Hi,
7411 N2: DAG.getShiftAmountConstant(Val: 16, VT: GRLenVT, DL));
7412 V = DAG.getNode(Opcode: ISD::OR, DL, VT: GRLenVT, N1: Lo, N2: Hi);
7413 } else if (UseLASX) {
7414 return SDValue();
7415 }
7416 }
7417
7418 if (!V) {
7419 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7420 V = DAG.getNode(Opcode: Opc, DL, VT: GRLenVT, Operand: Src);
7421 }
7422
7423 EVT T = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getVectorNumElements());
7424 V = DAG.getZExtOrTrunc(Op: V, DL, VT: T);
7425 return DAG.getBitcast(VT, V);
7426}
7427
7428static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7429 TargetLowering::DAGCombinerInfo &DCI,
7430 const LoongArchSubtarget &Subtarget) {
7431 MVT GRLenVT = Subtarget.getGRLenVT();
7432 EVT ValTy = N->getValueType(ResNo: 0);
7433 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
7434 ConstantSDNode *CN0, *CN1;
7435 SDLoc DL(N);
7436 unsigned ValBits = ValTy.getSizeInBits();
7437 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
7438 unsigned Shamt;
7439 bool SwapAndRetried = false;
7440
7441 // BSTRPICK requires the 32S feature.
7442 if (!Subtarget.has32S())
7443 return SDValue();
7444
7445 if (DCI.isBeforeLegalizeOps())
7446 return SDValue();
7447
7448 if (ValBits != 32 && ValBits != 64)
7449 return SDValue();
7450
7451Retry:
7452 // 1st pattern to match BSTRINS:
7453 // R = or (and X, mask0), (and (shl Y, lsb), mask1)
7454 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1
7455 // =>
7456 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
7457 if (N0.getOpcode() == ISD::AND &&
7458 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
7459 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
7460 N1.getOpcode() == ISD::AND && N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
7461 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7462 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
7463 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
7464 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7465 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
7466 (MaskIdx0 + MaskLen0 <= ValBits)) {
7467 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");
7468 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
7469 N2: N1.getOperand(i: 0).getOperand(i: 0),
7470 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
7471 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
7472 }
7473
7474 // 2nd pattern to match BSTRINS:
7475 // R = or (and X, mask0), (shl (and Y, mask1), lsb)
7476 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)
7477 // =>
7478 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
7479 if (N0.getOpcode() == ISD::AND &&
7480 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
7481 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
7482 N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
7483 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7484 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
7485 (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7486 isShiftedMask_64(Value: CN1->getZExtValue(), MaskIdx&: MaskIdx1, MaskLen&: MaskLen1) &&
7487 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
7488 (MaskIdx0 + MaskLen0 <= ValBits)) {
7489 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");
7490 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
7491 N2: N1.getOperand(i: 0).getOperand(i: 0),
7492 N3: DAG.getConstant(Val: (MaskIdx0 + MaskLen0 - 1), DL, VT: GRLenVT),
7493 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
7494 }
7495
7496 // 3rd pattern to match BSTRINS:
7497 // R = or (and X, mask0), (and Y, mask1)
7498 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0
7499 // =>
7500 // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb
7501 // where msb = lsb + size - 1
7502 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
7503 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
7504 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
7505 (MaskIdx0 + MaskLen0 <= 64) &&
7506 (CN1 = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1))) &&
7507 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
7508 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");
7509 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
7510 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0), N1,
7511 N2: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT)),
7512 N3: DAG.getConstant(Val: ValBits == 32
7513 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7514 : (MaskIdx0 + MaskLen0 - 1),
7515 DL, VT: GRLenVT),
7516 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
7517 }
7518
7519 // 4th pattern to match BSTRINS:
7520 // R = or (and X, mask), (shl Y, shamt)
7521 // where mask = (2**shamt - 1)
7522 // =>
7523 // R = BSTRINS X, Y, ValBits - 1, shamt
7524 // where ValBits = 32 or 64
7525 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&
7526 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
7527 isShiftedMask_64(Value: CN0->getZExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
7528 MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7529 (Shamt = CN1->getZExtValue()) == MaskLen0 &&
7530 (MaskIdx0 + MaskLen0 <= ValBits)) {
7531 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");
7532 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
7533 N2: N1.getOperand(i: 0),
7534 N3: DAG.getConstant(Val: (ValBits - 1), DL, VT: GRLenVT),
7535 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
7536 }
7537
7538 // 5th pattern to match BSTRINS:
7539 // R = or (and X, mask), const
7540 // where ~mask = (2**size - 1) << lsb, mask & const = 0
7541 // =>
7542 // R = BSTRINS X, (const >> lsb), msb, lsb
7543 // where msb = lsb + size - 1
7544 if (N0.getOpcode() == ISD::AND &&
7545 (CN0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) &&
7546 isShiftedMask_64(Value: ~CN0->getSExtValue(), MaskIdx&: MaskIdx0, MaskLen&: MaskLen0) &&
7547 (CN1 = dyn_cast<ConstantSDNode>(Val&: N1)) &&
7548 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
7549 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");
7550 return DAG.getNode(
7551 Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0.getOperand(i: 0),
7552 N2: DAG.getSignedConstant(Val: CN1->getSExtValue() >> MaskIdx0, DL, VT: ValTy),
7553 N3: DAG.getConstant(Val: ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7554 : (MaskIdx0 + MaskLen0 - 1),
7555 DL, VT: GRLenVT),
7556 N4: DAG.getConstant(Val: MaskIdx0, DL, VT: GRLenVT));
7557 }
7558
7559 // 6th pattern.
7560 // a = b | ((c & mask) << shamt), where all positions in b to be overwritten
7561 // by the incoming bits are known to be zero.
7562 // =>
7563 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt
7564 //
7565 // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th
7566 // pattern is more common than the 1st. So we put the 1st before the 6th in
7567 // order to match as many nodes as possible.
7568 ConstantSDNode *CNMask, *CNShamt;
7569 unsigned MaskIdx, MaskLen;
7570 if (N1.getOpcode() == ISD::SHL && N1.getOperand(i: 0).getOpcode() == ISD::AND &&
7571 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7572 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7573 MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7574 CNShamt->getZExtValue() + MaskLen <= ValBits) {
7575 Shamt = CNShamt->getZExtValue();
7576 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);
7577 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7578 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");
7579 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7580 N2: N1.getOperand(i: 0).getOperand(i: 0),
7581 N3: DAG.getConstant(Val: Shamt + MaskLen - 1, DL, VT: GRLenVT),
7582 N4: DAG.getConstant(Val: Shamt, DL, VT: GRLenVT));
7583 }
7584 }
7585
7586 // 7th pattern.
7587 // a = b | ((c << shamt) & shifted_mask), where all positions in b to be
7588 // overwritten by the incoming bits are known to be zero.
7589 // =>
7590 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx
7591 //
7592 // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd
7593 // before the 7th in order to match as many nodes as possible.
7594 if (N1.getOpcode() == ISD::AND &&
7595 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7596 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7597 N1.getOperand(i: 0).getOpcode() == ISD::SHL &&
7598 (CNShamt = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 0).getOperand(i: 1))) &&
7599 CNShamt->getZExtValue() == MaskIdx) {
7600 APInt ShMask(ValBits, CNMask->getZExtValue());
7601 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7602 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");
7603 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7604 N2: N1.getOperand(i: 0).getOperand(i: 0),
7605 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7606 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7607 }
7608 }
7609
7610 // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.
7611 if (!SwapAndRetried) {
7612 std::swap(a&: N0, b&: N1);
7613 SwapAndRetried = true;
7614 goto Retry;
7615 }
7616
7617 SwapAndRetried = false;
7618Retry2:
7619 // 8th pattern.
7620 // a = b | (c & shifted_mask), where all positions in b to be overwritten by
7621 // the incoming bits are known to be zero.
7622 // =>
7623 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx
7624 //
7625 // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So
7626 // we put it here in order to match as many nodes as possible or generate less
7627 // instructions.
7628 if (N1.getOpcode() == ISD::AND &&
7629 (CNMask = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) &&
7630 isShiftedMask_64(Value: CNMask->getZExtValue(), MaskIdx, MaskLen)) {
7631 APInt ShMask(ValBits, CNMask->getZExtValue());
7632 if (ShMask.isSubsetOf(RHS: DAG.computeKnownBits(Op: N0).Zero)) {
7633 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");
7634 return DAG.getNode(Opcode: LoongArchISD::BSTRINS, DL, VT: ValTy, N1: N0,
7635 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: N1->getValueType(ResNo: 0),
7636 N1: N1->getOperand(Num: 0),
7637 N2: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT)),
7638 N3: DAG.getConstant(Val: MaskIdx + MaskLen - 1, DL, VT: GRLenVT),
7639 N4: DAG.getConstant(Val: MaskIdx, DL, VT: GRLenVT));
7640 }
7641 }
7642 // Swap N0/N1 and retry.
7643 if (!SwapAndRetried) {
7644 std::swap(a&: N0, b&: N1);
7645 SwapAndRetried = true;
7646 goto Retry2;
7647 }
7648
7649 return SDValue();
7650}
7651
7652static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
7653 const LoongArchSubtarget &Subtarget) {
7654 EVT VT = N->getValueType(ResNo: 0);
7655 // LA64 promotes i32 integer operations to GPR-width operations during
7656 // legalization. Match the idiom before that promotion as well as after
7657 // it, otherwise common unsigned 32-bit bit tests miss this combine.
7658 if (VT != Subtarget.getGRLenVT() && !(Subtarget.is64Bit() && VT == MVT::i32))
7659 return SDValue();
7660
7661 SDValue XorOps[] = {N->getOperand(Num: 0), N->getOperand(Num: 1)};
7662 for (unsigned AndIdx = 0; AndIdx != 2; ++AndIdx) {
7663 SDValue And = XorOps[AndIdx];
7664 SDValue Mask = XorOps[1 - AndIdx];
7665 if (And.getOpcode() != ISD::AND || !And.hasOneUse())
7666 continue;
7667
7668 for (unsigned MaskIdx = 0; MaskIdx != 2; ++MaskIdx) {
7669 // Match by SDValue identity so that (X & M1) ^ M2 is never folded
7670 // unless M1 and M2 are the same value. Canonicalize before
7671 // instruction selection, where the AND operand may already have been
7672 // selected to a target machine node.
7673 if (And.getOperand(i: MaskIdx) != Mask)
7674 continue;
7675 SDLoc DL(N);
7676 SDValue X = And.getOperand(i: 1 - MaskIdx);
7677 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask, N2: DAG.getNOT(DL, Val: X, VT));
7678 }
7679 }
7680 return SDValue();
7681}
7682
7683static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {
7684 ExtType = ISD::NON_EXTLOAD;
7685
7686 switch (V.getNode()->getOpcode()) {
7687 case ISD::LOAD: {
7688 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: V.getNode());
7689 if ((LoadNode->getMemoryVT() == MVT::i8) ||
7690 (LoadNode->getMemoryVT() == MVT::i16)) {
7691 ExtType = LoadNode->getExtensionType();
7692 return true;
7693 }
7694 return false;
7695 }
7696 case ISD::AssertSext: {
7697 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7698 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7699 ExtType = ISD::SEXTLOAD;
7700 return true;
7701 }
7702 return false;
7703 }
7704 case ISD::AssertZext: {
7705 VTSDNode *TypeNode = cast<VTSDNode>(Val: V.getNode()->getOperand(Num: 1));
7706 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7707 ExtType = ISD::ZEXTLOAD;
7708 return true;
7709 }
7710 return false;
7711 }
7712 default:
7713 return false;
7714 }
7715
7716 return false;
7717}
7718
7719// Fold a lane-mask extraction that is only compared against zero into one of
7720// all-lane check instructions, which write the results into fcc register, so
7721// BCEQZ could use it directly.
7722//
7723// (VMSKLTZ X) != 0 -> VSETNEZ.V X
7724// (VMSKLTZ X) == 0 -> VSETEQZ.V X
7725//
7726// This is valid when X is the result of a vector compare instruction,
7727// so each lane of X has to be all-ones or all-zeros.
7728static SDValue foldVMskZeroTest(SDValue LHS, SDValue RHS, ISD::CondCode CC,
7729 const SDLoc &DL, SelectionDAG &DAG,
7730 const LoongArchSubtarget &Subtarget) {
7731 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7732 return SDValue();
7733 if (!isNullConstant(V: RHS))
7734 return SDValue();
7735
7736 unsigned MskOpc = LHS.getOpcode();
7737 if (MskOpc != LoongArchISD::VMSKLTZ && MskOpc != LoongArchISD::XVMSKLTZ)
7738 return SDValue();
7739 // Keeping the mask alive for another user would defeat the purpose.
7740 if (!LHS.hasOneUse())
7741 return SDValue();
7742
7743 SDValue Src = LHS.getOperand(i: 0);
7744 EVT SrcVT = Src.getValueType();
7745 // Make sure every lane is all-ones or all-zeros.
7746 if (!SrcVT.isVector() ||
7747 DAG.ComputeNumSignBits(Op: Src) != SrcVT.getScalarSizeInBits())
7748 return SDValue();
7749
7750 return DAG.getNode(Opcode: CC == ISD::SETNE ? LoongArchISD::VANYNONZERO
7751 : LoongArchISD::VALLZERO,
7752 DL, VT: Subtarget.getGRLenVT(), Operand: Src);
7753}
7754
7755// Eliminate redundant truncation and zero-extension nodes.
7756// * Case 1:
7757// +------------+ +------------+ +------------+
7758// | Input1 | | Input2 | | CC |
7759// +------------+ +------------+ +------------+
7760// | | |
7761// V V +----+
7762// +------------+ +------------+ |
7763// | TRUNCATE | | TRUNCATE | |
7764// +------------+ +------------+ |
7765// | | |
7766// V V |
7767// +------------+ +------------+ |
7768// | ZERO_EXT | | ZERO_EXT | |
7769// +------------+ +------------+ |
7770// | | |
7771// | +-------------+ |
7772// V V | |
7773// +----------------+ | |
7774// | AND | | |
7775// +----------------+ | |
7776// | | |
7777// +---------------+ | |
7778// | | |
7779// V V V
7780// +-------------+
7781// | CMP |
7782// +-------------+
7783// * Case 2:
7784// +------------+ +------------+ +-------------+ +------------+ +------------+
7785// | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
7786// +------------+ +------------+ +-------------+ +------------+ +------------+
7787// | | | | |
7788// V | | | |
7789// +------------+ | | | |
7790// | XOR |<---------------------+ | |
7791// +------------+ | | |
7792// | | | |
7793// V V +---------------+ |
7794// +------------+ +------------+ | |
7795// | TRUNCATE | | TRUNCATE | | +-------------------------+
7796// +------------+ +------------+ | |
7797// | | | |
7798// V V | |
7799// +------------+ +------------+ | |
7800// | ZERO_EXT | | ZERO_EXT | | |
7801// +------------+ +------------+ | |
7802// | | | |
7803// V V | |
7804// +----------------+ | |
7805// | AND | | |
7806// +----------------+ | |
7807// | | |
7808// +---------------+ | |
7809// | | |
7810// V V V
7811// +-------------+
7812// | CMP |
7813// +-------------+
7814static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
7815 TargetLowering::DAGCombinerInfo &DCI,
7816 const LoongArchSubtarget &Subtarget) {
7817 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
7818
7819 if (N->getValueType(ResNo: 0) == Subtarget.getGRLenVT())
7820 if (SDValue V = foldVMskZeroTest(LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), CC,
7821 DL: SDLoc(N), DAG, Subtarget))
7822 return V;
7823
7824 SDNode *AndNode = N->getOperand(Num: 0).getNode();
7825 if (AndNode->getOpcode() != ISD::AND)
7826 return SDValue();
7827
7828 SDValue AndInputValue2 = AndNode->getOperand(Num: 1);
7829 if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
7830 return SDValue();
7831
7832 SDValue CmpInputValue = N->getOperand(Num: 1);
7833 SDValue AndInputValue1 = AndNode->getOperand(Num: 0);
7834 if (AndInputValue1.getOpcode() == ISD::XOR) {
7835 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7836 return SDValue();
7837 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: AndInputValue1.getOperand(i: 1));
7838 if (!CN || !CN->isAllOnes())
7839 return SDValue();
7840 CN = dyn_cast<ConstantSDNode>(Val&: CmpInputValue);
7841 if (!CN || !CN->isZero())
7842 return SDValue();
7843 AndInputValue1 = AndInputValue1.getOperand(i: 0);
7844 if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
7845 return SDValue();
7846 } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
7847 if (AndInputValue2 != CmpInputValue)
7848 return SDValue();
7849 } else {
7850 return SDValue();
7851 }
7852
7853 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(Num: 0);
7854 if (TruncValue1.getOpcode() != ISD::TRUNCATE)
7855 return SDValue();
7856
7857 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(Num: 0);
7858 if (TruncValue2.getOpcode() != ISD::TRUNCATE)
7859 return SDValue();
7860
7861 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(Num: 0);
7862 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(Num: 0);
7863 ISD::LoadExtType ExtType1;
7864 ISD::LoadExtType ExtType2;
7865
7866 if (!checkValueWidth(V: TruncInputValue1, ExtType&: ExtType1) ||
7867 !checkValueWidth(V: TruncInputValue2, ExtType&: ExtType2))
7868 return SDValue();
7869
7870 if (TruncInputValue1->getValueType(ResNo: 0) != TruncInputValue2->getValueType(ResNo: 0) ||
7871 AndNode->getValueType(ResNo: 0) != TruncInputValue1->getValueType(ResNo: 0))
7872 return SDValue();
7873
7874 if ((ExtType2 != ISD::ZEXTLOAD) &&
7875 ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
7876 return SDValue();
7877
7878 // These truncation and zero-extension nodes are not necessary, remove them.
7879 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: AndNode->getValueType(ResNo: 0),
7880 N1: TruncInputValue1, N2: TruncInputValue2);
7881 SDValue NewSetCC =
7882 DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: NewAnd, RHS: TruncInputValue2, Cond: CC);
7883 DAG.ReplaceAllUsesWith(From: N, To: NewSetCC.getNode());
7884 return SDValue(N, 0);
7885}
7886
7887// Strip a single outer ISD::SIGN_EXTEND_INREG from \p V, if present, and
7888// return the inner value together with the narrow VT it was extending
7889// from. If no such node is present, returns \p V unchanged and an invalid
7890// EVT.
7891//
7892// i32 (and other sub-GRLen) arithmetic is legalized to operate on the full
7893// GRLen-width register, with a `sign_extend_inreg` re-normalizing the
7894// result back into the narrow type's range afterwards (see e.g. the
7895// `add i32` -> `add` + `sign_extend_inreg ..., i32` legalization). Any
7896// combine that reassociates such a binop must track and reapply this
7897// extension, otherwise the transformed code can produce a value whose
7898// high bits no longer match the narrow-type semantics.
7899static std::pair<SDValue, EVT> stripSignExtendInReg(SDValue V) {
7900 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
7901 return {V.getOperand(i: 0), cast<VTSDNode>(Val: V.getOperand(i: 1))->getVT()};
7902 return {V, EVT()};
7903}
7904
7905// Try to match \p BinV (after optionally stripping an outer
7906// sign_extend_inreg) as a supported binary operation that has \p X as one
7907// of its operands, returning the matched opcode, the other operand (the
7908// "delta"), and the narrow VT of the sign_extend_inreg that was stripped
7909// (invalid EVT if none was present).
7910//
7911// For commutative ops (add/or/xor), \p X may be either operand, since
7912// `binop(X, Y) == binop(Y, X)` and the identity element (0) works on
7913// either side.
7914//
7915// For `sub`, the operation is NOT commutative: `sub(X, Y) != sub(Y, X)`.
7916// Only `sub(X, Y)` (i.e. \p X is the *minuend*, the first operand) can be
7917// rewritten using the identity `X - 0 == X`. If \p X were the *subtrahend*
7918// (second operand, i.e. the pattern is actually `sub(Y, X)`), there is no
7919// way to express `cond ? (Y - X) : X` (or the symmetric case) as
7920// `X op (select ...)` without introducing an extra negation, so that case
7921// must be rejected instead of "optimized" into worse code.
7922static std::tuple<unsigned, SDValue, EVT>
7923matchBinOpWithSharedOperand(SDValue BinV, SDValue X) {
7924 auto [Inner, ExtVT] = stripSignExtendInReg(V: BinV);
7925
7926 unsigned Opc = Inner.getOpcode();
7927 switch (Opc) {
7928 case ISD::ADD:
7929 case ISD::OR:
7930 case ISD::XOR:
7931 if (Inner.getOperand(i: 0) == X)
7932 return {Opc, Inner.getOperand(i: 1), ExtVT};
7933 if (Inner.getOperand(i: 1) == X)
7934 return {Opc, Inner.getOperand(i: 0), ExtVT};
7935 return {0, SDValue(), EVT()};
7936 case ISD::SUB:
7937 // Only accept X as the minuend (first operand); see comment above.
7938 if (Inner.getOperand(i: 0) == X)
7939 return {Opc, Inner.getOperand(i: 1), ExtVT};
7940 return {0, SDValue(), EVT()};
7941 default:
7942 return {0, SDValue(), EVT()};
7943 }
7944}
7945
7946// Try to combine:
7947// select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
7948// select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
7949// for binop in {add, or, xor, sub}, where 0 is the identity element of the
7950// respective operation, additionally handling the common legalized form
7951// where the binop result is wrapped in a `sign_extend_inreg` (as happens
7952// for sub-GRLen types such as i32 on a 64-bit GRLen target). See
7953// matchBinOpWithSharedOperand() for the restrictions applied to
7954// non-commutative operations (currently only `sub`).
7955static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
7956 TargetLowering::DAGCombinerInfo &DCI,
7957 const LoongArchSubtarget &Subtarget) {
7958 if (DCI.isBeforeLegalizeOps())
7959 return SDValue();
7960
7961 EVT VT = N->getValueType(ResNo: 0);
7962 // Restrict to the scalar GRLen integer type that maskeqz/masknez operate
7963 // on; this also naturally excludes float and vector selects.
7964 if (VT != Subtarget.getGRLenVT())
7965 return SDValue();
7966
7967 SDValue Cond = N->getOperand(Num: 0);
7968 SDValue TrueV = N->getOperand(Num: 1);
7969 SDValue FalseV = N->getOperand(Num: 2);
7970 SDLoc DL(N);
7971
7972 auto TryFold = [&](SDValue BinV, SDValue SharedV,
7973 bool BinIsTrueArm) -> SDValue {
7974 auto [Opc, Delta, ExtVT] = matchBinOpWithSharedOperand(BinV, X: SharedV);
7975 if (!Opc)
7976 return SDValue();
7977
7978 // Avoid infinite combine loops: bail out if Delta is trivially the
7979 // same node we would otherwise be selecting on (shouldn't normally
7980 // happen, but guards against degenerate/self-referential IR).
7981 if (Delta.getNode() == N)
7982 return SDValue();
7983
7984 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
7985 SDValue NewSel = BinIsTrueArm ? DAG.getSelect(DL, VT, Cond, LHS: Delta, RHS: Zero)
7986 : DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: Delta);
7987 SDValue NewBin = DAG.getNode(Opcode: Opc, DL, VT, N1: SharedV, N2: NewSel);
7988
7989 // If the original binop result was normalized back into a narrower
7990 // type via sign_extend_inreg (e.g. i32 arithmetic on a 64-bit GRLen
7991 // target), the new binop must be re-normalized the same way: SharedV
7992 // is already known-sign-extended for that narrow type, but NewSel
7993 // (Delta or 0, selected) combined with SharedV via Opc can still
7994 // produce a 64-bit result whose high bits don't match the narrow
7995 // type's sign-extended representation.
7996 if (ExtVT != EVT())
7997 NewBin = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: NewBin,
7998 N2: DAG.getValueType(ExtVT));
7999
8000 return NewBin;
8001 };
8002
8003 if (SDValue R = TryFold(TrueV, FalseV, /*BinIsTrueArm=*/true))
8004 return R;
8005 if (SDValue R = TryFold(FalseV, TrueV, /*BinIsTrueArm=*/false))
8006 return R;
8007
8008 return SDValue();
8009}
8010
8011// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
8012static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,
8013 TargetLowering::DAGCombinerInfo &DCI,
8014 const LoongArchSubtarget &Subtarget) {
8015 if (DCI.isBeforeLegalizeOps())
8016 return SDValue();
8017
8018 SDValue Src = N->getOperand(Num: 0);
8019 if (Src.getOpcode() != LoongArchISD::REVB_2W)
8020 return SDValue();
8021
8022 return DAG.getNode(Opcode: LoongArchISD::BITREV_4B, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8023 Operand: Src.getOperand(i: 0));
8024}
8025
8026// Perform common combines for BR_CC and SELECT_CC conditions.
8027static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
8028 SelectionDAG &DAG, const LoongArchSubtarget &Subtarget) {
8029 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
8030
8031 // As far as arithmetic right shift always saves the sign,
8032 // shift can be omitted.
8033 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
8034 // setge (sra X, N), 0 -> setge X, 0
8035 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
8036 LHS.getOpcode() == ISD::SRA) {
8037 LHS = LHS.getOperand(i: 0);
8038 return true;
8039 }
8040
8041 if (!ISD::isIntEqualitySetCC(Code: CCVal))
8042 return false;
8043
8044 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
8045 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
8046 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
8047 LHS.getOperand(i: 0).getValueType() == Subtarget.getGRLenVT()) {
8048 // If we're looking for eq 0 instead of ne 0, we need to invert the
8049 // condition.
8050 bool Invert = CCVal == ISD::SETEQ;
8051 CCVal = LHS.getOperandCondCode(i: 2);
8052 if (Invert)
8053 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
8054
8055 RHS = LHS.getOperand(i: 1);
8056 LHS = LHS.getOperand(i: 0);
8057 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG);
8058
8059 CC = DAG.getCondCode(Cond: CCVal);
8060 return true;
8061 }
8062
8063 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, GRLen-1-C), 0, ge/lt)
8064 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
8065 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
8066 SDValue LHS0 = LHS.getOperand(i: 0);
8067 if (LHS0.getOpcode() == ISD::AND &&
8068 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
8069 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
8070 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
8071 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
8072 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
8073 CC = DAG.getCondCode(Cond: CCVal);
8074
8075 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
8076 LHS = LHS0.getOperand(i: 0);
8077 if (ShAmt != 0)
8078 LHS =
8079 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
8080 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
8081 return true;
8082 }
8083 }
8084 }
8085
8086 // (X, 1, setne) -> (X, 0, seteq) if we can prove X is 0/1.
8087 // This can occur when legalizing some floating point comparisons.
8088 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
8089 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
8090 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
8091 CC = DAG.getCondCode(Cond: CCVal);
8092 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
8093 return true;
8094 }
8095
8096 // Fold ((shl (extract_vector_elt X, I), GRLen - EleBits)), 0, eq/ne) ->
8097 // ((extract_vector_elt X, I), 0, eq/ne)
8098 if (isNullConstant(V: RHS) && (CCVal == ISD::SETEQ || CCVal == ISD::SETNE) &&
8099 LHS.getOpcode() == ISD::SHL && LHS.hasOneUse() &&
8100 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
8101 SDValue Ext = LHS.getOperand(i: 0);
8102 unsigned Sht = LHS.getConstantOperandVal(i: 1);
8103 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
8104 SDValue Vec = Ext.getOperand(i: 0);
8105 unsigned EleBits = Vec.getScalarValueSizeInBits();
8106 if ((EleBits + Sht) == Subtarget.getGRLen()) {
8107 LHS = Ext;
8108 return true;
8109 }
8110 }
8111 }
8112
8113 // Fold (C1, C2, cond) -> (0, 0, seteq/setne)
8114 if (isa<ConstantSDNode>(Val: LHS) && isa<ConstantSDNode>(Val: RHS)) {
8115 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
8116 EVT VT = LHS.getValueType();
8117 EVT SetCCResVT =
8118 TLI->getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8119 if (SDValue Folded = DAG.FoldSetCC(VT: SetCCResVT, N1: LHS, N2: RHS, Cond: CCVal, dl: DL)) {
8120 LHS = DAG.getConstant(Val: 0, DL, VT);
8121 RHS = DAG.getConstant(Val: 0, DL, VT);
8122 CC = DAG.getCondCode(Cond: !isNullConstant(V: Folded) ? ISD::SETEQ : ISD::SETNE);
8123 return true;
8124 }
8125 }
8126
8127 return false;
8128}
8129
8130static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG,
8131 TargetLowering::DAGCombinerInfo &DCI,
8132 const LoongArchSubtarget &Subtarget) {
8133 SDValue LHS = N->getOperand(Num: 1);
8134 SDValue RHS = N->getOperand(Num: 2);
8135 SDValue CC = N->getOperand(Num: 3);
8136 SDLoc DL(N);
8137
8138 // CC was folded into V, so always return ISD::SETNE is fine.
8139 if (SDValue V = foldVMskZeroTest(LHS, RHS, CC: cast<CondCodeSDNode>(Val&: CC)->get(),
8140 DL, DAG, Subtarget))
8141 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
8142 N1: N->getOperand(Num: 0), N2: V,
8143 N3: DAG.getConstant(Val: 0, DL, VT: Subtarget.getGRLenVT()),
8144 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: N->getOperand(Num: 4));
8145
8146 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
8147 return DAG.getNode(Opcode: LoongArchISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
8148 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
8149
8150 return SDValue();
8151}
8152
8153static SDValue performSELECT_CCCombine(SDNode *N, SelectionDAG &DAG,
8154 TargetLowering::DAGCombinerInfo &DCI,
8155 const LoongArchSubtarget &Subtarget) {
8156 // Transform
8157 SDValue LHS = N->getOperand(Num: 0);
8158 SDValue RHS = N->getOperand(Num: 1);
8159 SDValue CC = N->getOperand(Num: 2);
8160 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
8161 SDValue TrueV = N->getOperand(Num: 3);
8162 SDValue FalseV = N->getOperand(Num: 4);
8163 SDLoc DL(N);
8164 EVT VT = N->getValueType(ResNo: 0);
8165
8166 // If the True and False values are the same, we don't need a select_cc.
8167 if (TrueV == FalseV)
8168 return TrueV;
8169
8170 // (select (x < 0), y, z) -> x >> (GRLEN - 1) & (y - z) + z
8171 // (select (x >= 0), y, z) -> x >> (GRLEN - 1) & (z - y) + y
8172 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
8173 isNullConstant(V: RHS) &&
8174 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
8175 if (CCVal == ISD::CondCode::SETGE)
8176 std::swap(a&: TrueV, b&: FalseV);
8177
8178 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
8179 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
8180 // Only handle simm12, if it is not in this range, it can be considered as
8181 // register.
8182 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
8183 isInt<12>(x: TrueSImm - FalseSImm)) {
8184 SDValue SRA =
8185 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
8186 N2: DAG.getConstant(Val: Subtarget.getGRLen() - 1, DL, VT));
8187 SDValue AND =
8188 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
8189 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
8190 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
8191 }
8192
8193 if (CCVal == ISD::CondCode::SETGE)
8194 std::swap(a&: TrueV, b&: FalseV);
8195 }
8196
8197 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
8198 return DAG.getNode(Opcode: LoongArchISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
8199 Ops: {LHS, RHS, CC, TrueV, FalseV});
8200
8201 return SDValue();
8202}
8203
8204template <unsigned N>
8205static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp,
8206 SelectionDAG &DAG,
8207 const LoongArchSubtarget &Subtarget,
8208 bool IsSigned = false) {
8209 SDLoc DL(Node);
8210 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
8211 // Check the ImmArg.
8212 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
8213 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
8214 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
8215 ": argument out of range.");
8216 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: Subtarget.getGRLenVT());
8217 }
8218 return DAG.getConstant(Val: CImm->getZExtValue(), DL, VT: Subtarget.getGRLenVT());
8219}
8220
8221template <unsigned N>
8222static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,
8223 SelectionDAG &DAG, bool IsSigned = false) {
8224 SDLoc DL(Node);
8225 EVT ResTy = Node->getValueType(ResNo: 0);
8226 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
8227
8228 // Check the ImmArg.
8229 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
8230 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
8231 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
8232 ": argument out of range.");
8233 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
8234 }
8235 return DAG.getConstant(
8236 Val: APInt(ResTy.getScalarType().getSizeInBits(),
8237 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
8238 DL, VT: ResTy);
8239}
8240
8241static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG) {
8242 SDLoc DL(Node);
8243 EVT ResTy = Node->getValueType(ResNo: 0);
8244 SDValue Vec = Node->getOperand(Num: 2);
8245 SDValue Mask = DAG.getConstant(Val: Vec.getScalarValueSizeInBits() - 1, DL, VT: ResTy);
8246 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Vec, N2: Mask);
8247}
8248
8249static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG) {
8250 SDLoc DL(Node);
8251 EVT ResTy = Node->getValueType(ResNo: 0);
8252 SDValue One = DAG.getConstant(Val: 1, DL, VT: ResTy);
8253 SDValue Bit =
8254 DAG.getNode(Opcode: ISD::SHL, DL, VT: ResTy, N1: One, N2: truncateVecElts(Node, DAG));
8255
8256 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1),
8257 N2: DAG.getNOT(DL, Val: Bit, VT: ResTy));
8258}
8259
8260template <unsigned N>
8261static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG) {
8262 SDLoc DL(Node);
8263 EVT ResTy = Node->getValueType(ResNo: 0);
8264 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
8265 // Check the unsigned ImmArg.
8266 if (!isUInt<N>(CImm->getZExtValue())) {
8267 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
8268 ": argument out of range.");
8269 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
8270 }
8271
8272 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8273 SDValue Mask = DAG.getConstant(Val: ~BitImm, DL, VT: ResTy);
8274
8275 return DAG.getNode(Opcode: ISD::AND, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: Mask);
8276}
8277
8278template <unsigned N>
8279static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG) {
8280 SDLoc DL(Node);
8281 EVT ResTy = Node->getValueType(ResNo: 0);
8282 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
8283 // Check the unsigned ImmArg.
8284 if (!isUInt<N>(CImm->getZExtValue())) {
8285 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
8286 ": argument out of range.");
8287 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
8288 }
8289
8290 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8291 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
8292 return DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
8293}
8294
8295template <unsigned N>
8296static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG) {
8297 SDLoc DL(Node);
8298 EVT ResTy = Node->getValueType(ResNo: 0);
8299 auto *CImm = cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
8300 // Check the unsigned ImmArg.
8301 if (!isUInt<N>(CImm->getZExtValue())) {
8302 DAG.getContext()->emitError(ErrorStr: Node->getOperationName(G: 0) +
8303 ": argument out of range.");
8304 return DAG.getNode(Opcode: ISD::UNDEF, DL, VT: ResTy);
8305 }
8306
8307 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8308 SDValue BitImm = DAG.getConstant(Val: Imm, DL, VT: ResTy);
8309 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: Node->getOperand(Num: 1), N2: BitImm);
8310}
8311
8312template <unsigned W>
8313static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG,
8314 unsigned ResOp) {
8315 unsigned Imm = N->getConstantOperandVal(Num: 2);
8316 if (!isUInt<W>(Imm)) {
8317 const StringRef ErrorMsg = "argument out of range";
8318 DAG.getContext()->emitError(ErrorStr: N->getOperationName(G: 0) + ": " + ErrorMsg + ".");
8319 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
8320 }
8321 SDLoc DL(N);
8322 SDValue Vec = N->getOperand(Num: 1);
8323 SDValue Idx = DAG.getConstant(Val: Imm, DL, VT: MVT::i32);
8324 SDValue EltVT = DAG.getValueType(Vec.getValueType().getVectorElementType());
8325 return DAG.getNode(Opcode: ResOp, DL, VT: N->getValueType(ResNo: 0), N1: Vec, N2: Idx, N3: EltVT);
8326}
8327
8328static SDValue
8329performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
8330 TargetLowering::DAGCombinerInfo &DCI,
8331 const LoongArchSubtarget &Subtarget) {
8332 SDLoc DL(N);
8333 switch (N->getConstantOperandVal(Num: 0)) {
8334 default:
8335 break;
8336 case Intrinsic::loongarch_lsx_vadd_b:
8337 case Intrinsic::loongarch_lsx_vadd_h:
8338 case Intrinsic::loongarch_lsx_vadd_w:
8339 case Intrinsic::loongarch_lsx_vadd_d:
8340 case Intrinsic::loongarch_lasx_xvadd_b:
8341 case Intrinsic::loongarch_lasx_xvadd_h:
8342 case Intrinsic::loongarch_lasx_xvadd_w:
8343 case Intrinsic::loongarch_lasx_xvadd_d:
8344 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8345 N2: N->getOperand(Num: 2));
8346 case Intrinsic::loongarch_lsx_vaddi_bu:
8347 case Intrinsic::loongarch_lsx_vaddi_hu:
8348 case Intrinsic::loongarch_lsx_vaddi_wu:
8349 case Intrinsic::loongarch_lsx_vaddi_du:
8350 case Intrinsic::loongarch_lasx_xvaddi_bu:
8351 case Intrinsic::loongarch_lasx_xvaddi_hu:
8352 case Intrinsic::loongarch_lasx_xvaddi_wu:
8353 case Intrinsic::loongarch_lasx_xvaddi_du:
8354 return DAG.getNode(Opcode: ISD::ADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8355 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8356 case Intrinsic::loongarch_lsx_vsub_b:
8357 case Intrinsic::loongarch_lsx_vsub_h:
8358 case Intrinsic::loongarch_lsx_vsub_w:
8359 case Intrinsic::loongarch_lsx_vsub_d:
8360 case Intrinsic::loongarch_lasx_xvsub_b:
8361 case Intrinsic::loongarch_lasx_xvsub_h:
8362 case Intrinsic::loongarch_lasx_xvsub_w:
8363 case Intrinsic::loongarch_lasx_xvsub_d:
8364 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8365 N2: N->getOperand(Num: 2));
8366 case Intrinsic::loongarch_lsx_vsubi_bu:
8367 case Intrinsic::loongarch_lsx_vsubi_hu:
8368 case Intrinsic::loongarch_lsx_vsubi_wu:
8369 case Intrinsic::loongarch_lsx_vsubi_du:
8370 case Intrinsic::loongarch_lasx_xvsubi_bu:
8371 case Intrinsic::loongarch_lasx_xvsubi_hu:
8372 case Intrinsic::loongarch_lasx_xvsubi_wu:
8373 case Intrinsic::loongarch_lasx_xvsubi_du:
8374 return DAG.getNode(Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8375 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8376 case Intrinsic::loongarch_lsx_vneg_b:
8377 case Intrinsic::loongarch_lsx_vneg_h:
8378 case Intrinsic::loongarch_lsx_vneg_w:
8379 case Intrinsic::loongarch_lsx_vneg_d:
8380 case Intrinsic::loongarch_lasx_xvneg_b:
8381 case Intrinsic::loongarch_lasx_xvneg_h:
8382 case Intrinsic::loongarch_lasx_xvneg_w:
8383 case Intrinsic::loongarch_lasx_xvneg_d:
8384 return DAG.getNode(
8385 Opcode: ISD::SUB, DL, VT: N->getValueType(ResNo: 0),
8386 N1: DAG.getConstant(
8387 Val: APInt(N->getValueType(ResNo: 0).getScalarType().getSizeInBits(), 0,
8388 /*isSigned=*/true),
8389 DL: SDLoc(N), VT: N->getValueType(ResNo: 0)),
8390 N2: N->getOperand(Num: 1));
8391 case Intrinsic::loongarch_lsx_vmax_b:
8392 case Intrinsic::loongarch_lsx_vmax_h:
8393 case Intrinsic::loongarch_lsx_vmax_w:
8394 case Intrinsic::loongarch_lsx_vmax_d:
8395 case Intrinsic::loongarch_lasx_xvmax_b:
8396 case Intrinsic::loongarch_lasx_xvmax_h:
8397 case Intrinsic::loongarch_lasx_xvmax_w:
8398 case Intrinsic::loongarch_lasx_xvmax_d:
8399 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8400 N2: N->getOperand(Num: 2));
8401 case Intrinsic::loongarch_lsx_vmax_bu:
8402 case Intrinsic::loongarch_lsx_vmax_hu:
8403 case Intrinsic::loongarch_lsx_vmax_wu:
8404 case Intrinsic::loongarch_lsx_vmax_du:
8405 case Intrinsic::loongarch_lasx_xvmax_bu:
8406 case Intrinsic::loongarch_lasx_xvmax_hu:
8407 case Intrinsic::loongarch_lasx_xvmax_wu:
8408 case Intrinsic::loongarch_lasx_xvmax_du:
8409 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8410 N2: N->getOperand(Num: 2));
8411 case Intrinsic::loongarch_lsx_vmaxi_b:
8412 case Intrinsic::loongarch_lsx_vmaxi_h:
8413 case Intrinsic::loongarch_lsx_vmaxi_w:
8414 case Intrinsic::loongarch_lsx_vmaxi_d:
8415 case Intrinsic::loongarch_lasx_xvmaxi_b:
8416 case Intrinsic::loongarch_lasx_xvmaxi_h:
8417 case Intrinsic::loongarch_lasx_xvmaxi_w:
8418 case Intrinsic::loongarch_lasx_xvmaxi_d:
8419 return DAG.getNode(Opcode: ISD::SMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8420 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
8421 case Intrinsic::loongarch_lsx_vmaxi_bu:
8422 case Intrinsic::loongarch_lsx_vmaxi_hu:
8423 case Intrinsic::loongarch_lsx_vmaxi_wu:
8424 case Intrinsic::loongarch_lsx_vmaxi_du:
8425 case Intrinsic::loongarch_lasx_xvmaxi_bu:
8426 case Intrinsic::loongarch_lasx_xvmaxi_hu:
8427 case Intrinsic::loongarch_lasx_xvmaxi_wu:
8428 case Intrinsic::loongarch_lasx_xvmaxi_du:
8429 return DAG.getNode(Opcode: ISD::UMAX, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8430 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8431 case Intrinsic::loongarch_lsx_vmin_b:
8432 case Intrinsic::loongarch_lsx_vmin_h:
8433 case Intrinsic::loongarch_lsx_vmin_w:
8434 case Intrinsic::loongarch_lsx_vmin_d:
8435 case Intrinsic::loongarch_lasx_xvmin_b:
8436 case Intrinsic::loongarch_lasx_xvmin_h:
8437 case Intrinsic::loongarch_lasx_xvmin_w:
8438 case Intrinsic::loongarch_lasx_xvmin_d:
8439 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8440 N2: N->getOperand(Num: 2));
8441 case Intrinsic::loongarch_lsx_vmin_bu:
8442 case Intrinsic::loongarch_lsx_vmin_hu:
8443 case Intrinsic::loongarch_lsx_vmin_wu:
8444 case Intrinsic::loongarch_lsx_vmin_du:
8445 case Intrinsic::loongarch_lasx_xvmin_bu:
8446 case Intrinsic::loongarch_lasx_xvmin_hu:
8447 case Intrinsic::loongarch_lasx_xvmin_wu:
8448 case Intrinsic::loongarch_lasx_xvmin_du:
8449 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8450 N2: N->getOperand(Num: 2));
8451 case Intrinsic::loongarch_lsx_vmini_b:
8452 case Intrinsic::loongarch_lsx_vmini_h:
8453 case Intrinsic::loongarch_lsx_vmini_w:
8454 case Intrinsic::loongarch_lsx_vmini_d:
8455 case Intrinsic::loongarch_lasx_xvmini_b:
8456 case Intrinsic::loongarch_lasx_xvmini_h:
8457 case Intrinsic::loongarch_lasx_xvmini_w:
8458 case Intrinsic::loongarch_lasx_xvmini_d:
8459 return DAG.getNode(Opcode: ISD::SMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8460 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG, /*IsSigned=*/true));
8461 case Intrinsic::loongarch_lsx_vmini_bu:
8462 case Intrinsic::loongarch_lsx_vmini_hu:
8463 case Intrinsic::loongarch_lsx_vmini_wu:
8464 case Intrinsic::loongarch_lsx_vmini_du:
8465 case Intrinsic::loongarch_lasx_xvmini_bu:
8466 case Intrinsic::loongarch_lasx_xvmini_hu:
8467 case Intrinsic::loongarch_lasx_xvmini_wu:
8468 case Intrinsic::loongarch_lasx_xvmini_du:
8469 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8470 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8471 case Intrinsic::loongarch_lsx_vmul_b:
8472 case Intrinsic::loongarch_lsx_vmul_h:
8473 case Intrinsic::loongarch_lsx_vmul_w:
8474 case Intrinsic::loongarch_lsx_vmul_d:
8475 case Intrinsic::loongarch_lasx_xvmul_b:
8476 case Intrinsic::loongarch_lasx_xvmul_h:
8477 case Intrinsic::loongarch_lasx_xvmul_w:
8478 case Intrinsic::loongarch_lasx_xvmul_d:
8479 return DAG.getNode(Opcode: ISD::MUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8480 N2: N->getOperand(Num: 2));
8481 case Intrinsic::loongarch_lsx_vmadd_b:
8482 case Intrinsic::loongarch_lsx_vmadd_h:
8483 case Intrinsic::loongarch_lsx_vmadd_w:
8484 case Intrinsic::loongarch_lsx_vmadd_d:
8485 case Intrinsic::loongarch_lasx_xvmadd_b:
8486 case Intrinsic::loongarch_lasx_xvmadd_h:
8487 case Intrinsic::loongarch_lasx_xvmadd_w:
8488 case Intrinsic::loongarch_lasx_xvmadd_d: {
8489 EVT ResTy = N->getValueType(ResNo: 0);
8490 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
8491 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
8492 N2: N->getOperand(Num: 3)));
8493 }
8494 case Intrinsic::loongarch_lsx_vmsub_b:
8495 case Intrinsic::loongarch_lsx_vmsub_h:
8496 case Intrinsic::loongarch_lsx_vmsub_w:
8497 case Intrinsic::loongarch_lsx_vmsub_d:
8498 case Intrinsic::loongarch_lasx_xvmsub_b:
8499 case Intrinsic::loongarch_lasx_xvmsub_h:
8500 case Intrinsic::loongarch_lasx_xvmsub_w:
8501 case Intrinsic::loongarch_lasx_xvmsub_d: {
8502 EVT ResTy = N->getValueType(ResNo: 0);
8503 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 1),
8504 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N), VT: ResTy, N1: N->getOperand(Num: 2),
8505 N2: N->getOperand(Num: 3)));
8506 }
8507 case Intrinsic::loongarch_lsx_vdiv_b:
8508 case Intrinsic::loongarch_lsx_vdiv_h:
8509 case Intrinsic::loongarch_lsx_vdiv_w:
8510 case Intrinsic::loongarch_lsx_vdiv_d:
8511 case Intrinsic::loongarch_lasx_xvdiv_b:
8512 case Intrinsic::loongarch_lasx_xvdiv_h:
8513 case Intrinsic::loongarch_lasx_xvdiv_w:
8514 case Intrinsic::loongarch_lasx_xvdiv_d:
8515 return DAG.getNode(Opcode: ISD::SDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8516 N2: N->getOperand(Num: 2));
8517 case Intrinsic::loongarch_lsx_vdiv_bu:
8518 case Intrinsic::loongarch_lsx_vdiv_hu:
8519 case Intrinsic::loongarch_lsx_vdiv_wu:
8520 case Intrinsic::loongarch_lsx_vdiv_du:
8521 case Intrinsic::loongarch_lasx_xvdiv_bu:
8522 case Intrinsic::loongarch_lasx_xvdiv_hu:
8523 case Intrinsic::loongarch_lasx_xvdiv_wu:
8524 case Intrinsic::loongarch_lasx_xvdiv_du:
8525 return DAG.getNode(Opcode: ISD::UDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8526 N2: N->getOperand(Num: 2));
8527 case Intrinsic::loongarch_lsx_vmod_b:
8528 case Intrinsic::loongarch_lsx_vmod_h:
8529 case Intrinsic::loongarch_lsx_vmod_w:
8530 case Intrinsic::loongarch_lsx_vmod_d:
8531 case Intrinsic::loongarch_lasx_xvmod_b:
8532 case Intrinsic::loongarch_lasx_xvmod_h:
8533 case Intrinsic::loongarch_lasx_xvmod_w:
8534 case Intrinsic::loongarch_lasx_xvmod_d:
8535 return DAG.getNode(Opcode: ISD::SREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8536 N2: N->getOperand(Num: 2));
8537 case Intrinsic::loongarch_lsx_vmod_bu:
8538 case Intrinsic::loongarch_lsx_vmod_hu:
8539 case Intrinsic::loongarch_lsx_vmod_wu:
8540 case Intrinsic::loongarch_lsx_vmod_du:
8541 case Intrinsic::loongarch_lasx_xvmod_bu:
8542 case Intrinsic::loongarch_lasx_xvmod_hu:
8543 case Intrinsic::loongarch_lasx_xvmod_wu:
8544 case Intrinsic::loongarch_lasx_xvmod_du:
8545 return DAG.getNode(Opcode: ISD::UREM, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8546 N2: N->getOperand(Num: 2));
8547 case Intrinsic::loongarch_lsx_vand_v:
8548 case Intrinsic::loongarch_lasx_xvand_v:
8549 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8550 N2: N->getOperand(Num: 2));
8551 case Intrinsic::loongarch_lsx_vor_v:
8552 case Intrinsic::loongarch_lasx_xvor_v:
8553 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8554 N2: N->getOperand(Num: 2));
8555 case Intrinsic::loongarch_lsx_vxor_v:
8556 case Intrinsic::loongarch_lasx_xvxor_v:
8557 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8558 N2: N->getOperand(Num: 2));
8559 case Intrinsic::loongarch_lsx_vnor_v:
8560 case Intrinsic::loongarch_lasx_xvnor_v: {
8561 SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8562 N2: N->getOperand(Num: 2));
8563 return DAG.getNOT(DL, Val: Res, VT: Res->getValueType(ResNo: 0));
8564 }
8565 case Intrinsic::loongarch_lsx_vandi_b:
8566 case Intrinsic::loongarch_lasx_xvandi_b:
8567 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8568 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
8569 case Intrinsic::loongarch_lsx_vori_b:
8570 case Intrinsic::loongarch_lasx_xvori_b:
8571 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8572 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
8573 case Intrinsic::loongarch_lsx_vxori_b:
8574 case Intrinsic::loongarch_lasx_xvxori_b:
8575 return DAG.getNode(Opcode: ISD::XOR, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8576 N2: lowerVectorSplatImm<8>(Node: N, ImmOp: 2, DAG));
8577 case Intrinsic::loongarch_lsx_vsll_b:
8578 case Intrinsic::loongarch_lsx_vsll_h:
8579 case Intrinsic::loongarch_lsx_vsll_w:
8580 case Intrinsic::loongarch_lsx_vsll_d:
8581 case Intrinsic::loongarch_lasx_xvsll_b:
8582 case Intrinsic::loongarch_lasx_xvsll_h:
8583 case Intrinsic::loongarch_lasx_xvsll_w:
8584 case Intrinsic::loongarch_lasx_xvsll_d:
8585 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8586 N2: truncateVecElts(Node: N, DAG));
8587 case Intrinsic::loongarch_lsx_vslli_b:
8588 case Intrinsic::loongarch_lasx_xvslli_b:
8589 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8590 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
8591 case Intrinsic::loongarch_lsx_vslli_h:
8592 case Intrinsic::loongarch_lasx_xvslli_h:
8593 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8594 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
8595 case Intrinsic::loongarch_lsx_vslli_w:
8596 case Intrinsic::loongarch_lasx_xvslli_w:
8597 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8598 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8599 case Intrinsic::loongarch_lsx_vslli_d:
8600 case Intrinsic::loongarch_lasx_xvslli_d:
8601 return DAG.getNode(Opcode: ISD::SHL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8602 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8603 case Intrinsic::loongarch_lsx_vsrl_b:
8604 case Intrinsic::loongarch_lsx_vsrl_h:
8605 case Intrinsic::loongarch_lsx_vsrl_w:
8606 case Intrinsic::loongarch_lsx_vsrl_d:
8607 case Intrinsic::loongarch_lasx_xvsrl_b:
8608 case Intrinsic::loongarch_lasx_xvsrl_h:
8609 case Intrinsic::loongarch_lasx_xvsrl_w:
8610 case Intrinsic::loongarch_lasx_xvsrl_d:
8611 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8612 N2: truncateVecElts(Node: N, DAG));
8613 case Intrinsic::loongarch_lsx_vsrli_b:
8614 case Intrinsic::loongarch_lasx_xvsrli_b:
8615 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8616 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
8617 case Intrinsic::loongarch_lsx_vsrli_h:
8618 case Intrinsic::loongarch_lasx_xvsrli_h:
8619 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8620 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
8621 case Intrinsic::loongarch_lsx_vsrli_w:
8622 case Intrinsic::loongarch_lasx_xvsrli_w:
8623 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8624 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8625 case Intrinsic::loongarch_lsx_vsrli_d:
8626 case Intrinsic::loongarch_lasx_xvsrli_d:
8627 return DAG.getNode(Opcode: ISD::SRL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8628 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8629 case Intrinsic::loongarch_lsx_vsra_b:
8630 case Intrinsic::loongarch_lsx_vsra_h:
8631 case Intrinsic::loongarch_lsx_vsra_w:
8632 case Intrinsic::loongarch_lsx_vsra_d:
8633 case Intrinsic::loongarch_lasx_xvsra_b:
8634 case Intrinsic::loongarch_lasx_xvsra_h:
8635 case Intrinsic::loongarch_lasx_xvsra_w:
8636 case Intrinsic::loongarch_lasx_xvsra_d:
8637 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8638 N2: truncateVecElts(Node: N, DAG));
8639 case Intrinsic::loongarch_lsx_vsrai_b:
8640 case Intrinsic::loongarch_lasx_xvsrai_b:
8641 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8642 N2: lowerVectorSplatImm<3>(Node: N, ImmOp: 2, DAG));
8643 case Intrinsic::loongarch_lsx_vsrai_h:
8644 case Intrinsic::loongarch_lasx_xvsrai_h:
8645 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8646 N2: lowerVectorSplatImm<4>(Node: N, ImmOp: 2, DAG));
8647 case Intrinsic::loongarch_lsx_vsrai_w:
8648 case Intrinsic::loongarch_lasx_xvsrai_w:
8649 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8650 N2: lowerVectorSplatImm<5>(Node: N, ImmOp: 2, DAG));
8651 case Intrinsic::loongarch_lsx_vsrai_d:
8652 case Intrinsic::loongarch_lasx_xvsrai_d:
8653 return DAG.getNode(Opcode: ISD::SRA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8654 N2: lowerVectorSplatImm<6>(Node: N, ImmOp: 2, DAG));
8655 case Intrinsic::loongarch_lsx_vclz_b:
8656 case Intrinsic::loongarch_lsx_vclz_h:
8657 case Intrinsic::loongarch_lsx_vclz_w:
8658 case Intrinsic::loongarch_lsx_vclz_d:
8659 case Intrinsic::loongarch_lasx_xvclz_b:
8660 case Intrinsic::loongarch_lasx_xvclz_h:
8661 case Intrinsic::loongarch_lasx_xvclz_w:
8662 case Intrinsic::loongarch_lasx_xvclz_d:
8663 return DAG.getNode(Opcode: ISD::CTLZ, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
8664 case Intrinsic::loongarch_lsx_vpcnt_b:
8665 case Intrinsic::loongarch_lsx_vpcnt_h:
8666 case Intrinsic::loongarch_lsx_vpcnt_w:
8667 case Intrinsic::loongarch_lsx_vpcnt_d:
8668 case Intrinsic::loongarch_lasx_xvpcnt_b:
8669 case Intrinsic::loongarch_lasx_xvpcnt_h:
8670 case Intrinsic::loongarch_lasx_xvpcnt_w:
8671 case Intrinsic::loongarch_lasx_xvpcnt_d:
8672 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT: N->getValueType(ResNo: 0), Operand: N->getOperand(Num: 1));
8673 case Intrinsic::loongarch_lsx_vbitclr_b:
8674 case Intrinsic::loongarch_lsx_vbitclr_h:
8675 case Intrinsic::loongarch_lsx_vbitclr_w:
8676 case Intrinsic::loongarch_lsx_vbitclr_d:
8677 case Intrinsic::loongarch_lasx_xvbitclr_b:
8678 case Intrinsic::loongarch_lasx_xvbitclr_h:
8679 case Intrinsic::loongarch_lasx_xvbitclr_w:
8680 case Intrinsic::loongarch_lasx_xvbitclr_d:
8681 return lowerVectorBitClear(Node: N, DAG);
8682 case Intrinsic::loongarch_lsx_vbitclri_b:
8683 case Intrinsic::loongarch_lasx_xvbitclri_b:
8684 return lowerVectorBitClearImm<3>(Node: N, DAG);
8685 case Intrinsic::loongarch_lsx_vbitclri_h:
8686 case Intrinsic::loongarch_lasx_xvbitclri_h:
8687 return lowerVectorBitClearImm<4>(Node: N, DAG);
8688 case Intrinsic::loongarch_lsx_vbitclri_w:
8689 case Intrinsic::loongarch_lasx_xvbitclri_w:
8690 return lowerVectorBitClearImm<5>(Node: N, DAG);
8691 case Intrinsic::loongarch_lsx_vbitclri_d:
8692 case Intrinsic::loongarch_lasx_xvbitclri_d:
8693 return lowerVectorBitClearImm<6>(Node: N, DAG);
8694 case Intrinsic::loongarch_lsx_vbitset_b:
8695 case Intrinsic::loongarch_lsx_vbitset_h:
8696 case Intrinsic::loongarch_lsx_vbitset_w:
8697 case Intrinsic::loongarch_lsx_vbitset_d:
8698 case Intrinsic::loongarch_lasx_xvbitset_b:
8699 case Intrinsic::loongarch_lasx_xvbitset_h:
8700 case Intrinsic::loongarch_lasx_xvbitset_w:
8701 case Intrinsic::loongarch_lasx_xvbitset_d: {
8702 EVT VecTy = N->getValueType(ResNo: 0);
8703 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8704 return DAG.getNode(
8705 Opcode: ISD::OR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8706 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8707 }
8708 case Intrinsic::loongarch_lsx_vbitseti_b:
8709 case Intrinsic::loongarch_lasx_xvbitseti_b:
8710 return lowerVectorBitSetImm<3>(Node: N, DAG);
8711 case Intrinsic::loongarch_lsx_vbitseti_h:
8712 case Intrinsic::loongarch_lasx_xvbitseti_h:
8713 return lowerVectorBitSetImm<4>(Node: N, DAG);
8714 case Intrinsic::loongarch_lsx_vbitseti_w:
8715 case Intrinsic::loongarch_lasx_xvbitseti_w:
8716 return lowerVectorBitSetImm<5>(Node: N, DAG);
8717 case Intrinsic::loongarch_lsx_vbitseti_d:
8718 case Intrinsic::loongarch_lasx_xvbitseti_d:
8719 return lowerVectorBitSetImm<6>(Node: N, DAG);
8720 case Intrinsic::loongarch_lsx_vbitrev_b:
8721 case Intrinsic::loongarch_lsx_vbitrev_h:
8722 case Intrinsic::loongarch_lsx_vbitrev_w:
8723 case Intrinsic::loongarch_lsx_vbitrev_d:
8724 case Intrinsic::loongarch_lasx_xvbitrev_b:
8725 case Intrinsic::loongarch_lasx_xvbitrev_h:
8726 case Intrinsic::loongarch_lasx_xvbitrev_w:
8727 case Intrinsic::loongarch_lasx_xvbitrev_d: {
8728 EVT VecTy = N->getValueType(ResNo: 0);
8729 SDValue One = DAG.getConstant(Val: 1, DL, VT: VecTy);
8730 return DAG.getNode(
8731 Opcode: ISD::XOR, DL, VT: VecTy, N1: N->getOperand(Num: 1),
8732 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: VecTy, N1: One, N2: truncateVecElts(Node: N, DAG)));
8733 }
8734 case Intrinsic::loongarch_lsx_vbitrevi_b:
8735 case Intrinsic::loongarch_lasx_xvbitrevi_b:
8736 return lowerVectorBitRevImm<3>(Node: N, DAG);
8737 case Intrinsic::loongarch_lsx_vbitrevi_h:
8738 case Intrinsic::loongarch_lasx_xvbitrevi_h:
8739 return lowerVectorBitRevImm<4>(Node: N, DAG);
8740 case Intrinsic::loongarch_lsx_vbitrevi_w:
8741 case Intrinsic::loongarch_lasx_xvbitrevi_w:
8742 return lowerVectorBitRevImm<5>(Node: N, DAG);
8743 case Intrinsic::loongarch_lsx_vbitrevi_d:
8744 case Intrinsic::loongarch_lasx_xvbitrevi_d:
8745 return lowerVectorBitRevImm<6>(Node: N, DAG);
8746 case Intrinsic::loongarch_lsx_vfadd_s:
8747 case Intrinsic::loongarch_lsx_vfadd_d:
8748 case Intrinsic::loongarch_lasx_xvfadd_s:
8749 case Intrinsic::loongarch_lasx_xvfadd_d:
8750 return DAG.getNode(Opcode: ISD::FADD, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8751 N2: N->getOperand(Num: 2));
8752 case Intrinsic::loongarch_lsx_vfsub_s:
8753 case Intrinsic::loongarch_lsx_vfsub_d:
8754 case Intrinsic::loongarch_lasx_xvfsub_s:
8755 case Intrinsic::loongarch_lasx_xvfsub_d:
8756 return DAG.getNode(Opcode: ISD::FSUB, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8757 N2: N->getOperand(Num: 2));
8758 case Intrinsic::loongarch_lsx_vfmul_s:
8759 case Intrinsic::loongarch_lsx_vfmul_d:
8760 case Intrinsic::loongarch_lasx_xvfmul_s:
8761 case Intrinsic::loongarch_lasx_xvfmul_d:
8762 return DAG.getNode(Opcode: ISD::FMUL, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8763 N2: N->getOperand(Num: 2));
8764 case Intrinsic::loongarch_lsx_vfdiv_s:
8765 case Intrinsic::loongarch_lsx_vfdiv_d:
8766 case Intrinsic::loongarch_lasx_xvfdiv_s:
8767 case Intrinsic::loongarch_lasx_xvfdiv_d:
8768 return DAG.getNode(Opcode: ISD::FDIV, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8769 N2: N->getOperand(Num: 2));
8770 case Intrinsic::loongarch_lsx_vfmadd_s:
8771 case Intrinsic::loongarch_lsx_vfmadd_d:
8772 case Intrinsic::loongarch_lasx_xvfmadd_s:
8773 case Intrinsic::loongarch_lasx_xvfmadd_d:
8774 return DAG.getNode(Opcode: ISD::FMA, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 1),
8775 N2: N->getOperand(Num: 2), N3: N->getOperand(Num: 3));
8776 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
8777 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8778 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8779 N3: legalizeIntrinsicImmArg<4>(Node: N, ImmOp: 3, DAG, Subtarget));
8780 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
8781 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
8782 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8783 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8784 N3: legalizeIntrinsicImmArg<3>(Node: N, ImmOp: 3, DAG, Subtarget));
8785 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
8786 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
8787 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8788 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8789 N3: legalizeIntrinsicImmArg<2>(Node: N, ImmOp: 3, DAG, Subtarget));
8790 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
8791 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8792 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2),
8793 N3: legalizeIntrinsicImmArg<1>(Node: N, ImmOp: 3, DAG, Subtarget));
8794 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
8795 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
8796 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
8797 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
8798 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
8799 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
8800 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
8801 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
8802 return DAG.getNode(Opcode: LoongArchISD::VREPLGR2VR, DL, VT: N->getValueType(ResNo: 0),
8803 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8804 Operand: N->getOperand(Num: 1)));
8805 case Intrinsic::loongarch_lsx_vreplve_b:
8806 case Intrinsic::loongarch_lsx_vreplve_h:
8807 case Intrinsic::loongarch_lsx_vreplve_w:
8808 case Intrinsic::loongarch_lsx_vreplve_d:
8809 case Intrinsic::loongarch_lasx_xvreplve_b:
8810 case Intrinsic::loongarch_lasx_xvreplve_h:
8811 case Intrinsic::loongarch_lasx_xvreplve_w:
8812 case Intrinsic::loongarch_lasx_xvreplve_d:
8813 return DAG.getNode(Opcode: LoongArchISD::VREPLVE, DL, VT: N->getValueType(ResNo: 0),
8814 N1: N->getOperand(Num: 1),
8815 N2: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getGRLenVT(),
8816 Operand: N->getOperand(Num: 2)));
8817 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8818 if (!Subtarget.is64Bit())
8819 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8820 break;
8821 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8822 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8823 if (!Subtarget.is64Bit())
8824 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8825 break;
8826 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8827 if (!Subtarget.is64Bit())
8828 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_SEXT_ELT);
8829 break;
8830 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8831 if (!Subtarget.is64Bit())
8832 return lowerVectorPickVE2GR<4>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8833 break;
8834 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8835 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8836 if (!Subtarget.is64Bit())
8837 return lowerVectorPickVE2GR<3>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8838 break;
8839 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8840 if (!Subtarget.is64Bit())
8841 return lowerVectorPickVE2GR<2>(N, DAG, ResOp: LoongArchISD::VPICK_ZEXT_ELT);
8842 break;
8843 case Intrinsic::loongarch_lsx_bz_b:
8844 case Intrinsic::loongarch_lsx_bz_h:
8845 case Intrinsic::loongarch_lsx_bz_w:
8846 case Intrinsic::loongarch_lsx_bz_d:
8847 case Intrinsic::loongarch_lasx_xbz_b:
8848 case Intrinsic::loongarch_lasx_xbz_h:
8849 case Intrinsic::loongarch_lasx_xbz_w:
8850 case Intrinsic::loongarch_lasx_xbz_d:
8851 if (!Subtarget.is64Bit())
8852 return DAG.getNode(Opcode: LoongArchISD::VALL_ZERO, DL, VT: N->getValueType(ResNo: 0),
8853 Operand: N->getOperand(Num: 1));
8854 break;
8855 case Intrinsic::loongarch_lsx_bz_v:
8856 case Intrinsic::loongarch_lasx_xbz_v:
8857 if (!Subtarget.is64Bit())
8858 return DAG.getNode(Opcode: LoongArchISD::VANY_ZERO, DL, VT: N->getValueType(ResNo: 0),
8859 Operand: N->getOperand(Num: 1));
8860 break;
8861 case Intrinsic::loongarch_lsx_bnz_b:
8862 case Intrinsic::loongarch_lsx_bnz_h:
8863 case Intrinsic::loongarch_lsx_bnz_w:
8864 case Intrinsic::loongarch_lsx_bnz_d:
8865 case Intrinsic::loongarch_lasx_xbnz_b:
8866 case Intrinsic::loongarch_lasx_xbnz_h:
8867 case Intrinsic::loongarch_lasx_xbnz_w:
8868 case Intrinsic::loongarch_lasx_xbnz_d:
8869 if (!Subtarget.is64Bit())
8870 return DAG.getNode(Opcode: LoongArchISD::VALL_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8871 Operand: N->getOperand(Num: 1));
8872 break;
8873 case Intrinsic::loongarch_lsx_bnz_v:
8874 case Intrinsic::loongarch_lasx_xbnz_v:
8875 if (!Subtarget.is64Bit())
8876 return DAG.getNode(Opcode: LoongArchISD::VANY_NONZERO, DL, VT: N->getValueType(ResNo: 0),
8877 Operand: N->getOperand(Num: 1));
8878 break;
8879 case Intrinsic::loongarch_lasx_concat_128_s:
8880 case Intrinsic::loongarch_lasx_concat_128_d:
8881 case Intrinsic::loongarch_lasx_concat_128:
8882 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
8883 N1: N->getOperand(Num: 1), N2: N->getOperand(Num: 2));
8884 }
8885 return SDValue();
8886}
8887
8888static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG,
8889 TargetLowering::DAGCombinerInfo &DCI,
8890 const LoongArchSubtarget &Subtarget) {
8891 // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
8892 // conversion is unnecessary and can be replaced with the
8893 // MOVFR2GR_S_LA64 operand.
8894 SDValue Op0 = N->getOperand(Num: 0);
8895 if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8896 return Op0.getOperand(i: 0);
8897 return SDValue();
8898}
8899
8900static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG,
8901 TargetLowering::DAGCombinerInfo &DCI,
8902 const LoongArchSubtarget &Subtarget) {
8903 // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
8904 // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
8905 // operand.
8906 SDValue Op0 = N->getOperand(Num: 0);
8907 if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8908 assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
8909 "Unexpected value type!");
8910 return Op0.getOperand(i: 0);
8911 }
8912 return SDValue();
8913}
8914
8915static SDValue
8916performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG,
8917 TargetLowering::DAGCombinerInfo &DCI) {
8918 MVT VT = N->getSimpleValueType(ResNo: 0);
8919 unsigned NumBits = VT.getScalarSizeInBits();
8920
8921 // Simplify the inputs.
8922 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8923 APInt DemandedMask(APInt::getAllOnes(numBits: NumBits));
8924 if (TLI.SimplifyDemandedBits(Op: SDValue(N, 0), DemandedBits: DemandedMask, DCI))
8925 return SDValue(N, 0);
8926
8927 return SDValue();
8928}
8929
8930static SDValue
8931performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG,
8932 TargetLowering::DAGCombinerInfo &DCI,
8933 const LoongArchSubtarget &Subtarget) {
8934 SDValue Op0 = N->getOperand(Num: 0);
8935 SDLoc DL(N);
8936
8937 // If the input to SplitPairF64 is just BuildPairF64 then the operation is
8938 // redundant. Instead, use BuildPairF64's operands directly.
8939 if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8940 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
8941
8942 if (Op0->isUndef()) {
8943 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
8944 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
8945 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8946 }
8947
8948 // It's cheaper to materialise two 32-bit integers than to load a double
8949 // from the constant pool and transfer it to integer registers through the
8950 // stack.
8951 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
8952 APInt V = C->getValueAPF().bitcastToAPInt();
8953 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
8954 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
8955 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
8956 }
8957
8958 return SDValue();
8959}
8960
8961/// Do target-specific dag combines on LoongArchISD::VANDN nodes.
8962static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG,
8963 TargetLowering::DAGCombinerInfo &DCI,
8964 const LoongArchSubtarget &Subtarget) {
8965 SDValue N0 = N->getOperand(Num: 0);
8966 SDValue N1 = N->getOperand(Num: 1);
8967 MVT VT = N->getSimpleValueType(ResNo: 0);
8968 SDLoc DL(N);
8969
8970 // VANDN(undef, x) -> 0
8971 // VANDN(x, undef) -> 0
8972 if (N0.isUndef() || N1.isUndef())
8973 return DAG.getConstant(Val: 0, DL, VT);
8974
8975 // VANDN(0, x) -> x
8976 if (ISD::isBuildVectorAllZeros(V: N0))
8977 return N1;
8978
8979 // VANDN(x, 0) -> 0
8980 if (ISD::isBuildVectorAllZeros(V: N1))
8981 return DAG.getConstant(Val: 0, DL, VT);
8982
8983 // VANDN(x, -1) -> NOT(x) -> XOR(x, -1)
8984 if (ISD::isBuildVectorAllOnes(V: N1))
8985 return DAG.getNOT(DL, Val: N0, VT);
8986
8987 // Turn VANDN back to AND if input is inverted.
8988 if (SDValue Not = isNOT(V: N0, DAG))
8989 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: DAG.getBitcast(VT, V: Not), N2: N1);
8990
8991 // VANDN(bitcast(shl(splat(1), y)), x)
8992 // -> bitcast(VANDN(shl(splat(1), y), bitcast(x)))
8993 // Perform the VANDN in the element type of the shift so that it can be
8994 // selected as [x]vbitclr.{b/h/w/d}.
8995 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) {
8996 SDValue Shl = N0.getOperand(i: 0);
8997 EVT ShlVT = Shl.getValueType();
8998 APInt SplatVal;
8999 if (Shl.getOpcode() == ISD::SHL && ShlVT.isVector() && ShlVT.isInteger() &&
9000 ShlVT != VT && ShlVT.getSizeInBits() == VT.getSizeInBits() &&
9001 DAG.getTargetLoweringInfo().isTypeLegal(VT: ShlVT) &&
9002 ISD::isConstantSplatVector(V: Shl.getOperand(i: 0), SplatValue&: SplatVal) &&
9003 SplatVal.isOne()) {
9004 SDValue V = DAG.getNode(Opcode: LoongArchISD::VANDN, DL, VT: ShlVT, N1: Shl,
9005 N2: DAG.getBitcast(VT: ShlVT, V: N1));
9006 return DAG.getBitcast(VT, V);
9007 }
9008 }
9009
9010 // Folds for better commutativity:
9011 if (N1->hasOneUse()) {
9012 // VANDN(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
9013 if (SDValue Not = isNOT(V: N1, DAG))
9014 return DAG.getNOT(
9015 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)), VT);
9016
9017 // VANDN(x, SplatVector(Imm)) -> AND(NOT(x), NOT(SplatVector(~Imm)))
9018 // -> NOT(OR(x, SplatVector(-Imm))
9019 // Combination is performed only when VT is v16i8/v32i8, using `vnori.b` to
9020 // gain benefits.
9021 if (!DCI.isBeforeLegalizeOps() && (VT == MVT::v16i8 || VT == MVT::v32i8) &&
9022 N1.getOpcode() == ISD::BUILD_VECTOR) {
9023 if (SDValue SplatValue =
9024 cast<BuildVectorSDNode>(Val: N1.getNode())->getSplatValue()) {
9025 if (!N1->isOnlyUserOf(N: SplatValue.getNode()))
9026 return SDValue();
9027
9028 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatValue)) {
9029 uint8_t NCVal = static_cast<uint8_t>(~(C->getSExtValue()));
9030 SDValue Not =
9031 DAG.getSplat(VT, DL, Op: DAG.getTargetConstant(Val: NCVal, DL, VT: MVT::i8));
9032 return DAG.getNOT(
9033 DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: DAG.getBitcast(VT, V: Not)),
9034 VT);
9035 }
9036 }
9037 }
9038 }
9039
9040 return SDValue();
9041}
9042
9043static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp) {
9044 SDLoc DL(N);
9045 EVT VT = N->getValueType(ResNo: 0);
9046 SDValue Src = N->getOperand(Num: 0);
9047 EVT SrcVT = Src.getValueType();
9048
9049 unsigned DstElts = VT.getVectorNumElements();
9050 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9051 unsigned DstEltBits = VT.getScalarSizeInBits();
9052
9053 if (SrcEltBits >= DstEltBits)
9054 return SDValue();
9055
9056 MVT WidenEltVT = MVT::getIntegerVT(BitWidth: DstEltBits);
9057 EVT WidenSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: DstElts);
9058
9059 SDValue Extend = DAG.getNode(Opcode: ExtendOp, DL, VT: WidenSrcVT, Operand: Src);
9060 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Extend);
9061}
9062
9063// Merge two 64 to 32 convert instructions into one,
9064// e.g.
9065// vffint.s.l $vr0, $vr1, $vr2
9066// will convert 4 si64 into 4 float at once.
9067// or
9068// vftintrz.w.d $vr0, $vr1, $vr2
9069// which will convert 4 double into 4 si32 at once.
9070// also deal with their 256-bits LASX version.
9071static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode,
9072 unsigned BlockBits) {
9073 SDLoc DL(N);
9074 MVT DstVT = N->getSimpleValueType(ResNo: 0);
9075 SDValue Src = N->getOperand(Num: 0);
9076 MVT SrcVT = Src.getSimpleValueType();
9077 unsigned SrcBits = SrcVT.getSizeInBits();
9078
9079 SmallVector<SDValue, 4> Blocks;
9080 unsigned BlockNumElts = BlockBits / SrcVT.getScalarSizeInBits();
9081 MVT BlockVT = MVT::getVectorVT(VT: SrcVT.getScalarType(), NumElements: BlockNumElts);
9082 if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
9083 Src.getOperand(i: 0).getValueType() == BlockVT) {
9084 for (unsigned i = 0; i < Src.getNumOperands(); ++i)
9085 Blocks.push_back(Elt: Src.getOperand(i));
9086 } else if (SrcBits > BlockBits) {
9087 // Wider than one register: extract each BlockBits-wide sub-vector.
9088 for (unsigned i = 0; i < SrcBits / BlockBits; ++i)
9089 Blocks.push_back(
9090 Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: BlockVT, N1: Src,
9091 N2: DAG.getVectorIdxConstant(Val: i * BlockNumElts, DL)));
9092 } else {
9093 BlockBits = SrcBits;
9094 Blocks.push_back(Elt: Src);
9095 }
9096
9097 MVT NativeVecVT = MVT::getVectorVT(VT: DstVT.getScalarType(),
9098 NumElements: BlockBits / DstVT.getScalarSizeInBits());
9099 SmallVector<SDValue, 4> Parts;
9100 for (unsigned i = 0; i < Blocks.size(); i += 2) {
9101 SDValue Lo = Blocks[i];
9102 SDValue Hi = Blocks.size() > 1 ? Blocks[i + 1] : Lo;
9103 SDValue Res = DAG.getNode(Opcode, DL, VT: NativeVecVT, N1: Hi, N2: Lo);
9104
9105 if (BlockBits == 256) {
9106 SDValue Undef = DAG.getUNDEF(VT: NativeVecVT);
9107 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
9108 Res = DAG.getVectorShuffle(VT: NativeVecVT, dl: DL, N1: Res, N2: Undef, Mask);
9109 Res = DAG.getBitcast(VT: NativeVecVT, V: Res);
9110 }
9111
9112 Parts.push_back(Elt: Res);
9113 }
9114
9115 if (Blocks.size() == 1)
9116 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: DstVT, N1: Parts[0],
9117 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9118 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Parts);
9119}
9120
9121static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
9122 TargetLowering::DAGCombinerInfo &DCI,
9123 const LoongArchSubtarget &Subtarget) {
9124 SDLoc DL(N);
9125 EVT VT = N->getValueType(ResNo: 0);
9126 SDValue Src = N->getOperand(Num: 0);
9127 EVT SrcVT = Src.getValueType();
9128
9129 if (VT.isVector()) {
9130 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9131 unsigned DstEltBits = VT.getScalarSizeInBits();
9132 unsigned NumElts = VT.getVectorNumElements();
9133 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
9134
9135 // Sign-extend src to avoid scalarization.
9136 if (SrcEltBits <= DstEltBits)
9137 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::SIGN_EXTEND);
9138
9139 if (SrcEltBits != 64 || DstEltBits != 32 || !isPowerOf2_32(Value: NumElts))
9140 return SDValue();
9141
9142 if (!SrcVT.isSimple() || !VT.isSimple())
9143 return SDValue();
9144
9145 // [x]vffint.s.l needs LSX and at least one full 128-bit source register.
9146 if (!Subtarget.hasExtLSX() || SrcVT.getSizeInBits() < 128)
9147 return SDValue();
9148
9149 // Combine [x]vffint.s.l for vector si64 to float conversion.
9150 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFFINT, BlockBits);
9151 }
9152
9153 if (VT != MVT::f32 && VT != MVT::f64)
9154 return SDValue();
9155 if (VT == MVT::f32 && !Subtarget.hasBasicF())
9156 return SDValue();
9157 if (VT == MVT::f64 && !Subtarget.hasBasicD())
9158 return SDValue();
9159
9160 // Only optimize when the source and destination types have the same width.
9161 if (VT.getSizeInBits() != N->getOperand(Num: 0).getValueSizeInBits())
9162 return SDValue();
9163
9164 // If the result of an integer load is only used by an integer-to-float
9165 // conversion, use a fp load instead. This eliminates an integer-to-float-move
9166 // (movgr2fr) instruction.
9167 if (ISD::isNormalLoad(N: Src.getNode()) && Src.hasOneUse() &&
9168 // Do not change the width of a volatile load. This condition check is
9169 // inspired by AArch64.
9170 !cast<LoadSDNode>(Val&: Src)->isVolatile()) {
9171 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: Src);
9172 SDValue Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
9173 PtrInfo: LN0->getPointerInfo(), Alignment: LN0->getAlign(),
9174 MMOFlags: LN0->getMemOperand()->getFlags());
9175
9176 // Make sure successors of the original load stay after it by updating them
9177 // to use the new Chain.
9178 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: Load.getValue(R: 1));
9179 return DAG.getNode(Opcode: LoongArchISD::SITOF, DL: SDLoc(N), VT, Operand: Load);
9180 }
9181
9182 return SDValue();
9183}
9184
9185static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
9186 TargetLowering::DAGCombinerInfo &DCI,
9187 const LoongArchSubtarget &Subtarget) {
9188 SDLoc DL(N);
9189 EVT VT = N->getValueType(ResNo: 0);
9190
9191 // Zero-extend src to avoid scalarization.
9192 if (VT.isVector())
9193 return ExtendSrcToDst(N, DAG, ExtendOp: ISD::ZERO_EXTEND);
9194
9195 return SDValue();
9196}
9197
9198// Using [X]VFTINTRZ_W_D for double to signed 32-bit integer conversion.
9199// For example:
9200// v4i32 = fp_to_sint (concat_vectors v2f64, v2f64)
9201// Can be combined into:
9202// v4i32 = VFTINTRZ_W_D v2f64. v2f64
9203static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG,
9204 TargetLowering::DAGCombinerInfo &DCI,
9205 const LoongArchSubtarget &Subtarget) {
9206 if (!Subtarget.hasExtLSX())
9207 return SDValue();
9208
9209 SDLoc DL(N);
9210 EVT DstVT = N->getValueType(ResNo: 0);
9211 SDValue Src = N->getOperand(Num: 0);
9212 EVT SrcVT = Src.getValueType();
9213 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
9214
9215 if (!DstVT.isVector() || !DstVT.isSimple() || !SrcVT.isSimple())
9216 return SDValue();
9217
9218 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9219 unsigned SrcBits = SrcVT.getSizeInBits();
9220 unsigned DstEltBits = DstVT.getScalarSizeInBits();
9221 unsigned NumElts = DstVT.getVectorNumElements();
9222 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
9223
9224 if (!isPowerOf2_32(Value: NumElts) || !isPowerOf2_32(Value: DstEltBits))
9225 return SDValue();
9226
9227 if (SrcBits % BlockBits != 0 && SrcBits != 128)
9228 return SDValue();
9229
9230 if (DstEltBits < 32) {
9231 MVT PromoteVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 32), NumElements: NumElts);
9232 SDValue Conv = DAG.getNode(Opcode: N->getOpcode(), DL, VT: PromoteVT, Operand: Src);
9233 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Conv);
9234 }
9235
9236 if (SrcEltBits != 64 || DstEltBits != 32)
9237 return SDValue();
9238
9239 if (!IsSigned) {
9240 // LASX already has pattern for double convert to uint32.
9241 if (Subtarget.hasExtLASX())
9242 return SDValue();
9243 MVT TmpVT = MVT::getVectorVT(VT: MVT::i64, NumElements: NumElts);
9244 SDValue Tmp = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: TmpVT, Operand: Src);
9245 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Tmp);
9246 }
9247
9248 return MergeBlocksConvert(N, DAG, Opcode: LoongArchISD::VFTINTRZ, BlockBits);
9249}
9250
9251// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
9252// logical operations, like in the example below.
9253// or (and (truncate x, truncate y)),
9254// (xor (truncate z, build_vector (constants)))
9255// Given a target type \p VT, we generate
9256// or (and x, y), (xor z, zext(build_vector (constants)))
9257// given x, y and z are of type \p VT. We can do so, if operands are either
9258// truncates from VT types, the second operand is a vector of constants, can
9259// be recursively promoted or is an existing extension we can extend further.
9260static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
9261 SelectionDAG &DAG,
9262 const LoongArchSubtarget &Subtarget,
9263 unsigned Depth) {
9264 // Limit recursion to avoid excessive compile times.
9265 if (Depth >= SelectionDAG::MaxRecursionDepth)
9266 return SDValue();
9267
9268 if (!ISD::isBitwiseLogicOp(Opcode: N.getOpcode()))
9269 return SDValue();
9270
9271 SDValue N0 = N.getOperand(i: 0);
9272 SDValue N1 = N.getOperand(i: 1);
9273
9274 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9275 if (!TLI.isOperationLegalOrPromote(Op: N.getOpcode(), VT))
9276 return SDValue();
9277
9278 if (SDValue NN0 =
9279 PromoteMaskArithmetic(N: N0, DL, VT, DAG, Subtarget, Depth: Depth + 1))
9280 N0 = NN0;
9281 else {
9282 // The left side has to be a 'trunc'.
9283 bool LHSTrunc = N0.getOpcode() == ISD::TRUNCATE &&
9284 N0.getOperand(i: 0).getValueType() == VT;
9285 if (LHSTrunc)
9286 N0 = N0.getOperand(i: 0);
9287 else
9288 return SDValue();
9289 }
9290
9291 if (SDValue NN1 =
9292 PromoteMaskArithmetic(N: N1, DL, VT, DAG, Subtarget, Depth: Depth + 1))
9293 N1 = NN1;
9294 else {
9295 // The right side has to be a 'trunc', a (foldable) constant or an
9296 // existing extension we can extend further.
9297 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
9298 N1.getOperand(i: 0).getValueType() == VT;
9299 if (RHSTrunc)
9300 N1 = N1.getOperand(i: 0);
9301 else if (ISD::isExtVecInRegOpcode(Opcode: N1.getOpcode()) && VT.is256BitVector() &&
9302 Subtarget.hasExtLASX() && N1.hasOneUse())
9303 N1 = DAG.getNode(Opcode: N1.getOpcode(), DL, VT, Operand: N1.getOperand(i: 0));
9304 // On 32-bit platform, i64 is an illegal integer scalar type, and
9305 // FoldConstantArithmetic will fail for v4i64. This may be optimized in the
9306 // future.
9307 else if (SDValue Cst =
9308 DAG.FoldConstantArithmetic(Opcode: ISD::ZERO_EXTEND, DL, VT, Ops: {N1}))
9309 N1 = Cst;
9310 else
9311 return SDValue();
9312 }
9313
9314 return DAG.getNode(Opcode: N.getOpcode(), DL, VT, N1: N0, N2: N1);
9315}
9316
9317// On LASX the type v4i1/v8i1/v16i1 may be legalized to v4i32/v8i16/v16i8, which
9318// is LSX-sized register. In most cases we actually compare or select LASX-sized
9319// registers and mixing the two types creates horrible code. This method
9320// optimizes some of the transition sequences.
9321static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
9322 SelectionDAG &DAG,
9323 const LoongArchSubtarget &Subtarget) {
9324 EVT VT = N.getValueType();
9325 assert(VT.isVector() && "Expected vector type");
9326 assert((N.getOpcode() == ISD::ANY_EXTEND ||
9327 N.getOpcode() == ISD::ZERO_EXTEND ||
9328 N.getOpcode() == ISD::SIGN_EXTEND) &&
9329 "Invalid Node");
9330
9331 if (!Subtarget.hasExtLASX() || !VT.is256BitVector())
9332 return SDValue();
9333
9334 SDValue Narrow = N.getOperand(i: 0);
9335 EVT NarrowVT = Narrow.getValueType();
9336
9337 // Generate the wide operation.
9338 SDValue Op = PromoteMaskArithmetic(N: Narrow, DL, VT, DAG, Subtarget, Depth: 0);
9339 if (!Op)
9340 return SDValue();
9341 switch (N.getOpcode()) {
9342 default:
9343 llvm_unreachable("Unexpected opcode");
9344 case ISD::ANY_EXTEND:
9345 return Op;
9346 case ISD::ZERO_EXTEND:
9347 return DAG.getZeroExtendInReg(Op, DL, VT: NarrowVT);
9348 case ISD::SIGN_EXTEND:
9349 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
9350 N2: DAG.getValueType(NarrowVT));
9351 }
9352}
9353
9354static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG,
9355 TargetLowering::DAGCombinerInfo &DCI,
9356 const LoongArchSubtarget &Subtarget) {
9357 EVT VT = N->getValueType(ResNo: 0);
9358 SDLoc DL(N);
9359
9360 if (VT.isVector()) {
9361 if (SDValue R = PromoteMaskArithmetic(N: SDValue(N, 0), DL, DAG, Subtarget))
9362 return R;
9363
9364 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
9365 N->getValueSizeInBits(ResNo: 0) != N->getOperand(Num: 0).getValueSizeInBits() * 2)
9366 return SDValue();
9367
9368 if (SDValue R = matchHalfOf128BitLanes(N: N->getOperand(Num: 0), /*isLow=*/false)) {
9369 if (N->getOpcode() == ISD::SIGN_EXTEND)
9370 return DAG.getNode(Opcode: LoongArchISD::VEXTH, DL, VT, Operand: R);
9371 if (N->getOpcode() == ISD::ZERO_EXTEND)
9372 return DAG.getNode(Opcode: LoongArchISD::VEXTH_U, DL, VT, Operand: R);
9373 }
9374 }
9375
9376 return SDValue();
9377}
9378
9379static SDValue
9380performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
9381 TargetLowering::DAGCombinerInfo &DCI,
9382 const LoongArchSubtarget &Subtarget) {
9383 SDLoc DL(N);
9384 EVT VT = N->getValueType(ResNo: 0);
9385
9386 if (VT.isVector() && N->getNumOperands() == 2)
9387 if (SDValue R = combineFP_ROUND(N: SDValue(N, 0), DL, DAG, Subtarget))
9388 return R;
9389
9390 return SDValue();
9391}
9392
9393static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
9394 TargetLowering::DAGCombinerInfo &DCI,
9395 const LoongArchSubtarget &Subtarget) {
9396 if (DCI.isBeforeLegalizeOps())
9397 return SDValue();
9398
9399 EVT VT = N->getValueType(ResNo: 0);
9400 if (!VT.isVector())
9401 return SDValue();
9402
9403 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9404 return SDValue();
9405
9406 EVT EltVT = VT.getVectorElementType();
9407 if (!EltVT.isInteger())
9408 return SDValue();
9409
9410 SDValue Cond = N->getOperand(Num: 0);
9411 SDValue TrueVal = N->getOperand(Num: 1);
9412 SDValue FalseVal = N->getOperand(Num: 2);
9413
9414 // match:
9415 //
9416 // vselect (setcc shift, 0, seteq),
9417 // x,
9418 // rounded_shift
9419
9420 if (Cond.getOpcode() != ISD::SETCC)
9421 return SDValue();
9422
9423 if (!ISD::isConstantSplatVectorAllZeros(V: Cond.getOperand(i: 1)))
9424 return SDValue();
9425
9426 if (Cond.getOperandCondCode(i: 2) != ISD::SETEQ)
9427 return SDValue();
9428
9429 SDValue Shift = Cond.getOperand(i: 0);
9430
9431 // True branch must be original value:
9432 //
9433 // vselect cond, x, ...
9434
9435 SDValue X = TrueVal;
9436
9437 // Now match rounded shift pattern:
9438 //
9439 // add
9440 // (and
9441 // (srl X, shift-1)
9442 // 1)
9443 // (srl/sra X, shift)
9444
9445 if (FalseVal.getOpcode() != ISD::ADD)
9446 return SDValue();
9447
9448 SDValue Add0 = FalseVal.getOperand(i: 0);
9449 SDValue Add1 = FalseVal.getOperand(i: 1);
9450 SDValue And;
9451 SDValue Shr;
9452
9453 if (Add0.getOpcode() == ISD::AND) {
9454 And = Add0;
9455 Shr = Add1;
9456 } else if (Add1.getOpcode() == ISD::AND) {
9457 And = Add1;
9458 Shr = Add0;
9459 } else {
9460 return SDValue();
9461 }
9462
9463 // match:
9464 //
9465 // srl/sra X, shift
9466
9467 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
9468 return SDValue();
9469
9470 if (Shr.getOperand(i: 0) != X)
9471 return SDValue();
9472
9473 if (Shr.getOperand(i: 1) != Shift)
9474 return SDValue();
9475
9476 // match:
9477 //
9478 // and
9479 // (srl X, shift-1)
9480 // 1
9481
9482 SDValue Srl = And.getOperand(i: 0);
9483 SDValue One = And.getOperand(i: 1);
9484 APInt SplatVal;
9485
9486 if (Srl.getOpcode() != ISD::SRL)
9487 return SDValue();
9488
9489 One = peekThroughBitcasts(V: One);
9490 if (!isConstantSplatVector(N: One, SplatValue&: SplatVal, MinSizeInBits: EltVT.getSizeInBits()))
9491 return SDValue();
9492
9493 if (SplatVal != 1)
9494 return SDValue();
9495
9496 if (Srl.getOperand(i: 0) != X)
9497 return SDValue();
9498
9499 // match:
9500 //
9501 // shift-1
9502
9503 SDValue ShiftMinus1 = Srl.getOperand(i: 1);
9504
9505 if (ShiftMinus1.getOpcode() != ISD::ADD)
9506 return SDValue();
9507
9508 if (ShiftMinus1.getOperand(i: 0) != Shift)
9509 return SDValue();
9510
9511 if (!ISD::isConstantSplatVectorAllOnes(V: ShiftMinus1.getOperand(i: 1)))
9512 return SDValue();
9513
9514 // We matched a rounded right shift pattern and can lower it
9515 // to a single vector rounded shift instruction.
9516
9517 SDLoc DL(N);
9518 return DAG.getNode(Opcode: Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
9519 : LoongArchISD::VSRAR,
9520 DL, VT, N1: X, N2: Shift);
9521}
9522
9523static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
9524 TargetLowering::DAGCombinerInfo &DCI,
9525 const LoongArchSubtarget &Subtarget) {
9526 if (!DCI.isBeforeLegalize())
9527 return SDValue();
9528
9529 SDLoc DL(N);
9530 EVT ResTy = N->getValueType(ResNo: 0);
9531 SDValue N0 = N->getOperand(Num: 0);
9532 SDValue N1 = N->getOperand(Num: 1);
9533
9534 bool HasLSXOnly = Subtarget.hasExtLSX() && !Subtarget.hasExtLASX();
9535 if (ResTy != MVT::v8i16 && ResTy != MVT::v4i32 && ResTy != MVT::v2i64 &&
9536 ResTy != MVT::i128 && ResTy != MVT::v16i16 && ResTy != MVT::v8i32 &&
9537 ResTy != MVT::v4i64 && ResTy != MVT::v2i128)
9538 return SDValue();
9539 if (HasLSXOnly && (ResTy.is256BitVector() || ResTy == MVT::v2i128))
9540 return SDValue();
9541
9542 // Combine:
9543 // ti,tii,...,tx = extract_vector_elt t0, {0,2,4,.../1,3,5,...}
9544 // tj,tjj,...,ty = extract_vector_elt t1, {0,2,4,.../1,3,5,...}
9545 // tm = BUILD_VECTOR ti,tii,...,tx (Only when ResTy != MVT::i128)
9546 // tn = BUILD_VECTOR tj,tjj,...,ty (Only when ResTy != MVT::i128)
9547 // ta = {sign/zero}_extend tm
9548 // tb = {sign/zero}_extend tn
9549 // tr = mul ta, tb
9550 // to:
9551 // tr = VMULW{EV/OD}[U/US] t0, t1
9552 auto getExtType = [](unsigned Op0, unsigned Op1) -> int {
9553 if (Op0 == ISD::SIGN_EXTEND && Op1 == ISD::SIGN_EXTEND)
9554 return 0;
9555 if (Op0 == ISD::ZERO_EXTEND && Op1 == ISD::ZERO_EXTEND)
9556 return 1;
9557 if (Op0 == ISD::ZERO_EXTEND && Op1 == ISD::SIGN_EXTEND)
9558 return 2;
9559 if (Op0 == ISD::SIGN_EXTEND && Op1 == ISD::ZERO_EXTEND)
9560 return 3;
9561 return -1;
9562 };
9563
9564 int ExtType = getExtType(N0.getOpcode(), N1.getOpcode());
9565 if (ExtType < 0)
9566 return SDValue();
9567
9568 // Avoid the regressions on LA64 introduced by combining
9569 // i128-type multiplications.
9570 bool IsScalar = (ResTy == MVT::i128);
9571 if (IsScalar && ExtType < 2 && Subtarget.is64Bit())
9572 return SDValue();
9573
9574 SDValue Src0 = N0.getOperand(i: 0);
9575 SDValue Src1 = N1.getOperand(i: 0);
9576 if (IsScalar && (Src0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9577 Src1.getOpcode() != ISD::EXTRACT_VECTOR_ELT))
9578 return SDValue();
9579 if (!IsScalar && (Src0.getOpcode() != ISD::BUILD_VECTOR ||
9580 Src1.getOpcode() != ISD::BUILD_VECTOR))
9581 return SDValue();
9582
9583 unsigned ResBits = ResTy.getScalarSizeInBits();
9584 unsigned Src0Bits = Src0.getValueType().getScalarSizeInBits();
9585 unsigned Src1Bits = Src1.getValueType().getScalarSizeInBits();
9586 if (Src0Bits != Src1Bits || ResBits != Src0Bits * 2)
9587 return SDValue();
9588
9589 // Collect all EXTRACT_VECTOR_ELT.
9590 SmallVector<std::pair<SDValue, SDValue>> Elems;
9591 if (IsScalar) {
9592 Elems.emplace_back(Args&: Src0, Args&: Src1);
9593 } else {
9594 for (unsigned i = 0; i < Src0.getNumOperands(); ++i)
9595 Elems.emplace_back(Args: Src0.getOperand(i), Args: Src1.getOperand(i));
9596 }
9597
9598 unsigned Index;
9599 SDValue OrigN0, OrigN1;
9600 bool First = true;
9601 for (auto &[Op0, Op1] : Elems) {
9602 // Each element must be EXTRACT_VECTOR_ELT.
9603 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9604 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
9605 return SDValue();
9606
9607 // Check each EXTRACT_VECTOR_ELT's source vector and index.
9608 if (Op0.getOperand(i: 1) != Op1.getOperand(i: 1))
9609 return SDValue();
9610
9611 auto *IdxC = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
9612 if (!IdxC)
9613 return SDValue();
9614
9615 unsigned CurIdx = IdxC->getZExtValue();
9616 if (First) {
9617 if (CurIdx != 0 && CurIdx != 1)
9618 return SDValue();
9619 OrigN0 = Op0.getOperand(i: 0);
9620 OrigN1 = Op1.getOperand(i: 0);
9621 First = false;
9622 } else {
9623 if (CurIdx != Index + 2 || Op0.getOperand(i: 0) != OrigN0 ||
9624 Op1.getOperand(i: 0) != OrigN1)
9625 return SDValue();
9626 }
9627 Index = CurIdx;
9628 }
9629
9630 if (OrigN0.getValueType() != OrigN1.getValueType())
9631 return SDValue();
9632 if (OrigN0.getValueType().getVectorNumElements() !=
9633 (IsScalar ? 1 : ResTy.getVectorNumElements()) * 2)
9634 return SDValue();
9635
9636 SDValue Result;
9637 EVT OrigTy = OrigN0.getValueType();
9638 bool IsEven = (Index % 2 == 0);
9639
9640 static const unsigned OpcTable[3][2] = {
9641 {LoongArchISD::VMULWOD, LoongArchISD::VMULWEV},
9642 {LoongArchISD::VMULWODU, LoongArchISD::VMULWEVU},
9643 {LoongArchISD::VMULWODUS, LoongArchISD::VMULWEVUS}};
9644
9645 if (ExtType == 3)
9646 Result = DAG.getNode(Opcode: OpcTable[2][IsEven], DL, VT: OrigTy, N1: OrigN1, N2: OrigN0);
9647 else
9648 Result = DAG.getNode(Opcode: OpcTable[ExtType][IsEven], DL, VT: OrigTy, N1: OrigN0, N2: OrigN1);
9649
9650 return DAG.getBitcast(VT: ResTy, V: Result);
9651}
9652
9653static SDValue
9654performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
9655 const LoongArchSubtarget &Subtarget) {
9656 SelectionDAG &DAG = DCI.DAG;
9657 SDValue Src = N->getOperand(Num: 0);
9658 EVT VT = N->getValueType(ResNo: 0);
9659 EVT SrcVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
9660 unsigned Opc = Src.getOpcode();
9661 SDLoc DL(N);
9662
9663 // Leave the sign extension in place when it feeds an xor with another
9664 // sign-extended value; the generic xor-of-sext_inregs combine can then
9665 // fold both extensions into a single one.
9666 if (N->hasOneUse()) {
9667 SDNode *User = N->use_begin()->getUser();
9668 if (User->getOpcode() == ISD::XOR) {
9669 SDValue Other = User->getOperand(Num: 0).getNode() == N ? User->getOperand(Num: 1)
9670 : User->getOperand(Num: 0);
9671 if (Other.getOpcode() == ISD::SIGN_EXTEND_INREG)
9672 return SDValue();
9673 }
9674 }
9675
9676 // Do not hide a sign-extended boolean from a compare against zero: the
9677 // generic combiner folds (setcc (sext_inreg X, i1), 0, setlt) back to X.
9678 for (SDNode *User : N->users()) {
9679 if (User->getOpcode() != ISD::SETCC)
9680 continue;
9681 if ((User->getOperand(Num: 0).getNode() == N &&
9682 isNullConstant(V: User->getOperand(Num: 1))) ||
9683 (User->getOperand(Num: 1).getNode() == N &&
9684 isNullConstant(V: User->getOperand(Num: 0))))
9685 return SDValue();
9686 }
9687
9688 // Fold (sext_inreg (shl X, Y), i32) -> (sll.w X, Y) iff Y u< 32.
9689 if (DCI.isAfterLegalizeDAG() && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
9690 VT == MVT::i64 && Opc == ISD::SHL &&
9691 !isa<ConstantSDNode>(Val: Src.getOperand(i: 1)) &&
9692 DAG.computeKnownBits(Op: Src.getOperand(i: 1)).countMaxActiveBits() <= 5) {
9693 return DAG.getNode(Opcode: LoongArchISD::SLL_W, DL, VT, N1: Src.getOperand(i: 0),
9694 N2: Src.getOperand(i: 1));
9695 }
9696
9697 // The SETCC/XOR folds replace the sign extension itself. When the result
9698 // has several users they do not save an instruction and can duplicate the
9699 // compare; leave it to the generic combiner.
9700 if (!N->hasOneUse())
9701 return SDValue();
9702
9703 // Similarly, a sign-extended boolean that feeds a sign-bit extraction is
9704 // usually part of a compare that the generic combiner can fold away.
9705 for (SDNode *User : N->users()) {
9706 if ((User->getOpcode() == ISD::SRL || User->getOpcode() == ISD::SRA) &&
9707 isa<ConstantSDNode>(Val: User->getOperand(Num: 1)))
9708 return SDValue();
9709 }
9710
9711 // Fold (sext_inreg (setcc X, Y, CC), i1) -> (sub 0, (setcc X, Y, CC)) if CC
9712 // can be selected with a single instruction, or
9713 // (add (setcc X, Y, inv(CC)), -1) if the inverted CC is cheaper. LoongArch
9714 // selects setge/setle/setuge/setule as slt + xori, so inverting such an
9715 // integer compare avoids the extra xor. Inverting replaces the original
9716 // setcc, so only do it when it has a single use; otherwise (e.g. an FP
9717 // compare shared with another user) this would introduce a second compare.
9718 if (Opc == ISD::SETCC && SrcVT == MVT::i1) {
9719 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get();
9720 if (Src.hasOneUse() && Src.getOperand(i: 0).getValueType().isInteger()) {
9721 switch (CC) {
9722 case ISD::SETGE:
9723 case ISD::SETLE:
9724 case ISD::SETUGE:
9725 case ISD::SETULE:
9726 // (sub 0, X >= Y) == (add (X < Y), -1), etc.
9727 return DAG.getNode(
9728 Opcode: ISD::ADD, DL, VT,
9729 N1: DAG.getSetCC(
9730 DL: SDLoc(N), VT: Src.getValueType(), LHS: Src.getOperand(i: 0),
9731 RHS: Src.getOperand(i: 1),
9732 Cond: ISD::getSetCCInverse(Operation: CC, Type: Src.getOperand(i: 0).getValueType())),
9733 N2: DAG.getAllOnesConstant(DL, VT));
9734 default:
9735 break;
9736 }
9737 }
9738 return DAG.getNegative(Val: Src, DL, VT);
9739 }
9740
9741 // Fold (sext_inreg (xor (setcc), C), i1) -> (add (setcc), -1) where C is
9742 // either 1 or -1. Since setcc only produces 0 or 1, the xor only flips the
9743 // low bit, so the sign extension of that bit is -(1 - setcc) == setcc - 1.
9744 // The setcc may be wrapped in a truncate/any_extend when the boolean is
9745 // extended to VT (e.g. by a zext, which knows the upper bits are zero).
9746 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
9747 (isOneConstant(V: Src.getOperand(i: 1)) ||
9748 isAllOnesConstant(V: Src.getOperand(i: 1)))) {
9749 SDValue XorOp = Src.getOperand(i: 0);
9750 if (XorOp.getOpcode() == ISD::TRUNCATE ||
9751 XorOp.getOpcode() == ISD::ANY_EXTEND)
9752 XorOp = XorOp.getOperand(i: 0);
9753 if (XorOp.getOpcode() != ISD::SETCC)
9754 return SDValue();
9755 // The setcc may still have an i1 result type, in which case rebuild it
9756 // with VT so that both ADD operands have the same type. A setcc produces
9757 // a canonical 0/1 value, so this is exact while the any_extend leaves the
9758 // upper bits undefined.
9759 ISD::CondCode CC = cast<CondCodeSDNode>(Val: XorOp.getOperand(i: 2))->get();
9760 EVT OpVT = XorOp.getOperand(i: 0).getValueType();
9761
9762 // Inverting an FP compare is free, so this fold does not save anything
9763 // and can duplicate a shared fcmp. Keep the original form.
9764 if (OpVT.isFloatingPoint())
9765 return SDValue();
9766
9767 // Reusing the original compare is only possible when the setcc is
9768 // otherwise dead. Inverting a shared compare duplicates it (e.g. an FP
9769 // compare used by a zext as well).
9770 if (!XorOp.hasOneUse())
9771 return SDValue();
9772
9773 // A constant compare with a greater-than style condition code cannot use
9774 // the slti/sltui immediate patterns; leave it to the generic combiner,
9775 // which canonicalizes it to setlt/setult first.
9776 if (isa<ConstantSDNode>(Val: XorOp.getOperand(i: 1)) && CC != ISD::SETLT &&
9777 CC != ISD::SETULT)
9778 return SDValue();
9779
9780 // The generic (sub X, (0 - Y)) -> (add X, Y) fold is better than
9781 // rewriting the sign-extended boolean here.
9782 if (N->hasOneUse() && N->use_begin()->getUser()->getOpcode() == ISD::SUB)
9783 return SDValue();
9784
9785 // For integer compares, sext(!setcc CC) == -(setcc inv(CC)). Only keep
9786 // CC (as add -1) when CC itself is the single-instruction form; otherwise
9787 // negate the inverted setcc, which also CSEs with an existing one.
9788 if (OpVT.isInteger() && CC != ISD::SETLT && CC != ISD::SETGT &&
9789 CC != ISD::SETULT && CC != ISD::SETUGT)
9790 return DAG.getNegative(Val: DAG.getSetCC(DL, VT, LHS: XorOp.getOperand(i: 0),
9791 RHS: XorOp.getOperand(i: 1),
9792 Cond: ISD::getSetCCInverse(Operation: CC, Type: OpVT)),
9793 DL, VT);
9794 if (XorOp.getValueType() != VT)
9795 XorOp =
9796 DAG.getSetCC(DL, VT, LHS: XorOp.getOperand(i: 0), RHS: XorOp.getOperand(i: 1), Cond: CC);
9797 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: XorOp, N2: DAG.getAllOnesConstant(DL, VT));
9798 }
9799
9800 return SDValue();
9801}
9802
9803SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
9804 DAGCombinerInfo &DCI) const {
9805 SelectionDAG &DAG = DCI.DAG;
9806 switch (N->getOpcode()) {
9807 default:
9808 break;
9809 case ISD::ADD:
9810 return performADDCombine(N, DAG, DCI, Subtarget);
9811 case ISD::AND:
9812 return performANDCombine(N, DAG, DCI, Subtarget);
9813 case ISD::OR:
9814 return performORCombine(N, DAG, DCI, Subtarget);
9815 case ISD::XOR:
9816 return performXORCombine(N, DAG, Subtarget);
9817 case ISD::SETCC:
9818 return performSETCCCombine(N, DAG, DCI, Subtarget);
9819 case ISD::SELECT:
9820 return performSELECTCombine(N, DAG, DCI, Subtarget);
9821 case ISD::SHL:
9822 return performSHLCombine(N, DAG, DCI, Subtarget);
9823 case ISD::SRL:
9824 return performSRLCombine(N, DAG, DCI, Subtarget);
9825 case ISD::SUB:
9826 return performSUBCombine(N, DAG, DCI, Subtarget);
9827 case ISD::BITCAST:
9828 return performBITCASTCombine(N, DAG, DCI, Subtarget);
9829 case ISD::ANY_EXTEND:
9830 case ISD::ZERO_EXTEND:
9831 case ISD::SIGN_EXTEND:
9832 return performEXTENDCombine(N, DAG, DCI, Subtarget);
9833 case ISD::SIGN_EXTEND_INREG:
9834 return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
9835 case ISD::SINT_TO_FP:
9836 return performSINT_TO_FPCombine(N, DAG, DCI, Subtarget);
9837 case ISD::UINT_TO_FP:
9838 return performUINT_TO_FPCombine(N, DAG, DCI, Subtarget);
9839 case ISD::FP_TO_SINT:
9840 case ISD::FP_TO_UINT:
9841 return performFP_TO_INTCombine(N, DAG, DCI, Subtarget);
9842 case LoongArchISD::BITREV_W:
9843 return performBITREV_WCombine(N, DAG, DCI, Subtarget);
9844 case LoongArchISD::BR_CC:
9845 return performBR_CCCombine(N, DAG, DCI, Subtarget);
9846 case LoongArchISD::SELECT_CC:
9847 return performSELECT_CCCombine(N, DAG, DCI, Subtarget);
9848 case ISD::INTRINSIC_WO_CHAIN:
9849 return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
9850 case LoongArchISD::MOVGR2FR_W_LA64:
9851 return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
9852 case LoongArchISD::MOVFR2GR_S_LA64:
9853 return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
9854 case LoongArchISD::CRC_W_B_W:
9855 case LoongArchISD::CRC_W_H_W:
9856 case LoongArchISD::CRCC_W_B_W:
9857 case LoongArchISD::CRCC_W_H_W:
9858 case LoongArchISD::VMSKLTZ:
9859 case LoongArchISD::XVMSKLTZ:
9860 return performDemandedBitsCombine(N, DAG, DCI);
9861 case LoongArchISD::SPLIT_PAIR_F64:
9862 return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
9863 case LoongArchISD::VANDN:
9864 return performVANDNCombine(N, DAG, DCI, Subtarget);
9865 case ISD::CONCAT_VECTORS:
9866 return performCONCAT_VECTORSCombine(N, DAG, DCI, Subtarget);
9867 case ISD::VSELECT:
9868 return performVSELECTCombine(N, DAG, DCI, Subtarget);
9869 case LoongArchISD::VPACKEV:
9870 case LoongArchISD::VPERMI:
9871 if (SDValue Result =
9872 combineFP_ROUND(N: SDValue(N, 0), DL: SDLoc(N), DAG, Subtarget))
9873 return Result;
9874 break;
9875 case ISD::MUL:
9876 return performMULCombine(N, DAG, DCI, Subtarget);
9877 }
9878 return SDValue();
9879}
9880
9881static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI,
9882 MachineBasicBlock *MBB) {
9883 if (!ZeroDivCheck)
9884 return MBB;
9885
9886 // Build instructions:
9887 // MBB:
9888 // div(or mod) $dst, $dividend, $divisor
9889 // bne $divisor, $zero, SinkMBB
9890 // BreakMBB:
9891 // break 7 // BRK_DIVZERO
9892 // SinkMBB:
9893 // fallthrough
9894 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
9895 MachineFunction::iterator It = ++MBB->getIterator();
9896 MachineFunction *MF = MBB->getParent();
9897 auto BreakMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
9898 auto SinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
9899 MF->insert(MBBI: It, MBB: BreakMBB);
9900 MF->insert(MBBI: It, MBB: SinkMBB);
9901
9902 // Transfer the remainder of MBB and its successor edges to SinkMBB.
9903 SinkMBB->splice(Where: SinkMBB->end(), Other: MBB, From: std::next(x: MI.getIterator()), To: MBB->end());
9904 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
9905
9906 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
9907 DebugLoc DL = MI.getDebugLoc();
9908 MachineOperand &Divisor = MI.getOperand(i: 2);
9909 Register DivisorReg = Divisor.getReg();
9910
9911 // MBB:
9912 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BNE))
9913 .addReg(RegNo: DivisorReg, Flags: getKillRegState(B: Divisor.isKill()))
9914 .addReg(RegNo: LoongArch::R0)
9915 .addMBB(MBB: SinkMBB);
9916 MBB->addSuccessor(Succ: BreakMBB);
9917 MBB->addSuccessor(Succ: SinkMBB);
9918
9919 // BreakMBB:
9920 // See linux header file arch/loongarch/include/uapi/asm/break.h for the
9921 // definition of BRK_DIVZERO.
9922 BuildMI(BB: BreakMBB, MIMD: DL, MCID: TII.get(Opcode: LoongArch::BREAK)).addImm(Val: 7 /*BRK_DIVZERO*/);
9923 BreakMBB->addSuccessor(Succ: SinkMBB);
9924
9925 // Clear Divisor's kill flag.
9926 Divisor.setIsKill(false);
9927
9928 return SinkMBB;
9929}
9930
9931static MachineBasicBlock *
9932emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
9933 const LoongArchSubtarget &Subtarget) {
9934 unsigned InsOp;
9935 unsigned BroadcastOp;
9936 unsigned HalfSize;
9937 switch (MI.getOpcode()) {
9938 default:
9939 llvm_unreachable("Unexpected opcode");
9940 case LoongArch::PseudoXVINSGR2VR_B:
9941 HalfSize = 16;
9942 BroadcastOp = LoongArch::XVREPLGR2VR_B;
9943 InsOp = LoongArch::XVEXTRINS_B;
9944 break;
9945 case LoongArch::PseudoXVINSGR2VR_H:
9946 HalfSize = 8;
9947 BroadcastOp = LoongArch::XVREPLGR2VR_H;
9948 InsOp = LoongArch::XVEXTRINS_H;
9949 break;
9950 }
9951 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9952 const TargetRegisterClass *RC = &LoongArch::LASX256RegClass;
9953 const TargetRegisterClass *SubRC = &LoongArch::LSX128RegClass;
9954 DebugLoc DL = MI.getDebugLoc();
9955 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9956 // XDst = vector_insert XSrc, Elt, Idx
9957 Register XDst = MI.getOperand(i: 0).getReg();
9958 Register XSrc = MI.getOperand(i: 1).getReg();
9959 Register Elt = MI.getOperand(i: 2).getReg();
9960 unsigned Idx = MI.getOperand(i: 3).getImm();
9961
9962 if (XSrc.isVirtual() && MRI.getVRegDef(Reg: XSrc)->isImplicitDef() &&
9963 Idx < HalfSize) {
9964 Register ScratchSubReg1 = MRI.createVirtualRegister(RegClass: SubRC);
9965 Register ScratchSubReg2 = MRI.createVirtualRegister(RegClass: SubRC);
9966
9967 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::COPY), DestReg: ScratchSubReg1)
9968 .addReg(RegNo: XSrc, Flags: {}, SubReg: LoongArch::sub_128);
9969 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
9970 MCID: TII->get(Opcode: HalfSize == 8 ? LoongArch::VINSGR2VR_H
9971 : LoongArch::VINSGR2VR_B),
9972 DestReg: ScratchSubReg2)
9973 .addReg(RegNo: ScratchSubReg1)
9974 .addReg(RegNo: Elt)
9975 .addImm(Val: Idx);
9976
9977 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::SUBREG_TO_REG), DestReg: XDst)
9978 .addReg(RegNo: ScratchSubReg2)
9979 .addImm(Val: LoongArch::sub_128);
9980 } else {
9981 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
9982 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
9983
9984 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Elt);
9985
9986 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPERMI_Q), DestReg: ScratchReg2)
9987 .addReg(RegNo: ScratchReg1)
9988 .addReg(RegNo: XSrc)
9989 .addImm(Val: Idx >= HalfSize ? 48 : 18);
9990
9991 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InsOp), DestReg: XDst)
9992 .addReg(RegNo: XSrc)
9993 .addReg(RegNo: ScratchReg2)
9994 .addImm(Val: (Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
9995 }
9996
9997 MI.eraseFromParent();
9998 return BB;
9999}
10000
10001static MachineBasicBlock *emitPseudoCTPOP(MachineInstr &MI,
10002 MachineBasicBlock *BB,
10003 const LoongArchSubtarget &Subtarget) {
10004 assert(Subtarget.hasExtLSX());
10005 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
10006 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
10007 DebugLoc DL = MI.getDebugLoc();
10008 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
10009 Register Dst = MI.getOperand(i: 0).getReg();
10010 Register Src = MI.getOperand(i: 1).getReg();
10011
10012 unsigned BroadcastOp, CTOp, PickOp;
10013 switch (MI.getOpcode()) {
10014 default:
10015 llvm_unreachable("Unexpected opcode");
10016 case LoongArch::PseudoCTPOP_B:
10017 BroadcastOp = LoongArch::VREPLGR2VR_B;
10018 CTOp = LoongArch::VPCNT_B;
10019 PickOp = LoongArch::VPICKVE2GR_B;
10020 break;
10021 case LoongArch::PseudoCTPOP_H:
10022 case LoongArch::PseudoCTPOP_H_LA32:
10023 BroadcastOp = LoongArch::VREPLGR2VR_H;
10024 CTOp = LoongArch::VPCNT_H;
10025 PickOp = LoongArch::VPICKVE2GR_H;
10026 break;
10027 case LoongArch::PseudoCTPOP_W:
10028 case LoongArch::PseudoCTPOP_W_LA32:
10029 BroadcastOp = LoongArch::VREPLGR2VR_W;
10030 CTOp = LoongArch::VPCNT_W;
10031 PickOp = LoongArch::VPICKVE2GR_W;
10032 break;
10033 case LoongArch::PseudoCTPOP_D:
10034 BroadcastOp = LoongArch::VREPLGR2VR_D;
10035 CTOp = LoongArch::VPCNT_D;
10036 PickOp = LoongArch::VPICKVE2GR_D;
10037 break;
10038 }
10039
10040 Register ScratchReg1 = MRI.createVirtualRegister(RegClass: RC);
10041 Register ScratchReg2 = MRI.createVirtualRegister(RegClass: RC);
10042 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BroadcastOp), DestReg: ScratchReg1).addReg(RegNo: Src);
10043 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: CTOp), DestReg: ScratchReg2).addReg(RegNo: ScratchReg1);
10044 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PickOp), DestReg: Dst).addReg(RegNo: ScratchReg2).addImm(Val: 0);
10045
10046 MI.eraseFromParent();
10047 return BB;
10048}
10049
10050static MachineBasicBlock *
10051emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
10052 const LoongArchSubtarget &Subtarget) {
10053 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
10054 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
10055 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
10056 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
10057 Register Dst = MI.getOperand(i: 0).getReg();
10058 Register Src = MI.getOperand(i: 1).getReg();
10059 DebugLoc DL = MI.getDebugLoc();
10060 unsigned EleBits = 8;
10061 unsigned NotOpc = 0;
10062 unsigned MskOpc;
10063
10064 switch (MI.getOpcode()) {
10065 default:
10066 llvm_unreachable("Unexpected opcode");
10067 case LoongArch::PseudoVMSKLTZ_B:
10068 MskOpc = LoongArch::VMSKLTZ_B;
10069 break;
10070 case LoongArch::PseudoVMSKLTZ_H:
10071 MskOpc = LoongArch::VMSKLTZ_H;
10072 EleBits = 16;
10073 break;
10074 case LoongArch::PseudoVMSKLTZ_W:
10075 MskOpc = LoongArch::VMSKLTZ_W;
10076 EleBits = 32;
10077 break;
10078 case LoongArch::PseudoVMSKLTZ_D:
10079 MskOpc = LoongArch::VMSKLTZ_D;
10080 EleBits = 64;
10081 break;
10082 case LoongArch::PseudoVMSKGEZ_B:
10083 MskOpc = LoongArch::VMSKGEZ_B;
10084 break;
10085 case LoongArch::PseudoVMSKEQZ_B:
10086 MskOpc = LoongArch::VMSKNZ_B;
10087 NotOpc = LoongArch::VNOR_V;
10088 break;
10089 case LoongArch::PseudoVMSKNEZ_B:
10090 MskOpc = LoongArch::VMSKNZ_B;
10091 break;
10092 case LoongArch::PseudoXVMSKLTZ_B:
10093 MskOpc = LoongArch::XVMSKLTZ_B;
10094 RC = &LoongArch::LASX256RegClass;
10095 break;
10096 case LoongArch::PseudoXVMSKLTZ_H:
10097 MskOpc = LoongArch::XVMSKLTZ_H;
10098 RC = &LoongArch::LASX256RegClass;
10099 EleBits = 16;
10100 break;
10101 case LoongArch::PseudoXVMSKLTZ_W:
10102 MskOpc = LoongArch::XVMSKLTZ_W;
10103 RC = &LoongArch::LASX256RegClass;
10104 EleBits = 32;
10105 break;
10106 case LoongArch::PseudoXVMSKLTZ_D:
10107 MskOpc = LoongArch::XVMSKLTZ_D;
10108 RC = &LoongArch::LASX256RegClass;
10109 EleBits = 64;
10110 break;
10111 case LoongArch::PseudoXVMSKGEZ_B:
10112 MskOpc = LoongArch::XVMSKGEZ_B;
10113 RC = &LoongArch::LASX256RegClass;
10114 break;
10115 case LoongArch::PseudoXVMSKEQZ_B:
10116 MskOpc = LoongArch::XVMSKNZ_B;
10117 NotOpc = LoongArch::XVNOR_V;
10118 RC = &LoongArch::LASX256RegClass;
10119 break;
10120 case LoongArch::PseudoXVMSKNEZ_B:
10121 MskOpc = LoongArch::XVMSKNZ_B;
10122 RC = &LoongArch::LASX256RegClass;
10123 break;
10124 }
10125
10126 Register Msk = MRI.createVirtualRegister(RegClass: RC);
10127 if (NotOpc) {
10128 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
10129 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Tmp).addReg(RegNo: Src);
10130 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: NotOpc), DestReg: Msk).addReg(RegNo: Tmp).addReg(RegNo: Tmp);
10131 } else {
10132 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MskOpc), DestReg: Msk).addReg(RegNo: Src);
10133 }
10134
10135 if (TRI->getRegSizeInBits(RC: *RC) > 128) {
10136 Register Lo = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10137 Register Hi = MRI.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10138 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Lo)
10139 .addReg(RegNo: Msk)
10140 .addImm(Val: 0);
10141 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::XVPICKVE2GR_WU), DestReg: Hi)
10142 .addReg(RegNo: Msk)
10143 .addImm(Val: 4);
10144 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
10145 MCID: TII->get(Opcode: Subtarget.is64Bit() ? LoongArch::BSTRINS_D
10146 : LoongArch::BSTRINS_W),
10147 DestReg: Dst)
10148 .addReg(RegNo: Lo)
10149 .addReg(RegNo: Hi)
10150 .addImm(Val: 256 / EleBits - 1)
10151 .addImm(Val: 128 / EleBits);
10152 } else {
10153 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::VPICKVE2GR_HU), DestReg: Dst)
10154 .addReg(RegNo: Msk)
10155 .addImm(Val: 0);
10156 }
10157
10158 MI.eraseFromParent();
10159 return BB;
10160}
10161
10162static MachineBasicBlock *
10163emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
10164 const LoongArchSubtarget &Subtarget) {
10165 assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
10166 "Unexpected instruction");
10167
10168 MachineFunction &MF = *BB->getParent();
10169 DebugLoc DL = MI.getDebugLoc();
10170 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
10171 Register LoReg = MI.getOperand(i: 0).getReg();
10172 Register HiReg = MI.getOperand(i: 1).getReg();
10173 Register SrcReg = MI.getOperand(i: 2).getReg();
10174
10175 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFR2GR_S_64), DestReg: LoReg).addReg(RegNo: SrcReg);
10176 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVFRH2GR_S), DestReg: HiReg)
10177 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
10178 MI.eraseFromParent(); // The pseudo instruction is gone now.
10179 return BB;
10180}
10181
10182static MachineBasicBlock *
10183emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
10184 const LoongArchSubtarget &Subtarget) {
10185 assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
10186 "Unexpected instruction");
10187
10188 MachineFunction &MF = *BB->getParent();
10189 DebugLoc DL = MI.getDebugLoc();
10190 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
10191 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
10192 Register TmpReg = MRI.createVirtualRegister(RegClass: &LoongArch::FPR64RegClass);
10193 Register DstReg = MI.getOperand(i: 0).getReg();
10194 Register LoReg = MI.getOperand(i: 1).getReg();
10195 Register HiReg = MI.getOperand(i: 2).getReg();
10196
10197 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FR_W_64), DestReg: TmpReg)
10198 .addReg(RegNo: LoReg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()));
10199 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: LoongArch::MOVGR2FRH_W), DestReg: DstReg)
10200 .addReg(RegNo: TmpReg)
10201 .addReg(RegNo: HiReg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
10202 MI.eraseFromParent(); // The pseudo instruction is gone now.
10203 return BB;
10204}
10205
10206static bool isSelectPseudo(MachineInstr &MI) {
10207 switch (MI.getOpcode()) {
10208 default:
10209 return false;
10210 case LoongArch::Select_GPR_Using_CC_GPR:
10211 return true;
10212 }
10213}
10214
10215static MachineBasicBlock *
10216emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB,
10217 const LoongArchSubtarget &Subtarget) {
10218 // To "insert" Select_* instructions, we actually have to insert the triangle
10219 // control-flow pattern. The incoming instructions know the destination vreg
10220 // to set, the condition code register to branch on, the true/false values to
10221 // select between, and the condcode to use to select the appropriate branch.
10222 //
10223 // We produce the following control flow:
10224 // HeadMBB
10225 // | \
10226 // | IfFalseMBB
10227 // | /
10228 // TailMBB
10229 //
10230 // When we find a sequence of selects we attempt to optimize their emission
10231 // by sharing the control flow. Currently we only handle cases where we have
10232 // multiple selects with the exact same condition (same LHS, RHS and CC).
10233 // The selects may be interleaved with other instructions if the other
10234 // instructions meet some requirements we deem safe:
10235 // - They are not pseudo instructions.
10236 // - They are debug instructions. Otherwise,
10237 // - They do not have side-effects, do not access memory and their inputs do
10238 // not depend on the results of the select pseudo-instructions.
10239 // The TrueV/FalseV operands of the selects cannot depend on the result of
10240 // previous selects in the sequence.
10241 // These conditions could be further relaxed. See the X86 target for a
10242 // related approach and more information.
10243
10244 Register LHS = MI.getOperand(i: 1).getReg();
10245 Register RHS;
10246 if (MI.getOperand(i: 2).isReg())
10247 RHS = MI.getOperand(i: 2).getReg();
10248 auto CC = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
10249
10250 SmallVector<MachineInstr *, 4> SelectDebugValues;
10251 SmallSet<Register, 4> SelectDests;
10252 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
10253
10254 MachineInstr *LastSelectPseudo = &MI;
10255 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
10256 SequenceMBBI != E; ++SequenceMBBI) {
10257 if (SequenceMBBI->isDebugInstr())
10258 continue;
10259 if (isSelectPseudo(MI&: *SequenceMBBI)) {
10260 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
10261 !SequenceMBBI->getOperand(i: 2).isReg() ||
10262 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
10263 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
10264 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
10265 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
10266 break;
10267 LastSelectPseudo = &*SequenceMBBI;
10268 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
10269 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
10270 continue;
10271 }
10272 if (SequenceMBBI->hasUnmodeledSideEffects() ||
10273 SequenceMBBI->mayLoadOrStore() ||
10274 SequenceMBBI->usesCustomInsertionHook())
10275 break;
10276 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
10277 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
10278 }))
10279 break;
10280 }
10281
10282 const LoongArchInstrInfo &TII = *Subtarget.getInstrInfo();
10283 const BasicBlock *LLVM_BB = BB->getBasicBlock();
10284 DebugLoc DL = MI.getDebugLoc();
10285 MachineFunction::iterator I = ++BB->getIterator();
10286
10287 MachineBasicBlock *HeadMBB = BB;
10288 MachineFunction *F = BB->getParent();
10289 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
10290 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
10291
10292 F->insert(MBBI: I, MBB: IfFalseMBB);
10293 F->insert(MBBI: I, MBB: TailMBB);
10294
10295 // Set the call frame size on entry to the new basic blocks.
10296 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
10297 IfFalseMBB->setCallFrameSize(CallFrameSize);
10298 TailMBB->setCallFrameSize(CallFrameSize);
10299
10300 // Transfer debug instructions associated with the selects to TailMBB.
10301 for (MachineInstr *DebugInstr : SelectDebugValues) {
10302 TailMBB->push_back(MI: DebugInstr->removeFromParent());
10303 }
10304
10305 // Move all instructions after the sequence to TailMBB.
10306 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
10307 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
10308 // Update machine-CFG edges by transferring all successors of the current
10309 // block to the new block which will contain the Phi nodes for the selects.
10310 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
10311 // Set the successors for HeadMBB.
10312 HeadMBB->addSuccessor(Succ: IfFalseMBB);
10313 HeadMBB->addSuccessor(Succ: TailMBB);
10314
10315 // Insert appropriate branch.
10316 if (MI.getOperand(i: 2).isImm())
10317 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC))
10318 .addReg(RegNo: LHS)
10319 .addImm(Val: MI.getOperand(i: 2).getImm())
10320 .addMBB(MBB: TailMBB);
10321 else
10322 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: CC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: TailMBB);
10323
10324 // IfFalseMBB just falls through to TailMBB.
10325 IfFalseMBB->addSuccessor(Succ: TailMBB);
10326
10327 // Create PHIs for all of the select pseudo-instructions.
10328 auto SelectMBBI = MI.getIterator();
10329 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
10330 auto InsertionPoint = TailMBB->begin();
10331 while (SelectMBBI != SelectEnd) {
10332 auto Next = std::next(x: SelectMBBI);
10333 if (isSelectPseudo(MI&: *SelectMBBI)) {
10334 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
10335 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
10336 MCID: TII.get(Opcode: LoongArch::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
10337 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
10338 .addMBB(MBB: HeadMBB)
10339 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
10340 .addMBB(MBB: IfFalseMBB);
10341 SelectMBBI->eraseFromParent();
10342 }
10343 SelectMBBI = Next;
10344 }
10345
10346 F->getProperties().resetNoPHIs();
10347 return TailMBB;
10348}
10349
10350MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
10351 MachineInstr &MI, MachineBasicBlock *BB) const {
10352 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
10353 DebugLoc DL = MI.getDebugLoc();
10354
10355 switch (MI.getOpcode()) {
10356 default:
10357 llvm_unreachable("Unexpected instr type to insert");
10358 case LoongArch::DIV_W:
10359 case LoongArch::DIV_WU:
10360 case LoongArch::MOD_W:
10361 case LoongArch::MOD_WU:
10362 case LoongArch::DIV_D:
10363 case LoongArch::DIV_DU:
10364 case LoongArch::MOD_D:
10365 case LoongArch::MOD_DU:
10366 return insertDivByZeroTrap(MI, MBB: BB);
10367 break;
10368 case LoongArch::WRFCSR: {
10369 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVGR2FCSR),
10370 DestReg: LoongArch::FCSR0 + MI.getOperand(i: 0).getImm())
10371 .addReg(RegNo: MI.getOperand(i: 1).getReg());
10372 MI.eraseFromParent();
10373 return BB;
10374 }
10375 case LoongArch::RDFCSR: {
10376 MachineInstr *ReadFCSR =
10377 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoongArch::MOVFCSR2GR),
10378 DestReg: MI.getOperand(i: 0).getReg())
10379 .addReg(RegNo: LoongArch::FCSR0 + MI.getOperand(i: 1).getImm());
10380 ReadFCSR->getOperand(i: 1).setIsUndef();
10381 MI.eraseFromParent();
10382 return BB;
10383 }
10384 case LoongArch::Select_GPR_Using_CC_GPR:
10385 return emitSelectPseudo(MI, BB, Subtarget);
10386 case LoongArch::BuildPairF64Pseudo:
10387 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
10388 case LoongArch::SplitPairF64Pseudo:
10389 return emitSplitPairF64Pseudo(MI, BB, Subtarget);
10390 case LoongArch::PseudoXVINSGR2VR_B:
10391 case LoongArch::PseudoXVINSGR2VR_H:
10392 return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
10393 case LoongArch::PseudoCTPOP_B:
10394 case LoongArch::PseudoCTPOP_H:
10395 case LoongArch::PseudoCTPOP_W:
10396 case LoongArch::PseudoCTPOP_D:
10397 case LoongArch::PseudoCTPOP_H_LA32:
10398 case LoongArch::PseudoCTPOP_W_LA32:
10399 return emitPseudoCTPOP(MI, BB, Subtarget);
10400 case LoongArch::PseudoVMSKLTZ_B:
10401 case LoongArch::PseudoVMSKLTZ_H:
10402 case LoongArch::PseudoVMSKLTZ_W:
10403 case LoongArch::PseudoVMSKLTZ_D:
10404 case LoongArch::PseudoVMSKGEZ_B:
10405 case LoongArch::PseudoVMSKEQZ_B:
10406 case LoongArch::PseudoVMSKNEZ_B:
10407 case LoongArch::PseudoXVMSKLTZ_B:
10408 case LoongArch::PseudoXVMSKLTZ_H:
10409 case LoongArch::PseudoXVMSKLTZ_W:
10410 case LoongArch::PseudoXVMSKLTZ_D:
10411 case LoongArch::PseudoXVMSKGEZ_B:
10412 case LoongArch::PseudoXVMSKEQZ_B:
10413 case LoongArch::PseudoXVMSKNEZ_B:
10414 return emitPseudoVMSKCOND(MI, BB, Subtarget);
10415 case TargetOpcode::STATEPOINT:
10416 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
10417 // while bl call instruction (where statepoint will be lowered at the
10418 // end) has implicit def. This def is early-clobber as it will be set at
10419 // the moment of the call and earlier than any use is read.
10420 // Add this implicit dead def here as a workaround.
10421 MI.addOperand(MF&: *MI.getMF(),
10422 Op: MachineOperand::CreateReg(
10423 Reg: LoongArch::R1, /*isDef*/ true,
10424 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
10425 /*isUndef*/ false, /*isEarlyClobber*/ true));
10426 if (!Subtarget.is64Bit())
10427 report_fatal_error(reason: "STATEPOINT is only supported on 64-bit targets");
10428 return emitPatchPoint(MI, MBB: BB);
10429 case LoongArch::PROBED_STACKALLOC_DYN:
10430 return emitDynamicProbedAlloc(MI, MBB: BB);
10431 }
10432}
10433
10434bool LoongArchTargetLowering::allowsMisalignedMemoryAccesses(
10435 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10436 unsigned *Fast) const {
10437 if (!Subtarget.hasUAL())
10438 return false;
10439
10440 // TODO: set reasonable speed number.
10441 if (Fast)
10442 *Fast = 1;
10443 return true;
10444}
10445
10446//===----------------------------------------------------------------------===//
10447// Calling Convention Implementation
10448//===----------------------------------------------------------------------===//
10449
10450// Eight general-purpose registers a0-a7 used for passing integer arguments,
10451// with a0-a1 reused to return values. Generally, the GPRs are used to pass
10452// fixed-point arguments, and floating-point arguments when no FPR is available
10453// or with soft float ABI.
10454const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6,
10455 LoongArch::R7, LoongArch::R8, LoongArch::R9,
10456 LoongArch::R10, LoongArch::R11};
10457
10458// PreserveNone calling convention:
10459// Arguments may be passed in any general-purpose registers except:
10460// - R1 : return address register
10461// - R22 : frame pointer
10462// - R31 : base pointer
10463//
10464// All general-purpose registers are treated as caller-saved,
10465// except R1 (RA) and R22 (FP).
10466//
10467// Non-volatile registers are allocated first so that a function
10468// can call normal functions without having to spill and reload
10469// argument registers.
10470const MCPhysReg PreserveNoneArgGPRs[] = {
10471 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
10472 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
10473 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
10474 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
10475 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
10476 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
10477 LoongArch::R20};
10478
10479// Eight floating-point registers fa0-fa7 used for passing floating-point
10480// arguments, and fa0-fa1 are also used to return values.
10481const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2,
10482 LoongArch::F3, LoongArch::F4, LoongArch::F5,
10483 LoongArch::F6, LoongArch::F7};
10484// FPR32 and FPR64 alias each other.
10485const MCPhysReg ArgFPR64s[] = {
10486 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
10487 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
10488
10489const MCPhysReg ArgVRs[] = {LoongArch::VR0, LoongArch::VR1, LoongArch::VR2,
10490 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
10491 LoongArch::VR6, LoongArch::VR7};
10492
10493const MCPhysReg ArgXRs[] = {LoongArch::XR0, LoongArch::XR1, LoongArch::XR2,
10494 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
10495 LoongArch::XR6, LoongArch::XR7};
10496
10497static Register allocateArgGPR(CCState &State) {
10498 switch (State.getCallingConv()) {
10499 case CallingConv::PreserveNone:
10500 if (!State.isVarArg())
10501 return State.AllocateReg(Regs: PreserveNoneArgGPRs);
10502 [[fallthrough]];
10503 default:
10504 return State.AllocateReg(Regs: ArgGPRs);
10505 }
10506}
10507
10508// Pass a 2*GRLen argument that has been split into two GRLen values through
10509// registers or the stack as necessary.
10510static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
10511 CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1,
10512 unsigned ValNo2, MVT ValVT2, MVT LocVT2,
10513 ISD::ArgFlagsTy ArgFlags2) {
10514 unsigned GRLenInBytes = GRLen / 8;
10515 if (Register Reg = allocateArgGPR(State)) {
10516 // At least one half can be passed via register.
10517 State.addLoc(V: CCValAssign::getReg(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(), Reg,
10518 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
10519 } else {
10520 // Both halves must be passed on the stack, with proper alignment.
10521 Align StackAlign =
10522 std::max(a: Align(GRLenInBytes), b: ArgFlags1.getNonZeroOrigAlign());
10523 State.addLoc(
10524 V: CCValAssign::getMem(ValNo: VA1.getValNo(), ValVT: VA1.getValVT(),
10525 Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: StackAlign),
10526 LocVT: VA1.getLocVT(), HTP: CCValAssign::Full));
10527 State.addLoc(V: CCValAssign::getMem(
10528 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
10529 LocVT: LocVT2, HTP: CCValAssign::Full));
10530 return false;
10531 }
10532 if (Register Reg = allocateArgGPR(State)) {
10533 // The second half can also be passed via register.
10534 State.addLoc(
10535 V: CCValAssign::getReg(ValNo: ValNo2, ValVT: ValVT2, Reg, LocVT: LocVT2, HTP: CCValAssign::Full));
10536 } else {
10537 // The second half is passed via the stack, without additional alignment.
10538 State.addLoc(V: CCValAssign::getMem(
10539 ValNo: ValNo2, ValVT: ValVT2, Offset: State.AllocateStack(Size: GRLenInBytes, Alignment: Align(GRLenInBytes)),
10540 LocVT: LocVT2, HTP: CCValAssign::Full));
10541 }
10542 return false;
10543}
10544
10545// Implements the LoongArch calling convention. Returns true upon failure.
10546static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
10547 unsigned ValNo, MVT ValVT,
10548 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
10549 CCState &State, bool IsRet, Type *OrigTy) {
10550 unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
10551 assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
10552 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
10553 MVT LocVT = ValVT;
10554
10555 // Any return value split into more than two values can't be returned
10556 // directly.
10557 if (IsRet && ValNo > 1)
10558 return true;
10559
10560 // If passing a variadic argument, or if no FPR is available.
10561 bool UseGPRForFloat = true;
10562
10563 switch (ABI) {
10564 default:
10565 llvm_unreachable("Unexpected ABI");
10566 break;
10567 case LoongArchABI::ABI_ILP32F:
10568 case LoongArchABI::ABI_LP64F:
10569 case LoongArchABI::ABI_ILP32D:
10570 case LoongArchABI::ABI_LP64D:
10571 UseGPRForFloat = ArgFlags.isVarArg();
10572 break;
10573 case LoongArchABI::ABI_ILP32S:
10574 case LoongArchABI::ABI_LP64S:
10575 break;
10576 }
10577
10578 // If this is a variadic argument, the LoongArch calling convention requires
10579 // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
10580 // byte alignment. An aligned register should be used regardless of whether
10581 // the original argument was split during legalisation or not. The argument
10582 // will not be passed by registers if the original type is larger than
10583 // 2*GRLen, so the register alignment rule does not apply.
10584 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
10585 if (ArgFlags.isVarArg() &&
10586 ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
10587 DL.getTypeAllocSize(Ty: OrigTy) == TwoGRLenInBytes) {
10588 unsigned RegIdx = State.getFirstUnallocated(Regs: ArgGPRs);
10589 // Skip 'odd' register if necessary.
10590 if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1)
10591 State.AllocateReg(Regs: ArgGPRs);
10592 }
10593
10594 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
10595 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
10596 State.getPendingArgFlags();
10597
10598 assert(PendingLocs.size() == PendingArgFlags.size() &&
10599 "PendingLocs and PendingArgFlags out of sync");
10600
10601 // FPR32 and FPR64 alias each other.
10602 if (State.getFirstUnallocated(Regs: ArgFPR32s) == std::size(ArgFPR32s))
10603 UseGPRForFloat = true;
10604
10605 if (UseGPRForFloat && ValVT == MVT::f32) {
10606 LocVT = GRLenVT;
10607 LocInfo = CCValAssign::BCvt;
10608 } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
10609 LocVT = MVT::i64;
10610 LocInfo = CCValAssign::BCvt;
10611 } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
10612 // Handle passing f64 on LA32D with a soft float ABI or when floating point
10613 // registers are exhausted.
10614 assert(PendingLocs.empty() && "Can't lower f64 if it is split");
10615 // Depending on available argument GPRS, f64 may be passed in a pair of
10616 // GPRs, split between a GPR and the stack, or passed completely on the
10617 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
10618 // cases.
10619 MCRegister Reg = allocateArgGPR(State);
10620 if (!Reg) {
10621 int64_t StackOffset = State.AllocateStack(Size: 8, Alignment: Align(8));
10622 State.addLoc(
10623 V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
10624 return false;
10625 }
10626 LocVT = MVT::i32;
10627 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10628 MCRegister HiReg = allocateArgGPR(State);
10629 if (HiReg) {
10630 State.addLoc(
10631 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: HiReg, LocVT, HTP: LocInfo));
10632 } else {
10633 int64_t StackOffset = State.AllocateStack(Size: 4, Alignment: Align(4));
10634 State.addLoc(
10635 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
10636 }
10637 return false;
10638 }
10639
10640 // Split arguments might be passed indirectly, so keep track of the pending
10641 // values.
10642 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
10643 LocVT = GRLenVT;
10644 LocInfo = CCValAssign::Indirect;
10645 PendingLocs.push_back(
10646 Elt: CCValAssign::getPending(ValNo, ValVT, LocVT, HTP: LocInfo));
10647 PendingArgFlags.push_back(Elt: ArgFlags);
10648 if (!ArgFlags.isSplitEnd()) {
10649 return false;
10650 }
10651 }
10652
10653 // If the split argument only had two elements, it should be passed directly
10654 // in registers or on the stack.
10655 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
10656 PendingLocs.size() <= 2) {
10657 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
10658 // Apply the normal calling convention rules to the first half of the
10659 // split argument.
10660 CCValAssign VA = PendingLocs[0];
10661 ISD::ArgFlagsTy AF = PendingArgFlags[0];
10662 PendingLocs.clear();
10663 PendingArgFlags.clear();
10664 return CC_LoongArchAssign2GRLen(GRLen, State, VA1: VA, ArgFlags1: AF, ValNo2: ValNo, ValVT2: ValVT, LocVT2: LocVT,
10665 ArgFlags2: ArgFlags);
10666 }
10667
10668 // Allocate to a register if possible, or else a stack slot.
10669 Register Reg;
10670 unsigned StoreSizeBytes = GRLen / 8;
10671 Align StackAlign = Align(GRLen / 8);
10672
10673 if (ValVT == MVT::f32 && !UseGPRForFloat) {
10674 Reg = State.AllocateReg(Regs: ArgFPR32s);
10675 } else if (ValVT == MVT::f64 && !UseGPRForFloat) {
10676 Reg = State.AllocateReg(Regs: ArgFPR64s);
10677 } else if (ValVT.is128BitVector()) {
10678 Reg = State.AllocateReg(Regs: ArgVRs);
10679 UseGPRForFloat = false;
10680 StoreSizeBytes = 16;
10681 StackAlign = Align(16);
10682 } else if (ValVT.is256BitVector()) {
10683 Reg = State.AllocateReg(Regs: ArgXRs);
10684 UseGPRForFloat = false;
10685 StoreSizeBytes = 32;
10686 StackAlign = Align(32);
10687 } else {
10688 Reg = allocateArgGPR(State);
10689 }
10690
10691 unsigned StackOffset =
10692 Reg ? 0 : State.AllocateStack(Size: StoreSizeBytes, Alignment: StackAlign);
10693
10694 // If we reach this point and PendingLocs is non-empty, we must be at the
10695 // end of a split argument that must be passed indirectly.
10696 if (!PendingLocs.empty()) {
10697 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
10698 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
10699 for (auto &It : PendingLocs) {
10700 if (Reg)
10701 It.convertToReg(Reg);
10702 else
10703 It.convertToMem(Offset: StackOffset);
10704 State.addLoc(V: It);
10705 }
10706 PendingLocs.clear();
10707 PendingArgFlags.clear();
10708 return false;
10709 }
10710 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
10711 "Expected an GRLenVT at this stage");
10712
10713 if (Reg) {
10714 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10715 return false;
10716 }
10717
10718 // When a floating-point value is passed on the stack, no bit-cast is needed.
10719 if (ValVT.isFloatingPoint()) {
10720 LocVT = ValVT;
10721 LocInfo = CCValAssign::Full;
10722 }
10723
10724 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset: StackOffset, LocVT, HTP: LocInfo));
10725 return false;
10726}
10727
10728void LoongArchTargetLowering::analyzeInputArgs(
10729 MachineFunction &MF, CCState &CCInfo,
10730 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
10731 LoongArchCCAssignFn Fn) const {
10732 FunctionType *FType = MF.getFunction().getFunctionType();
10733 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10734 MVT ArgVT = Ins[i].VT;
10735 Type *ArgTy = nullptr;
10736 if (IsRet)
10737 ArgTy = FType->getReturnType();
10738 else if (Ins[i].isOrigArg())
10739 ArgTy = FType->getParamType(i: Ins[i].getOrigArgIndex());
10740 LoongArchABI::ABI ABI =
10741 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10742 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
10743 CCInfo, IsRet, ArgTy)) {
10744 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
10745 << '\n');
10746 llvm_unreachable("");
10747 }
10748 }
10749}
10750
10751void LoongArchTargetLowering::analyzeOutputArgs(
10752 MachineFunction &MF, CCState &CCInfo,
10753 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
10754 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const {
10755 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10756 MVT ArgVT = Outs[i].VT;
10757 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
10758 LoongArchABI::ABI ABI =
10759 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10760 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
10761 CCInfo, IsRet, OrigTy)) {
10762 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
10763 << "\n");
10764 llvm_unreachable("");
10765 }
10766 }
10767}
10768
10769// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
10770// values.
10771static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
10772 const CCValAssign &VA, const SDLoc &DL) {
10773 switch (VA.getLocInfo()) {
10774 default:
10775 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10776 case CCValAssign::Full:
10777 case CCValAssign::Indirect:
10778 break;
10779 case CCValAssign::BCvt:
10780 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10781 Val = DAG.getNode(Opcode: LoongArchISD::MOVGR2FR_W_LA64, DL, VT: MVT::f32, Operand: Val);
10782 else
10783 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
10784 break;
10785 }
10786 return Val;
10787}
10788
10789static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
10790 const CCValAssign &VA, const SDLoc &DL,
10791 const ISD::InputArg &In,
10792 const LoongArchTargetLowering &TLI) {
10793 MachineFunction &MF = DAG.getMachineFunction();
10794 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10795 EVT LocVT = VA.getLocVT();
10796 SDValue Val;
10797 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
10798 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
10799 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
10800 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
10801
10802 // If input is sign extended from 32 bits, note it for the OptW pass.
10803 if (In.isOrigArg()) {
10804 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
10805 if (OrigArg->getType()->isIntegerTy()) {
10806 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
10807 // An input zero extended from i31 can also be considered sign extended.
10808 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
10809 (BitWidth < 32 && In.Flags.isZExt())) {
10810 LoongArchMachineFunctionInfo *LAFI =
10811 MF.getInfo<LoongArchMachineFunctionInfo>();
10812 LAFI->addSExt32Register(Reg: VReg);
10813 }
10814 }
10815 }
10816
10817 return convertLocVTToValVT(DAG, Val, VA, DL);
10818}
10819
10820// The caller is responsible for loading the full value if the argument is
10821// passed with CCValAssign::Indirect.
10822static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
10823 const CCValAssign &VA, const SDLoc &DL) {
10824 MachineFunction &MF = DAG.getMachineFunction();
10825 MachineFrameInfo &MFI = MF.getFrameInfo();
10826 EVT ValVT = VA.getValVT();
10827 int FI = MFI.CreateFixedObject(Size: ValVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
10828 /*IsImmutable=*/true);
10829 SDValue FIN = DAG.getFrameIndex(
10830 FI, VT: MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0)));
10831
10832 ISD::LoadExtType ExtType;
10833 switch (VA.getLocInfo()) {
10834 default:
10835 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10836 case CCValAssign::Full:
10837 case CCValAssign::Indirect:
10838 case CCValAssign::BCvt:
10839 ExtType = ISD::NON_EXTLOAD;
10840 break;
10841 }
10842 return DAG.getExtLoad(
10843 ExtType, dl: DL, VT: VA.getLocVT(), Chain, Ptr: FIN,
10844 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT: ValVT);
10845}
10846
10847static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain,
10848 const CCValAssign &VA,
10849 const CCValAssign &HiVA,
10850 const SDLoc &DL) {
10851 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
10852 "Unexpected VA");
10853 MachineFunction &MF = DAG.getMachineFunction();
10854 MachineFrameInfo &MFI = MF.getFrameInfo();
10855 MachineRegisterInfo &RegInfo = MF.getRegInfo();
10856
10857 assert(VA.isRegLoc() && "Expected register VA assignment");
10858
10859 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10860 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
10861 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
10862 SDValue Hi;
10863 if (HiVA.isMemLoc()) {
10864 // Second half of f64 is passed on the stack.
10865 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
10866 /*IsImmutable=*/true);
10867 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
10868 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
10869 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
10870 } else {
10871 // Second half of f64 is passed in another GPR.
10872 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
10873 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
10874 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
10875 }
10876 return DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
10877}
10878
10879static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
10880 const CCValAssign &VA, const SDLoc &DL) {
10881 EVT LocVT = VA.getLocVT();
10882
10883 switch (VA.getLocInfo()) {
10884 default:
10885 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10886 case CCValAssign::Full:
10887 break;
10888 case CCValAssign::BCvt:
10889 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10890 Val = DAG.getNode(Opcode: LoongArchISD::MOVFR2GR_S_LA64, DL, VT: MVT::i64, Operand: Val);
10891 else
10892 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
10893 break;
10894 }
10895 return Val;
10896}
10897
10898static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
10899 CCValAssign::LocInfo LocInfo,
10900 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
10901 CCState &State) {
10902 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
10903 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
10904 // s0 s1 s2 s3 s4 s5 s6 s7 s8
10905 static const MCPhysReg GPRList[] = {
10906 LoongArch::R23, LoongArch::R24, LoongArch::R25,
10907 LoongArch::R26, LoongArch::R27, LoongArch::R28,
10908 LoongArch::R29, LoongArch::R30, LoongArch::R31};
10909 if (MCRegister Reg = State.AllocateReg(Regs: GPRList)) {
10910 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10911 return false;
10912 }
10913 }
10914
10915 if (LocVT == MVT::f32) {
10916 // Pass in STG registers: F1, F2, F3, F4
10917 // fs0,fs1,fs2,fs3
10918 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
10919 LoongArch::F26, LoongArch::F27};
10920 if (MCRegister Reg = State.AllocateReg(Regs: FPR32List)) {
10921 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10922 return false;
10923 }
10924 }
10925
10926 if (LocVT == MVT::f64) {
10927 // Pass in STG registers: D1, D2, D3, D4
10928 // fs4,fs5,fs6,fs7
10929 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
10930 LoongArch::F30_64, LoongArch::F31_64};
10931 if (MCRegister Reg = State.AllocateReg(Regs: FPR64List)) {
10932 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
10933 return false;
10934 }
10935 }
10936
10937 report_fatal_error(reason: "No registers left in GHC calling convention");
10938 return true;
10939}
10940
10941// Transform physical registers into virtual registers.
10942SDValue LoongArchTargetLowering::LowerFormalArguments(
10943 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10944 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
10945 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
10946
10947 MachineFunction &MF = DAG.getMachineFunction();
10948
10949 switch (CallConv) {
10950 default:
10951 llvm_unreachable("Unsupported calling convention");
10952 case CallingConv::C:
10953 case CallingConv::Fast:
10954 case CallingConv::PreserveNone:
10955 case CallingConv::PreserveMost:
10956 break;
10957 case CallingConv::GHC:
10958 if (!MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicF) ||
10959 !MF.getSubtarget().hasFeature(Feature: LoongArch::FeatureBasicD))
10960 report_fatal_error(
10961 reason: "GHC calling convention requires the F and D extensions");
10962 }
10963
10964 const Function &Func = MF.getFunction();
10965 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
10966 MVT GRLenVT = Subtarget.getGRLenVT();
10967 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
10968
10969 // Check if this function has any musttail calls. If so, incoming indirect
10970 // arg pointers must be saved in virtual registers so they survive across
10971 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
10972 // when needed to avoid adding register pressure to non-musttail functions.
10973 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
10974 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
10975 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
10976 return CI->isMustTailCall();
10977 return false;
10978 });
10979 });
10980 // Used with varargs to acumulate store chains.
10981 std::vector<SDValue> OutChains;
10982
10983 // Assign locations to all of the incoming arguments.
10984 SmallVector<CCValAssign> ArgLocs;
10985 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10986
10987 if (CallConv == CallingConv::GHC)
10988 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_LoongArch_GHC);
10989 else
10990 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, Fn: CC_LoongArch);
10991
10992 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10993 CCValAssign &VA = ArgLocs[i];
10994 SDValue ArgValue;
10995 // Passing f64 on LA32D with a soft float ABI must be handled as a special
10996 // case.
10997 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10998 assert(VA.needsCustom());
10999 ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
11000 } else if (VA.isRegLoc())
11001 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
11002 else
11003 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
11004 if (VA.getLocInfo() == CCValAssign::Indirect) {
11005 // If the original argument was split and passed by reference, we need to
11006 // load all parts of it here (using the same address).
11007 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
11008 PtrInfo: MachinePointerInfo()));
11009 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
11010 if (HasMusttail) {
11011 LoongArchMachineFunctionInfo *LAFI =
11012 MF.getInfo<LoongArchMachineFunctionInfo>();
11013 Register VReg =
11014 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
11015 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
11016 LAFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
11017 }
11018 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
11019 assert(ArgPartOffset == 0);
11020 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
11021 CCValAssign &PartVA = ArgLocs[i + 1];
11022 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
11023 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
11024 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
11025 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
11026 PtrInfo: MachinePointerInfo()));
11027 ++i;
11028 ++InsIdx;
11029 }
11030 continue;
11031 }
11032 InVals.push_back(Elt: ArgValue);
11033 }
11034
11035 if (IsVarArg) {
11036 ArrayRef<MCPhysReg> ArgRegs = ArrayRef(ArgGPRs);
11037 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
11038 const TargetRegisterClass *RC = &LoongArch::GPRRegClass;
11039 MachineFrameInfo &MFI = MF.getFrameInfo();
11040 MachineRegisterInfo &RegInfo = MF.getRegInfo();
11041 auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>();
11042
11043 // Offset of the first variable argument from stack pointer, and size of
11044 // the vararg save area. For now, the varargs save area is either zero or
11045 // large enough to hold a0-a7.
11046 int VaArgOffset, VarArgsSaveSize;
11047
11048 // If all registers are allocated, then all varargs must be passed on the
11049 // stack and we don't need to save any argregs.
11050 if (ArgRegs.size() == Idx) {
11051 VaArgOffset = CCInfo.getStackSize();
11052 VarArgsSaveSize = 0;
11053 } else {
11054 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx);
11055 VaArgOffset = -VarArgsSaveSize;
11056 }
11057
11058 // Record the frame index of the first variable argument
11059 // which is a value necessary to VASTART.
11060 int FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
11061 LoongArchFI->setVarArgsFrameIndex(FI);
11062
11063 // If saving an odd number of registers then create an extra stack slot to
11064 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures
11065 // offsets to even-numbered registered remain 2*GRLen-aligned.
11066 if (Idx % 2) {
11067 MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset - (int)GRLenInBytes,
11068 IsImmutable: true);
11069 VarArgsSaveSize += GRLenInBytes;
11070 }
11071
11072 // Copy the integer registers that may have been used for passing varargs
11073 // to the vararg save area.
11074 for (unsigned I = Idx; I < ArgRegs.size();
11075 ++I, VaArgOffset += GRLenInBytes) {
11076 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
11077 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
11078 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: GRLenVT);
11079 FI = MFI.CreateFixedObject(Size: GRLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
11080 SDValue PtrOff = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
11081 SDValue Store = DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: PtrOff,
11082 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
11083 cast<StoreSDNode>(Val: Store.getNode())
11084 ->getMemOperand()
11085 ->setValue((Value *)nullptr);
11086 OutChains.push_back(x: Store);
11087 }
11088 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
11089 }
11090
11091 // All stores are grouped in one node to allow the matching between
11092 // the size of Ins and InVals. This only happens for vararg functions.
11093 if (!OutChains.empty()) {
11094 OutChains.push_back(x: Chain);
11095 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
11096 }
11097
11098 return Chain;
11099}
11100
11101bool LoongArchTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
11102 return CI->isTailCall();
11103}
11104
11105// Check if the return value is used as only a return value, as otherwise
11106// we can't perform a tail-call.
11107bool LoongArchTargetLowering::isUsedByReturnOnly(SDNode *N,
11108 SDValue &Chain) const {
11109 if (N->getNumValues() != 1)
11110 return false;
11111 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
11112 return false;
11113
11114 SDNode *Copy = *N->user_begin();
11115 if (Copy->getOpcode() != ISD::CopyToReg)
11116 return false;
11117
11118 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
11119 // isn't safe to perform a tail call.
11120 if (Copy->getGluedNode())
11121 return false;
11122
11123 // The copy must be used by a LoongArchISD::RET, and nothing else.
11124 bool HasRet = false;
11125 for (SDNode *Node : Copy->users()) {
11126 if (Node->getOpcode() != LoongArchISD::RET)
11127 return false;
11128 HasRet = true;
11129 }
11130
11131 if (!HasRet)
11132 return false;
11133
11134 Chain = Copy->getOperand(Num: 0);
11135 return true;
11136}
11137
11138// Check whether the call is eligible for tail call optimization.
11139bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
11140 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
11141 const SmallVectorImpl<CCValAssign> &ArgLocs) const {
11142
11143 auto CalleeCC = CLI.CallConv;
11144 auto &Outs = CLI.Outs;
11145 auto &Caller = MF.getFunction();
11146 auto CallerCC = Caller.getCallingConv();
11147
11148 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
11149
11150 // Byval parameters hand the function a pointer directly into the stack area
11151 // we want to reuse during a tail call. Working around this *is* possible
11152 // but less efficient and uglier in LowerCall. For musttail, there is no
11153 // workaround today: a byval arg requires a local copy that becomes invalid
11154 // after the tail call deallocates the caller's frame, so rejecting here
11155 // (and triggering reportFatalInternalError in LowerCall) is safer than
11156 // miscompiling.
11157 for (auto &Arg : Outs)
11158 if (Arg.Flags.isByVal())
11159 return false;
11160
11161 // musttail bypasses the remaining checks: the checks either reject cases
11162 // we handle specially (indirect args are forwarded via incoming pointers,
11163 // stack-passed args reuse the matching incoming layout, sret is forwarded
11164 // like any other pointer arg) or are optimizations not applicable to
11165 // mandatory tail calls.
11166 if (IsMustTail)
11167 return true;
11168
11169 // Do not tail call opt if the stack is used to pass parameters.
11170 if (CCInfo.getStackSize() != 0)
11171 return false;
11172
11173 // Do not tail call opt if any parameters need to be passed indirectly.
11174 for (auto &VA : ArgLocs)
11175 if (VA.getLocInfo() == CCValAssign::Indirect)
11176 return false;
11177
11178 // Do not tail call opt if either caller or callee uses struct return
11179 // semantics.
11180 auto IsCallerStructRet = Caller.hasStructRetAttr();
11181 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
11182 if (IsCallerStructRet || IsCalleeStructRet)
11183 return false;
11184
11185 // The callee has to preserve all registers the caller needs to preserve.
11186 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
11187 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
11188 if (CalleeCC != CallerCC) {
11189 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
11190 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
11191 return false;
11192 }
11193 return true;
11194}
11195
11196static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
11197 return DAG.getDataLayout().getPrefTypeAlign(
11198 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
11199}
11200
11201// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
11202// and output parameter nodes.
11203SDValue
11204LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
11205 SmallVectorImpl<SDValue> &InVals) const {
11206 SelectionDAG &DAG = CLI.DAG;
11207 SDLoc &DL = CLI.DL;
11208 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
11209 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
11210 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
11211 SDValue Chain = CLI.Chain;
11212 SDValue Callee = CLI.Callee;
11213 CallingConv::ID CallConv = CLI.CallConv;
11214 bool IsVarArg = CLI.IsVarArg;
11215 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
11216 MVT GRLenVT = Subtarget.getGRLenVT();
11217 bool &IsTailCall = CLI.IsTailCall;
11218
11219 MachineFunction &MF = DAG.getMachineFunction();
11220
11221 // Analyze the operands of the call, assigning locations to each operand.
11222 SmallVector<CCValAssign> ArgLocs;
11223 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
11224
11225 if (CallConv == CallingConv::GHC)
11226 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_LoongArch_GHC);
11227 else
11228 analyzeOutputArgs(MF, CCInfo&: ArgCCInfo, Outs, /*IsRet=*/false, CLI: &CLI, Fn: CC_LoongArch);
11229
11230 // Check if it's really possible to do a tail call.
11231 if (IsTailCall)
11232 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
11233
11234 if (IsTailCall)
11235 ++NumTailCalls;
11236 else if (CLI.CB && CLI.CB->isMustTailCall())
11237 report_fatal_error(reason: "failed to perform tail call elimination on a call "
11238 "site marked musttail");
11239
11240 // Get a count of how many bytes are to be pushed on the stack.
11241 unsigned NumBytes = ArgCCInfo.getStackSize();
11242
11243 // Create local copies for byval args.
11244 SmallVector<SDValue> ByValArgs;
11245 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
11246 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11247 if (!Flags.isByVal())
11248 continue;
11249
11250 SDValue Arg = OutVals[i];
11251 unsigned Size = Flags.getByValSize();
11252 Align Alignment = Flags.getNonZeroByValAlign();
11253
11254 int FI =
11255 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
11256 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
11257 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: GRLenVT);
11258
11259 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
11260 /*IsVolatile=*/isVol: false,
11261 /*AlwaysInline=*/false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
11262 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
11263 ByValArgs.push_back(Elt: FIPtr);
11264 }
11265
11266 if (!IsTailCall)
11267 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
11268
11269 // Copy argument values to their designated locations.
11270 SmallVector<std::pair<Register, SDValue>> RegsToPass;
11271 SmallVector<SDValue> MemOpChains;
11272 SDValue StackPtr;
11273 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
11274 ++i, ++OutIdx) {
11275 CCValAssign &VA = ArgLocs[i];
11276 SDValue ArgValue = OutVals[OutIdx];
11277 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
11278
11279 // Handle passing f64 on LA32D with a soft float ABI as a special case.
11280 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11281 assert(VA.isRegLoc() && "Expected register VA assignment");
11282 assert(VA.needsCustom());
11283 SDValue SplitF64 =
11284 DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
11285 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
11286 SDValue Lo = SplitF64.getValue(R: 0);
11287 SDValue Hi = SplitF64.getValue(R: 1);
11288
11289 Register RegLo = VA.getLocReg();
11290 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
11291
11292 // Get the CCValAssign for the Hi part.
11293 CCValAssign &HiVA = ArgLocs[++i];
11294
11295 if (HiVA.isMemLoc()) {
11296 // Second half of f64 is passed on the stack.
11297 if (!StackPtr.getNode())
11298 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
11299 SDValue Address =
11300 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
11301 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
11302 // Emit the store.
11303 MemOpChains.push_back(Elt: DAG.getStore(
11304 Chain, dl: DL, Val: Hi, Ptr: Address,
11305 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
11306 } else {
11307 // Second half of f64 is passed in another GPR.
11308 Register RegHigh = HiVA.getLocReg();
11309 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
11310 }
11311 continue;
11312 }
11313
11314 // Promote the value if needed.
11315 // For now, only handle fully promoted and indirect arguments.
11316 if (VA.getLocInfo() == CCValAssign::Indirect) {
11317 // For musttail calls, reuse incoming indirect pointers instead of
11318 // creating new stack temporaries. The incoming pointers point to the
11319 // caller's caller's frame, which remains valid after a tail call.
11320 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
11321 LoongArchMachineFunctionInfo *LAFI =
11322 MF.getInfo<LoongArchMachineFunctionInfo>();
11323 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
11324
11325 // Resolve which formal parameter is being passed at this call
11326 // position.
11327 //
11328 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
11329 // but Outs[].OrigArgIndex is an index into a filtered arg list
11330 // (empty types removed, via CallLoweringInfo in the target-
11331 // independent layer). IncomingIndirectArgs is keyed by the
11332 // caller's unfiltered Argument::getArgNo(), so we have to walk
11333 // the caller's formals (same filter) to translate the index.
11334 // This target-independent asymmetry should be normalized so
11335 // backends do not need to re-derive the mapping.
11336 //
11337 // Steps:
11338 // 1. Find the call operand at filtered position CallArgIdx.
11339 // 2. If it is an Argument, use getArgNo() directly (same filter
11340 // for caller formals and call operands).
11341 // 3. Otherwise (computed value), walk the caller's formals and
11342 // skip empty types to map the filtered index to getArgNo().
11343 const Argument *FormalArg = nullptr;
11344 unsigned FilteredIdx = 0;
11345 for (const auto &CallArg : CLI.CB->args()) {
11346 if (CallArg->getType()->isEmptyTy())
11347 continue;
11348 if (FilteredIdx == CallArgIdx) {
11349 FormalArg = dyn_cast<Argument>(Val: CallArg);
11350 break;
11351 }
11352 ++FilteredIdx;
11353 }
11354
11355 // For forwarded args, getArgNo() gives the unfiltered index directly.
11356 // For computed args, walk the caller's formals to resolve it.
11357 unsigned FormalArgIdx = CallArgIdx;
11358 if (FormalArg) {
11359 FormalArgIdx = FormalArg->getArgNo();
11360 } else {
11361 FilteredIdx = 0;
11362 for (const auto &Arg : MF.getFunction().args()) {
11363 if (Arg.getType()->isEmptyTy())
11364 continue;
11365 if (FilteredIdx == CallArgIdx) {
11366 FormalArgIdx = Arg.getArgNo();
11367 break;
11368 }
11369 ++FilteredIdx;
11370 }
11371 }
11372
11373 Register VReg = LAFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
11374 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
11375 // Thread the CopyFromReg output chain through MemOpChains so the
11376 // TokenFactor below sequences the copy with any stores we emit
11377 // for this argument.
11378 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
11379 SDValue IncomingPtr = CopyOp;
11380
11381 if (!FormalArg) {
11382 // Computed value: store into the incoming indirect pointer for the
11383 // same-position formal parameter (musttail guarantees matching
11384 // prototypes, so types match). The pointer survives the tail call
11385 // since it points to the caller's caller's frame.
11386 //
11387 // The data-flow edge through IncomingPtr already prevents the
11388 // store from being scheduled before the CopyFromReg. Threading
11389 // CopyOp.getValue(1) (the copy's output chain) into the store
11390 // makes that ordering explicit on the chain edge as well, which
11391 // is the convention for memory ops chaining off their producers.
11392 MemOpChains.push_back(
11393 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
11394 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
11395 // Store any split parts at their respective offsets.
11396 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11397 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11398 SDValue PartValue = OutVals[OutIdx + 1];
11399 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11400 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
11401 SDValue Addr =
11402 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
11403 MemOpChains.push_back(
11404 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
11405 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
11406 ++i;
11407 ++OutIdx;
11408 }
11409 }
11410 ArgValue = IncomingPtr;
11411
11412 // Skip any remaining split parts (for forwarded args, they are
11413 // covered by the forwarded pointer).
11414 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11415 ++i;
11416 ++OutIdx;
11417 }
11418 } else {
11419 // Store the argument in a stack slot and pass its address.
11420 Align StackAlign =
11421 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
11422 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
11423 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
11424 // If the original argument was split and passed by reference, we need
11425 // to store the required parts of it here (and pass just one address).
11426 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
11427 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11428 assert(ArgPartOffset == 0);
11429 // Calculate the total size to store. We don't have access to what we're
11430 // actually storing other than performing the loop and collecting the
11431 // info.
11432 SmallVector<std::pair<SDValue, SDValue>> Parts;
11433 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
11434 SDValue PartValue = OutVals[OutIdx + 1];
11435 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11436 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
11437 EVT PartVT = PartValue.getValueType();
11438 StoredSize += PartVT.getStoreSize();
11439 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
11440 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
11441 ++i;
11442 ++OutIdx;
11443 }
11444 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
11445 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
11446 MemOpChains.push_back(
11447 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
11448 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
11449 for (const auto &Part : Parts) {
11450 SDValue PartValue = Part.first;
11451 SDValue PartOffset = Part.second;
11452 SDValue Address =
11453 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
11454 MemOpChains.push_back(
11455 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
11456 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
11457 }
11458 ArgValue = SpillSlot;
11459 }
11460 } else {
11461 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL);
11462 }
11463
11464 // Use local copy if it is a byval arg.
11465 if (Flags.isByVal())
11466 ArgValue = ByValArgs[j++];
11467
11468 if (VA.isRegLoc()) {
11469 // Queue up the argument copies and emit them at the end.
11470 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
11471 } else {
11472 assert(VA.isMemLoc() && "Argument not register or memory");
11473 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
11474 "Tail call not allowed if stack is used for passing parameters");
11475
11476 // Work out the address of the stack slot.
11477 if (!StackPtr.getNode())
11478 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoongArch::R3, VT: PtrVT);
11479 SDValue Address =
11480 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
11481 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
11482
11483 // Emit the store.
11484 MemOpChains.push_back(
11485 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
11486 }
11487 }
11488
11489 // Join the stores, which are independent of one another.
11490 if (!MemOpChains.empty())
11491 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
11492
11493 SDValue Glue;
11494
11495 // Build a sequence of copy-to-reg nodes, chained and glued together.
11496 for (auto &Reg : RegsToPass) {
11497 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
11498 Glue = Chain.getValue(R: 1);
11499 }
11500
11501 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
11502 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
11503 // split it and then direct call can be matched by PseudoCALL_SMALL.
11504 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
11505 const GlobalValue *GV = S->getGlobal();
11506 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV)
11507 ? LoongArchII::MO_CALL
11508 : LoongArchII::MO_CALL_PLT;
11509 Callee = DAG.getTargetGlobalAddress(GV: S->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: OpFlags);
11510 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
11511 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV: nullptr)
11512 ? LoongArchII::MO_CALL
11513 : LoongArchII::MO_CALL_PLT;
11514 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: OpFlags);
11515 }
11516
11517 // The first call operand is the chain and the second is the target address.
11518 SmallVector<SDValue> Ops;
11519 Ops.push_back(Elt: Chain);
11520 Ops.push_back(Elt: Callee);
11521
11522 // Add argument registers to the end of the list so that they are
11523 // known live into the call.
11524 for (auto &Reg : RegsToPass)
11525 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
11526
11527 if (!IsTailCall) {
11528 // Add a register mask operand representing the call-preserved registers.
11529 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
11530 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
11531 assert(Mask && "Missing call preserved mask for calling convention");
11532 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
11533 }
11534
11535 // Glue the call to the argument copies, if any.
11536 if (Glue.getNode())
11537 Ops.push_back(Elt: Glue);
11538
11539 // Emit the call.
11540 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
11541 unsigned Op;
11542 switch (DAG.getTarget().getCodeModel()) {
11543 default:
11544 report_fatal_error(reason: "Unsupported code model");
11545 case CodeModel::Small:
11546 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
11547 break;
11548 case CodeModel::Medium:
11549 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
11550 break;
11551 case CodeModel::Large:
11552 assert(Subtarget.is64Bit() && "Large code model requires LA64");
11553 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
11554 break;
11555 }
11556
11557 if (IsTailCall) {
11558 MF.getFrameInfo().setHasTailCall();
11559 SDValue Ret = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
11560 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
11561 return Ret;
11562 }
11563
11564 Chain = DAG.getNode(Opcode: Op, DL, VTList: NodeTys, Ops);
11565 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
11566 Glue = Chain.getValue(R: 1);
11567
11568 // Mark the end of the call, which is glued to the call itself.
11569 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
11570 Glue = Chain.getValue(R: 1);
11571
11572 // Assign locations to each value returned by this call.
11573 SmallVector<CCValAssign> RVLocs;
11574 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
11575 analyzeInputArgs(MF, CCInfo&: RetCCInfo, Ins, /*IsRet=*/true, Fn: CC_LoongArch);
11576
11577 // Copy all of the result registers out of their specified physreg.
11578 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
11579 auto &VA = RVLocs[i];
11580 // Copy the value out.
11581 SDValue RetValue =
11582 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
11583 // Glue the RetValue to the end of the call sequence.
11584 Chain = RetValue.getValue(R: 1);
11585 Glue = RetValue.getValue(R: 2);
11586
11587 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11588 assert(VA.needsCustom());
11589 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
11590 VT: MVT::i32, Glue);
11591 Chain = RetValue2.getValue(R: 1);
11592 Glue = RetValue2.getValue(R: 2);
11593 RetValue = DAG.getNode(Opcode: LoongArchISD::BUILD_PAIR_F64, DL, VT: MVT::f64,
11594 N1: RetValue, N2: RetValue2);
11595 } else
11596 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL);
11597
11598 InVals.push_back(Elt: RetValue);
11599 }
11600
11601 return Chain;
11602}
11603
11604bool LoongArchTargetLowering::CanLowerReturn(
11605 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
11606 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
11607 const Type *RetTy) const {
11608 SmallVector<CCValAssign> RVLocs;
11609 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
11610
11611 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
11612 LoongArchABI::ABI ABI =
11613 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
11614 if (CC_LoongArch(DL: MF.getDataLayout(), ABI, ValNo: i, ValVT: Outs[i].VT, LocInfo: CCValAssign::Full,
11615 ArgFlags: Outs[i].Flags, State&: CCInfo, /*IsRet=*/true, OrigTy: nullptr))
11616 return false;
11617 }
11618 return true;
11619}
11620
11621SDValue LoongArchTargetLowering::LowerReturn(
11622 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
11623 const SmallVectorImpl<ISD::OutputArg> &Outs,
11624 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
11625 SelectionDAG &DAG) const {
11626 // Stores the assignment of the return value to a location.
11627 SmallVector<CCValAssign> RVLocs;
11628
11629 // Info about the registers and stack slot.
11630 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
11631 *DAG.getContext());
11632
11633 analyzeOutputArgs(MF&: DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
11634 CLI: nullptr, Fn: CC_LoongArch);
11635 if (CallConv == CallingConv::GHC && !RVLocs.empty())
11636 report_fatal_error(reason: "GHC functions return void only");
11637 SDValue Glue;
11638 SmallVector<SDValue, 4> RetOps(1, Chain);
11639
11640 // Copy the result values into the output registers.
11641 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
11642 SDValue Val = OutVals[OutIdx];
11643 CCValAssign &VA = RVLocs[i];
11644 assert(VA.isRegLoc() && "Can only return in registers!");
11645
11646 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11647 // Handle returning f64 on LA32D with a soft float ABI.
11648 assert(VA.isRegLoc() && "Expected return via registers");
11649 assert(VA.needsCustom());
11650 SDValue SplitF64 = DAG.getNode(Opcode: LoongArchISD::SPLIT_PAIR_F64, DL,
11651 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
11652 SDValue Lo = SplitF64.getValue(R: 0);
11653 SDValue Hi = SplitF64.getValue(R: 1);
11654 Register RegLo = VA.getLocReg();
11655 Register RegHi = RVLocs[++i].getLocReg();
11656
11657 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
11658 Glue = Chain.getValue(R: 1);
11659 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
11660 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
11661 Glue = Chain.getValue(R: 1);
11662 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
11663 } else {
11664 // Handle a 'normal' return.
11665 Val = convertValVTToLocVT(DAG, Val, VA, DL);
11666 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
11667
11668 // Guarantee that all emitted copies are stuck together.
11669 Glue = Chain.getValue(R: 1);
11670 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
11671 }
11672 }
11673
11674 RetOps[0] = Chain; // Update chain.
11675
11676 // Add the glue node if we have it.
11677 if (Glue.getNode())
11678 RetOps.push_back(Elt: Glue);
11679
11680 return DAG.getNode(Opcode: LoongArchISD::RET, DL, VT: MVT::Other, Ops: RetOps);
11681}
11682
11683// Check if a constant splat can be generated using [x]vldi, where imm[12] == 1.
11684// Note: The following prefixes are excluded:
11685// imm[11:8] == 4'b0000, 4'b0100, 4'b1000
11686// as they can be represented using [x]vrepli.[whb]
11687std::pair<bool, uint64_t> LoongArchTargetLowering::isImmVLDILegalForMode1(
11688 const APInt &SplatValue, const unsigned SplatBitSize) const {
11689 uint64_t RequiredImm = 0;
11690 uint64_t V = SplatValue.getZExtValue();
11691 if (SplatBitSize == 16 && !(V & 0x00FF)) {
11692 // 4'b0101
11693 RequiredImm = (0b10101 << 8) | (V >> 8);
11694 return {true, RequiredImm};
11695 } else if (SplatBitSize == 32) {
11696 // 4'b0001
11697 if (!(V & 0xFFFF00FF)) {
11698 RequiredImm = (0b10001 << 8) | (V >> 8);
11699 return {true, RequiredImm};
11700 }
11701 // 4'b0010
11702 if (!(V & 0xFF00FFFF)) {
11703 RequiredImm = (0b10010 << 8) | (V >> 16);
11704 return {true, RequiredImm};
11705 }
11706 // 4'b0011
11707 if (!(V & 0x00FFFFFF)) {
11708 RequiredImm = (0b10011 << 8) | (V >> 24);
11709 return {true, RequiredImm};
11710 }
11711 // 4'b0110
11712 if ((V & 0xFFFF00FF) == 0xFF) {
11713 RequiredImm = (0b10110 << 8) | (V >> 8);
11714 return {true, RequiredImm};
11715 }
11716 // 4'b0111
11717 if ((V & 0xFF00FFFF) == 0xFFFF) {
11718 RequiredImm = (0b10111 << 8) | (V >> 16);
11719 return {true, RequiredImm};
11720 }
11721 // 4'b1010
11722 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
11723 RequiredImm =
11724 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11725 return {true, RequiredImm};
11726 }
11727 } else if (SplatBitSize == 64) {
11728 // 4'b1011
11729 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
11730 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
11731 RequiredImm =
11732 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11733 return {true, RequiredImm};
11734 }
11735 // 4'b1100
11736 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
11737 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
11738 RequiredImm =
11739 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
11740 return {true, RequiredImm};
11741 }
11742 // 4'b1001
11743 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
11744 uint8_t res = 0;
11745 for (int i = 0; i < 8; ++i) {
11746 uint8_t byte = x & 0xFF;
11747 if (byte == 0 || byte == 0xFF)
11748 res |= ((byte & 1) << i);
11749 else
11750 return {false, 0};
11751 x >>= 8;
11752 }
11753 return {true, res};
11754 };
11755 auto [IsSame, Suffix] = sameBitsPreByte(V);
11756 if (IsSame) {
11757 RequiredImm = (0b11001 << 8) | Suffix;
11758 return {true, RequiredImm};
11759 }
11760 }
11761 return {false, RequiredImm};
11762}
11763
11764bool LoongArchTargetLowering::isFPImmVLDILegal(const APFloat &Imm,
11765 EVT VT) const {
11766 if (!Subtarget.hasExtLSX())
11767 return false;
11768
11769 if (VT == MVT::f32) {
11770 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
11771 return (masked == 0x3e000000 || masked == 0x40000000);
11772 }
11773
11774 if (VT == MVT::f64) {
11775 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
11776 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
11777 }
11778
11779 return false;
11780}
11781
11782// Only form FP_TO_SINT_SAT from clamp/NaN-guard idioms when it will be lowered
11783// to a single [X]VFTINTRZ/FTINTRZ, otherwise the generic expansion is worse
11784// than the original sequence.
11785bool LoongArchTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
11786 EVT VT) const {
11787 if (!isOperationLegalOrCustom(Op, VT))
11788 return false;
11789 if (VT.isVector())
11790 return FPVT.getScalarSizeInBits() == VT.getScalarSizeInBits();
11791 if (FPVT == MVT::f32)
11792 return VT.getSizeInBits() == 32 || Subtarget.hasBasicD();
11793 return FPVT == MVT::f64 && Subtarget.hasBasicD();
11794}
11795
11796TargetLowering::FPImmAction
11797LoongArchTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
11798 bool ForCodeSize) const {
11799 // TODO: Maybe need more checks here after vector extension is supported.
11800 if (VT == MVT::f32 && !Subtarget.hasBasicF())
11801 return FPImmAction::ConstantPool;
11802 if (VT == MVT::f64 && !Subtarget.hasBasicD())
11803 return FPImmAction::ConstantPool;
11804 if (Imm.isZero() || isFPImmVLDILegal(Imm, VT))
11805 return FPImmAction::Legal;
11806
11807 // Immediates the size of a GPR are built from their bits. Others are built in
11808 // lowerConstantFP.
11809 if (VT.getSizeInBits() == Subtarget.getGRLen() && isFPImmCheapInGPR(Imm, VT))
11810 return FPImmAction::Bitcast;
11811 return Imm.isOne() ? FPImmAction::Legal : FPImmAction::ConstantPool;
11812}
11813
11814bool LoongArchTargetLowering::isFPImmCheapInGPR(const APFloat &Imm,
11815 EVT VT) const {
11816 // Count the instructions to generate the bits and move them to a float
11817 // register.
11818 auto Seq =
11819 LoongArchMatInt::generateInstSeq(Val: Imm.bitcastToAPInt().getSExtValue());
11820 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
11821 return InsNum <= MaterializeFPImmInsNum || Imm.isOne();
11822}
11823
11824bool LoongArchTargetLowering::isCheapToSpeculateCttz(Type *) const {
11825 return true;
11826}
11827
11828bool LoongArchTargetLowering::isCheapToSpeculateCtlz(Type *) const {
11829 return true;
11830}
11831
11832bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
11833 const Instruction *I) const {
11834 if (!Subtarget.is64Bit())
11835 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
11836
11837 if (isa<LoadInst>(Val: I))
11838 return true;
11839
11840 // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
11841 // require fences beacuse we can use amswap_db.[w/d].
11842 Type *Ty = I->getOperand(i: 0)->getType();
11843 if (isa<StoreInst>(Val: I) && Ty->isIntegerTy()) {
11844 unsigned Size = Ty->getIntegerBitWidth();
11845 return (Size == 8 || Size == 16);
11846 }
11847
11848 return false;
11849}
11850
11851EVT LoongArchTargetLowering::getSetCCResultType(const DataLayout &DL,
11852 LLVMContext &Context,
11853 EVT VT) const {
11854 if (!VT.isVector())
11855 return getPointerTy(DL);
11856 return VT.changeVectorElementTypeToInteger();
11857}
11858
11859bool LoongArchTargetLowering::canMergeStoresTo(
11860 unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const {
11861 // Do not merge to float value size (128 or 256 bits) if no implicit
11862 // float attribute is set.
11863 bool NoFloat = MF.getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat);
11864 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
11865 if (NoFloat)
11866 return MemVT.getSizeInBits() <= MaxIntSize;
11867
11868 // Make sure we don't merge greater than our maximum supported vector width.
11869 if (Subtarget.hasExtLASX())
11870 MaxIntSize = 256;
11871 else if (Subtarget.hasExtLSX())
11872 MaxIntSize = 128;
11873
11874 return MemVT.getSizeInBits() <= MaxIntSize;
11875}
11876
11877bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
11878 EVT VT = Y.getValueType();
11879
11880 if (VT.isVector())
11881 return Subtarget.hasExtLSX() && VT.isInteger();
11882
11883 return VT.isScalarInteger() && !isa<ConstantSDNode>(Val: Y);
11884}
11885
11886void LoongArchTargetLowering::getTgtMemIntrinsic(
11887 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
11888 MachineFunction &MF, unsigned Intrinsic) const {
11889 switch (Intrinsic) {
11890 default:
11891 return;
11892 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
11893 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
11894 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
11895 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
11896 IntrinsicInfo Info;
11897 Info.opc = ISD::INTRINSIC_W_CHAIN;
11898 Info.memVT = MVT::i32;
11899 Info.ptrVal = I.getArgOperand(i: 0);
11900 Info.offset = 0;
11901 Info.align = Align(4);
11902 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
11903 MachineMemOperand::MOVolatile;
11904 Infos.push_back(Elt: Info);
11905 return;
11906 // TODO: Add more Intrinsics later.
11907 }
11908 }
11909}
11910
11911// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
11912// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
11913// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
11914// regression, we need to implement it manually.
11915void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
11916 AtomicRMWInst::BinOp Op = AI->getOperation();
11917
11918 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
11919 Op == AtomicRMWInst::And) &&
11920 "Unable to expand");
11921 unsigned MinWordSize = 4;
11922
11923 IRBuilder<> Builder(AI);
11924 LLVMContext &Ctx = Builder.getContext();
11925 const DataLayout &DL = AI->getDataLayout();
11926 Type *ValueType = AI->getType();
11927 Type *WordType = Type::getIntNTy(C&: Ctx, N: MinWordSize * 8);
11928
11929 Value *Addr = AI->getPointerOperand();
11930 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
11931 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
11932
11933 Value *AlignedAddr = Builder.CreateIntrinsic(
11934 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
11935 Args: {Addr, ConstantInt::get(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))}, FMFSource: nullptr,
11936 Name: "AlignedAddr");
11937
11938 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
11939 Value *PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
11940 Value *ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
11941 ShiftAmt = Builder.CreateTrunc(V: ShiftAmt, DestTy: WordType, Name: "ShiftAmt");
11942 Value *Mask = Builder.CreateShl(
11943 LHS: ConstantInt::get(Ty: WordType,
11944 V: (1 << (DL.getTypeStoreSize(Ty: ValueType) * 8)) - 1),
11945 RHS: ShiftAmt, Name: "Mask");
11946 Value *Inv_Mask = Builder.CreateNot(V: Mask, Name: "Inv_Mask");
11947 Value *ValOperand_Shifted =
11948 Builder.CreateShl(LHS: Builder.CreateZExt(V: AI->getValOperand(), DestTy: WordType),
11949 RHS: ShiftAmt, Name: "ValOperand_Shifted");
11950 Value *NewOperand;
11951 if (Op == AtomicRMWInst::And)
11952 NewOperand = Builder.CreateOr(LHS: ValOperand_Shifted, RHS: Inv_Mask, Name: "AndOperand");
11953 else
11954 NewOperand = ValOperand_Shifted;
11955
11956 AtomicRMWInst *NewAI =
11957 Builder.CreateAtomicRMW(Op, Ptr: AlignedAddr, Val: NewOperand, Align: Align(MinWordSize),
11958 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
11959
11960 Value *Shift = Builder.CreateLShr(LHS: NewAI, RHS: ShiftAmt, Name: "shifted");
11961 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: ValueType, Name: "extracted");
11962 Value *FinalOldResult = Builder.CreateBitCast(V: Trunc, DestTy: ValueType);
11963 AI->replaceAllUsesWith(V: FinalOldResult);
11964 AI->eraseFromParent();
11965}
11966
11967TargetLowering::AtomicExpansionKind
11968LoongArchTargetLowering::shouldExpandAtomicRMWInIR(
11969 const AtomicRMWInst *AI) const {
11970 // TODO: Add more AtomicRMWInst that needs to be extended.
11971
11972 // Since floating-point operation requires a non-trivial set of data
11973 // operations, use CmpXChg to expand.
11974 if (AI->isFloatingPointOperation() ||
11975 AI->getOperation() == AtomicRMWInst::UIncWrap ||
11976 AI->getOperation() == AtomicRMWInst::UDecWrap ||
11977 AI->getOperation() == AtomicRMWInst::USubCond ||
11978 AI->getOperation() == AtomicRMWInst::USubSat)
11979 return AtomicExpansionKind::CmpXChg;
11980
11981 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
11982 (AI->getOperation() == AtomicRMWInst::Xchg ||
11983 AI->getOperation() == AtomicRMWInst::Add ||
11984 AI->getOperation() == AtomicRMWInst::Sub)) {
11985 return AtomicExpansionKind::None;
11986 }
11987
11988 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11989 if (Subtarget.hasLAMCAS()) {
11990 if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
11991 AI->getOperation() == AtomicRMWInst::Or ||
11992 AI->getOperation() == AtomicRMWInst::Xor))
11993 return AtomicExpansionKind::CustomExpand;
11994 if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
11995 return AtomicExpansionKind::CmpXChg;
11996 }
11997
11998 if (Size == 8 || Size == 16)
11999 return AtomicExpansionKind::MaskedIntrinsic;
12000 return AtomicExpansionKind::None;
12001}
12002
12003static Intrinsic::ID
12004getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen,
12005 AtomicRMWInst::BinOp BinOp) {
12006 if (GRLen == 64) {
12007 switch (BinOp) {
12008 default:
12009 llvm_unreachable("Unexpected AtomicRMW BinOp");
12010 case AtomicRMWInst::Xchg:
12011 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
12012 case AtomicRMWInst::Add:
12013 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
12014 case AtomicRMWInst::Sub:
12015 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
12016 case AtomicRMWInst::Nand:
12017 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
12018 case AtomicRMWInst::UMax:
12019 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
12020 case AtomicRMWInst::UMin:
12021 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
12022 case AtomicRMWInst::Max:
12023 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
12024 case AtomicRMWInst::Min:
12025 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
12026 // TODO: support other AtomicRMWInst.
12027 }
12028 }
12029
12030 if (GRLen == 32) {
12031 switch (BinOp) {
12032 default:
12033 llvm_unreachable("Unexpected AtomicRMW BinOp");
12034 case AtomicRMWInst::Xchg:
12035 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
12036 case AtomicRMWInst::Add:
12037 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
12038 case AtomicRMWInst::Sub:
12039 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
12040 case AtomicRMWInst::Nand:
12041 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
12042 case AtomicRMWInst::UMax:
12043 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
12044 case AtomicRMWInst::UMin:
12045 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
12046 case AtomicRMWInst::Max:
12047 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
12048 case AtomicRMWInst::Min:
12049 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
12050 // TODO: support other AtomicRMWInst.
12051 }
12052 }
12053
12054 llvm_unreachable("Unexpected GRLen\n");
12055}
12056
12057TargetLowering::AtomicExpansionKind
12058LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
12059 const AtomicCmpXchgInst *CI) const {
12060
12061 if (Subtarget.hasLAMCAS())
12062 return AtomicExpansionKind::None;
12063
12064 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
12065 if (Size == 8 || Size == 16)
12066 return AtomicExpansionKind::MaskedIntrinsic;
12067 return AtomicExpansionKind::None;
12068}
12069
12070Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
12071 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
12072 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
12073 unsigned GRLen = Subtarget.getGRLen();
12074 AtomicOrdering FailOrd = CI->getFailureOrdering();
12075 Value *FailureOrdering =
12076 Builder.getIntN(N: Subtarget.getGRLen(), C: static_cast<uint64_t>(FailOrd));
12077 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
12078 if (GRLen == 64) {
12079 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
12080 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
12081 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
12082 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
12083 }
12084 Type *Tys[] = {AlignedAddr->getType()};
12085 Value *Result = Builder.CreateIntrinsic(
12086 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
12087 if (GRLen == 64)
12088 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
12089 return Result;
12090}
12091
12092Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic(
12093 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
12094 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
12095 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
12096 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
12097 // mask, as this produces better code than the LL/SC loop emitted by
12098 // int_loongarch_masked_atomicrmw_xchg.
12099 if (AI->getOperation() == AtomicRMWInst::Xchg &&
12100 isa<ConstantInt>(Val: AI->getValOperand())) {
12101 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
12102 if (CVal->isZero())
12103 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
12104 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
12105 Align: AI->getAlign(), Ordering: Ord);
12106 if (CVal->isMinusOne())
12107 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
12108 Align: AI->getAlign(), Ordering: Ord);
12109 }
12110
12111 unsigned GRLen = Subtarget.getGRLen();
12112 Value *Ordering =
12113 Builder.getIntN(N: GRLen, C: static_cast<uint64_t>(AI->getOrdering()));
12114 Type *Tys[] = {AlignedAddr->getType()};
12115 Function *LlwOpScwLoop = Intrinsic::getOrInsertDeclaration(
12116 M: AI->getModule(),
12117 id: getIntrinsicForMaskedAtomicRMWBinOp(GRLen, BinOp: AI->getOperation()), OverloadTys: Tys);
12118
12119 if (GRLen == 64) {
12120 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
12121 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
12122 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
12123 }
12124
12125 Value *Result;
12126
12127 // Must pass the shift amount needed to sign extend the loaded value prior
12128 // to performing a signed comparison for min/max. ShiftAmt is the number of
12129 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which
12130 // is the number of bits to left+right shift the value in order to
12131 // sign-extend.
12132 if (AI->getOperation() == AtomicRMWInst::Min ||
12133 AI->getOperation() == AtomicRMWInst::Max) {
12134 const DataLayout &DL = AI->getDataLayout();
12135 unsigned ValWidth =
12136 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
12137 Value *SextShamt =
12138 Builder.CreateSub(LHS: Builder.getIntN(N: GRLen, C: GRLen - ValWidth), RHS: ShiftAmt);
12139 Result = Builder.CreateCall(Callee: LlwOpScwLoop,
12140 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
12141 } else {
12142 Result =
12143 Builder.CreateCall(Callee: LlwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
12144 }
12145
12146 if (GRLen == 64)
12147 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
12148 return Result;
12149}
12150
12151bool LoongArchTargetLowering::isFMAFasterThanFMulAndFAdd(
12152 const MachineFunction &MF, EVT VT) const {
12153 VT = VT.getScalarType();
12154
12155 if (!VT.isSimple())
12156 return false;
12157
12158 switch (VT.getSimpleVT().SimpleTy) {
12159 case MVT::f32:
12160 case MVT::f64:
12161 return true;
12162 default:
12163 break;
12164 }
12165
12166 return false;
12167}
12168
12169Register LoongArchTargetLowering::getExceptionPointerRegister(
12170 ExceptionHandling EH, const Constant *PersonalityFn) const {
12171 return LoongArch::R4;
12172}
12173
12174Register LoongArchTargetLowering::getExceptionSelectorRegister(
12175 ExceptionHandling EH, const Constant *PersonalityFn) const {
12176 return LoongArch::R5;
12177}
12178
12179//===----------------------------------------------------------------------===//
12180// Target Optimization Hooks
12181//===----------------------------------------------------------------------===//
12182
12183static int getEstimateRefinementSteps(EVT VT,
12184 const LoongArchSubtarget &Subtarget) {
12185 // Feature FRECIPE instructions relative accuracy is 2^-14.
12186 // IEEE float has 23 digits and double has 52 digits.
12187 int RefinementSteps = VT.getScalarType() == MVT::f64 ? 2 : 1;
12188 return RefinementSteps;
12189}
12190
12191static bool
12192isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget) {
12193 assert(Subtarget.hasFrecipe() &&
12194 "Reciprocal estimate queried on unsupported target");
12195
12196 if (!VT.isSimple())
12197 return false;
12198
12199 switch (VT.getSimpleVT().SimpleTy) {
12200 case MVT::f32:
12201 // f32 is the base type for reciprocal estimate instructions.
12202 return true;
12203
12204 case MVT::f64:
12205 return Subtarget.hasBasicD();
12206
12207 case MVT::v4f32:
12208 case MVT::v2f64:
12209 return Subtarget.hasExtLSX();
12210
12211 case MVT::v8f32:
12212 case MVT::v4f64:
12213 return Subtarget.hasExtLASX();
12214
12215 default:
12216 return false;
12217 }
12218}
12219
12220SDValue LoongArchTargetLowering::getSqrtEstimate(SDValue Operand,
12221 SelectionDAG &DAG, int Enabled,
12222 int &RefinementSteps,
12223 bool &UseOneConstNR,
12224 bool Reciprocal) const {
12225 assert(Enabled != ReciprocalEstimate::Disabled &&
12226 "Enabled should never be Disabled here");
12227
12228 if (!Subtarget.hasFrecipe())
12229 return SDValue();
12230
12231 SDLoc DL(Operand);
12232 EVT VT = Operand.getValueType();
12233
12234 // Check supported types.
12235 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
12236 return SDValue();
12237
12238 // Handle refinement steps.
12239 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12240 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12241
12242 // LoongArch only has FRSQRTE which is 1.0 / sqrt(x).
12243 UseOneConstNR = false;
12244 SDValue Rsqrt = DAG.getNode(Opcode: LoongArchISD::FRSQRTE, DL, VT, Operand);
12245
12246 // If the caller wants 1.0 / sqrt(x), or if further refinement steps
12247 // are needed (which rely on the reciprocal form), return the raw reciprocal
12248 // estimate.
12249 if (Reciprocal || RefinementSteps > 0)
12250 return Rsqrt;
12251
12252 // Otherwise, return sqrt(x) by multiplying with the operand.
12253 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Operand, N2: Rsqrt);
12254}
12255
12256SDValue LoongArchTargetLowering::getRecipEstimate(SDValue Operand,
12257 SelectionDAG &DAG,
12258 int Enabled,
12259 int &RefinementSteps) const {
12260 assert(Enabled != ReciprocalEstimate::Disabled &&
12261 "Enabled should never be Disabled here");
12262
12263 if (!Subtarget.hasFrecipe())
12264 return SDValue();
12265
12266 SDLoc DL(Operand);
12267 EVT VT = Operand.getValueType();
12268
12269 // Check supported types.
12270 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
12271 return SDValue();
12272
12273 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12274 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12275
12276 // FRECIPE computes 1.0 / x.
12277 return DAG.getNode(Opcode: LoongArchISD::FRECIPE, DL, VT, Operand);
12278}
12279
12280//===----------------------------------------------------------------------===//
12281// LoongArch Inline Assembly Support
12282//===----------------------------------------------------------------------===//
12283
12284LoongArchTargetLowering::ConstraintType
12285LoongArchTargetLowering::getConstraintType(StringRef Constraint) const {
12286 // LoongArch specific constraints in GCC: config/loongarch/constraints.md
12287 //
12288 // 'f': A floating-point register (if available).
12289 // 'k': A memory operand whose address is formed by a base register and
12290 // (optionally scaled) index register.
12291 // 'l': A signed 16-bit constant.
12292 // 'm': A memory operand whose address is formed by a base register and
12293 // offset that is suitable for use in instructions with the same
12294 // addressing mode as st.w and ld.w.
12295 // 'q': A general-purpose register except for $r0 and $r1 (for the csrxchg
12296 // instruction)
12297 // 'I': A signed 12-bit constant (for arithmetic instructions).
12298 // 'J': Integer zero.
12299 // 'K': An unsigned 12-bit constant (for logic instructions).
12300 // "ZB": An address that is held in a general-purpose register. The offset is
12301 // zero.
12302 // "ZC": A memory operand whose address is formed by a base register and
12303 // offset that is suitable for use in instructions with the same
12304 // addressing mode as ll.w and sc.w.
12305 if (Constraint.size() == 1) {
12306 switch (Constraint[0]) {
12307 default:
12308 break;
12309 case 'f':
12310 case 'q':
12311 return C_RegisterClass;
12312 case 'l':
12313 case 'I':
12314 case 'J':
12315 case 'K':
12316 return C_Immediate;
12317 case 'k':
12318 return C_Memory;
12319 }
12320 }
12321
12322 if (Constraint == "ZC" || Constraint == "ZB")
12323 return C_Memory;
12324
12325 // 'm' is handled here.
12326 return TargetLowering::getConstraintType(Constraint);
12327}
12328
12329InlineAsm::ConstraintCode LoongArchTargetLowering::getInlineAsmMemConstraint(
12330 StringRef ConstraintCode) const {
12331 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
12332 .Case(S: "k", Value: InlineAsm::ConstraintCode::k)
12333 .Case(S: "ZB", Value: InlineAsm::ConstraintCode::ZB)
12334 .Case(S: "ZC", Value: InlineAsm::ConstraintCode::ZC)
12335 .Default(Value: TargetLowering::getInlineAsmMemConstraint(ConstraintCode));
12336}
12337
12338std::pair<unsigned, const TargetRegisterClass *>
12339LoongArchTargetLowering::getRegForInlineAsmConstraint(
12340 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
12341 // First, see if this is a constraint that directly corresponds to a LoongArch
12342 // register class.
12343 if (Constraint.size() == 1) {
12344 switch (Constraint[0]) {
12345 case 'r':
12346 // TODO: Support fixed vectors up to GRLen?
12347 if (VT.isVector())
12348 break;
12349 return std::make_pair(x: 0U, y: &LoongArch::GPRRegClass);
12350 case 'q':
12351 return std::make_pair(x: 0U, y: &LoongArch::GPRNoR0R1RegClass);
12352 case 'f':
12353 if (Subtarget.hasBasicF() && VT == MVT::f32)
12354 return std::make_pair(x: 0U, y: &LoongArch::FPR32RegClass);
12355 if (Subtarget.hasBasicD() && VT == MVT::f64)
12356 return std::make_pair(x: 0U, y: &LoongArch::FPR64RegClass);
12357 if (Subtarget.hasExtLSX() &&
12358 TRI->isTypeLegalForClass(RC: LoongArch::LSX128RegClass, T: VT))
12359 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
12360 if (Subtarget.hasExtLSX() && VT == MVT::i128)
12361 return std::make_pair(x: 0U, y: &LoongArch::LSX128RegClass);
12362 if (Subtarget.hasExtLASX() &&
12363 TRI->isTypeLegalForClass(RC: LoongArch::LASX256RegClass, T: VT))
12364 return std::make_pair(x: 0U, y: &LoongArch::LASX256RegClass);
12365 break;
12366 default:
12367 break;
12368 }
12369 }
12370
12371 // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen
12372 // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm
12373 // constraints while the official register name is prefixed with a '$'. So we
12374 // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.)
12375 // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is
12376 // case insensitive, so no need to convert the constraint to upper case here.
12377 //
12378 // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly
12379 // decode the usage of register name aliases into their official names. And
12380 // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use
12381 // official register names.
12382 if (Constraint.starts_with(Prefix: "{$r") || Constraint.starts_with(Prefix: "{$f") ||
12383 Constraint.starts_with(Prefix: "{$vr") || Constraint.starts_with(Prefix: "{$xr")) {
12384 bool IsFP = Constraint[2] == 'f';
12385 std::pair<StringRef, StringRef> Temp = Constraint.split(Separator: '$');
12386 std::pair<unsigned, const TargetRegisterClass *> R;
12387 R = TargetLowering::getRegForInlineAsmConstraint(
12388 TRI, Constraint: join_items(Separator: "", Items&: Temp.first, Items&: Temp.second), VT);
12389 // Match those names to the widest floating point register type available.
12390 if (IsFP) {
12391 unsigned RegNo = R.first;
12392 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
12393 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
12394 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
12395 return std::make_pair(x&: DReg, y: &LoongArch::FPR64RegClass);
12396 }
12397 }
12398 }
12399 return R;
12400 }
12401
12402 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
12403}
12404
12405void LoongArchTargetLowering::LowerAsmOperandForConstraint(
12406 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
12407 SelectionDAG &DAG) const {
12408 // Currently only support length 1 constraints.
12409 if (Constraint.size() == 1) {
12410 switch (Constraint[0]) {
12411 case 'l':
12412 // Validate & create a 16-bit signed immediate operand.
12413 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
12414 uint64_t CVal = C->getSExtValue();
12415 if (isInt<16>(x: CVal))
12416 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
12417 VT: Subtarget.getGRLenVT()));
12418 }
12419 return;
12420 case 'I':
12421 // Validate & create a 12-bit signed immediate operand.
12422 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
12423 uint64_t CVal = C->getSExtValue();
12424 if (isInt<12>(x: CVal))
12425 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
12426 VT: Subtarget.getGRLenVT()));
12427 }
12428 return;
12429 case 'J':
12430 // Validate & create an integer zero operand.
12431 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
12432 if (C->getZExtValue() == 0)
12433 Ops.push_back(
12434 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
12435 return;
12436 case 'K':
12437 // Validate & create a 12-bit unsigned immediate operand.
12438 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
12439 uint64_t CVal = C->getZExtValue();
12440 if (isUInt<12>(x: CVal))
12441 Ops.push_back(
12442 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getGRLenVT()));
12443 }
12444 return;
12445 default:
12446 break;
12447 }
12448 }
12449 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
12450}
12451
12452#define GET_REGISTER_MATCHER
12453#include "LoongArchGenAsmMatcher.inc"
12454
12455Register
12456LoongArchTargetLowering::getRegisterByName(const char *RegName, LLT VT,
12457 const MachineFunction &MF) const {
12458 std::pair<StringRef, StringRef> Name = StringRef(RegName).split(Separator: '$');
12459 std::string NewRegName = Name.second.str();
12460 Register Reg = MatchRegisterAltName(Name: NewRegName);
12461 if (!Reg)
12462 Reg = MatchRegisterName(Name: NewRegName);
12463 if (!Reg)
12464 return Reg;
12465 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
12466 if (!ReservedRegs.test(Idx: Reg))
12467 report_fatal_error(reason: Twine("Trying to obtain non-reserved register \"" +
12468 StringRef(RegName) + "\"."));
12469 return Reg;
12470}
12471
12472bool LoongArchTargetLowering::decomposeMulByConstant(LLVMContext &Context,
12473 EVT VT, SDValue C) const {
12474 // TODO: Support vectors.
12475 if (!VT.isScalarInteger())
12476 return false;
12477
12478 // Omit the optimization if the data size exceeds GRLen.
12479 if (VT.getSizeInBits() > Subtarget.getGRLen())
12480 return false;
12481
12482 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
12483 const APInt &Imm = ConstNode->getAPIntValue();
12484 // Break MUL into (SLLI + ADD/SUB) or ALSL.
12485 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
12486 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
12487 return true;
12488 // Break MUL into (ALSL x, (SLLI x, imm0), imm1).
12489 if (ConstNode->hasOneUse() &&
12490 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
12491 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
12492 return true;
12493 // Break (MUL x, imm) into (ADD (SLLI x, s0), (SLLI x, s1)),
12494 // in which the immediate has two set bits. Or Break (MUL x, imm)
12495 // into (SUB (SLLI x, s0), (SLLI x, s1)), in which the immediate
12496 // equals to (1 << s0) - (1 << s1).
12497 if (ConstNode->hasOneUse() && !(Imm.sge(RHS: -2048) && Imm.sle(RHS: 4095))) {
12498 unsigned Shifts = Imm.countr_zero();
12499 // Reject immediates which can be composed via a single LUI.
12500 if (Shifts >= 12)
12501 return false;
12502 // Reject multiplications can be optimized to
12503 // (SLLI (ALSL x, x, 1/2/3/4), s).
12504 APInt ImmPop = Imm.ashr(ShiftAmt: Shifts);
12505 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
12506 return false;
12507 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`,
12508 // since it needs one more instruction than other 3 cases.
12509 APInt ImmSmall = APInt(Imm.getBitWidth(), 1ULL << Shifts, true);
12510 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
12511 (ImmSmall - Imm).isPowerOf2())
12512 return true;
12513 }
12514 }
12515
12516 return false;
12517}
12518
12519bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
12520 const AddrMode &AM,
12521 Type *Ty, unsigned AS,
12522 Instruction *I) const {
12523 // LoongArch has four basic addressing modes:
12524 // 1. reg
12525 // 2. reg + 12-bit signed offset
12526 // 3. reg + 14-bit signed offset left-shifted by 2
12527 // 4. reg1 + reg2
12528 // TODO: Add more checks after support vector extension.
12529
12530 // No global is ever allowed as a base.
12531 if (AM.BaseGV)
12532 return false;
12533
12534 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
12535 // with `UAL` feature.
12536 if (!isInt<12>(x: AM.BaseOffs) &&
12537 !(isShiftedInt<14, 2>(x: AM.BaseOffs) && Subtarget.hasUAL()))
12538 return false;
12539
12540 switch (AM.Scale) {
12541 case 0:
12542 // "r+i" or just "i", depending on HasBaseReg.
12543 break;
12544 case 1:
12545 // "r+r+i" is not allowed.
12546 if (AM.HasBaseReg && AM.BaseOffs)
12547 return false;
12548 // Otherwise we have "r+r" or "r+i".
12549 break;
12550 case 2:
12551 // "2*r+r" or "2*r+i" is not allowed.
12552 if (AM.HasBaseReg || AM.BaseOffs)
12553 return false;
12554 // Allow "2*r" as "r+r".
12555 break;
12556 default:
12557 return false;
12558 }
12559
12560 return true;
12561}
12562
12563bool LoongArchTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
12564 return isInt<12>(x: Imm);
12565}
12566
12567bool LoongArchTargetLowering::isLegalAddImmediate(int64_t Imm) const {
12568 return isInt<12>(x: Imm);
12569}
12570
12571bool LoongArchTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
12572 // Zexts are free if they can be combined with a load.
12573 // Don't advertise i32->i64 zextload as being free for LA64. It interacts
12574 // poorly with type legalization of compares preferring sext.
12575 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
12576 EVT MemVT = LD->getMemoryVT();
12577 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
12578 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
12579 LD->getExtensionType() == ISD::ZEXTLOAD))
12580 return true;
12581 }
12582
12583 return TargetLowering::isZExtFree(Val, VT2);
12584}
12585
12586bool LoongArchTargetLowering::isSExtCheaperThanZExt(EVT SrcVT,
12587 EVT DstVT) const {
12588 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
12589}
12590
12591bool LoongArchTargetLowering::signExtendConstant(const ConstantInt *CI) const {
12592 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
12593}
12594
12595bool LoongArchTargetLowering::hasAndNotCompare(SDValue Y) const {
12596 // TODO: Support vectors.
12597 if (Y.getValueType().isVector())
12598 return false;
12599
12600 return true;
12601}
12602
12603ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
12604 // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
12605 return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
12606}
12607
12608bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
12609 Type *Ty, bool IsSigned) const {
12610 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
12611 return true;
12612
12613 return IsSigned;
12614}
12615
12616bool LoongArchTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
12617 // Return false to suppress the unnecessary extensions if the LibCall
12618 // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
12619 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
12620 Type.getSizeInBits() < Subtarget.getGRLen()))
12621 return false;
12622 return true;
12623}
12624
12625// memcpy, and other memory intrinsics, typically tries to use wider load/store
12626// if the source/dest is aligned and the copy size is large enough. We therefore
12627// want to align such objects passed to memory intrinsics.
12628bool LoongArchTargetLowering::shouldAlignPointerArgs(CallInst *CI,
12629 unsigned &MinSize,
12630 Align &PrefAlign) const {
12631 if (!isa<MemIntrinsic>(Val: CI))
12632 return false;
12633
12634 if (Subtarget.is64Bit()) {
12635 MinSize = 8;
12636 PrefAlign = Align(8);
12637 } else {
12638 MinSize = 4;
12639 PrefAlign = Align(4);
12640 }
12641
12642 return true;
12643}
12644
12645TargetLoweringBase::LegalizeTypeAction
12646LoongArchTargetLowering::getPreferredVectorAction(MVT VT) const {
12647 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
12648 VT.getVectorElementType() != MVT::i1)
12649 return TypeWidenVector;
12650
12651 return TargetLoweringBase::getPreferredVectorAction(VT);
12652}
12653
12654bool LoongArchTargetLowering::splitValueIntoRegisterParts(
12655 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
12656 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
12657 bool IsABIRegCopy = CC.has_value();
12658 EVT ValueVT = Val.getValueType();
12659
12660 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12661 PartVT == MVT::f32) {
12662 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
12663 // nan, and cast to f32.
12664 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
12665 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
12666 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
12667 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
12668 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Val);
12669 Parts[0] = Val;
12670 return true;
12671 }
12672
12673 return false;
12674}
12675
12676SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
12677 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
12678 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
12679 bool IsABIRegCopy = CC.has_value();
12680
12681 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12682 PartVT == MVT::f32) {
12683 SDValue Val = Parts[0];
12684
12685 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
12686 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
12687 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
12688 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
12689 return Val;
12690 }
12691
12692 return SDValue();
12693}
12694
12695MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
12696 CallingConv::ID CC,
12697 EVT VT) const {
12698 // Use f32 to pass f16.
12699 if (VT == MVT::f16 && Subtarget.hasBasicF())
12700 return MVT::f32;
12701
12702 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
12703}
12704
12705unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
12706 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
12707 // Use f32 to pass f16.
12708 if (VT == MVT::f16 && Subtarget.hasBasicF())
12709 return 1;
12710
12711 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
12712}
12713
12714void LoongArchTargetLowering::computeKnownBitsForTargetNode(
12715 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
12716 const SelectionDAG &DAG, unsigned Depth) const {
12717 unsigned Opc = Op.getOpcode();
12718 Known.resetAll();
12719 switch (Opc) {
12720 default:
12721 break;
12722 case LoongArchISD::VANYNONZERO:
12723 case LoongArchISD::VALLZERO:
12724 case LoongArchISD::VALL_ZERO:
12725 case LoongArchISD::VANY_ZERO:
12726 case LoongArchISD::VALL_NONZERO:
12727 case LoongArchISD::VANY_NONZERO: {
12728 // MOVCF2GR zero-extend the i1 cond to GPR.
12729 Known.Zero.setBitsFrom(1);
12730 break;
12731 }
12732 case LoongArchISD::VPICK_ZEXT_ELT: {
12733 assert(isa<VTSDNode>(Op->getOperand(2)) && "Unexpected operand!");
12734 EVT VT = cast<VTSDNode>(Val: Op->getOperand(Num: 2))->getVT();
12735 unsigned VTBits = VT.getScalarSizeInBits();
12736 assert(Known.getBitWidth() >= VTBits && "Unexpected width!");
12737 Known.Zero.setBitsFrom(VTBits);
12738 break;
12739 }
12740 case LoongArchISD::BSTRINS: {
12741 // BSTRINS X, Y, msb, lsb: replace X[msb:lsb] with Y[msb-lsb:0].
12742 unsigned Msb = Op.getConstantOperandVal(i: 2);
12743 unsigned Lsb = Op.getConstantOperandVal(i: 3);
12744 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
12745 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
12746 Known.insertBits(SubBits: Known2.trunc(BitWidth: Msb - Lsb + 1), BitPosition: Lsb);
12747 break;
12748 }
12749 case LoongArchISD::BSTRPICK: {
12750 // BSTRPICK X, msb, lsb: zero-extend X[msb:lsb].
12751 unsigned BitWidth = Known.getBitWidth();
12752 unsigned Msb = Op.getConstantOperandVal(i: 1);
12753 unsigned Lsb = Op.getConstantOperandVal(i: 2);
12754 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
12755 Known = Known.extractBits(NumBits: Msb - Lsb + 1, BitPosition: Lsb).zext(BitWidth);
12756 break;
12757 }
12758 }
12759}
12760
12761unsigned LoongArchTargetLowering::ComputeNumSignBitsForTargetNode(
12762 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
12763 unsigned Depth) const {
12764 switch (Op.getOpcode()) {
12765 default:
12766 break;
12767 case LoongArchISD::BSTRINS: {
12768 unsigned BitWidth = Op.getScalarValueSizeInBits();
12769 unsigned Msb = Op.getConstantOperandVal(i: 2);
12770 unsigned Lsb = Op.getConstantOperandVal(i: 3);
12771 // The bits above msb come from X.
12772 if (Msb != BitWidth - 1) {
12773 unsigned Tmp = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
12774 return std::min(a: Tmp, b: BitWidth - 1 - Msb);
12775 }
12776 // The high bits come from Y shifted left by lsb.
12777 unsigned Tmp = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
12778 return Tmp > Lsb ? Tmp - Lsb : 1;
12779 }
12780 }
12781 return 1;
12782}
12783
12784bool LoongArchTargetLowering::SimplifyDemandedBitsForTargetNode(
12785 SDValue Op, const APInt &OriginalDemandedBits,
12786 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
12787 unsigned Depth) const {
12788 EVT VT = Op.getValueType();
12789 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
12790 unsigned Opc = Op.getOpcode();
12791 switch (Opc) {
12792 default:
12793 break;
12794 case LoongArchISD::CRC_W_B_W:
12795 case LoongArchISD::CRC_W_H_W:
12796 case LoongArchISD::CRCC_W_B_W:
12797 case LoongArchISD::CRCC_W_H_W: {
12798 KnownBits KnownSrc;
12799 APInt DemandedSrcBits =
12800 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: (Opc == LoongArchISD::CRC_W_B_W ||
12801 Opc == LoongArchISD::CRCC_W_B_W)
12802 ? 8
12803 : 16);
12804 return SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: DemandedSrcBits,
12805 DemandedElts: OriginalDemandedElts, Known&: KnownSrc, TLO, Depth: Depth + 1);
12806 }
12807 case LoongArchISD::VMSKLTZ:
12808 case LoongArchISD::XVMSKLTZ: {
12809 SDValue Src = Op.getOperand(i: 0);
12810 MVT SrcVT = Src.getSimpleValueType();
12811 unsigned SrcBits = SrcVT.getScalarSizeInBits();
12812 unsigned NumElts = SrcVT.getVectorNumElements();
12813
12814 // If we don't need the sign bits at all just return zero.
12815 if (OriginalDemandedBits.countr_zero() >= NumElts)
12816 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
12817
12818 // Only demand the vector elements of the sign bits we need.
12819 APInt KnownUndef, KnownZero;
12820 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(width: NumElts);
12821 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedElts, KnownUndef, KnownZero,
12822 TLO, Depth: Depth + 1))
12823 return true;
12824
12825 Known.Zero = KnownZero.zext(width: BitWidth);
12826 Known.Zero.setHighBits(BitWidth - NumElts);
12827
12828 // [X]VMSKLTZ only uses the MSB from each vector element.
12829 KnownBits KnownSrc;
12830 APInt DemandedSrcBits = APInt::getSignMask(BitWidth: SrcBits);
12831 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, Known&: KnownSrc, TLO,
12832 Depth: Depth + 1))
12833 return true;
12834
12835 if (KnownSrc.One[SrcBits - 1])
12836 Known.One.setLowBits(NumElts);
12837 else if (KnownSrc.Zero[SrcBits - 1])
12838 Known.Zero.setLowBits(NumElts);
12839
12840 // Attempt to avoid multi-use ops if we don't need anything from it.
12841 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
12842 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
12843 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT, Operand: NewSrc));
12844 return false;
12845 }
12846 }
12847
12848 return TargetLowering::SimplifyDemandedBitsForTargetNode(
12849 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
12850}
12851
12852bool LoongArchTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
12853 unsigned Opc = VecOp.getOpcode();
12854
12855 // Assume target opcodes can't be scalarized.
12856 // TODO - do we have any exceptions?
12857 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
12858 return false;
12859
12860 // If the vector op is not supported, try to convert to scalar.
12861 EVT VecVT = VecOp.getValueType();
12862 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
12863 return true;
12864
12865 // If the vector op is supported, but the scalar op is not, the transform may
12866 // not be worthwhile.
12867 EVT ScalarVT = VecVT.getScalarType();
12868 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
12869}
12870
12871TargetLowering::ExtractSubvectorCost
12872LoongArchTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
12873 unsigned Index) const {
12874 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
12875 return ExtractSubvectorCost::Expensive;
12876
12877 // Extract a 128-bit subvector from index 0 of a 256-bit vector is free.
12878 if (Index == 0)
12879 return ExtractSubvectorCost::Free;
12880 return ExtractSubvectorCost::Expensive;
12881}
12882
12883bool LoongArchTargetLowering::isExtractVecEltCheap(EVT VT,
12884 unsigned Index) const {
12885 EVT EltVT = VT.getScalarType();
12886
12887 // Extract a scalar FP value from index 0 of a vector is free.
12888 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
12889}
12890
12891bool LoongArchTargetLowering::hasInlineStackProbe(
12892 const MachineFunction &MF) const {
12893
12894 // If the function specifically requests inline stack probes, emit them.
12895 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
12896 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
12897 "inline-asm";
12898
12899 return false;
12900}
12901
12902unsigned LoongArchTargetLowering::getStackProbeSize(const MachineFunction &MF,
12903 Align StackAlign) const {
12904 // The default stack probe size is 4096 if the function has no
12905 // stack-probe-size attribute.
12906 const Function &Fn = MF.getFunction();
12907 unsigned StackProbeSize =
12908 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
12909 // Round down to the stack alignment.
12910 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
12911 return StackProbeSize ? StackProbeSize : StackAlign.value();
12912}
12913
12914SDValue
12915LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
12916 SelectionDAG &DAG) const {
12917 MachineFunction &MF = DAG.getMachineFunction();
12918 if (!hasInlineStackProbe(MF))
12919 return SDValue();
12920
12921 const MVT GRLenVT = Subtarget.getGRLenVT();
12922 // Get the inputs.
12923 SDValue Chain = Op.getOperand(i: 0);
12924 SDValue Size = Op.getOperand(i: 1);
12925
12926 const MaybeAlign Align =
12927 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
12928 const SDLoc dl(Op);
12929 const EVT VT = Op.getValueType();
12930
12931 // Construct the new SP value in a GPR.
12932 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: LoongArch::R3, VT: GRLenVT);
12933 Chain = SP.getValue(R: 1);
12934 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: GRLenVT, N1: SP, N2: Size);
12935 if (Align)
12936 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
12937 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
12938
12939 // Set the real SP to the new value with a probing loop.
12940 Chain = DAG.getNode(Opcode: LoongArchISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
12941 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
12942}
12943
12944MachineBasicBlock *
12945LoongArchTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
12946 MachineBasicBlock *MBB) const {
12947 MachineFunction &MF = *MBB->getParent();
12948 MachineBasicBlock::iterator MBBI = MI.getIterator();
12949 DebugLoc DL = MBB->findDebugLoc(MBBI);
12950 const Register TargetReg = MI.getOperand(i: 0).getReg();
12951
12952 const LoongArchInstrInfo *TII = Subtarget.getInstrInfo();
12953 const bool IsLA64 = Subtarget.is64Bit();
12954 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
12955 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
12956 const uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
12957
12958 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
12959 MachineBasicBlock *const LoopTestMBB =
12960 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12961 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
12962 MachineBasicBlock *const ExitMBB =
12963 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
12964 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
12965 const Register SPReg = LoongArch::R3;
12966 const Register ScratchReg =
12967 MF.getRegInfo().createVirtualRegister(RegClass: &LoongArch::GPRRegClass);
12968
12969 // ScratchReg = ProbeSize
12970 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
12971
12972 // LoopTest:
12973 // sub.{w/d} $sp, $sp, ScratchReg
12974 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12975 MCID: TII->get(Opcode: IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W), DestReg: SPReg)
12976 .addReg(RegNo: SPReg)
12977 .addReg(RegNo: ScratchReg);
12978
12979 // st.{w/d} $zero, $sp, 0
12980 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
12981 MCID: TII->get(Opcode: IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
12982 .addReg(RegNo: LoongArch::R0)
12983 .addReg(RegNo: SPReg)
12984 .addImm(Val: 0);
12985
12986 // bltu TargetReg, $sp, LoopTest
12987 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::BLTU))
12988 .addReg(RegNo: TargetReg)
12989 .addReg(RegNo: SPReg)
12990 .addMBB(MBB: LoopTestMBB);
12991
12992 // move $sp, TargetReg
12993 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: LoongArch::OR), DestReg: SPReg)
12994 .addReg(RegNo: TargetReg)
12995 .addReg(RegNo: LoongArch::R0);
12996
12997 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
12998 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
12999
13000 LoopTestMBB->addSuccessor(Succ: ExitMBB);
13001 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
13002 MBB->addSuccessor(Succ: LoopTestMBB);
13003
13004 MI.eraseFromParent();
13005 MF.getInfo<LoongArchMachineFunctionInfo>()->setDynamicAllocation();
13006 return ExitMBB->begin()->getParent();
13007}
13008