1//===-- HexagonISelLoweringHLX.cpp --- Lowering HLX operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelLowering.h"
10#include "HexagonRegisterInfo.h"
11#include "HexagonSubtarget.h"
12#include "llvm/ADT/SmallVector.h"
13#include "llvm/CodeGen/MachineRegisterInfo.h"
14#include "llvm/CodeGen/TargetInstrInfo.h"
15#include "llvm/IR/IntrinsicsHexagon.h"
16
17using namespace llvm;
18
19// 2048B HLX types
20static const MVT AllTypes2048B[] = {MVT::v2048i8, MVT::v1024i16, MVT::v512i32,
21 MVT::v4096i8, MVT::v2048i16, MVT::v1024i32};
22static const MVT LegalTypes2048B[] = {MVT::v1024i16, MVT::v2048i16,
23 MVT::v1024i32};
24static const MVT LegalPredTypes2048B[] = {MVT::v1024i1, MVT::v2048i1};
25
26void HexagonTargetLowering::initializeHLXLowering() {
27 ArrayRef<MVT> AllTypes = AllTypes2048B;
28 ArrayRef<MVT> LegalTypes = LegalTypes2048B;
29 ArrayRef<MVT> LegalPredTypes = LegalPredTypes2048B;
30 MVT HalfWordPairTy = MVT::v2048i16;
31
32 // TODO: Remove i32 types for single HLX register
33 // ISA only supports halfword, byte operations on single register
34 addRegisterClass(VT: MVT::v2048i8, RC: &Hexagon::HlxLRRegClass);
35 addRegisterClass(VT: MVT::v1024i16, RC: &Hexagon::HlxLRRegClass);
36 addRegisterClass(VT: MVT::v512i32, RC: &Hexagon::HlxLRRegClass);
37 addRegisterClass(VT: MVT::v1024i1, RC: &Hexagon::HlxLRRegClass);
38 addRegisterClass(VT: MVT::v4096i8, RC: &Hexagon::HlxLLRRegClass);
39 addRegisterClass(VT: MVT::v2048i16, RC: &Hexagon::HlxLLRRegClass);
40 addRegisterClass(VT: MVT::v1024i32, RC: &Hexagon::HlxLLRRegClass);
41 addRegisterClass(VT: MVT::v2048i1, RC: &Hexagon::HlxLLRRegClass);
42
43 // Route element-sensitive operations unsupported by a type to
44 // LowerHlxOperation.
45 static const unsigned UnsupportedOnByteAndSingleWord[] = {
46 ISD::ADD, ISD::SUB, ISD::UADDSAT, ISD::SADDSAT,
47 ISD::USUBSAT, ISD::SSUBSAT, ISD::MUL, ISD::SHL,
48 ISD::SRL, ISD::SRA, ISD::SPLAT_VECTOR, ISD::VSELECT,
49 ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
50 ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::AVGFLOORS,
51 ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU};
52 SmallVector<MVT, 8> UnsupportedTypes;
53 for (MVT T : AllTypes)
54 if (!llvm::is_contained(Range&: LegalTypes, Element: T))
55 UnsupportedTypes.push_back(Elt: T);
56 setOperationAction(Ops: UnsupportedOnByteAndSingleWord, VTs: UnsupportedTypes, Action: Custom);
57
58 // Catch VSELECT before type legalization widens an unsupported predicate.
59 for (MVT T : UnsupportedTypes) {
60 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
61 if (!llvm::is_contained(Range&: LegalPredTypes, Element: PredTy))
62 setOperationAction(Op: ISD::VSELECT, VT: PredTy, Action: Custom);
63 }
64
65 // Reject unsupported halfword-pair operations in LowerHlxOperation.
66 for (unsigned Opc : {ISD::MUL, ISD::ABDS, ISD::ABDU, ISD::AVGFLOORS,
67 ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU})
68 setOperationAction(Op: Opc, VT: HalfWordPairTy, Action: Custom);
69
70 for (MVT T : UnsupportedTypes) {
71 setCondCodeAction(CCs: ISD::SETGT, VT: T, Action: Custom);
72 setCondCodeAction(CCs: ISD::SETUGT, VT: T, Action: Custom);
73 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Custom);
74 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Custom);
75 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Custom);
76 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Custom);
77 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Custom);
78 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Custom);
79 setCondCodeAction(CCs: ISD::SETEQ, VT: T, Action: Custom);
80 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Custom);
81 }
82
83 for (MVT T : LegalTypes) {
84 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
85 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
86 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
87 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
88 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
89 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
90 }
91
92 setOperationAction(Op: ISD::MUL, VT: MVT::v1024i16, Action: Legal);
93 setOperationAction(Op: ISD::MUL, VT: MVT::v1024i32, Action: Legal);
94
95 setOperationAction(Op: ISD::MULHU, VT: MVT::v1024i16, Action: Legal);
96 setOperationAction(Op: ISD::MULHS, VT: MVT::v1024i16, Action: Legal);
97
98 for (MVT T : AllTypes) {
99 setOperationAction(Op: ISD::AND, VT: T, Action: Legal);
100 setOperationAction(Op: ISD::OR, VT: T, Action: Legal);
101 setOperationAction(Op: ISD::XOR, VT: T, Action: Legal);
102 }
103
104 setOperationAction(Op: ISD::SRA, VT: MVT::v1024i16, Action: Legal);
105 setOperationAction(Op: ISD::SRA, VT: MVT::v1024i32, Action: Legal);
106 setOperationAction(Op: ISD::SRL, VT: MVT::v1024i16, Action: Legal);
107 setOperationAction(Op: ISD::SRL, VT: MVT::v1024i32, Action: Legal);
108 setOperationAction(Op: ISD::SHL, VT: MVT::v1024i16, Action: Legal);
109 setOperationAction(Op: ISD::SHL, VT: MVT::v1024i32, Action: Legal);
110
111 for (MVT T : LegalTypes) {
112 // GT comparisons are legal
113 setCondCodeAction(CCs: ISD::SETGT, VT: T, Action: Legal);
114 setCondCodeAction(CCs: ISD::SETUGT, VT: T, Action: Legal);
115 // Expand all non-GT comparisons
116 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand);
117 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand);
118 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand);
119 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand);
120 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand);
121 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand);
122 // Expand equality comparisons
123 setCondCodeAction(CCs: ISD::SETEQ, VT: T, Action: Expand);
124 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand);
125 }
126
127 // Add SETCC operation for predicate result types
128 for (MVT T : LegalPredTypes)
129 setOperationAction(Op: ISD::SETCC, VT: T, Action: Legal);
130
131 for (MVT T : LegalTypes) {
132 setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal);
133 setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal);
134 setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal);
135 setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal);
136 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
137 }
138
139 // absdiff and average lack halfword-pair instructions.
140 MVT HalfWordSingleTy = MVT::v1024i16;
141 MVT WordPairTy = MVT::v1024i32;
142 for (MVT T : {HalfWordSingleTy, WordPairTy}) {
143 setOperationAction(Op: ISD::ABDS, VT: T, Action: Legal);
144 setOperationAction(Op: ISD::ABDU, VT: T, Action: Legal);
145 setOperationAction(Op: ISD::AVGFLOORS, VT: T, Action: Legal);
146 setOperationAction(Op: ISD::AVGFLOORU, VT: T, Action: Legal);
147 setOperationAction(Op: ISD::AVGCEILS, VT: T, Action: Legal);
148 setOperationAction(Op: ISD::AVGCEILU, VT: T, Action: Legal);
149 }
150
151 setOperationAction(Op: ISD::SELECT, VT: MVT::v1024i16, Action: Custom);
152 setOperationAction(Op: ISD::SELECT, VT: MVT::v1024i32, Action: Custom);
153
154 // Splat supports halfword singles and word pairs only.
155 for (MVT T : LegalTypes)
156 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
157
158 setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i8, Action: Legal);
159 setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i16, Action: Legal);
160 setOperationAction(Op: ISD::LOAD, VT: MVT::v512i32, Action: Legal);
161 setOperationAction(Op: ISD::STORE, VT: MVT::v2048i8, Action: Legal);
162 setOperationAction(Op: ISD::STORE, VT: MVT::v1024i16, Action: Legal);
163 setOperationAction(Op: ISD::STORE, VT: MVT::v512i32, Action: Legal);
164
165 setOperationAction(Op: ISD::MLOAD, VT: MVT::v1024i16, Action: Custom);
166 setOperationAction(Op: ISD::MSTORE, VT: MVT::v1024i16, Action: Custom);
167
168 setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v1024i32, Action: Legal);
169 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v1024i32, Action: Legal);
170 setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v2048i16, Action: Legal);
171
172 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v1024i16, Action: Legal);
173 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2048i8, Action: Legal);
174}
175
176bool HexagonTargetLowering::isHlxSingleTy(MVT Ty) const {
177 return Subtarget.isHLXVectorType(VecTy: Ty) &&
178 Ty.getSizeInBits() == 8 * Subtarget.getHLXVectorLength();
179}
180
181bool HexagonTargetLowering::isHlxPairTy(MVT Ty) const {
182 return Subtarget.isHLXVectorType(VecTy: Ty) &&
183 Ty.getSizeInBits() == 16 * Subtarget.getHLXVectorLength();
184}
185
186bool HexagonTargetLowering::isHlxOperation(SDNode *N, SelectionDAG &DAG) const {
187 if (!Subtarget.useHLXOps())
188 return false;
189 // If the type of any result, or any operand type are HLX vector types,
190 // this is an HLX operation.
191 auto IsHLXTy = [this](EVT Ty) {
192 return Ty.isSimple() && Subtarget.isHLXVectorType(VecTy: Ty.getSimpleVT());
193 };
194 auto IsHLXOp = [this](SDValue Op) {
195 return Op.getValueType().isSimple() && Subtarget.isHLXVectorType(VecTy: ty(Op));
196 };
197 if (llvm::any_of(Range: N->values(), P: IsHLXTy) || llvm::any_of(Range: N->ops(), P: IsHLXOp))
198 return true;
199
200 // No widening for now
201 return false;
202}
203
204bool HexagonTargetLowering::allowsHlxMemoryAccess(
205 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
206 // Disallow double vector stores (to prevent unnecessary
207 // store widening in DAG combiner).
208 if (VecTy.getSizeInBits() > 8 * Subtarget.getHLXVectorLength())
209 return false;
210 if (!Subtarget.isHLXVectorType(VecTy))
211 return false;
212 if (Fast)
213 *Fast = 1;
214 return true;
215}
216
217HexagonTargetLowering::TypePair
218HexagonTargetLowering::typeHlxSplit(MVT VecTy) const {
219 assert(VecTy.isVector());
220 unsigned NumElem = VecTy.getVectorNumElements();
221 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
222 MVT ElemTy = VecTy.getVectorElementType();
223 MVT HalfTy;
224 if (ElemTy == MVT::i32)
225 HalfTy = VecTy.changeVectorElementType(EltVT: MVT::i16);
226 else
227 HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem / 2);
228 return {HalfTy, HalfTy};
229}
230
231// Reload a stack-passed HLX pair argument: two contiguous sub-vector loads
232// recombined with COMBINELL. Calling-convention use only; user pair loads
233// through a pointer are unsupported (QTOOL-134974).
234SDValue HexagonTargetLowering::splitHlxPairArgLoad(
235 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue BaseAddr,
236 MVT PairTy, MachinePointerInfo PtrInfo) const {
237 unsigned HlxVLen = Subtarget.getHLXVectorLength();
238 MVT SingleTy = typeHlxSplit(VecTy: PairTy).first;
239 SDValue Base1 =
240 DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl);
241 MachineFunction &MF = DAG.getMachineFunction();
242 Align HalfAlign = DAG.getEVTAlign(MemoryVT: SingleTy);
243 MachineMemOperand *MOp0 = MF.getMachineMemOperand(
244 PtrInfo, F: MachineMemOperand::MOLoad, Size: HlxVLen, BaseAlignment: HalfAlign);
245 MachineMemOperand *MOp1 =
246 MF.getMachineMemOperand(PtrInfo: PtrInfo.getWithOffset(O: HlxVLen),
247 F: MachineMemOperand::MOLoad, Size: HlxVLen, BaseAlignment: HalfAlign);
248 SDValue Load0 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: BaseAddr, MMO: MOp0);
249 SDValue Load1 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: Base1, MMO: MOp1);
250 return DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: PairTy, N1: Load1, N2: Load0);
251}
252
253// Spill a stack-passed HLX pair argument: extract vsub_lo/vsub_hi and store
254// them to contiguous slots, appending both chains to MemOpChains.
255// Calling-convention use only (QTOOL-134974).
256void HexagonTargetLowering::splitHlxPairArgStore(
257 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue StoreVal,
258 SDValue BaseAddr, MVT PairTy, MachinePointerInfo PtrInfo,
259 SmallVectorImpl<SDValue> &MemOpChains) const {
260 unsigned HlxVLen = Subtarget.getHLXVectorLength();
261 MVT SingleTy = typeHlxSplit(VecTy: PairTy).first;
262 SDValue Base1 =
263 DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl);
264 SDValue Lo =
265 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: StoreVal);
266 SDValue Hi =
267 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: StoreVal);
268 MemOpChains.push_back(Elt: DAG.getStore(Chain, dl, Val: Lo, Ptr: BaseAddr, PtrInfo));
269 MemOpChains.push_back(
270 Elt: DAG.getStore(Chain, dl, Val: Hi, Ptr: Base1, PtrInfo: PtrInfo.getWithOffset(O: HlxVLen)));
271}
272
273// Shared helper for splitting pair memory intrinsics into two single-vector
274// operations. Used by LowerHlxPairIntrinsic for the LX_LL16b_tmp /
275// LX_SS16b_new pair load/store intrinsics.
276static SDValue SplitHlxPairMemHelper(SDValue Op, SelectionDAG &DAG,
277 const HexagonSubtarget &ST,
278 SDValue BaseAddr,
279 SDValue StoreVal = SDValue()) {
280 auto *MemN = cast<MemSDNode>(Val: Op.getNode());
281 MVT PairTy = MemN->getMemoryVT().getSimpleVT();
282 unsigned HlxVLen = ST.getHLXVectorLength();
283 assert(PairTy.getSizeInBits() == 16 * HlxVLen &&
284 PairTy.getVectorElementType() == MVT::i32 &&
285 "Expected HLX vector pair type");
286 MVT SingleTy = PairTy.changeVectorElementType(EltVT: MVT::i16);
287 SDValue Chain = MemN->getChain();
288 const SDLoc &dl(Op);
289 SDValue Base1 =
290 DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl);
291
292 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
293 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
294 MachineFunction &MF = DAG.getMachineFunction();
295 MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: HlxVLen);
296 MOp1 = MF.getMachineMemOperand(MMO, Offset: HlxVLen, Size: HlxVLen);
297 }
298
299 if (StoreVal) {
300 // Store: split value into two halves, emit two stores, combine chains.
301 SDValue Lo =
302 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: StoreVal);
303 SDValue Hi =
304 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: StoreVal);
305 SDValue Store0 = DAG.getStore(Chain, dl, Val: Lo, Ptr: BaseAddr, MMO: MOp0);
306 SDValue Store1 = DAG.getStore(Chain, dl, Val: Hi, Ptr: Base1, MMO: MOp1);
307 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1);
308 }
309
310 // Load: emit two single-vector loads, combine with COMBINELL.
311 SDValue Load0 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: BaseAddr, MMO: MOp0);
312 SDValue Load1 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: Base1, MMO: MOp1);
313 return DAG.getMergeValues(
314 Ops: {DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: PairTy, N1: Load1, N2: Load0),
315 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1),
316 N2: Load1.getValue(R: 1))},
317 dl);
318}
319
320// Lower pair load/store intrinsics (LX_LL16b_tmp, LX_SS16b_new) by
321// splitting into two single-vector operations via SplitHlxPairMemHelper.
322SDValue HexagonTargetLowering::LowerHlxPairIntrinsic(SDValue Op,
323 SelectionDAG &DAG) const {
324 unsigned IntNo = Op.getConstantOperandVal(i: 1);
325 switch (IntNo) {
326 case Intrinsic::hexagon_LX_LL16b_tmp:
327 return SplitHlxPairMemHelper(Op, DAG, ST: Subtarget, BaseAddr: Op.getOperand(i: 2));
328 case Intrinsic::hexagon_LX_SS16b_new:
329 return SplitHlxPairMemHelper(Op, DAG, ST: Subtarget, BaseAddr: Op.getOperand(i: 2),
330 StoreVal: Op.getOperand(i: 3));
331 default:
332 return Op;
333 }
334}
335
336// Lower masked load/store operations to HLX instructions.
337//
338// HLX does not have native masked load/store instructions. Instead, we
339// lower them to a sequence of regular load/store + VSELECT operations:
340//
341// MLOAD(base, mask, passthru) => VSELECT(mask, LOAD(base), passthru)
342// MSTORE(base, value, mask) => STORE(base, VSELECT(mask, value,
343// LOAD(base)))
344//
345// For MLOAD: Load the full vector, then use VSELECT to choose between
346// loaded values (where mask is true) and passthru values (where mask is
347// false).
348//
349// For MSTORE: Load the existing memory contents, use VSELECT to merge the
350// new value (where mask is true) with old contents (where mask is false),
351// then store the result. This preserves unmasked lanes.
352SDValue HexagonTargetLowering::LowerHlxMaskedOp(SDValue Op,
353 SelectionDAG &DAG) const {
354 auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode());
355 unsigned HwLen = Subtarget.getHLXVectorLength();
356
357 // HLX has no native masked load/store. We emulate them with a full-width
358 // load (and, for stores, a load-modify-store), which reads/writes the
359 // inactive lanes. That is only sound for ordinary memory accesses: a
360 // volatile or atomic access must not touch inactive lanes, so bail out
361 // rather than silently change the program's memory semantics.
362 if (MaskN->isVolatile() || !MaskN->getMemOperand()->isUnordered()) {
363 DAG.getContext()->emitError(
364 ErrorStr: "Volatile or atomic masked load/store of HLX vector type is not "
365 "supported");
366 return SDValue();
367 }
368
369 if (MaskN->getAlign().value() % HwLen != 0) {
370 DAG.getContext()->emitError(
371 ErrorStr: "Masked load/store of HLX vector type, requires alignment of " +
372 Twine(HwLen) + " bytes for lowering to HLX");
373 return SDValue();
374 }
375
376 const SDLoc &dl(Op);
377 MachineFunction &MF = DAG.getMachineFunction();
378 SDValue Mask = MaskN->getMask();
379 SDValue Chain = MaskN->getChain();
380 SDValue Base = MaskN->getBasePtr();
381 MachineMemOperand *MemOp =
382 MaskN->getMemOperand()
383 ? MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen)
384 : nullptr;
385
386 unsigned Opc = Op->getOpcode();
387
388 switch (Opc) {
389 case ISD::MLOAD: {
390 MVT ValTy = ty(Op);
391 SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp);
392 SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru();
393 if (isUndef(Op: Thru))
394 return Load;
395 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru);
396 return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl);
397 }
398 case ISD::MSTORE: {
399 SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue();
400 auto PtrInfo = (MemOp && MemOp->getValue())
401 ? MachinePointerInfo(MemOp->getValue())
402 : MachinePointerInfo();
403 SDValue Load =
404 DAG.getLoad(VT: Value.getSimpleValueType(), dl, Chain, Ptr: Base, PtrInfo,
405 Alignment: Align(HwLen), MMOFlags: MachineMemOperand::MOLoad);
406 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: Value.getSimpleValueType(),
407 N1: Mask, N2: Value, N3: Load);
408 SDValue Store = DAG.getStore(Chain: Load.getValue(R: 1), dl, Val: VSel, Ptr: Base, MMO: MemOp);
409 return Store;
410 }
411 default: {
412 [[maybe_unused]] std::string Name =
413 "Unexpected masked operation: " + Op->getOperationName(G: &DAG);
414 llvm_unreachable(Name.c_str());
415 }
416 }
417}
418
419// Lowers ternary operations of the following types:
420// HLX_Vector = (int)? HLX_Vector: HLX_Vector
421// HLX_VectorPair = (int)? HLX_VectorPair: HLX_VectorPair
422SDValue HexagonTargetLowering::LowerHlxSelect(SDValue Op,
423 SelectionDAG &DAG) const {
424 SDValue CondVec = Op.getOperand(i: 0); // i1
425 SDValue True = Op.getOperand(i: 1);
426 SDValue False = Op.getOperand(i: 2);
427
428 const SDLoc &dl(Op);
429 MVT ValTy = ty(Op);
430
431 // Always splat condition into single-vector v1024i16 (HlxLR), not pair.
432 // LX_mux_h only needs a single HlxLR condition register, and for pairs
433 // the scalar i1 condition is uniform so the same register is reused for
434 // both halves.
435 MVT CondTy = MVT::getVectorVT(VT: MVT::i16, NumElements: ValTy.getVectorNumElements());
436 if (ty(Op: CondVec) == MVT::i1) {
437 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i16, Operand: CondVec);
438 CondVec = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: CondTy, Operand: Ext);
439 }
440
441 if (!isHlxPairTy(Ty: ValTy))
442 return DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: CondVec, N2: True, N3: False);
443
444 TypePair HalfTys = typeHlxSplit(VecTy: ValTy);
445 MVT SingleTy = HalfTys.first; // v1024i16
446
447 SDValue TrueLo =
448 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: True);
449 SDValue TrueHi =
450 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: True);
451 SDValue FalseLo =
452 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: False);
453 SDValue FalseHi =
454 DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: False);
455
456 SDValue LoSelect =
457 DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: SingleTy, N1: CondVec, N2: TrueLo, N3: FalseLo);
458 SDValue HiSelect =
459 DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: SingleTy, N1: CondVec, N2: TrueHi, N3: FalseHi);
460
461 return DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: ValTy, N1: HiSelect, N2: LoSelect);
462}
463
464SDValue HexagonTargetLowering::LowerHlxOperation(SDValue Op,
465 SelectionDAG &DAG) const {
466 unsigned Opc = Op.getOpcode();
467 switch (Opc) {
468 default:
469 break;
470 case ISD::INTRINSIC_W_CHAIN:
471 case ISD::INTRINSIC_VOID:
472 return LowerHlxPairIntrinsic(Op, DAG);
473 case ISD::INTRINSIC_WO_CHAIN:
474 return Op;
475 case ISD::MLOAD:
476 case ISD::MSTORE:
477 return LowerHlxMaskedOp(Op, DAG);
478 case ISD::SELECT:
479 return LowerHlxSelect(Op, DAG);
480 case ISD::ADD:
481 case ISD::SUB:
482 case ISD::UADDSAT:
483 case ISD::SADDSAT:
484 case ISD::USUBSAT:
485 case ISD::SSUBSAT:
486 case ISD::MUL:
487 case ISD::SHL:
488 case ISD::SRL:
489 case ISD::SRA:
490 case ISD::SMIN:
491 case ISD::SMAX:
492 case ISD::UMIN:
493 case ISD::UMAX:
494 case ISD::ABS:
495 case ISD::ABDS:
496 case ISD::ABDU:
497 case ISD::AVGFLOORS:
498 case ISD::AVGFLOORU:
499 case ISD::AVGCEILS:
500 case ISD::AVGCEILU:
501 case ISD::SPLAT_VECTOR:
502 case ISD::VSELECT:
503 case ISD::SETCC: {
504 MVT VT = Opc == ISD::SETCC ? ty(Op: Op.getOperand(i: 0)) : ty(Op);
505 DAG.getContext()->emitError(
506 ErrorStr: "HLX does not support '" + Op->getOperationName(G: &DAG) +
507 "' on vector type " + EVT(VT).getEVTString() +
508 "; use an element type and vector length supported by the HLX ISA");
509 return DAG.getUNDEF(VT: ty(Op));
510 }
511 }
512#ifndef NDEBUG
513 Op.dumpr(&DAG);
514#endif
515 llvm_unreachable("Unhandled HLX operation");
516}
517