| 1 | //===-- HexagonISelLoweringHLX.cpp --- Lowering HLX operations ------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "HexagonISelLowering.h" |
| 10 | #include "HexagonRegisterInfo.h" |
| 11 | #include "HexagonSubtarget.h" |
| 12 | #include "llvm/ADT/SmallVector.h" |
| 13 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 14 | #include "llvm/CodeGen/TargetInstrInfo.h" |
| 15 | #include "llvm/IR/IntrinsicsHexagon.h" |
| 16 | |
| 17 | using namespace llvm; |
| 18 | |
| 19 | // 2048B HLX types |
| 20 | static const MVT AllTypes2048B[] = {MVT::v2048i8, MVT::v1024i16, MVT::v512i32, |
| 21 | MVT::v4096i8, MVT::v2048i16, MVT::v1024i32}; |
| 22 | static const MVT LegalTypes2048B[] = {MVT::v1024i16, MVT::v2048i16, |
| 23 | MVT::v1024i32}; |
| 24 | static const MVT LegalPredTypes2048B[] = {MVT::v1024i1, MVT::v2048i1}; |
| 25 | |
| 26 | void HexagonTargetLowering::initializeHLXLowering() { |
| 27 | ArrayRef<MVT> AllTypes = AllTypes2048B; |
| 28 | ArrayRef<MVT> LegalTypes = LegalTypes2048B; |
| 29 | ArrayRef<MVT> LegalPredTypes = LegalPredTypes2048B; |
| 30 | MVT HalfWordPairTy = MVT::v2048i16; |
| 31 | |
| 32 | // TODO: Remove i32 types for single HLX register |
| 33 | // ISA only supports halfword, byte operations on single register |
| 34 | addRegisterClass(VT: MVT::v2048i8, RC: &Hexagon::HlxLRRegClass); |
| 35 | addRegisterClass(VT: MVT::v1024i16, RC: &Hexagon::HlxLRRegClass); |
| 36 | addRegisterClass(VT: MVT::v512i32, RC: &Hexagon::HlxLRRegClass); |
| 37 | addRegisterClass(VT: MVT::v1024i1, RC: &Hexagon::HlxLRRegClass); |
| 38 | addRegisterClass(VT: MVT::v4096i8, RC: &Hexagon::HlxLLRRegClass); |
| 39 | addRegisterClass(VT: MVT::v2048i16, RC: &Hexagon::HlxLLRRegClass); |
| 40 | addRegisterClass(VT: MVT::v1024i32, RC: &Hexagon::HlxLLRRegClass); |
| 41 | addRegisterClass(VT: MVT::v2048i1, RC: &Hexagon::HlxLLRRegClass); |
| 42 | |
| 43 | // Route element-sensitive operations unsupported by a type to |
| 44 | // LowerHlxOperation. |
| 45 | static const unsigned UnsupportedOnByteAndSingleWord[] = { |
| 46 | ISD::ADD, ISD::SUB, ISD::UADDSAT, ISD::SADDSAT, |
| 47 | ISD::USUBSAT, ISD::SSUBSAT, ISD::MUL, ISD::SHL, |
| 48 | ISD::SRL, ISD::SRA, ISD::SPLAT_VECTOR, ISD::VSELECT, |
| 49 | ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, |
| 50 | ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::AVGFLOORS, |
| 51 | ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU}; |
| 52 | SmallVector<MVT, 8> UnsupportedTypes; |
| 53 | for (MVT T : AllTypes) |
| 54 | if (!llvm::is_contained(Range&: LegalTypes, Element: T)) |
| 55 | UnsupportedTypes.push_back(Elt: T); |
| 56 | setOperationAction(Ops: UnsupportedOnByteAndSingleWord, VTs: UnsupportedTypes, Action: Custom); |
| 57 | |
| 58 | // Catch VSELECT before type legalization widens an unsupported predicate. |
| 59 | for (MVT T : UnsupportedTypes) { |
| 60 | MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements()); |
| 61 | if (!llvm::is_contained(Range&: LegalPredTypes, Element: PredTy)) |
| 62 | setOperationAction(Op: ISD::VSELECT, VT: PredTy, Action: Custom); |
| 63 | } |
| 64 | |
| 65 | // Reject unsupported halfword-pair operations in LowerHlxOperation. |
| 66 | for (unsigned Opc : {ISD::MUL, ISD::ABDS, ISD::ABDU, ISD::AVGFLOORS, |
| 67 | ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU}) |
| 68 | setOperationAction(Op: Opc, VT: HalfWordPairTy, Action: Custom); |
| 69 | |
| 70 | for (MVT T : UnsupportedTypes) { |
| 71 | setCondCodeAction(CCs: ISD::SETGT, VT: T, Action: Custom); |
| 72 | setCondCodeAction(CCs: ISD::SETUGT, VT: T, Action: Custom); |
| 73 | setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Custom); |
| 74 | setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Custom); |
| 75 | setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Custom); |
| 76 | setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Custom); |
| 77 | setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Custom); |
| 78 | setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Custom); |
| 79 | setCondCodeAction(CCs: ISD::SETEQ, VT: T, Action: Custom); |
| 80 | setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Custom); |
| 81 | } |
| 82 | |
| 83 | for (MVT T : LegalTypes) { |
| 84 | setOperationAction(Op: ISD::ADD, VT: T, Action: Legal); |
| 85 | setOperationAction(Op: ISD::SUB, VT: T, Action: Legal); |
| 86 | setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal); |
| 87 | setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal); |
| 88 | setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal); |
| 89 | setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal); |
| 90 | } |
| 91 | |
| 92 | setOperationAction(Op: ISD::MUL, VT: MVT::v1024i16, Action: Legal); |
| 93 | setOperationAction(Op: ISD::MUL, VT: MVT::v1024i32, Action: Legal); |
| 94 | |
| 95 | setOperationAction(Op: ISD::MULHU, VT: MVT::v1024i16, Action: Legal); |
| 96 | setOperationAction(Op: ISD::MULHS, VT: MVT::v1024i16, Action: Legal); |
| 97 | |
| 98 | for (MVT T : AllTypes) { |
| 99 | setOperationAction(Op: ISD::AND, VT: T, Action: Legal); |
| 100 | setOperationAction(Op: ISD::OR, VT: T, Action: Legal); |
| 101 | setOperationAction(Op: ISD::XOR, VT: T, Action: Legal); |
| 102 | } |
| 103 | |
| 104 | setOperationAction(Op: ISD::SRA, VT: MVT::v1024i16, Action: Legal); |
| 105 | setOperationAction(Op: ISD::SRA, VT: MVT::v1024i32, Action: Legal); |
| 106 | setOperationAction(Op: ISD::SRL, VT: MVT::v1024i16, Action: Legal); |
| 107 | setOperationAction(Op: ISD::SRL, VT: MVT::v1024i32, Action: Legal); |
| 108 | setOperationAction(Op: ISD::SHL, VT: MVT::v1024i16, Action: Legal); |
| 109 | setOperationAction(Op: ISD::SHL, VT: MVT::v1024i32, Action: Legal); |
| 110 | |
| 111 | for (MVT T : LegalTypes) { |
| 112 | // GT comparisons are legal |
| 113 | setCondCodeAction(CCs: ISD::SETGT, VT: T, Action: Legal); |
| 114 | setCondCodeAction(CCs: ISD::SETUGT, VT: T, Action: Legal); |
| 115 | // Expand all non-GT comparisons |
| 116 | setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand); |
| 117 | setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand); |
| 118 | setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand); |
| 119 | setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand); |
| 120 | setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand); |
| 121 | setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand); |
| 122 | // Expand equality comparisons |
| 123 | setCondCodeAction(CCs: ISD::SETEQ, VT: T, Action: Expand); |
| 124 | setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand); |
| 125 | } |
| 126 | |
| 127 | // Add SETCC operation for predicate result types |
| 128 | for (MVT T : LegalPredTypes) |
| 129 | setOperationAction(Op: ISD::SETCC, VT: T, Action: Legal); |
| 130 | |
| 131 | for (MVT T : LegalTypes) { |
| 132 | setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal); |
| 133 | setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal); |
| 134 | setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal); |
| 135 | setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal); |
| 136 | setOperationAction(Op: ISD::ABS, VT: T, Action: Legal); |
| 137 | } |
| 138 | |
| 139 | // absdiff and average lack halfword-pair instructions. |
| 140 | MVT HalfWordSingleTy = MVT::v1024i16; |
| 141 | MVT WordPairTy = MVT::v1024i32; |
| 142 | for (MVT T : {HalfWordSingleTy, WordPairTy}) { |
| 143 | setOperationAction(Op: ISD::ABDS, VT: T, Action: Legal); |
| 144 | setOperationAction(Op: ISD::ABDU, VT: T, Action: Legal); |
| 145 | setOperationAction(Op: ISD::AVGFLOORS, VT: T, Action: Legal); |
| 146 | setOperationAction(Op: ISD::AVGFLOORU, VT: T, Action: Legal); |
| 147 | setOperationAction(Op: ISD::AVGCEILS, VT: T, Action: Legal); |
| 148 | setOperationAction(Op: ISD::AVGCEILU, VT: T, Action: Legal); |
| 149 | } |
| 150 | |
| 151 | setOperationAction(Op: ISD::SELECT, VT: MVT::v1024i16, Action: Custom); |
| 152 | setOperationAction(Op: ISD::SELECT, VT: MVT::v1024i32, Action: Custom); |
| 153 | |
| 154 | // Splat supports halfword singles and word pairs only. |
| 155 | for (MVT T : LegalTypes) |
| 156 | setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal); |
| 157 | |
| 158 | setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i8, Action: Legal); |
| 159 | setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i16, Action: Legal); |
| 160 | setOperationAction(Op: ISD::LOAD, VT: MVT::v512i32, Action: Legal); |
| 161 | setOperationAction(Op: ISD::STORE, VT: MVT::v2048i8, Action: Legal); |
| 162 | setOperationAction(Op: ISD::STORE, VT: MVT::v1024i16, Action: Legal); |
| 163 | setOperationAction(Op: ISD::STORE, VT: MVT::v512i32, Action: Legal); |
| 164 | |
| 165 | setOperationAction(Op: ISD::MLOAD, VT: MVT::v1024i16, Action: Custom); |
| 166 | setOperationAction(Op: ISD::MSTORE, VT: MVT::v1024i16, Action: Custom); |
| 167 | |
| 168 | setOperationAction(Op: ISD::SIGN_EXTEND, VT: MVT::v1024i32, Action: Legal); |
| 169 | setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v1024i32, Action: Legal); |
| 170 | setOperationAction(Op: ISD::ZERO_EXTEND, VT: MVT::v2048i16, Action: Legal); |
| 171 | |
| 172 | setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v1024i16, Action: Legal); |
| 173 | setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2048i8, Action: Legal); |
| 174 | } |
| 175 | |
| 176 | bool HexagonTargetLowering::isHlxSingleTy(MVT Ty) const { |
| 177 | return Subtarget.isHLXVectorType(VecTy: Ty) && |
| 178 | Ty.getSizeInBits() == 8 * Subtarget.getHLXVectorLength(); |
| 179 | } |
| 180 | |
| 181 | bool HexagonTargetLowering::isHlxPairTy(MVT Ty) const { |
| 182 | return Subtarget.isHLXVectorType(VecTy: Ty) && |
| 183 | Ty.getSizeInBits() == 16 * Subtarget.getHLXVectorLength(); |
| 184 | } |
| 185 | |
| 186 | bool HexagonTargetLowering::isHlxOperation(SDNode *N, SelectionDAG &DAG) const { |
| 187 | if (!Subtarget.useHLXOps()) |
| 188 | return false; |
| 189 | // If the type of any result, or any operand type are HLX vector types, |
| 190 | // this is an HLX operation. |
| 191 | auto IsHLXTy = [this](EVT Ty) { |
| 192 | return Ty.isSimple() && Subtarget.isHLXVectorType(VecTy: Ty.getSimpleVT()); |
| 193 | }; |
| 194 | auto IsHLXOp = [this](SDValue Op) { |
| 195 | return Op.getValueType().isSimple() && Subtarget.isHLXVectorType(VecTy: ty(Op)); |
| 196 | }; |
| 197 | if (llvm::any_of(Range: N->values(), P: IsHLXTy) || llvm::any_of(Range: N->ops(), P: IsHLXOp)) |
| 198 | return true; |
| 199 | |
| 200 | // No widening for now |
| 201 | return false; |
| 202 | } |
| 203 | |
| 204 | bool HexagonTargetLowering::allowsHlxMemoryAccess( |
| 205 | MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const { |
| 206 | // Disallow double vector stores (to prevent unnecessary |
| 207 | // store widening in DAG combiner). |
| 208 | if (VecTy.getSizeInBits() > 8 * Subtarget.getHLXVectorLength()) |
| 209 | return false; |
| 210 | if (!Subtarget.isHLXVectorType(VecTy)) |
| 211 | return false; |
| 212 | if (Fast) |
| 213 | *Fast = 1; |
| 214 | return true; |
| 215 | } |
| 216 | |
| 217 | HexagonTargetLowering::TypePair |
| 218 | HexagonTargetLowering::typeHlxSplit(MVT VecTy) const { |
| 219 | assert(VecTy.isVector()); |
| 220 | unsigned NumElem = VecTy.getVectorNumElements(); |
| 221 | assert((NumElem % 2) == 0 && "Expecting even-sized vector type" ); |
| 222 | MVT ElemTy = VecTy.getVectorElementType(); |
| 223 | MVT HalfTy; |
| 224 | if (ElemTy == MVT::i32) |
| 225 | HalfTy = VecTy.changeVectorElementType(EltVT: MVT::i16); |
| 226 | else |
| 227 | HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem / 2); |
| 228 | return {HalfTy, HalfTy}; |
| 229 | } |
| 230 | |
| 231 | // Reload a stack-passed HLX pair argument: two contiguous sub-vector loads |
| 232 | // recombined with COMBINELL. Calling-convention use only; user pair loads |
| 233 | // through a pointer are unsupported (QTOOL-134974). |
| 234 | SDValue HexagonTargetLowering::splitHlxPairArgLoad( |
| 235 | SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue BaseAddr, |
| 236 | MVT PairTy, MachinePointerInfo PtrInfo) const { |
| 237 | unsigned HlxVLen = Subtarget.getHLXVectorLength(); |
| 238 | MVT SingleTy = typeHlxSplit(VecTy: PairTy).first; |
| 239 | SDValue Base1 = |
| 240 | DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl); |
| 241 | MachineFunction &MF = DAG.getMachineFunction(); |
| 242 | Align HalfAlign = DAG.getEVTAlign(MemoryVT: SingleTy); |
| 243 | MachineMemOperand *MOp0 = MF.getMachineMemOperand( |
| 244 | PtrInfo, F: MachineMemOperand::MOLoad, Size: HlxVLen, BaseAlignment: HalfAlign); |
| 245 | MachineMemOperand *MOp1 = |
| 246 | MF.getMachineMemOperand(PtrInfo: PtrInfo.getWithOffset(O: HlxVLen), |
| 247 | F: MachineMemOperand::MOLoad, Size: HlxVLen, BaseAlignment: HalfAlign); |
| 248 | SDValue Load0 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: BaseAddr, MMO: MOp0); |
| 249 | SDValue Load1 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: Base1, MMO: MOp1); |
| 250 | return DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: PairTy, N1: Load1, N2: Load0); |
| 251 | } |
| 252 | |
| 253 | // Spill a stack-passed HLX pair argument: extract vsub_lo/vsub_hi and store |
| 254 | // them to contiguous slots, appending both chains to MemOpChains. |
| 255 | // Calling-convention use only (QTOOL-134974). |
| 256 | void HexagonTargetLowering::splitHlxPairArgStore( |
| 257 | SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue StoreVal, |
| 258 | SDValue BaseAddr, MVT PairTy, MachinePointerInfo PtrInfo, |
| 259 | SmallVectorImpl<SDValue> &MemOpChains) const { |
| 260 | unsigned HlxVLen = Subtarget.getHLXVectorLength(); |
| 261 | MVT SingleTy = typeHlxSplit(VecTy: PairTy).first; |
| 262 | SDValue Base1 = |
| 263 | DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl); |
| 264 | SDValue Lo = |
| 265 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: StoreVal); |
| 266 | SDValue Hi = |
| 267 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: StoreVal); |
| 268 | MemOpChains.push_back(Elt: DAG.getStore(Chain, dl, Val: Lo, Ptr: BaseAddr, PtrInfo)); |
| 269 | MemOpChains.push_back( |
| 270 | Elt: DAG.getStore(Chain, dl, Val: Hi, Ptr: Base1, PtrInfo: PtrInfo.getWithOffset(O: HlxVLen))); |
| 271 | } |
| 272 | |
| 273 | // Shared helper for splitting pair memory intrinsics into two single-vector |
| 274 | // operations. Used by LowerHlxPairIntrinsic for the LX_LL16b_tmp / |
| 275 | // LX_SS16b_new pair load/store intrinsics. |
| 276 | static SDValue SplitHlxPairMemHelper(SDValue Op, SelectionDAG &DAG, |
| 277 | const HexagonSubtarget &ST, |
| 278 | SDValue BaseAddr, |
| 279 | SDValue StoreVal = SDValue()) { |
| 280 | auto *MemN = cast<MemSDNode>(Val: Op.getNode()); |
| 281 | MVT PairTy = MemN->getMemoryVT().getSimpleVT(); |
| 282 | unsigned HlxVLen = ST.getHLXVectorLength(); |
| 283 | assert(PairTy.getSizeInBits() == 16 * HlxVLen && |
| 284 | PairTy.getVectorElementType() == MVT::i32 && |
| 285 | "Expected HLX vector pair type" ); |
| 286 | MVT SingleTy = PairTy.changeVectorElementType(EltVT: MVT::i16); |
| 287 | SDValue Chain = MemN->getChain(); |
| 288 | const SDLoc &dl(Op); |
| 289 | SDValue Base1 = |
| 290 | DAG.getMemBasePlusOffset(Base: BaseAddr, Offset: TypeSize::getFixed(ExactSize: HlxVLen), DL: dl); |
| 291 | |
| 292 | MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr; |
| 293 | if (MachineMemOperand *MMO = MemN->getMemOperand()) { |
| 294 | MachineFunction &MF = DAG.getMachineFunction(); |
| 295 | MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: HlxVLen); |
| 296 | MOp1 = MF.getMachineMemOperand(MMO, Offset: HlxVLen, Size: HlxVLen); |
| 297 | } |
| 298 | |
| 299 | if (StoreVal) { |
| 300 | // Store: split value into two halves, emit two stores, combine chains. |
| 301 | SDValue Lo = |
| 302 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: StoreVal); |
| 303 | SDValue Hi = |
| 304 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: StoreVal); |
| 305 | SDValue Store0 = DAG.getStore(Chain, dl, Val: Lo, Ptr: BaseAddr, MMO: MOp0); |
| 306 | SDValue Store1 = DAG.getStore(Chain, dl, Val: Hi, Ptr: Base1, MMO: MOp1); |
| 307 | return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1); |
| 308 | } |
| 309 | |
| 310 | // Load: emit two single-vector loads, combine with COMBINELL. |
| 311 | SDValue Load0 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: BaseAddr, MMO: MOp0); |
| 312 | SDValue Load1 = DAG.getLoad(VT: SingleTy, dl, Chain, Ptr: Base1, MMO: MOp1); |
| 313 | return DAG.getMergeValues( |
| 314 | Ops: {DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: PairTy, N1: Load1, N2: Load0), |
| 315 | DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1), |
| 316 | N2: Load1.getValue(R: 1))}, |
| 317 | dl); |
| 318 | } |
| 319 | |
| 320 | // Lower pair load/store intrinsics (LX_LL16b_tmp, LX_SS16b_new) by |
| 321 | // splitting into two single-vector operations via SplitHlxPairMemHelper. |
| 322 | SDValue HexagonTargetLowering::LowerHlxPairIntrinsic(SDValue Op, |
| 323 | SelectionDAG &DAG) const { |
| 324 | unsigned IntNo = Op.getConstantOperandVal(i: 1); |
| 325 | switch (IntNo) { |
| 326 | case Intrinsic::hexagon_LX_LL16b_tmp: |
| 327 | return SplitHlxPairMemHelper(Op, DAG, ST: Subtarget, BaseAddr: Op.getOperand(i: 2)); |
| 328 | case Intrinsic::hexagon_LX_SS16b_new: |
| 329 | return SplitHlxPairMemHelper(Op, DAG, ST: Subtarget, BaseAddr: Op.getOperand(i: 2), |
| 330 | StoreVal: Op.getOperand(i: 3)); |
| 331 | default: |
| 332 | return Op; |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | // Lower masked load/store operations to HLX instructions. |
| 337 | // |
| 338 | // HLX does not have native masked load/store instructions. Instead, we |
| 339 | // lower them to a sequence of regular load/store + VSELECT operations: |
| 340 | // |
| 341 | // MLOAD(base, mask, passthru) => VSELECT(mask, LOAD(base), passthru) |
| 342 | // MSTORE(base, value, mask) => STORE(base, VSELECT(mask, value, |
| 343 | // LOAD(base))) |
| 344 | // |
| 345 | // For MLOAD: Load the full vector, then use VSELECT to choose between |
| 346 | // loaded values (where mask is true) and passthru values (where mask is |
| 347 | // false). |
| 348 | // |
| 349 | // For MSTORE: Load the existing memory contents, use VSELECT to merge the |
| 350 | // new value (where mask is true) with old contents (where mask is false), |
| 351 | // then store the result. This preserves unmasked lanes. |
| 352 | SDValue HexagonTargetLowering::LowerHlxMaskedOp(SDValue Op, |
| 353 | SelectionDAG &DAG) const { |
| 354 | auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode()); |
| 355 | unsigned HwLen = Subtarget.getHLXVectorLength(); |
| 356 | |
| 357 | // HLX has no native masked load/store. We emulate them with a full-width |
| 358 | // load (and, for stores, a load-modify-store), which reads/writes the |
| 359 | // inactive lanes. That is only sound for ordinary memory accesses: a |
| 360 | // volatile or atomic access must not touch inactive lanes, so bail out |
| 361 | // rather than silently change the program's memory semantics. |
| 362 | if (MaskN->isVolatile() || !MaskN->getMemOperand()->isUnordered()) { |
| 363 | DAG.getContext()->emitError( |
| 364 | ErrorStr: "Volatile or atomic masked load/store of HLX vector type is not " |
| 365 | "supported" ); |
| 366 | return SDValue(); |
| 367 | } |
| 368 | |
| 369 | if (MaskN->getAlign().value() % HwLen != 0) { |
| 370 | DAG.getContext()->emitError( |
| 371 | ErrorStr: "Masked load/store of HLX vector type, requires alignment of " + |
| 372 | Twine(HwLen) + " bytes for lowering to HLX" ); |
| 373 | return SDValue(); |
| 374 | } |
| 375 | |
| 376 | const SDLoc &dl(Op); |
| 377 | MachineFunction &MF = DAG.getMachineFunction(); |
| 378 | SDValue Mask = MaskN->getMask(); |
| 379 | SDValue Chain = MaskN->getChain(); |
| 380 | SDValue Base = MaskN->getBasePtr(); |
| 381 | MachineMemOperand *MemOp = |
| 382 | MaskN->getMemOperand() |
| 383 | ? MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen) |
| 384 | : nullptr; |
| 385 | |
| 386 | unsigned Opc = Op->getOpcode(); |
| 387 | |
| 388 | switch (Opc) { |
| 389 | case ISD::MLOAD: { |
| 390 | MVT ValTy = ty(Op); |
| 391 | SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp); |
| 392 | SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru(); |
| 393 | if (isUndef(Op: Thru)) |
| 394 | return Load; |
| 395 | SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru); |
| 396 | return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl); |
| 397 | } |
| 398 | case ISD::MSTORE: { |
| 399 | SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue(); |
| 400 | auto PtrInfo = (MemOp && MemOp->getValue()) |
| 401 | ? MachinePointerInfo(MemOp->getValue()) |
| 402 | : MachinePointerInfo(); |
| 403 | SDValue Load = |
| 404 | DAG.getLoad(VT: Value.getSimpleValueType(), dl, Chain, Ptr: Base, PtrInfo, |
| 405 | Alignment: Align(HwLen), MMOFlags: MachineMemOperand::MOLoad); |
| 406 | SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: Value.getSimpleValueType(), |
| 407 | N1: Mask, N2: Value, N3: Load); |
| 408 | SDValue Store = DAG.getStore(Chain: Load.getValue(R: 1), dl, Val: VSel, Ptr: Base, MMO: MemOp); |
| 409 | return Store; |
| 410 | } |
| 411 | default: { |
| 412 | [[maybe_unused]] std::string Name = |
| 413 | "Unexpected masked operation: " + Op->getOperationName(G: &DAG); |
| 414 | llvm_unreachable(Name.c_str()); |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | // Lowers ternary operations of the following types: |
| 420 | // HLX_Vector = (int)? HLX_Vector: HLX_Vector |
| 421 | // HLX_VectorPair = (int)? HLX_VectorPair: HLX_VectorPair |
| 422 | SDValue HexagonTargetLowering::LowerHlxSelect(SDValue Op, |
| 423 | SelectionDAG &DAG) const { |
| 424 | SDValue CondVec = Op.getOperand(i: 0); // i1 |
| 425 | SDValue True = Op.getOperand(i: 1); |
| 426 | SDValue False = Op.getOperand(i: 2); |
| 427 | |
| 428 | const SDLoc &dl(Op); |
| 429 | MVT ValTy = ty(Op); |
| 430 | |
| 431 | // Always splat condition into single-vector v1024i16 (HlxLR), not pair. |
| 432 | // LX_mux_h only needs a single HlxLR condition register, and for pairs |
| 433 | // the scalar i1 condition is uniform so the same register is reused for |
| 434 | // both halves. |
| 435 | MVT CondTy = MVT::getVectorVT(VT: MVT::i16, NumElements: ValTy.getVectorNumElements()); |
| 436 | if (ty(Op: CondVec) == MVT::i1) { |
| 437 | SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i16, Operand: CondVec); |
| 438 | CondVec = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: CondTy, Operand: Ext); |
| 439 | } |
| 440 | |
| 441 | if (!isHlxPairTy(Ty: ValTy)) |
| 442 | return DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: CondVec, N2: True, N3: False); |
| 443 | |
| 444 | TypePair HalfTys = typeHlxSplit(VecTy: ValTy); |
| 445 | MVT SingleTy = HalfTys.first; // v1024i16 |
| 446 | |
| 447 | SDValue TrueLo = |
| 448 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: True); |
| 449 | SDValue TrueHi = |
| 450 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: True); |
| 451 | SDValue FalseLo = |
| 452 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: False); |
| 453 | SDValue FalseHi = |
| 454 | DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: False); |
| 455 | |
| 456 | SDValue LoSelect = |
| 457 | DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: SingleTy, N1: CondVec, N2: TrueLo, N3: FalseLo); |
| 458 | SDValue HiSelect = |
| 459 | DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: SingleTy, N1: CondVec, N2: TrueHi, N3: FalseHi); |
| 460 | |
| 461 | return DAG.getNode(Opcode: HexagonISD::COMBINELL, DL: dl, VT: ValTy, N1: HiSelect, N2: LoSelect); |
| 462 | } |
| 463 | |
| 464 | SDValue HexagonTargetLowering::LowerHlxOperation(SDValue Op, |
| 465 | SelectionDAG &DAG) const { |
| 466 | unsigned Opc = Op.getOpcode(); |
| 467 | switch (Opc) { |
| 468 | default: |
| 469 | break; |
| 470 | case ISD::INTRINSIC_W_CHAIN: |
| 471 | case ISD::INTRINSIC_VOID: |
| 472 | return LowerHlxPairIntrinsic(Op, DAG); |
| 473 | case ISD::INTRINSIC_WO_CHAIN: |
| 474 | return Op; |
| 475 | case ISD::MLOAD: |
| 476 | case ISD::MSTORE: |
| 477 | return LowerHlxMaskedOp(Op, DAG); |
| 478 | case ISD::SELECT: |
| 479 | return LowerHlxSelect(Op, DAG); |
| 480 | case ISD::ADD: |
| 481 | case ISD::SUB: |
| 482 | case ISD::UADDSAT: |
| 483 | case ISD::SADDSAT: |
| 484 | case ISD::USUBSAT: |
| 485 | case ISD::SSUBSAT: |
| 486 | case ISD::MUL: |
| 487 | case ISD::SHL: |
| 488 | case ISD::SRL: |
| 489 | case ISD::SRA: |
| 490 | case ISD::SMIN: |
| 491 | case ISD::SMAX: |
| 492 | case ISD::UMIN: |
| 493 | case ISD::UMAX: |
| 494 | case ISD::ABS: |
| 495 | case ISD::ABDS: |
| 496 | case ISD::ABDU: |
| 497 | case ISD::AVGFLOORS: |
| 498 | case ISD::AVGFLOORU: |
| 499 | case ISD::AVGCEILS: |
| 500 | case ISD::AVGCEILU: |
| 501 | case ISD::SPLAT_VECTOR: |
| 502 | case ISD::VSELECT: |
| 503 | case ISD::SETCC: { |
| 504 | MVT VT = Opc == ISD::SETCC ? ty(Op: Op.getOperand(i: 0)) : ty(Op); |
| 505 | DAG.getContext()->emitError( |
| 506 | ErrorStr: "HLX does not support '" + Op->getOperationName(G: &DAG) + |
| 507 | "' on vector type " + EVT(VT).getEVTString() + |
| 508 | "; use an element type and vector length supported by the HLX ISA" ); |
| 509 | return DAG.getUNDEF(VT: ty(Op)); |
| 510 | } |
| 511 | } |
| 512 | #ifndef NDEBUG |
| 513 | Op.dumpr(&DAG); |
| 514 | #endif |
| 515 | llvm_unreachable("Unhandled HLX operation" ); |
| 516 | } |
| 517 | |