1//===- AArch64GlobalISelUtils.cpp --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file Implementations of AArch64-specific helper functions used in the
9/// GlobalISel pipeline.
10//===----------------------------------------------------------------------===//
11#include "AArch64GlobalISelUtils.h"
12#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
13#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
14#include "llvm/CodeGen/GlobalISel/Utils.h"
15#include "llvm/CodeGen/TargetLowering.h"
16#include "llvm/IR/InstrTypes.h"
17
18using namespace llvm;
19using namespace MIPatternMatch;
20
21bool AArch64GISelUtils::isVShiftRImm(Register Reg,
22 const MachineRegisterInfo &MRI, LLT Ty,
23 int64_t &Cnt) {
24 assert(Ty.isVector() && "vector shift count is not a vector type");
25 int64_t ElementBits = Ty.getScalarSizeInBits();
26 assert(ElementBits <= 64 && "expected <= 64bit shift type");
27 MachineInstr *MI = MRI.getVRegDef(Reg);
28 std::optional<APInt> C;
29 if (std::optional<Register> Splat = getVectorSplat(MI: *MI, MRI)) {
30 C = getIConstantVRegVal(VReg: *Splat, MRI);
31 } else if (auto *BV = dyn_cast<GBuildVector>(Val: MI)) {
32 for (unsigned I = 0, E = BV->getNumSources(); I < E && !C; ++I)
33 C = getIConstantVRegVal(VReg: BV->getSourceReg(I), MRI);
34 if (C && !isBuildVectorConstantSplat(Reg, MRI, SplatValue: *C, /*AllowUndef=*/true))
35 return false;
36 }
37 if (!C)
38 return false;
39 Cnt = C->getSExtValue();
40 return Cnt >= 1 && Cnt <= ElementBits;
41}
42
43bool AArch64GISelUtils::isCMN(const MachineInstr *MaybeSub,
44 const CmpInst::Predicate &Pred,
45 const MachineRegisterInfo &MRI) {
46 // Match:
47 //
48 // %sub = G_SUB 0, %y
49 // %cmp = G_ICMP eq/ne, %sub, %z
50 //
51 // Or
52 //
53 // %sub = G_SUB 0, %y
54 // %cmp = G_ICMP eq/ne, %z, %sub
55 // or with signed comparisons with the no-signed-wrap flag set
56 if (!MaybeSub || MaybeSub->getOpcode() != TargetOpcode::G_SUB ||
57 (!CmpInst::isEquality(pred: Pred) &&
58 !(CmpInst::isSigned(Pred) && MaybeSub->getFlag(Flag: MachineInstr::NoSWrap))))
59 return false;
60 auto MaybeZero =
61 getIConstantVRegValWithLookThrough(VReg: MaybeSub->getOperand(i: 1).getReg(), MRI);
62 return MaybeZero && MaybeZero->Value.getZExtValue() == 0;
63}
64
65void AArch64GISelUtils::applyEmitBZero(MachineInstr &MI,
66 MachineIRBuilder &MIRBuilder) {
67 assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
68
69 MIRBuilder.setInstrAndDebugLoc(MI);
70 MIRBuilder
71 .buildInstr(Opc: TargetOpcode::G_BZERO, DstOps: {},
72 SrcOps: {MI.getOperand(i: 0), MI.getOperand(i: 2)})
73 .addImm(Val: MI.getOperand(i: 3).getImm())
74 .addMemOperand(MMO: *MI.memoperands_begin());
75 MI.eraseFromParent();
76}
77
78bool AArch64GISelUtils::matchEmitBZero(const MachineInstr &MI,
79 const MachineRegisterInfo &MRI,
80 const LibcallLoweringInfo &Libcalls,
81 bool MinSize) {
82 assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
83 if (Libcalls.getLibcallImpl(Call: RTLIB::BZERO) == RTLIB::Unsupported)
84 return false;
85
86 auto Zero =
87 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 1).getReg(), MRI);
88 if (!Zero || Zero->Value.getSExtValue() != 0)
89 return false;
90
91 // It's not faster to use bzero rather than memset for sizes <= 256.
92 // However, it *does* save us a mov from wzr, so if we're going for
93 // minsize, use bzero even if it's slower.
94 if (!MinSize) {
95 // If the size is known, check it. If it is not known, assume using bzero is
96 // better.
97 if (auto Size = getIConstantVRegValWithLookThrough(
98 VReg: MI.getOperand(i: 2).getReg(), MRI)) {
99 if (Size->Value.getSExtValue() <= 256)
100 return false;
101 }
102 }
103 return true;
104}
105
106std::tuple<uint16_t, Register>
107AArch64GISelUtils::extractPtrauthBlendDiscriminators(Register Disc,
108 MachineRegisterInfo &MRI) {
109 Register AddrDisc = Disc;
110 uint16_t ConstDisc = 0;
111
112 if (auto ConstDiscVal = getIConstantVRegVal(VReg: Disc, MRI)) {
113 if (isUInt<16>(x: ConstDiscVal->getZExtValue())) {
114 ConstDisc = ConstDiscVal->getZExtValue();
115 AddrDisc = Register();
116 }
117 return std::make_tuple(args&: ConstDisc, args&: AddrDisc);
118 }
119
120 Register BlendAddrDisc, BlendConstDisc;
121 if (!mi_match(R: Disc, MRI,
122 P: m_GIntrinsic<Intrinsic::ptrauth_blend>(Ops: m_Reg(R&: BlendAddrDisc),
123 Ops: m_Reg(R&: BlendConstDisc))))
124 return std::make_tuple(args&: ConstDisc, args&: AddrDisc);
125
126 if (auto ConstDiscVal = getIConstantVRegVal(VReg: BlendConstDisc, MRI)) {
127 if (isUInt<16>(x: ConstDiscVal->getZExtValue())) {
128 ConstDisc = ConstDiscVal->getZExtValue();
129 AddrDisc = BlendAddrDisc;
130 }
131 }
132 return std::make_tuple(args&: ConstDisc, args&: AddrDisc);
133}
134
135void AArch64GISelUtils::changeFCMPPredToAArch64CC(
136 const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
137 AArch64CC::CondCode &CondCode2) {
138 CondCode2 = AArch64CC::AL;
139 switch (P) {
140 default:
141 llvm_unreachable("Unknown FP condition!");
142 case CmpInst::FCMP_OEQ:
143 CondCode = AArch64CC::EQ;
144 break;
145 case CmpInst::FCMP_OGT:
146 CondCode = AArch64CC::GT;
147 break;
148 case CmpInst::FCMP_OGE:
149 CondCode = AArch64CC::GE;
150 break;
151 case CmpInst::FCMP_OLT:
152 CondCode = AArch64CC::MI;
153 break;
154 case CmpInst::FCMP_OLE:
155 CondCode = AArch64CC::LS;
156 break;
157 case CmpInst::FCMP_ONE:
158 CondCode = AArch64CC::MI;
159 CondCode2 = AArch64CC::GT;
160 break;
161 case CmpInst::FCMP_ORD:
162 CondCode = AArch64CC::VC;
163 break;
164 case CmpInst::FCMP_UNO:
165 CondCode = AArch64CC::VS;
166 break;
167 case CmpInst::FCMP_UEQ:
168 CondCode = AArch64CC::EQ;
169 CondCode2 = AArch64CC::VS;
170 break;
171 case CmpInst::FCMP_UGT:
172 CondCode = AArch64CC::HI;
173 break;
174 case CmpInst::FCMP_UGE:
175 CondCode = AArch64CC::PL;
176 break;
177 case CmpInst::FCMP_ULT:
178 CondCode = AArch64CC::LT;
179 break;
180 case CmpInst::FCMP_ULE:
181 CondCode = AArch64CC::LE;
182 break;
183 case CmpInst::FCMP_UNE:
184 CondCode = AArch64CC::NE;
185 break;
186 case CmpInst::FCMP_TRUE:
187 CondCode = AArch64CC::AL;
188 break;
189 case CmpInst::FCMP_FALSE:
190 CondCode = AArch64CC::NV;
191 break;
192 }
193}
194
195void AArch64GISelUtils::changeVectorFCMPPredToAArch64CC(
196 const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
197 AArch64CC::CondCode &CondCode2, bool &Invert) {
198 Invert = false;
199 switch (P) {
200 default:
201 // Mostly the scalar mappings work fine.
202 changeFCMPPredToAArch64CC(P, CondCode, CondCode2);
203 break;
204 case CmpInst::FCMP_UNO:
205 Invert = true;
206 [[fallthrough]];
207 case CmpInst::FCMP_ORD:
208 CondCode = AArch64CC::MI;
209 CondCode2 = AArch64CC::GE;
210 break;
211 case CmpInst::FCMP_UEQ:
212 case CmpInst::FCMP_ULT:
213 case CmpInst::FCMP_ULE:
214 case CmpInst::FCMP_UGT:
215 case CmpInst::FCMP_UGE:
216 // All of the compare-mask comparisons are ordered, but we can switch
217 // between the two by a double inversion. E.g. ULE == !OGT.
218 Invert = true;
219 changeFCMPPredToAArch64CC(P: CmpInst::getInversePredicate(pred: P), CondCode,
220 CondCode2);
221 break;
222 }
223}
224