1//===--------------------- HexagonPostRAHandleQFP.cpp --------------------------
2//===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===---------------------------------------------------------------------===//
9// For v79 and above, we generate qf operations for HVX which includes vadd,
10// vsub and vmpy instructions. These qf operations with qf operands are fast,
11// maintain similar accuracy as IEEE and saves power.
12//
13// However, these qf operands should always be converted back to IEEE format
14// when used in non-HVX instructions. This is because of how the qf values
15// are stored in memory. qf operands have 4 extra bits. If used in non-HVX
16// operations, these bits get dropped resulting in incorrect value being
17// used. So, before use in any non-HVX operation we need to convert these
18// qf values to IEEE format.
19//
20// During register allocation, when no more physical registers are available
21// the qf operands may be spilled to memory. This instantly causes loss of
22// accuracy. This pass prevents that by:
23// 1. Inserting qf type to IEEE type conversion instructions before the spill.
24// 2. Iterating over the uses of qf def (created before the spill) and
25// changing their opcodes to handle IEEE type operands for saturating
26// instructions. This is because, the refills will use IEEE type operands, but
27// the instructions will still assume qf operands. For non-saturating
28// instructions which uses qf, we incorporate a conversion to IEEE before that.
29// 3. Iterating over the uses of qf def created by the spill and replacing
30// them with appropiate opcode (which uses IEEE operands) for saturating
31// instructions. For non-saturating instructions which uses qf,
32// we incorporate a conversion to IEEE before that.
33// 4. Iterating over the copy instructions and checking their uses,
34// inserting conversions from qf to IEEE whenever required. The conversions
35// are inserted after their reaching def since there can be multiple defs
36// for use in non-SSA form.
37//
38// To get the use-def chains, we make use of Register DataFlow Graph (RDF),
39// since after register allocation SSA form is lost. This can be done during
40// spills and fills during Frame Lowering for register allocation. However,
41// that was abandoned due to the intermediate state of the code.
42// Liveness is preserved in this pass.
43//
44// NOTE:
45// Saturating instructions: Instructions for which transformation involves
46// only changing the opcode. Eg. vmpy(qf32, sf) saturates to vmpy(sf, sf) when
47// we see that the first operand is now a sf type.
48// Non-Saturating instructions: Instructions for which conversion(s) have
49// to be inserted. Eg. Vd.f8=Vu.qf16. If the use operand is now hf type,
50// we have to insert a conversion qf16 = hf before this instruction.
51//
52// FIXME tags have been added for potential errors, along with the underlying
53// assumption.
54// FIXME Implement v81 specific optimizations as below. At the moment, we add
55// converts.
56// Vd.qf16=Vu.hf
57// Vd.qf16=Vu.qf16
58// Vd.qf32=Vu.qf32
59// Vd.qf32=Vu.sf
60//===---------------------------------------------------------------------===//
61
62#include "HexagonTargetMachine.h"
63#include "llvm/CodeGen/LiveInterval.h"
64#include "llvm/CodeGen/LiveIntervals.h"
65#include "llvm/CodeGen/LivePhysRegs.h"
66#include "llvm/CodeGen/MachineDominanceFrontier.h"
67#include "llvm/CodeGen/MachineDominators.h"
68#include "llvm/CodeGen/MachineFunction.h"
69#include "llvm/CodeGen/MachineInstrBuilder.h"
70#include "llvm/CodeGen/MachineRegisterInfo.h"
71#include "llvm/CodeGen/Passes.h"
72#include "llvm/CodeGen/RDFGraph.h"
73#include "llvm/CodeGen/RDFLiveness.h"
74#include "llvm/CodeGen/RDFRegisters.h"
75#include "llvm/CodeGen/TargetInstrInfo.h"
76#include "llvm/CodeGen/TargetRegisterInfo.h"
77#include "llvm/InitializePasses.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Debug.h"
80#include "llvm/Support/raw_ostream.h"
81#include "llvm/Target/TargetMachine.h"
82
83#define DEBUG_TYPE "handle-qfp"
84
85using namespace llvm;
86using namespace rdf;
87
88extern cl::opt<QFloatMode> QFloatModeValue;
89
90cl::opt<bool> DisablePostRAHandleQFloat(
91 "disable-handle-qfp", cl::init(Val: false),
92 cl::desc("Disable handling of Qfloat spills/refills after register "
93 "allocation."));
94
95static cl::opt<bool> EnablePostRAXqfCompliance(
96 "enable-postra-xqf-check", cl::init(Val: false),
97 cl::desc("Enable ABI compliance for xqf operands post regalloc."));
98
99namespace llvm {
100FunctionPass *createHexagonPostRAHandleQFP();
101void initializeHexagonPostRAHandleQFPPass(PassRegistry &);
102} // namespace llvm
103
104// QF Instructions list which need to be analyzed.
105// The value of the key denotes a pair
106// pair.first|pair.second = True if IEEE type, false otherwise.
107// We only need to change the opcode to handling qf/sf
108// misuses for these, or these instructions can be 'saturated'.
109DenseMap<unsigned short, std::pair<bool, bool>> QFPSatInstsMap{
110 {Hexagon::V6_vadd_qf16_mix, {false, true}},
111 {Hexagon::V6_vadd_qf16, {false, false}},
112 {Hexagon::V6_vadd_qf32_mix, {false, true}},
113 {Hexagon::V6_vadd_qf32, {false, false}},
114 {Hexagon::V6_vsub_qf16_mix, {false, true}},
115 {Hexagon::V6_vsub_hf_mix, {true, false}},
116 {Hexagon::V6_vsub_qf16, {false, false}},
117 {Hexagon::V6_vsub_qf32_mix, {false, true}},
118 {Hexagon::V6_vsub_sf_mix, {true, false}},
119 {Hexagon::V6_vsub_qf32, {false, false}},
120 {Hexagon::V6_vmpy_qf16_mix_hf, {false, true}},
121 {Hexagon::V6_vmpy_qf16, {false, false}},
122 {Hexagon::V6_vmpy_qf32_mix_hf, {false, true}},
123 {Hexagon::V6_vmpy_qf32_qf16, {false, false}},
124 {Hexagon::V6_vmpy_qf32, {false, false}},
125 {Hexagon::V6_vmpy_rt_qf16, {false, true}},
126 // These opcodes take a single operand only.
127 // Second placeholder op is true always.
128 {Hexagon::V6_vabs_qf32_qf32, {false, true}},
129 {Hexagon::V6_vabs_qf16_qf16, {false, true}},
130 {Hexagon::V6_vneg_qf32_qf32, {false, true}},
131 {Hexagon::V6_vneg_qf16_qf16, {false, true}},
132 {Hexagon::V6_vilog2_qf32, {false, true}},
133 {Hexagon::V6_vilog2_qf16, {false, true}},
134 {Hexagon::V6_vconv_qf32_qf32, {false, true}},
135 {Hexagon::V6_vconv_qf16_qf16, {false, true}},
136};
137
138// This holds the instruction opcodes for which there are
139// no 'saturating' opcodes. The only way is to insert
140// convert instructions before them.
141SmallVector<unsigned short, 5> QFNonSatInstr{
142 Hexagon::V6_vconv_hf_qf16, Hexagon::V6_vconv_hf_qf32,
143 Hexagon::V6_vconv_sf_qf32,
144 // v81 instructions
145 Hexagon::V6_vconv_bf_qf32, Hexagon::V6_vconv_f8_qf16};
146
147namespace {
148class HexagonPostRAHandleQFP : public MachineFunctionPass {
149public:
150 static char ID;
151 HexagonPostRAHandleQFP() : MachineFunctionPass(ID) {
152 PassRegistry &R = *PassRegistry::getPassRegistry();
153 initializeHexagonPostRAHandleQFPPass(R);
154 }
155 StringRef getPassName() const override {
156 return "Hexagon handle QFloat spills and refills post RA.";
157 }
158 void getAnalysisUsage(AnalysisUsage &AU) const override {
159 MachineFunctionPass::getAnalysisUsage(AU);
160 AU.addRequired<MachineDominatorTreeWrapperPass>();
161 AU.addRequired<MachineDominanceFrontierWrapperPass>();
162 AU.setPreservesCFG();
163 }
164 bool runOnMachineFunction(MachineFunction &MF) override;
165
166private:
167 // QFUses collects the instructions which uses QF operands.
168 // These have to be deleted and transformed to opcodes
169 // to denote usage of IEEE operands.
170 // It might involve changing the order of the Register operands.
171 using QFUses = std::map<MachineInstr *, std::pair<bool, bool>>;
172 QFUses QFUsesMap;
173
174 // Holds the Register Dataglow Graph.
175 DataFlowGraph *DFG = nullptr;
176
177 // Stores spill nodes and their reaching definition instructions
178 // which generates the qf operand to be stored.
179 std::vector<std::pair<MachineInstr *, NodeAddr<DefNode *>>> SpillMIs;
180 // Stores the refill nodes consisting of load instructions.
181 std::vector<NodeAddr<DefNode *>> RefillMIs;
182
183 // Stores the type of op.
184 enum ConvOperand {
185 Undefined = 0x0,
186 Lo = 0x1,
187 Hi = 0x2,
188 HiLo = 0x3,
189 };
190 // Stores the convert instructions which take qf operands.
191 MapVector<MachineInstr *, unsigned> QFNonSatMIs;
192
193 // Stores the qf-generating vmul/vadd/etc. nodes with mutiple reaching defs
194 std::set<NodeId> PossibleMultiReachDefs;
195 // Qf generating instructions to ignore. Do not insert conversion instruction
196 // to sf/hf from qf, if the instr is present in this list; since that means
197 // a conversion has already been inserted after the instruction.
198 SmallPtrSet<MachineInstr *, 4> IgnoreInsertConvList;
199
200 // Register type
201 enum class RegType { qf32, qf16, qf32_double, qf16_double, ieee, undefined };
202 // Stores the copy instructions which their reaching def, along with the op
203 // type
204 std::map<std::pair<NodeId, NodeId>, RegType> QFCopys;
205
206 // Stores the reaching defs of copies whose result has to be converted to IEEE
207 DenseMap<MachineInstr *, RegType> ReachDefOfCopies;
208
209 // Stores copies which need to be converted back to qf. The uses of these
210 // copies feed to qf type instructions and hence can be converted back to qf
211 // type.
212 DenseMap<MachineInstr *, std::pair<NodeAddr<DefNode *>, RegType>>
213 ConvertToQfCopies;
214
215 // Subregister kill set for a doubletype use. The pair of bool,bool
216 // represents the hi and lo subregisters of the double register.
217 DenseMap<MachineInstr *, std::pair<bool, bool>> SubRegKillSet;
218
219 const HexagonInstrInfo *HII = nullptr;
220 const HexagonRegisterInfo *HRI = nullptr;
221 MachineRegisterInfo *MRI = nullptr;
222 Liveness *LV = nullptr;
223 const HexagonSubtarget *HST = nullptr;
224
225 void collectQFPStackSpill(NodeAddr<StmtNode *> *);
226 void collectQFPStackRefill(NodeAddr<StmtNode *> *);
227 void collectCopies(NodeAddr<StmtNode *> *);
228 bool HandleRefills();
229 bool HandleSpills();
230 bool HandleCopies();
231 bool HandleNonSatInstr();
232 bool HandleMultiReachingDefs();
233 bool HandleReachDefOfCopies();
234 bool HandleConvertToQfCopies();
235 RegType HasQfUses(NodeAddr<DefNode *>, MachineInstr *);
236 void collectConvQFInstr(NodeAddr<DefNode *> &);
237 void collectQFUses(NodeAddr<DefNode *>, MachineInstr *DefMI);
238 void conditionallyInsert(MachineInstr &, Register &);
239
240 // Helper functions
241 unsigned short getreplacedQFOpcode(unsigned, bool, bool);
242 MCPhysReg findAllocatableReg(MachineInstr *MI) const;
243 void insertIEEEToQF(MachineInstr *, Register, MachineOperand, bool is32bit);
244 void collectLivenessForSubregs(NodeAddr<UseNode *> &);
245 void insertInstr(MachineInstr *, unsigned, unsigned, unsigned, RegState);
246};
247} // namespace
248
249// This class handles spurious vector instrutions which do not
250// follow the ABI. For eg, vcombine(qf,qf) takes qf operands
251// instead of IEEE type. This diagnostic pass can be used
252// as a final verifier for XQF implementation. Turned off by
253// default
254class XqfPostRADiagnosis {
255public:
256 XqfPostRADiagnosis(DataFlowGraph &G, Liveness &L, const HexagonInstrInfo *HII)
257 : G(&G), L(&L), HII(HII) {}
258 // Deleting default constructor to handle misconstruction
259 XqfPostRADiagnosis() = delete;
260
261 void runCompliance() const;
262 void print_warning(Twine &, MachineInstr *, MachineInstr *) const;
263
264private:
265 DataFlowGraph *G = nullptr;
266 Liveness *L = nullptr;
267 const HexagonInstrInfo *HII = nullptr;
268};
269
270void XqfPostRADiagnosis::print_warning(Twine &wstr, MachineInstr *DefMI,
271 MachineInstr *UseMI) const {
272#ifndef NDEBUG
273 dbgs() << wstr;
274 dbgs() << "\n\tDef:";
275 DefMI->dump();
276 // dbgs() << "\t" << DefMI->getParent()->getName();
277 dbgs() << "\tUse:";
278 UseMI->dump();
279 // dbgs() << "\t" << UseMI->getParent()->getName();
280#endif // NDEBUG
281}
282
283// This static function gets all reached uses of a def.
284// When it encounters a phi node, it goes over the
285// reached uses of the phi node too.
286static void getAllRealUses(NodeAddr<DefNode *> DA, NodeSet &UNodeSet,
287 Liveness *L, DataFlowGraph *G,
288 bool comprehensive = false) {
289 RegisterRef DR = DA.Addr->getRegRef(G: *G);
290 NodeAddr<StmtNode *> DefStmt = DA.Addr->getOwner(G: *G);
291 MachineInstr *Instr = DefStmt.Addr->getCode();
292 auto UseSet = L->getAllReachedUses(RefRR: DR, DefA: DA);
293
294 for (auto UI : UseSet) {
295 NodeAddr<UseNode *> UA = G->addr<UseNode *>(N: UI);
296
297 MachineFunction *MF = Instr->getMF();
298 const auto &HRI = MF->getSubtarget<HexagonSubtarget>().getRegisterInfo();
299 Register RR = UA.Addr->getRegRef(G: *G).Id;
300 if (HRI->isFakeReg(Reg: RR))
301 continue;
302
303 if (UA.Addr->getFlags() & NodeAttrs::PhiRef) {
304 NodeAddr<PhiNode *> PA = UA.Addr->getOwner(G: *G);
305 NodeId id = PA.Id;
306 const Liveness::RefMap &phiUse = L->getRealUses(P: id);
307 for (auto I : phiUse) {
308 if (!G->getPRI().alias(RA: RegisterRef(I.first), RB: DR))
309 continue;
310 auto phiUseSet = I.second;
311 for (auto phiUI : phiUseSet) {
312 NodeAddr<UseNode *> phiUA = G->addr<UseNode *>(N: phiUI.first);
313 UNodeSet.insert(x: phiUA.Id);
314 }
315 }
316 } else {
317 // FIXME Due to bug in RDF, check if the reaching def of the use
318 // reaches this instruction
319 if (comprehensive) {
320 UNodeSet.insert(x: UA.Id);
321 continue;
322 }
323 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *G);
324 for (NodeAddr<UseNode *> UA : UseStmt.Addr->members_if(P: G->IsUse, G: *G)) {
325 NodeId QFPDefNode = UA.Addr->getReachingDef();
326 NodeAddr<DefNode *> RegDef = G->addr<DefNode *>(N: QFPDefNode);
327 // FIXME Reaching def computation error
328 if (QFPDefNode == 0)
329 continue;
330 NodeAddr<StmtNode *> RegStmt = RegDef.Addr->getOwner(G: *G);
331 MachineInstr *ReachDefInstr = RegStmt.Addr->getCode();
332 if (ReachDefInstr && ReachDefInstr == Instr)
333 UNodeSet.insert(x: UA.Id);
334 }
335 }
336 }
337}
338
339void XqfPostRADiagnosis::runCompliance() const {
340 NodeAddr<FuncNode *> FA = G->getFunc();
341 for (NodeAddr<BlockNode *> BA : FA.Addr->members(G: *G)) {
342 for (auto IA : BA.Addr->members(G: *G)) {
343 if (!G->IsCode<NodeAttrs::Stmt>(BA: IA))
344 continue;
345 NodeAddr<StmtNode *> SA = IA;
346 MachineInstr *DefMI = SA.Addr->getCode();
347 if (DefMI->isDebugInstr() || DefMI->isInlineAsm())
348 continue;
349 auto NodeBase = SA.Addr->members_if(P: G->IsDef, G: *G);
350 if (NodeBase.empty())
351 continue;
352 NodeAddr<DefNode *> DfNode = NodeBase.front();
353
354 NodeSet UseSet;
355 getAllRealUses(DA: DfNode, UNodeSet&: UseSet, L, G, comprehensive: true);
356 for (auto UI : UseSet) {
357 NodeAddr<UseNode *> UA = G->addr<UseNode *>(N: UI);
358 if (UA.Addr->getFlags() & NodeAttrs::PhiRef)
359 continue;
360 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *G);
361 MachineInstr *UseMI = UseStmt.Addr->getCode();
362 if (UseMI->isDebugInstr() || UseMI->isInlineAsm())
363 continue;
364 unsigned OpNo = UA.Addr->getOp().getOperandNo();
365 if (HII->usesQF32Operand(MI: UseMI, Index: OpNo) && !HII->isQFP32Instr(MI: DefMI)) {
366 Twine wstr(Twine("Mismatch: sf type used as qf32 at operand ")
367 .concat(Suffix: Twine(OpNo)));
368 print_warning(wstr, DefMI, UseMI);
369 } else if (!HII->usesQF32Operand(MI: UseMI, Index: OpNo) &&
370 HII->isQFP32Instr(MI: DefMI)) {
371 Twine wstr(Twine("Mismatch: qf32 type used as sf at operand ")
372 .concat(Suffix: Twine(OpNo)));
373 print_warning(wstr, DefMI, UseMI);
374 } else if (HII->usesQF16Operand(MI: UseMI, Index: OpNo) &&
375 !HII->isQFP16Instr(MI: DefMI)) {
376 Twine wstr(Twine("Mismatch: hf type used as qf16 at operand ")
377 .concat(Suffix: Twine(OpNo)));
378 print_warning(wstr, DefMI, UseMI);
379 } else if (!HII->usesQF16Operand(MI: UseMI, Index: OpNo) &&
380 HII->isQFP16Instr(MI: DefMI)) {
381 Twine wstr(Twine("Mismatch: qf16 type used as hf at operand ")
382 .concat(Suffix: Twine(OpNo)));
383 print_warning(wstr, DefMI, UseMI);
384 }
385 }
386 }
387 }
388}
389
390char HexagonPostRAHandleQFP::ID = 0;
391
392namespace llvm {
393char &HexagonPostRAHandleQFPID = HexagonPostRAHandleQFP::ID;
394}
395
396// Check whether the instruction is added already, if not add it
397// along with the Register values and qf type.
398// If already added, then check the register values and edit them.
399void HexagonPostRAHandleQFP::conditionallyInsert(MachineInstr &MI,
400 Register &DefReg) {
401 LLVM_DEBUG(dbgs() << "\nCollecting instruction using QF: "; MI.dump());
402 // check if the key exists.
403 Register Reg1 = MI.getOperand(i: 1).getReg();
404
405 // If the use is a unary operation, make second register point to Defreg
406 // This ensures that secondOp is always true
407 Register Reg2 = MI.getNumOperands() == 2 ? DefReg : MI.getOperand(i: 2).getReg();
408
409 if (QFUsesMap.find(x: &MI) != QFUsesMap.end()) {
410 auto Entry = QFUsesMap[&MI];
411 bool firstOp = ((Reg1 == DefReg) ? true : false) | Entry.first;
412 bool secondOp = ((Reg2 == DefReg) ? true : false) | Entry.second;
413 QFUsesMap[&MI] = std::make_pair(x&: firstOp, y&: secondOp);
414
415 } else { // encountered first time.
416 // Get the default type of the operand:
417 // True : IEEE type
418 // False : QF type
419 auto defaultPair = QFPSatInstsMap[MI.getOpcode()];
420 bool firstOp = (Reg1 == DefReg) ? true : defaultPair.first;
421 bool secondOp = (Reg2 == DefReg) ? true : defaultPair.second;
422 QFUsesMap[&MI] = std::make_pair(x&: firstOp, y&: secondOp);
423 }
424}
425
426unsigned short HexagonPostRAHandleQFP::getreplacedQFOpcode(unsigned srcOpcode,
427 bool firstOp,
428 bool secondOp) {
429 if (firstOp && secondOp) {
430 switch (srcOpcode) {
431 case Hexagon::V6_vadd_qf32:
432 case Hexagon::V6_vadd_qf32_mix:
433 return Hexagon::V6_vadd_sf;
434 case Hexagon::V6_vadd_qf16:
435 case Hexagon::V6_vadd_qf16_mix:
436 return Hexagon::V6_vadd_hf;
437
438 case Hexagon::V6_vsub_qf32:
439 case Hexagon::V6_vsub_qf32_mix:
440 case Hexagon::V6_vsub_sf_mix:
441 return Hexagon::V6_vsub_sf;
442 case Hexagon::V6_vsub_qf16:
443 case Hexagon::V6_vsub_qf16_mix:
444 case Hexagon::V6_vsub_hf_mix:
445 return Hexagon::V6_vsub_hf;
446
447 case Hexagon::V6_vmpy_qf32:
448 return Hexagon::V6_vmpy_qf32_sf;
449 case Hexagon::V6_vmpy_qf16:
450 case Hexagon::V6_vmpy_qf16_mix_hf:
451 return Hexagon::V6_vmpy_qf16_hf;
452 case Hexagon::V6_vmpy_qf32_qf16:
453 case Hexagon::V6_vmpy_qf32_mix_hf:
454 return Hexagon::V6_vmpy_qf32_hf;
455
456 case Hexagon::V6_vmpy_rt_qf16:
457 return Hexagon::V6_vmpy_rt_hf;
458 // v81 opcodes start
459 case Hexagon::V6_vabs_qf32_qf32:
460 return Hexagon::V6_vabs_qf32_sf;
461 case Hexagon::V6_vabs_qf16_qf16:
462 return Hexagon::V6_vabs_qf16_hf;
463 case Hexagon::V6_vneg_qf32_qf32:
464 return Hexagon::V6_vneg_qf32_sf;
465 case Hexagon::V6_vneg_qf16_qf16:
466 return Hexagon::V6_vneg_qf16_hf;
467 case Hexagon::V6_vilog2_qf32:
468 return Hexagon::V6_vilog2_sf;
469 case Hexagon::V6_vilog2_qf16:
470 return Hexagon::V6_vilog2_hf;
471 case Hexagon::V6_vconv_qf32_qf32:
472 return Hexagon::V6_vconv_qf32_sf;
473 case Hexagon::V6_vconv_qf16_qf16:
474 return Hexagon::V6_vconv_qf16_hf;
475 // v81 opcodes end
476
477 default:
478 llvm_unreachable("Invalid qf opcode in this scenario!");
479 }
480 } else if (firstOp) {
481 switch (srcOpcode) {
482 case Hexagon::V6_vadd_qf32:
483 return Hexagon::V6_vadd_qf32_mix; // interchange reqd
484 case Hexagon::V6_vadd_qf16:
485 return Hexagon::V6_vadd_qf16_mix; // interchange reqd
486
487 case Hexagon::V6_vsub_qf32:
488 if (HST->useHVXV81Ops())
489 return Hexagon::V6_vsub_sf_mix;
490 else if (HST->useHVXV79Ops())
491 return Hexagon::V6_vsub_sf; // conv reqd
492 else
493 llvm_unreachable("Invalid Hexagon Arch for this scenario!");
494 case Hexagon::V6_vsub_qf16:
495 if (HST->useHVXV81Ops())
496 return Hexagon::V6_vsub_hf_mix;
497 else if (HST->useHVXV79Ops())
498 return Hexagon::V6_vsub_hf; // conv reqd
499 else
500 llvm_unreachable("Invalid Hexagon Arch for this scenario!");
501 case Hexagon::V6_vsub_qf32_mix:
502 return Hexagon::V6_vsub_sf;
503 case Hexagon::V6_vsub_qf16_mix:
504 return Hexagon::V6_vsub_hf;
505
506 // This opcode does not have a mixed type. Hence if one
507 // of op1 or op2 is IEEE type and another qf type,
508 // send the opcode which takes in both as IEEE type.
509 case Hexagon::V6_vmpy_qf32:
510 return Hexagon::V6_vmpy_qf32_sf; // conv reqd
511 case Hexagon::V6_vmpy_qf16:
512 return Hexagon::V6_vmpy_qf16_mix_hf; // interchange reqd
513 case Hexagon::V6_vmpy_qf32_qf16:
514 return Hexagon::V6_vmpy_qf32_mix_hf; // interchange reqd
515
516 default:
517 return srcOpcode;
518 }
519 } else if (secondOp) {
520 switch (srcOpcode) {
521 case Hexagon::V6_vadd_qf32:
522 return Hexagon::V6_vadd_qf32_mix;
523 case Hexagon::V6_vadd_qf16:
524 return Hexagon::V6_vadd_qf16_mix;
525
526 case Hexagon::V6_vsub_qf32:
527 return Hexagon::V6_vsub_qf32_mix;
528 case Hexagon::V6_vsub_qf16:
529 return Hexagon::V6_vsub_qf16_mix;
530 case Hexagon::V6_vsub_sf_mix:
531 return Hexagon::V6_vsub_sf;
532 case Hexagon::V6_vsub_hf_mix:
533 return Hexagon::V6_vsub_hf;
534
535 case Hexagon::V6_vmpy_qf32:
536 return Hexagon::V6_vmpy_qf32_sf; // conv reqd
537
538 case Hexagon::V6_vmpy_qf16:
539 return Hexagon::V6_vmpy_qf16_mix_hf;
540 case Hexagon::V6_vmpy_qf32_qf16:
541 return Hexagon::V6_vmpy_qf32_mix_hf;
542
543 default:
544 return srcOpcode;
545 }
546 } else
547 return srcOpcode;
548}
549
550// Insert IEEE to Qf conversion instructions
551// is32bit: If true, SrcReg holds sf type, else a hf type
552void HexagonPostRAHandleQFP::insertIEEEToQF(MachineInstr *MI, Register SrcReg,
553 MachineOperand SrcOp,
554 bool is32bit = false) {
555
556 auto MBB = MI->getParent();
557 MachineInstrBuilder MIB;
558 const DebugLoc &DL = MI->getDebugLoc();
559
560 if (HST->useHVXV81Ops()) {
561 auto Op = is32bit ? Hexagon::V6_vconv_qf32_sf : Hexagon::V6_vconv_qf16_hf;
562 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: SrcReg)
563 .addReg(RegNo: SrcReg, Flags: RegState::Renamable | RegState::Kill);
564 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
565 MIB.getInstr()->dump());
566
567 } else if (HST->useHVXV79Ops()) {
568 // Get an available register
569 auto V0_Reg = findAllocatableReg(MI);
570
571 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vd0), DestReg: V0_Reg);
572 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
573 MIB.getInstr()->dump());
574 auto Op = is32bit ? Hexagon::V6_vadd_sf : Hexagon::V6_vadd_hf;
575 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: SrcReg)
576 .addReg(RegNo: SrcReg, Flags: RegState::Renamable | RegState::Kill)
577 .addReg(RegNo: V0_Reg, Flags: RegState::Kill);
578 LLVM_DEBUG(dbgs() << "Inserting new instruction: "; MIB.getInstr()->dump());
579 } else
580 llvm_unreachable("Not possible to insert qf = hf/sf for this unknown\
581 subtarget!");
582}
583
584// Create a new instruction which handle sf/hf types to replace
585// qf type handling instructions.
586bool HexagonPostRAHandleQFP::HandleRefills() {
587
588 bool Changed = false;
589 LLVM_DEBUG(dbgs() << "HandleRefills: ");
590 std::vector<MachineInstr *> eraseList;
591
592 for (auto It : QFUsesMap) {
593
594 // Separately handle unary qf opcodes
595 MachineInstr *MI = It.first;
596 auto SrcOpcode = MI->getOpcode();
597 auto Pair = It.second;
598 auto SrcOp1 = MI->getOperand(i: 1);
599 Register DestReg = MI->getOperand(i: 0).getReg();
600 auto MBB = MI->getParent();
601 MachineInstrBuilder MIB;
602 LLVM_DEBUG(dbgs() << "\nProcessing: "; MI->dump());
603 const DebugLoc &DL = MI->getDebugLoc();
604
605 // lambda to handle unary qf operations
606 // ieee: True if the 1st operand is sf/hf type, false if qf type
607 auto HandleUnaryRefill = [&](MachineInstr *MI, bool isIeee) -> bool {
608 if (isIeee) {
609 auto finalOpcode = getreplacedQFOpcode(srcOpcode: SrcOpcode, firstOp: true, secondOp: true);
610 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
611 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1));
612 Changed |= true;
613 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
614 MIB.getInstr()->dump());
615 } else
616 eraseList.push_back(x: MI);
617 return Changed;
618 };
619
620 if (MI->getNumOperands() == 2) {
621 Changed |= HandleUnaryRefill(It.first, It.second.first);
622 continue;
623 }
624 auto SrcOp2 = MI->getOperand(i: 2);
625
626 // lambda to handle mixed type vsub instructions for v79
627 auto HandleSub = [&](auto srcOpcode) -> bool {
628 auto ConvOp = (srcOpcode == Hexagon::V6_vsub_qf32)
629 ? Hexagon::V6_vconv_sf_qf32
630 : Hexagon::V6_vconv_hf_qf16;
631 auto SubOp = (ConvOp == Hexagon::V6_vconv_sf_qf32) ? Hexagon::V6_vsub_sf
632 : Hexagon::V6_vsub_hf;
633
634 Register SrcOp2Reg = SrcOp2.getReg();
635 MIB = BuildMI(*MBB, *MI, DL, HII->get(Opcode: ConvOp), SrcOp2Reg)
636 .addReg(SrcOp2Reg, getRegState(RegOp: SrcOp2) | RegState::Kill);
637 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
638 MIB.getInstr()->dump());
639 MIB = BuildMI(*MBB, *MI, DL, HII->get(Opcode: SubOp), DestReg)
640 .addReg(SrcOp1.getReg(), getRegState(RegOp: SrcOp1))
641 .addReg(SrcOp2Reg, getRegState(RegOp: SrcOp2));
642 // If Op2 is not killed, it is used after this instruction.
643 // convert it back to original qf form.
644 if (!SrcOp2.isKill())
645 insertIEEEToQF(MI: &*(++MI->getIterator()), SrcReg: SrcOp2.getReg(), SrcOp: SrcOp2);
646 return true;
647 };
648
649 // If both operands are sf type, we only need to replace the opcode.
650 if (Pair.first == true && Pair.second == true) {
651 auto finalOpcode = getreplacedQFOpcode(srcOpcode: SrcOpcode, firstOp: true, secondOp: true);
652 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
653 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1))
654 .addReg(RegNo: SrcOp2.getReg(), Flags: getRegState(RegOp: SrcOp2));
655 Changed |= true;
656 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
657 MIB.getInstr()->dump());
658
659 } else if (Pair.first == true && Pair.second == false) {
660 auto finalOpcode = getreplacedQFOpcode(srcOpcode: SrcOpcode, firstOp: true, secondOp: false);
661
662 // If 2nd op is qf, first op is sf, convert the 2nd
663 // op to sf before inserting the vmpy instruction.
664 if (SrcOpcode == Hexagon::V6_vmpy_qf32) {
665 Register SrcOp2Reg = SrcOp2.getReg();
666 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32),
667 DestReg: SrcOp2Reg)
668 .addReg(RegNo: SrcOp2Reg, Flags: getRegState(RegOp: SrcOp2) | RegState::Kill);
669 LLVM_DEBUG(dbgs() << "\nInserting new instruction before: ";
670 MIB.getInstr()->dump());
671 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
672 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1))
673 .addReg(RegNo: SrcOp2Reg, Flags: getRegState(RegOp: SrcOp2));
674 // If Op2 is not killed convert back to qf, since there
675 // are uses for this qf op.
676 if (!SrcOp2.isKill())
677 insertIEEEToQF(MI: &*(++MI->getIterator()), SrcReg: SrcOp2.getReg(), SrcOp: SrcOp2,
678 is32bit: true /* sf type reg */);
679
680 // if the opcode is mixed type, we use Op2 as first operand
681 // since that takes in qf type. Op1 is taken as second op.
682 } else if (finalOpcode == Hexagon::V6_vadd_qf16_mix ||
683 finalOpcode == Hexagon::V6_vadd_qf32_mix ||
684 finalOpcode == Hexagon::V6_vmpy_qf16_mix_hf ||
685 finalOpcode == Hexagon::V6_vmpy_qf32_mix_hf) {
686 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
687 .addReg(RegNo: SrcOp2.getReg(), Flags: getRegState(RegOp: SrcOp2))
688 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1));
689
690 // Subtracting is not associative, so if Op1 is sf/hf type and
691 // Op2 is qf type, we cannot interchange the operands.
692 // For v79, we convert Op2 to IEEE and use the non-mix type
693 // instruction for the subtraction.
694 // For v81, we have an appropiate opcode with vsub(sf/hf, qf) type
695 } else if ((SrcOpcode == Hexagon::V6_vsub_qf32 ||
696 SrcOpcode == Hexagon::V6_vsub_qf16) &&
697 HST->useHVXV79Ops()) {
698 Changed |= HandleSub(SrcOpcode);
699
700 } else {
701 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
702 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1))
703 .addReg(RegNo: SrcOp2.getReg(), Flags: getRegState(RegOp: SrcOp2));
704 }
705 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
706 MIB.getInstr()->dump());
707 Changed |= true;
708 } else if (Pair.first == false && Pair.second == true) {
709
710 auto finalOpcode = getreplacedQFOpcode(srcOpcode: SrcOpcode, firstOp: false, secondOp: true);
711 // If 2nd op is sf, first op is qf, convert the 1st
712 // op to sf before inserting the vmpy instruction.
713 if (SrcOpcode == Hexagon::V6_vmpy_qf32) {
714 Register SrcOp1Reg = SrcOp1.getReg();
715 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32),
716 DestReg: SrcOp1Reg)
717 .addReg(RegNo: SrcOp1Reg, Flags: getRegState(RegOp: SrcOp1) | RegState::Kill);
718 LLVM_DEBUG(dbgs() << "\nInserting new instruction before: ";
719 MIB.getInstr()->dump());
720 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
721 .addReg(RegNo: SrcOp1Reg, Flags: getRegState(RegOp: SrcOp1))
722 .addReg(RegNo: SrcOp2.getReg(), Flags: getRegState(RegOp: SrcOp2));
723 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
724 MIB.getInstr()->dump());
725 // If Op1 is not killed convert back to qf, since there
726 // are uses for this qf op.
727 if (!SrcOp1.isKill())
728 insertIEEEToQF(MI: &*(++MI->getIterator()), SrcReg: SrcOp1.getReg(), SrcOp: SrcOp1,
729 is32bit: true /*sf type reg*/);
730 } else {
731
732 MIB = BuildMI(BB&: *MBB, I&: *MI, MIMD: DL, MCID: HII->get(Opcode: finalOpcode), DestReg)
733 .addReg(RegNo: SrcOp1.getReg(), Flags: getRegState(RegOp: SrcOp1))
734 .addReg(RegNo: SrcOp2.getReg(), Flags: getRegState(RegOp: SrcOp2));
735 LLVM_DEBUG(dbgs() << "\nInserting new instruction: ";
736 MIB.getInstr()->dump());
737 }
738 Changed |= true;
739 } else {
740 // Both the operands of this instructions are valid, so no use of
741 // this instruction is to be modified. We need to remove this
742 // instruction from the action map QFUsesMap.
743 eraseList.push_back(x: MI);
744 }
745 }
746
747 for (MachineInstr *delMI : eraseList)
748 QFUsesMap.erase(x: delMI);
749
750 return Changed;
751}
752
753// Insert a new instruction.
754void HexagonPostRAHandleQFP::insertInstr(MachineInstr *MI, unsigned MIOpcode,
755 unsigned SrcReg, unsigned DstReg,
756 RegState Flags) {
757
758 MachineInstrBuilder MIB;
759 MachineBasicBlock *MBB = MI->getParent();
760 DebugLoc DL = MI->getDebugLoc();
761 MachineBasicBlock::iterator MIt = MI;
762 auto MINext = ++MI->getIterator();
763 if (++MIt == MBB->end())
764 MIB = BuildMI(BB: MBB, MIMD: DL, MCID: HII->get(Opcode: MIOpcode), DestReg: DstReg).addReg(RegNo: SrcReg, Flags);
765 else
766 MIB = BuildMI(BB&: *MBB, I: MINext, MIMD: DL, MCID: HII->get(Opcode: MIOpcode), DestReg: DstReg)
767 .addReg(RegNo: SrcReg, Flags);
768 LLVM_DEBUG(dbgs() << "\t\tInserting after conv: "; MIB.getInstr()->dump());
769}
770
771// Find an available vector register to store 0x0. We have reserved vector
772// register v30 to be exempted from being used during register allocation
773// for this purpose.
774MCPhysReg HexagonPostRAHandleQFP::findAllocatableReg(MachineInstr *MI) const {
775 LLVM_DEBUG(dbgs() << "\tUsing V30 register to store a vector of zeroes!");
776 return Hexagon::V30;
777}
778
779// Insert qf = sf/hf conversions before non-saturating instructions
780bool HexagonPostRAHandleQFP::HandleNonSatInstr() {
781
782 for (auto It : QFNonSatMIs) {
783 MachineInstr *MI = It.first;
784 auto MIOpcode = MI->getOpcode();
785 auto Op = MI->getOperand(i: 1);
786 Register DefReg = Op.getReg();
787 LLVM_DEBUG(dbgs() << "Analyzing convert instruction: "; MI->dump());
788 // Handle hf = qf16.
789 // Handle f8 = qf16
790 if (MIOpcode == Hexagon::V6_vconv_hf_qf16 ||
791 MIOpcode == Hexagon::V6_vconv_f8_qf16) {
792
793 insertIEEEToQF(MI, SrcReg: DefReg, SrcOp: Op);
794 // TODO Check if there are any reaching def which is qf generating type.
795 // That op should be converted to sf/hf
796 if (!Op.isKill())
797 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_hf_qf16, SrcReg: DefReg, DstReg: DefReg,
798 Flags: getRegState(RegOp: Op) | RegState::Kill);
799
800 // Handle hf = qf.qf.
801 // Handle bf = qf.qf
802 } else if (MIOpcode == Hexagon::V6_vconv_hf_qf32 ||
803 MIOpcode == Hexagon::V6_vconv_bf_qf32) {
804 Register DefLo = HRI->getSubReg(Reg: DefReg, Idx: Hexagon::vsub_lo);
805 Register DefHi = HRI->getSubReg(Reg: DefReg, Idx: Hexagon::vsub_hi);
806
807 if (It.second == ConvOperand::HiLo) {
808 insertIEEEToQF(MI, SrcReg: DefLo, SrcOp: Op, is32bit: true /* sf type */);
809 insertIEEEToQF(MI, SrcReg: DefHi, SrcOp: Op, is32bit: true /* sf type */);
810
811 // Check which subregister is live and convert it
812 // and according insert conversion for that subreg
813 auto KillState = SubRegKillSet[MI];
814 if (!KillState.first)
815 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefHi, DstReg: DefHi,
816 Flags: getRegState(RegOp: Op) | RegState::Kill);
817
818 if (!KillState.second)
819 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefLo, DstReg: DefLo,
820 Flags: getRegState(RegOp: Op) | RegState::Kill);
821
822 } else if (It.second == ConvOperand::Hi) {
823 insertIEEEToQF(MI, SrcReg: DefHi, SrcOp: Op, is32bit: true /* sf type */);
824 if (!Op.isKill())
825 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefHi, DstReg: DefHi,
826 Flags: getRegState(RegOp: Op) | RegState::Kill);
827
828 } else { // It.second == ConvOperand::Lo
829 insertIEEEToQF(MI, SrcReg: DefLo, SrcOp: Op, is32bit: true /* sf type */);
830 if (!Op.isKill())
831 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefLo, DstReg: DefLo,
832 Flags: getRegState(RegOp: Op) | RegState::Kill);
833 }
834 // Handle sf = qf32.
835 } else if (MIOpcode == Hexagon::V6_vconv_sf_qf32) {
836 insertIEEEToQF(MI, SrcReg: DefReg, SrcOp: Op, is32bit: true /* sf type */);
837 if (!Op.isKill())
838 insertInstr(MI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefReg, DstReg: DefReg,
839 Flags: getRegState(RegOp: Op) | RegState::Kill);
840
841 } else {
842 llvm_unreachable("Unhandled non-saturating instruction!");
843 }
844 }
845
846 if (QFNonSatMIs.empty())
847 return false;
848 return true;
849}
850
851// Calculates the liveness of subregisters (whether killed or not)
852// when double register is used. This is necessary because RDF
853// carries liveness of the superreg and not the subregisters individually
854void HexagonPostRAHandleQFP::collectLivenessForSubregs(
855 NodeAddr<UseNode *> &UsedNode) {
856 RegisterRef UR = UsedNode.Addr->getRegRef(G: *DFG);
857 NodeAddr<StmtNode *> UseStmt = UsedNode.Addr->getOwner(G: *DFG);
858 MachineInstr *UseInstr = UseStmt.Addr->getCode();
859 auto UseOp = UseInstr->getOperand(i: 1);
860 Register UseDefLo = HRI->getSubReg(Reg: UseOp.getReg(), Idx: Hexagon::vsub_lo);
861 Register UseDefHi = HRI->getSubReg(Reg: UseOp.getReg(), Idx: Hexagon::vsub_hi);
862
863 NodeSet Visited, Defs;
864 bool isHiSubRegKilled = true, isLoSubRegKilled = true;
865 const auto &P = LV->getAllReachingDefsRec(RefRR: UR, RefA: UsedNode, Visited, Defs);
866
867 if (!P.second)
868 return;
869
870 for (auto RD : P.first) {
871 NodeAddr<DefNode *> RegDef = DFG->addr<DefNode *>(N: RD);
872 Register RR = RegDef.Addr->getRegRef(G: *DFG).Id;
873 if (HRI->isFakeReg(Reg: RR))
874 continue;
875 NodeAddr<StmtNode *> RegStmt = RegDef.Addr->getOwner(G: *DFG);
876 MachineInstr *ReachDefInstr = RegStmt.Addr->getCode();
877 if (ReachDefInstr == nullptr)
878 continue;
879
880 // If the reaching def is WReg, then the kill flag in the use is correct
881 // since there is no subreg
882 Register DefReg = ReachDefInstr->getOperand(i: 0).getReg();
883 if (Hexagon::HvxWRRegClass.contains(Reg: DefReg)) {
884 if (!UseOp.isKill())
885 isHiSubRegKilled = isLoSubRegKilled = false;
886
887 // If the reaching ref is VReg, the liveness might be different between
888 // each of the subreg. Handle them individually.
889 // Find the other uses after this use for the reaching def. If it exists,
890 // the subregister is live after the use.
891 // NOTE: Assumption: The uses are in order in RDF.
892 } else {
893 NodeSet UseSet;
894 getAllRealUses(DA: RegDef, UNodeSet&: UseSet, L: LV, G: DFG);
895 for (auto UIntr : UseSet) {
896 NodeAddr<UseNode *> UA = DFG->addr<UseNode *>(N: UIntr);
897 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *DFG);
898 MachineInstr *UseMI = UseStmt.Addr->getCode();
899 if (UseMI == nullptr)
900 continue;
901 // When we reach the use set a flag to see if there are other uses
902 // after this. If yes, then the register is not killed.
903 if (UseMI == UseInstr)
904 continue;
905 if (HII->isMIBefore(A: UseInstr, B: UseMI) && DefReg == UseDefLo) {
906 isLoSubRegKilled = false;
907 break;
908 }
909 if (HII->isMIBefore(A: UseInstr, B: UseMI) && DefReg == UseDefHi) {
910 isHiSubRegKilled = false;
911 break;
912 }
913 }
914 }
915 }
916 SubRegKillSet[UseInstr] = std::make_pair(x&: isHiSubRegKilled, y&: isLoSubRegKilled);
917}
918
919// Store all refill instructions.
920void HexagonPostRAHandleQFP::collectQFPStackRefill(
921 NodeAddr<StmtNode *> *StNode) {
922 NodeAddr<DefNode *> DfNode =
923 StNode->Addr->members_if(P: DFG->IsDef, G: *DFG).front();
924 MachineInstr *MI = StNode->Addr->getCode();
925 // Check if operand to this instruction is a frame index.
926 const MachineOperand &OpFI = MI->getOperand(i: 1);
927 if (!OpFI.isFI())
928 return;
929
930 // LLVM_DEBUG(dbgs() << "\n[Stack Refill]: Collecting: "; MI->dump());
931 RefillMIs.push_back(x: DfNode);
932}
933
934// Iterate over the uses of the qf generating instruction in RDG graph
935// If we get a qf to IEEE convert instruction, add it to a list.
936void HexagonPostRAHandleQFP::collectConvQFInstr(NodeAddr<DefNode *> &RegDef) {
937
938 NodeSet UseSet;
939 NodeAddr<StmtNode *> DefStmt = RegDef.Addr->getOwner(G: *DFG);
940 MachineInstr *DefInstr = DefStmt.Addr->getCode();
941 getAllRealUses(DA: RegDef, UNodeSet&: UseSet, L: LV, G: DFG);
942 for (auto UI : UseSet) {
943 NodeAddr<UseNode *> UA = DFG->addr<UseNode *>(N: UI);
944 if (UA.Addr->getFlags() & NodeAttrs::PhiRef)
945 continue;
946 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *DFG);
947 MachineInstr *QFConvInstr = UseStmt.Addr->getCode();
948 if (std::find(first: QFNonSatInstr.begin(), last: QFNonSatInstr.end(),
949 val: QFConvInstr->getOpcode()) != QFNonSatInstr.end()) {
950
951 // The use is a double register type. But the def can be hi/lo or double
952 // type. So conversion needs to be inserted only for the type
953 // which is in IEEE form.
954 auto UseReg = QFConvInstr->getOperand(i: 1).getReg();
955 auto DefReg = DefInstr->getOperand(i: 0).getReg();
956 if (Hexagon::HvxWRRegClass.contains(Reg: UseReg)) {
957
958 collectLivenessForSubregs(UsedNode&: UA);
959 unsigned Op = ConvOperand::Undefined;
960 if (QFNonSatMIs.contains(Key: QFConvInstr))
961 Op = QFNonSatMIs[QFConvInstr];
962
963 // Def is double type
964 if (Hexagon::HvxWRRegClass.contains(Reg: DefReg))
965 Op = ConvOperand::HiLo;
966 // Def is lo of double type
967 else if (DefReg == HRI->getSubReg(Reg: UseReg, Idx: Hexagon::vsub_lo))
968 Op |= ConvOperand::Lo;
969 // Def is hi of double type
970 else
971 Op |= ConvOperand::Hi;
972 QFNonSatMIs[QFConvInstr] = Op;
973 } else // for other def-use, BothOp is used as default
974 QFNonSatMIs[QFConvInstr] = ConvOperand::HiLo;
975
976 IgnoreInsertConvList.insert(Ptr: DefInstr);
977 LLVM_DEBUG(std::string OpType = ""; switch (QFNonSatMIs[QFConvInstr]) {
978 case ConvOperand::HiLo:
979 OpType = "HiLo Op";
980 break;
981 case ConvOperand::Lo:
982 OpType = "Lo Op";
983 break;
984 case ConvOperand::Hi:
985 OpType = "Hi Op";
986 break;
987 default:
988 OpType = "Undefined";
989 } dbgs() << "Collecting convert instruction with type "
990 << OpType << " : ";
991 QFConvInstr->dump());
992 }
993 }
994}
995
996// Check if the COPY statements use came from a def which generates
997// a qf type. If yes, collect it in a vector. Also, collect copies
998// with reaching def other copies (nested copies).
999void HexagonPostRAHandleQFP::collectCopies(NodeAddr<StmtNode *> *StNode) {
1000
1001 NodeAddr<DefNode *> CopyDef =
1002 StNode->Addr->members_if(P: DFG->IsDef, G: *DFG).front();
1003 MachineInstr *CopyInstr = StNode->Addr->getCode();
1004 LLVM_DEBUG(dbgs() << "\nAnalyzing copy: "; StNode->Addr->getCode()->dump());
1005
1006 for (NodeAddr<UseNode *> UA : StNode->Addr->members_if(P: DFG->IsUse, G: *DFG)) {
1007 RegisterRef UR = UA.Addr->getRegRef(G: *DFG);
1008 NodeSet Visited, Defs;
1009 const auto &P = LV->getAllReachingDefsRec(RefRR: UR, RefA: UA, Visited, Defs);
1010 if (!P.second) {
1011 LLVM_DEBUG({
1012 dbgs() << "*** Unable to collect all reaching defs for use ***\n"
1013 << PrintNode<UseNode *>(UA, *DFG) << '\n';
1014 });
1015 continue;
1016 }
1017
1018 // Note: there can be multiple reaching defs of the copy
1019 for (auto RD : P.first) {
1020 NodeAddr<DefNode *> RegDef = DFG->addr<DefNode *>(N: RD);
1021 Register RR = RegDef.Addr->getRegRef(G: *DFG).Id;
1022 if (HRI->isFakeReg(Reg: RR))
1023 continue;
1024 NodeAddr<StmtNode *> RegStmt = RegDef.Addr->getOwner(G: *DFG);
1025 MachineInstr *ReachDefInstr = RegStmt.Addr->getCode();
1026 if (ReachDefInstr == nullptr)
1027 continue;
1028 LLVM_DEBUG(dbgs() << "\t[Reaching Def]: "; ReachDefInstr->dump());
1029
1030 // If the reaching def is a COPY,collect it with reg type ieee
1031 if (ReachDefInstr->getOpcode() == TargetOpcode::COPY) {
1032 auto pairKey = std::make_pair(x&: CopyDef.Id, y&: RegDef.Id);
1033 QFCopys[pairKey] = RegType::ieee;
1034 continue;
1035 }
1036
1037 // If the reaching def is a qf instr, collect the copy.
1038 // reg type is selected based on the op
1039 auto RegT = RegType::undefined;
1040 if (HII->isQFPInstr(MI: ReachDefInstr)) {
1041 if (HII->isQFP32Instr(MI: ReachDefInstr)) {
1042 // check whether the copies register is hvxWR or hvxVR type
1043 // NOTE: Assumption: A copy's reaching def shall not be 2,
1044 // i.e., for each of the subregister.
1045 if (Hexagon::HvxWRRegClass.contains(
1046 Reg: ReachDefInstr->getOperand(i: 0).getReg()))
1047 RegT = RegType::qf32_double;
1048 else
1049 RegT = RegType::qf32;
1050 } else if (HII->isQFP16Instr(MI: ReachDefInstr)) {
1051 // Check if qf16 instruction outputs double-wide register
1052 if (Hexagon::HvxWRRegClass.contains(
1053 Reg: ReachDefInstr->getOperand(i: 0).getReg())) {
1054 RegT = RegType::qf16_double;
1055 } else {
1056 RegT = RegType::qf16;
1057 }
1058 }
1059 } else {
1060 // if the copy involves non-qf vector registers collect it too
1061 Register CopyReg = CopyInstr->getOperand(i: 1).getReg();
1062 if (Hexagon::HvxWRRegClass.contains(Reg: CopyReg) ||
1063 Hexagon::HvxVRRegClass.contains(Reg: CopyReg))
1064 RegT = RegType::ieee;
1065 else
1066 continue;
1067 }
1068 auto pairKey = std::make_pair(x&: CopyDef.Id, y&: RegDef.Id);
1069 QFCopys[pairKey] = RegT;
1070 }
1071 }
1072}
1073
1074// Inserts an qf instruction to a list. These instruction
1075// values are spilled to the stack.
1076void HexagonPostRAHandleQFP::collectQFPStackSpill(
1077 NodeAddr<StmtNode *> *StNode) {
1078
1079 MachineInstr *MI = StNode->Addr->getCode();
1080 LLVM_DEBUG(dbgs() << "\n[Stack Spill]: Analyzing: "; MI->dump());
1081 // Check if operand to this instruction is a frame index.
1082 const MachineOperand &OpFI = MI->getOperand(i: 0);
1083 if (!OpFI.isFI())
1084 return;
1085
1086 // Pre-RegAlloc
1087 //%46:hvxwr = V6_vmpy_qf32_hf %7:hvxvr, %10:hvxvr
1088 // PS_vstorerw_ai %stack.3, 0, %46:hvxwr :: (store (s2048) into %stack.3,
1089 // align 128)
1090 // Post-RegAlloc
1091 // renamable $w4 = V6_vmpy_qf32_hf killed renamable $v1, renamable $v0
1092 // PS_vstorerw_ai %stack.3, 0, renamable $w4 :: (store (s2048) into %stack.3,
1093 // align 128)
1094
1095 if (!MI->getOperand(i: 2).isReg())
1096 return;
1097
1098 // Iterate over the operands of the store instruction to get their reaching
1099 // defs
1100 NodeId QFPDefNode = 0;
1101 for (NodeAddr<UseNode *> UA : StNode->Addr->members_if(P: DFG->IsUse, G: *DFG)) {
1102 QFPDefNode = UA.Addr->getReachingDef();
1103
1104 // Get the defining instruction node(s)
1105 NodeAddr<DefNode *> RegDef = DFG->addr<DefNode *>(N: QFPDefNode);
1106 assert(QFPDefNode != 0 && "Reaching def computation error");
1107 NodeAddr<StmtNode *> RegStmt = RegDef.Addr->getOwner(G: *DFG);
1108 MachineInstr *ReachDefInstr = RegStmt.Addr->getCode();
1109 if (ReachDefInstr == nullptr)
1110 continue;
1111 LLVM_DEBUG(dbgs() << "[Stack Spill]:\tReaching Def of operand:";
1112 ReachDefInstr->dump());
1113 // Reaching Def cannot be a phi instruction.
1114 if (RegDef.Addr->getFlags() & NodeAttrs::PhiRef)
1115 continue;
1116
1117 if (!HII->isQFPInstr(MI: ReachDefInstr))
1118 continue;
1119
1120 auto RR = RegDef.Addr->getRegRef(G: *DFG).Id;
1121 if (HRI->isFakeReg(Reg: RR))
1122 continue;
1123
1124 LLVM_DEBUG(dbgs() << "Found a QFPStackSpill via \n"; MI->dump();
1125 dbgs() << "The corresponding XQF instruction is:\n";
1126 ReachDefInstr->dump());
1127
1128 // Collect the spills.
1129 SpillMIs.push_back(x: std::make_pair(x&: MI, y&: RegDef));
1130 }
1131}
1132
1133// Find the uses of qf generating instructions and conditionally add them
1134// to a list.
1135void HexagonPostRAHandleQFP::collectQFUses(NodeAddr<DefNode *> RegDef,
1136 MachineInstr *DefMI) {
1137
1138 NodeSet UseSet;
1139 LLVM_DEBUG(dbgs() << " Finding uses of: "; DefMI->dump(););
1140 getAllRealUses(DA: RegDef, UNodeSet&: UseSet, L: LV, G: DFG);
1141
1142 for (auto UI : UseSet) {
1143 NodeAddr<UseNode *> UA = DFG->addr<UseNode *>(N: UI);
1144 if (UA.Addr->getFlags() & NodeAttrs::PhiRef)
1145 continue;
1146 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *DFG);
1147 MachineInstr *UseMI = UseStmt.Addr->getCode();
1148 LLVM_DEBUG(dbgs() << "\t\t\t[Reached Use of QF operand]: "; UseMI->dump());
1149
1150 Register UsedReg = UA.Addr->getRegRef(G: *DFG).Id;
1151 if (QFPSatInstsMap.find(Val: UseMI->getOpcode()) != QFPSatInstsMap.end()) {
1152 if (PossibleMultiReachDefs.count(x: UseStmt.Id) == 0) {
1153 PossibleMultiReachDefs.insert(x: UseStmt.Id);
1154 LLVM_DEBUG(dbgs() << "\n[Collect instr with possible multidef]:";
1155 UseMI->dump());
1156 }
1157 conditionallyInsert(MI&: *UseMI, DefReg&: UsedReg);
1158 }
1159 }
1160}
1161
1162// Process the list which can have multiple definitions. A possible case
1163// can be reaching defs to be a copy and a qf-generating instr respectively.
1164// Only handle the qf-generating instruction by inserting convert to sf/hf
1165// after it. Additionally, then handle the reached uses of this reaching
1166// def since the type has changed to sf/hf from qf after the conversion.
1167bool HexagonPostRAHandleQFP::HandleMultiReachingDefs() {
1168
1169 bool Changed = false;
1170 // Note: It may seem this loop can further add to PossibleMultiReachDefs.
1171 // But it is not expected to since if any instruction has multiple
1172 // definitions it should already be present in it.
1173 for (auto It : PossibleMultiReachDefs) {
1174 NodeAddr<StmtNode *> Stmt = DFG->addr<StmtNode *>(N: It);
1175 MachineInstr *Instr = Stmt.Addr->getCode();
1176 // get the op type for the original instruction.
1177 // True is sf/hf, false is qf
1178 auto Pair = QFUsesMap[Instr];
1179
1180 unsigned short UseNo = 1;
1181 // Iterate over the operands
1182 for (NodeAddr<UseNode *> UA : Stmt.Addr->members_if(P: DFG->IsUse, G: *DFG)) {
1183
1184 // If the type is qf for the operand,
1185 // we skip since there is no scope for mismatch
1186 if ((UseNo == 1 && Pair.first == false) ||
1187 (UseNo == 2 && Pair.second == false)) {
1188 ++UseNo;
1189 continue;
1190 }
1191
1192 RegisterRef UR = UA.Addr->getRegRef(G: *DFG);
1193 NodeSet Visited, Defs;
1194 const auto &P = LV->getAllReachingDefsRec(RefRR: UR, RefA: UA, Visited, Defs);
1195 if (!P.second) {
1196 LLVM_DEBUG({
1197 dbgs() << "*** Unable to collect all reaching defs for use ***\n"
1198 << PrintNode<UseNode *>(UA, *DFG) << '\n';
1199 });
1200 continue;
1201 }
1202
1203 // Iterate over the reaching defs and process the ones which
1204 // generate qf. Ignore the ones which have already been handled
1205 for (auto RD : P.first) {
1206 NodeAddr<DefNode *> RegDef = DFG->addr<DefNode *>(N: RD);
1207
1208 // Ignore fake reaches
1209 auto RR = RegDef.Addr->getRegRef(G: *DFG).Id;
1210 if (HRI->isFakeReg(Reg: RR))
1211 continue;
1212
1213 NodeAddr<StmtNode *> RegStmt = RegDef.Addr->getOwner(G: *DFG);
1214 MachineInstr *ReachDefInstr = RegStmt.Addr->getCode();
1215
1216 if (ReachDefInstr == nullptr)
1217 continue;
1218
1219 if (!HII->isQFPInstr(MI: ReachDefInstr))
1220 continue;
1221 if (IgnoreInsertConvList.find(Ptr: ReachDefInstr) !=
1222 IgnoreInsertConvList.end())
1223 continue;
1224 LLVM_DEBUG(dbgs() << "[Multidef] Handling reaching def:";
1225 ReachDefInstr->dump());
1226
1227 auto *MBB = ReachDefInstr->getParent();
1228 auto &dl = ReachDefInstr->getDebugLoc();
1229 auto NextReachMI = ++ReachDefInstr->getIterator();
1230 auto DefOp = ReachDefInstr->getOperand(i: 0);
1231 Register OpReg = DefOp.getReg();
1232 MachineInstrBuilder MIB;
1233
1234 // For double vector regs, two conversions are inserted. Single
1235 // conversion for qf32 type
1236 if (HII->isQFP32Instr(MI: ReachDefInstr)) {
1237 // if the reaching def is a qf double type
1238 if (Hexagon::HvxWRRegClass.contains(
1239 Reg: ReachDefInstr->getOperand(i: 0).getReg())) {
1240 Register RegLo = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_lo);
1241 Register RegHi = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_hi);
1242 MIB = BuildMI(BB&: *MBB, I: NextReachMI, MIMD: dl,
1243 MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: RegLo)
1244 .addReg(RegNo: RegLo, Flags: RegState::Renamable | RegState::Kill);
1245 LLVM_DEBUG(dbgs() << "[MultiDef] Inserting convert instruction: ";
1246 MIB.getInstr()->dump());
1247 MIB = BuildMI(BB&: *MBB, I: NextReachMI, MIMD: dl,
1248 MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: RegHi)
1249 .addReg(RegNo: RegHi, Flags: RegState::Renamable | RegState::Kill);
1250 } else { // If the reaching def is a qf type
1251 MIB = BuildMI(BB&: *MBB, I: NextReachMI, MIMD: dl,
1252 MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: OpReg)
1253 .addReg(RegNo: OpReg, Flags: RegState::Renamable | RegState::Kill);
1254 }
1255 }
1256 if (HII->isQFP16Instr(MI: ReachDefInstr)) {
1257 MIB = BuildMI(BB&: *MBB, I: NextReachMI, MIMD: dl,
1258 MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: OpReg)
1259 .addReg(RegNo: OpReg, Flags: RegState::Renamable | RegState::Kill);
1260 }
1261 LLVM_DEBUG(dbgs() << "[MultiDef] Inserting convert instruction: ";
1262 MIB.getInstr()->dump(); dbgs() << "\tafter instruction: ";
1263 ReachDefInstr->dump());
1264
1265 // find the uses of the newly transformed to sf/hf and handle
1266 // accordingly. Uses can be vmul/vadd/etc. types or converts which take
1267 // in qf types.
1268 collectQFUses(RegDef, DefMI: ReachDefInstr);
1269 collectConvQFInstr(RegDef);
1270 IgnoreInsertConvList.insert(Ptr: ReachDefInstr);
1271 Changed = true;
1272 }
1273 UseNo++;
1274 }
1275 }
1276 return Changed;
1277}
1278
1279bool HexagonPostRAHandleQFP::HandleConvertToQfCopies() {
1280 if (ConvertToQfCopies.empty())
1281 return false;
1282
1283 LLVM_DEBUG(
1284 dbgs() << "\n*** Inserting convert to qf for selected copies ***\n");
1285
1286 // Any reached use of the copy should not already be collected to be
1287 // converted to IEEE. If present, it means that the reached use has
1288 // other reaching def with type IEEE, other than this copy.
1289 auto CanTransform = [&](MachineInstr *MI, unsigned OpNo) -> bool {
1290 if (QFUsesMap.find(x: MI) != QFUsesMap.end()) {
1291 auto Entry = QFUsesMap[MI];
1292 if (OpNo == 1 && Entry.first == true)
1293 return false;
1294 if (OpNo == 2 && Entry.second == true)
1295 return false;
1296 }
1297 return true;
1298 };
1299
1300 for (auto It : ConvertToQfCopies) {
1301 NodeSet UseSet;
1302 getAllRealUses(DA: It.second.first, UNodeSet&: UseSet, L: LV, G: DFG);
1303
1304 bool transform = true;
1305 for (auto UI : UseSet) {
1306 NodeAddr<UseNode *> UA = DFG->addr<UseNode *>(N: UI);
1307 if (UA.Addr->getFlags() & NodeAttrs::PhiRef)
1308 continue;
1309 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *DFG);
1310 MachineInstr *UseMI = UseStmt.Addr->getCode();
1311 unsigned OpNo = UA.Addr->getOp().getOperandNo();
1312
1313 if (!CanTransform(UseMI, OpNo)) {
1314 transform = false;
1315 break;
1316 }
1317 }
1318
1319 if (transform) {
1320
1321 LLVM_DEBUG(dbgs() << "\n[HandleConvertToQfCopies]\tProcessing Copy:";
1322 It.first->dump());
1323 auto CopyOp = It.first->getOperand(i: 0);
1324 auto NextMIIter = std::next(x: It.first->getIterator());
1325 switch (It.second.second) {
1326 case RegType::qf32_double: {
1327 Register DefLo = HRI->getSubReg(Reg: CopyOp.getReg(), Idx: Hexagon::vsub_lo);
1328 Register DefHi = HRI->getSubReg(Reg: CopyOp.getReg(), Idx: Hexagon::vsub_hi);
1329 insertIEEEToQF(MI: &*NextMIIter, SrcReg: DefLo, SrcOp: CopyOp, /*is32bit=*/true);
1330 insertIEEEToQF(MI: &*NextMIIter, SrcReg: DefHi, SrcOp: CopyOp, /*is32bit=*/true);
1331 break;
1332 }
1333 case RegType::qf16_double: {
1334 Register DefLo = HRI->getSubReg(Reg: CopyOp.getReg(), Idx: Hexagon::vsub_lo);
1335 Register DefHi = HRI->getSubReg(Reg: CopyOp.getReg(), Idx: Hexagon::vsub_hi);
1336 insertIEEEToQF(MI: &*NextMIIter, SrcReg: DefLo, SrcOp: CopyOp, /*is32bit=*/false);
1337 insertIEEEToQF(MI: &*NextMIIter, SrcReg: DefHi, SrcOp: CopyOp, /*is32bit=*/false);
1338 break;
1339 }
1340 case RegType::qf16:
1341 insertIEEEToQF(MI: &*NextMIIter, SrcReg: CopyOp.getReg(), SrcOp: CopyOp,
1342 /*is32bit=*/false);
1343 break;
1344 case RegType::qf32:
1345 insertIEEEToQF(MI: &*NextMIIter, SrcReg: CopyOp.getReg(), SrcOp: CopyOp, /*is32bit=*/true);
1346 break;
1347 default:
1348 break;
1349 }
1350 } else {
1351 collectQFUses(RegDef: It.second.first, DefMI: It.first);
1352 collectConvQFInstr(RegDef&: It.second.first);
1353 }
1354 }
1355 return true;
1356}
1357
1358bool HexagonPostRAHandleQFP::HandleReachDefOfCopies() {
1359 if (ReachDefOfCopies.empty())
1360 return false;
1361
1362 MachineInstrBuilder MIB;
1363 for (auto It : ReachDefOfCopies) {
1364 auto *MBB = It.first->getParent();
1365 auto &dl = It.first->getDebugLoc();
1366 auto NextMI = ++(It.first)->getIterator();
1367 auto RegOp = It.first->getOperand(i: 0);
1368 Register OpReg = RegOp.getReg();
1369
1370 if (It.second == RegType::qf32)
1371 MIB =
1372 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: OpReg)
1373 .addReg(RegNo: OpReg, Flags: RegState::Renamable | RegState::Kill);
1374 else if (It.second == RegType::qf16)
1375 MIB =
1376 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: OpReg)
1377 .addReg(RegNo: OpReg, Flags: RegState::Renamable | RegState::Kill);
1378 else if (It.second == RegType::qf32_double) {
1379 Register RegLo = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_lo);
1380 Register RegHi = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_hi);
1381 MIB =
1382 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: RegLo)
1383 .addReg(RegNo: RegLo, Flags: RegState::Renamable | RegState::Kill);
1384 LLVM_DEBUG(dbgs() << "Inserting convert instruction: ";
1385 MIB.getInstr()->dump());
1386 MIB =
1387 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: RegHi)
1388 .addReg(RegNo: RegHi, Flags: RegState::Renamable | RegState::Kill);
1389 } else if (It.second == RegType::qf16_double) {
1390 Register RegLo = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_lo);
1391 Register RegHi = HRI->getSubReg(Reg: OpReg, Idx: Hexagon::vsub_hi);
1392 MIB =
1393 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: RegLo)
1394 .addReg(RegNo: RegLo, Flags: RegState::Renamable | RegState::Kill);
1395 LLVM_DEBUG(dbgs() << "Inserting convert instruction: ";
1396 MIB.getInstr()->dump());
1397 MIB =
1398 BuildMI(BB&: *MBB, I: NextMI, MIMD: dl, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: RegHi)
1399 .addReg(RegNo: RegHi, Flags: RegState::Renamable | RegState::Kill);
1400 }
1401 LLVM_DEBUG(dbgs() << "Inserting convert instruction: ";
1402 MIB.getInstr()->dump(); dbgs() << "\tafter instruction: ";
1403 It.first->dump());
1404 }
1405 return true;
1406}
1407
1408HexagonPostRAHandleQFP::RegType
1409HexagonPostRAHandleQFP::HasQfUses(NodeAddr<DefNode *> CopyDef,
1410 MachineInstr *CopyMI) {
1411 NodeSet UseSet;
1412 getAllRealUses(DA: CopyDef, UNodeSet&: UseSet, L: LV, G: DFG);
1413
1414 if (UseSet.size() == 0)
1415 return RegType::undefined;
1416
1417 bool hasQf16Use = false;
1418 bool hasQf32Use = false;
1419
1420 LLVM_DEBUG(dbgs() << "[COPY]\nUses of the copy are: ");
1421 for (auto UI : UseSet) {
1422 NodeAddr<UseNode *> UA = DFG->addr<UseNode *>(N: UI);
1423 if (UA.Addr->getFlags() & NodeAttrs::PhiRef)
1424 continue;
1425 NodeAddr<StmtNode *> UseStmt = UA.Addr->getOwner(G: *DFG);
1426 MachineInstr *UseMI = UseStmt.Addr->getCode();
1427 unsigned OpNo = UA.Addr->getOp().getOperandNo();
1428
1429 LLVM_DEBUG(dbgs() << "\nCopy's use: "; UseMI->dump());
1430 // Any reached use should not be a non-qf instruction
1431 if (!HII->usesQFOperand(MI: UseMI, Index: OpNo))
1432 return RegType::ieee;
1433
1434 // Determine the qf type from the use
1435 if (HII->usesQF16Operand(MI: UseMI, Index: OpNo))
1436 hasQf16Use = true;
1437 else if (HII->usesQF32Operand(MI: UseMI, Index: OpNo))
1438 hasQf32Use = true;
1439
1440 // Any reached use should not already be converted to IEEE.
1441 // If present, it means that the reached use has other reaching def
1442 // other than the copy.
1443 if (QFUsesMap.find(x: UseMI) != QFUsesMap.end()) {
1444 auto Entry = QFUsesMap[UseMI];
1445 if (OpNo == 1 && Entry.first == true)
1446 return RegType::ieee;
1447 if (OpNo == 2 && Entry.second == true)
1448 return RegType::ieee;
1449 }
1450 }
1451
1452 // Set the output type based on uses
1453 if (hasQf16Use) {
1454 // Check if copy destination is double-wide
1455 if (Hexagon::HvxWRRegClass.contains(Reg: CopyMI->getOperand(i: 0).getReg()))
1456 return RegType::qf16_double;
1457 else
1458 return RegType::qf16;
1459 } else if (hasQf32Use) {
1460 if (Hexagon::HvxWRRegClass.contains(Reg: CopyMI->getOperand(i: 0).getReg()))
1461 return RegType::qf32_double;
1462 else
1463 return RegType::qf32;
1464 }
1465
1466 return RegType::undefined;
1467}
1468
1469// Go through the collected copies and insert conversion to sf/hf
1470// conditionally *after their reaching defs*. This is done because there
1471// can be mutliple reaching defs of the copies. Also, check for the uses
1472// of the reaching def and handle qf uses too by changing opcode or
1473// inserting converts.
1474// Additionally, check for the uses of the copy
1475// and handle them via changing opcode or inserting converts.
1476bool HexagonPostRAHandleQFP::HandleCopies() {
1477
1478 bool Changed = false;
1479
1480 // If a convert is inserted after a reaching def, add it to ignorelist.
1481 // This is because this reaching def can be reaching def of other copies
1482 // due to non-SSA form.
1483 for (auto It : QFCopys) {
1484
1485 // Get details of the copy node
1486 NodeAddr<DefNode *> CopyNode = DFG->addr<DefNode *>(N: It.first.first);
1487 NodeAddr<StmtNode *> StNode = CopyNode.Addr->getOwner(G: *DFG);
1488 [[maybe_unused]] auto *CopyMI = StNode.Addr->getCode();
1489 LLVM_DEBUG(dbgs() << "\nHandling Reaching Defs of COPY: "; CopyMI->dump();
1490 std::string Type; switch (It.second) {
1491 case RegType::qf32_double:
1492 Type = "qf32_double";
1493 break;
1494 case RegType::qf32:
1495 Type = "qf32";
1496 break;
1497 case RegType::qf16:
1498 Type = "qf16";
1499 break;
1500 case RegType::qf16_double:
1501 Type = "qf16_double";
1502 break;
1503 default:
1504 Type = "ieee";
1505 } dbgs() << "\t Type: "
1506 << Type << "\n");
1507
1508 // insert convert to IEEE after the reaching def if it generates qf type
1509 RegType RTy = It.second;
1510 if (RTy != RegType::ieee) {
1511
1512 // get details of the reaching def node
1513 NodeAddr<DefNode *> ReachDefNode = DFG->addr<DefNode *>(N: It.first.second);
1514 NodeAddr<StmtNode *> StNode = ReachDefNode.Addr->getOwner(G: *DFG);
1515 auto *ReachingDef = StNode.Addr->getCode();
1516
1517 if (IgnoreInsertConvList.find(Ptr: ReachingDef) != IgnoreInsertConvList.end())
1518 continue;
1519
1520 // Collect the reaching defs to be processed later.
1521 ReachDefOfCopies.insert(KV: std::make_pair(x&: ReachingDef, y&: RTy));
1522
1523 // Process the reached uses of the reaching def now for
1524 // incorrect usage, since the register type has changed
1525 // following the conversion.
1526 LLVM_DEBUG(dbgs() << "\n[COPY]\tAnalyzing uses of the reaching defs \
1527 of the copy...");
1528 collectQFUses(RegDef: ReachDefNode, DefMI: ReachingDef);
1529 collectConvQFInstr(RegDef&: ReachDefNode);
1530 IgnoreInsertConvList.insert(Ptr: ReachingDef);
1531 Changed = true;
1532 }
1533 }
1534
1535 // Loop through copies with qf uses
1536 for (auto It : QFCopys) {
1537
1538 // Get details of the copy node
1539 NodeAddr<DefNode *> CopyNode = DFG->addr<DefNode *>(N: It.first.first);
1540 NodeAddr<StmtNode *> StNode = CopyNode.Addr->getOwner(G: *DFG);
1541 auto *CopyMI = StNode.Addr->getCode();
1542 LLVM_DEBUG(dbgs() << "\nHandling COPY: "; CopyMI->dump());
1543 RegType RTy = It.second;
1544
1545 // Process the reached uses of the copy to find any incorrect
1546 // qf uses. If the copy's uses are all qf types, we need to convert
1547 // its result back to qf
1548 // FIXME: don't include the copy if its the last instruction since
1549 // it is *probably* not possible to insert via BuildMI at the end of BB
1550 RTy = HasQfUses(CopyDef: CopyNode, CopyMI);
1551 if (RTy != RegType::ieee && RTy != RegType::undefined &&
1552 (++CopyMI->getIterator() != CopyMI->getParent()->end())) {
1553 if (!ConvertToQfCopies.contains(Val: CopyMI)) {
1554 ConvertToQfCopies[CopyMI] = std::make_pair(x&: CopyNode, y&: RTy);
1555 LLVM_DEBUG(dbgs() << "\n[ConvertToQfCopies]\tAdded copy: ";
1556 CopyMI->dump(); std::string Type; switch (RTy) {
1557 case RegType::qf32_double:
1558 Type = "qf32_double";
1559 break;
1560 case RegType::qf32:
1561 Type = "qf32";
1562 break;
1563 case RegType::qf16:
1564 Type = "qf16";
1565 break;
1566 case RegType::qf16_double:
1567 Type = "qf16_double";
1568 break;
1569 default:
1570 Type = "ieee";
1571 } dbgs() << "\t Type: "
1572 << Type << "\n");
1573 }
1574 continue;
1575 }
1576 LLVM_DEBUG(dbgs() << "\n[COPY]\tAnalyzing uses of the copy...");
1577 collectQFUses(RegDef: CopyNode, DefMI: CopyMI);
1578 collectConvQFInstr(RegDef&: CopyNode);
1579 }
1580
1581 Changed |= HandleReachDefOfCopies();
1582 Changed |= HandleMultiReachingDefs();
1583 Changed |= HandleConvertToQfCopies();
1584
1585 return Changed;
1586}
1587
1588// Inserts conversion instruction sf/hf = qf before spilling
1589// Uses the same physical register for conversion.
1590// Additinally checks for the uses of the register; and
1591// conditionally store them to handle later.
1592bool HexagonPostRAHandleQFP::HandleSpills() {
1593
1594 LLVM_DEBUG(dbgs() << "\n[Handling Spill]\n");
1595 bool Changed = false;
1596 for (auto It : SpillMIs) {
1597
1598 MachineInstr *MI = It.first;
1599 auto OpC = MI->getOpcode();
1600
1601 auto NodeDef = It.second;
1602 NodeAddr<StmtNode *> Stmt = NodeDef.Addr->getOwner(G: *DFG);
1603 MachineInstr *DefMI = Stmt.Addr->getCode();
1604 auto RegOp = MI->getOperand(i: 2);
1605 Register DefR = RegOp.getReg();
1606
1607 // handles widened qf16/qf32 instructions.
1608 if (OpC == Hexagon::PS_vstorerw_ai) {
1609 if (!Hexagon::HvxWRRegClass.contains(Reg: DefR))
1610 assert(false && " Unhandled Vector Register class passed\n");
1611 // Walk through the uses of DefLo and DefHi and if there is QFP
1612 // instructions, the instruction needs to be updated to use sf operands
1613 // instead of qf operands.
1614 collectQFUses(RegDef: NodeDef, DefMI);
1615
1616 if (IgnoreInsertConvList.find(Ptr: DefMI) != IgnoreInsertConvList.end())
1617 continue;
1618
1619 // Collect the reached uses of ReachDefInstr
1620 // which are sf/hf = qf conversion instructions.
1621 collectConvQFInstr(RegDef&: NodeDef);
1622 Register DefLo = HRI->getSubReg(Reg: DefR, Idx: Hexagon::vsub_lo);
1623 Register DefHi = HRI->getSubReg(Reg: DefR, Idx: Hexagon::vsub_hi);
1624
1625 // Create two copy instructions, one each for Hi and Lo conditionally.
1626 // Liveness is the same is for the store instruction for the register.
1627 // If both are double registers, two insertions are done.
1628 // If one of the subregs are reaching to the store, conversion is done
1629 // for that subreg.
1630 Register DReg = DefMI->getOperand(i: 0).getReg();
1631 if (HII->isQFP16Instr(MI: DefMI)) {
1632 if (DefLo == DReg || Hexagon::HvxWRRegClass.contains(Reg: DReg))
1633 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_hf_qf16, SrcReg: DefLo, DstReg: DefLo,
1634 Flags: getRegState(RegOp) | RegState::Kill);
1635
1636 if (DefHi == DReg || Hexagon::HvxWRRegClass.contains(Reg: DReg))
1637 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_hf_qf16, SrcReg: DefHi, DstReg: DefHi,
1638 Flags: getRegState(RegOp) | RegState::Kill);
1639 } else if (HII->isQFP32Instr(MI: DefMI)) {
1640 if (DefLo == DReg || Hexagon::HvxWRRegClass.contains(Reg: DReg))
1641 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefLo, DstReg: DefLo,
1642 Flags: getRegState(RegOp) | RegState::Kill);
1643
1644 if (DefHi == DReg || Hexagon::HvxWRRegClass.contains(Reg: DReg))
1645 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefHi, DstReg: DefHi,
1646 Flags: getRegState(RegOp) | RegState::Kill);
1647 }
1648 IgnoreInsertConvList.insert(Ptr: DefMI);
1649 Changed = true;
1650
1651 // Handles instructions which output qf32 type.
1652 } else if (OpC == Hexagon::PS_vstorerv_ai && HII->isQFP32Instr(MI: DefMI)) {
1653 collectQFUses(RegDef: NodeDef, DefMI);
1654 if (IgnoreInsertConvList.find(Ptr: DefMI) != IgnoreInsertConvList.end())
1655 continue;
1656 collectConvQFInstr(RegDef&: NodeDef);
1657
1658 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_sf_qf32, SrcReg: DefR, DstReg: DefR,
1659 Flags: getRegState(RegOp) | RegState::Kill);
1660
1661 IgnoreInsertConvList.insert(Ptr: DefMI);
1662 Changed = true;
1663
1664 // Handles instructions which output qf16 type.
1665 } else if (OpC == Hexagon::PS_vstorerv_ai && HII->isQFP16Instr(MI: DefMI)) {
1666 collectQFUses(RegDef: NodeDef, DefMI);
1667 if (IgnoreInsertConvList.find(Ptr: DefMI) != IgnoreInsertConvList.end())
1668 continue;
1669 collectConvQFInstr(RegDef&: NodeDef);
1670
1671 insertInstr(MI: DefMI, MIOpcode: Hexagon::V6_vconv_hf_qf16, SrcReg: DefR, DstReg: DefR,
1672 Flags: getRegState(RegOp) | RegState::Kill);
1673
1674 IgnoreInsertConvList.insert(Ptr: DefMI);
1675 Changed = true;
1676 } else {
1677 LLVM_DEBUG(MI->dump());
1678 llvm_unreachable("This case is not handled. Look above for MI\n");
1679 }
1680 }
1681 return Changed;
1682}
1683
1684bool HexagonPostRAHandleQFP::runOnMachineFunction(MachineFunction &MF) {
1685
1686 if (DisablePostRAHandleQFloat)
1687 return false;
1688
1689 LLVM_DEBUG(
1690 dbgs() << "\n=== Entering Hexagon Fixup QF spills and refills pass ===\n"
1691 << "Mode: ";
1692 switch (QFloatModeValue) {
1693 case QFloatMode::StrictIEEE:
1694 dbgs() << "Strict IEEE";
1695 break;
1696 case QFloatMode::IEEE:
1697 dbgs() << "IEEE";
1698 break;
1699 case QFloatMode::Lossy:
1700 dbgs() << "Lossy";
1701 break;
1702 default:
1703 dbgs() << "Legacy";
1704 break;
1705 };
1706 dbgs() << "\n";);
1707 bool Changed = false;
1708
1709 auto &_HST = MF.getSubtarget<HexagonSubtarget>();
1710 if (!_HST.useHVXOps())
1711 return false;
1712
1713 HII = _HST.getInstrInfo();
1714
1715 // If the mode is legacy, the function may not contain qf instructions
1716 // check if this pass is required to run for legacy mode.
1717 if (QFloatModeValue == QFloatMode::Legacy)
1718 if (!HII->hasQFPInstrs(MF))
1719 return false;
1720
1721 HRI = _HST.getRegisterInfo();
1722 MRI = &MF.getRegInfo();
1723 const auto &MDF = getAnalysis<MachineDominanceFrontierWrapperPass>().getMDF();
1724 MachineDominatorTree *MDT =
1725 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
1726 HST = &_HST;
1727
1728 // We need Register Dataflow Graph(RDG) to calculate reaching definitions
1729 // since the Machine code is not in SSA.
1730 // DDG holds the graph on which we iterate for the nodes.
1731 DataFlowGraph G(MF, *HII, *HRI, *MDT, MDF);
1732 G.build();
1733 DFG = &G;
1734
1735 Liveness L(*MRI, *DFG);
1736 L.computePhiInfo();
1737 LV = &L;
1738
1739 // Find and save the list of QFP stack spills.
1740 // For refills store all refill instructions to process conditionally later.
1741 NodeAddr<FuncNode *> FA = DFG->getFunc();
1742 LLVM_DEBUG(dbgs() << "==== [RefMap#]=====:\n "
1743 << Print<NodeAddr<FuncNode *>>(FA, *DFG) << "\n");
1744 for (NodeAddr<BlockNode *> BA : FA.Addr->members(G: *DFG)) {
1745 for (auto IA : BA.Addr->members(G: *DFG)) {
1746
1747 if (!DFG->IsCode<NodeAttrs::Stmt>(BA: IA))
1748 continue;
1749
1750 // 'SA' holds the Statement node which contains the machine instruction.
1751 NodeAddr<StmtNode *> SA = IA;
1752 MachineInstr *I = SA.Addr->getCode();
1753
1754 switch (I->getOpcode()) {
1755 case Hexagon::PS_vstorerw_ai:
1756 case Hexagon::PS_vstorerv_ai:
1757 collectQFPStackSpill(StNode: &SA);
1758 break;
1759 case Hexagon::PS_vloadrw_ai:
1760 case Hexagon::PS_vloadrv_ai:
1761 collectQFPStackRefill(StNode: &SA);
1762 break;
1763 case TargetOpcode::COPY:
1764 collectCopies(StNode: &SA);
1765 break;
1766 default:
1767 break;
1768 }
1769 }
1770 }
1771
1772 // Walk through the spills and insert converts when necessary.
1773 // Additionally, walk though the uses of the converts and
1774 // store them conditionally for later processing.
1775 LLVM_DEBUG(dbgs() << "\nHandling spills....");
1776 Changed |= HandleSpills();
1777 SpillMIs.clear();
1778
1779 // Walk through the uses of the refill instructions.
1780 // Process them if they are used as qf operands.
1781 LLVM_DEBUG(dbgs() << "\nCollecting refills....\n");
1782 for (NodeAddr<DefNode *> DfNode : RefillMIs) {
1783
1784 NodeAddr<StmtNode *> Stmt = DfNode.Addr->getOwner(G: *DFG);
1785 MachineInstr *DefMI = Stmt.Addr->getCode();
1786 collectQFUses(RegDef: DfNode, DefMI);
1787 collectConvQFInstr(RegDef&: DfNode);
1788 }
1789 RefillMIs.clear();
1790
1791 LLVM_DEBUG(dbgs() << "\nHandling copies....");
1792 Changed |= HandleCopies();
1793 QFCopys.clear();
1794 PossibleMultiReachDefs.clear();
1795 ReachDefOfCopies.clear();
1796 ConvertToQfCopies.clear();
1797
1798 LLVM_DEBUG(dbgs() << "\n === QF Uses map === "; for (auto It : QFUsesMap) {
1799 dbgs() << "\nInstruction: ";
1800 It.first->dump();
1801 dbgs() << "\t Property: " << It.second.first << " ," << It.second.second;
1802 });
1803
1804 // Insert new opcodes as applicable for the refill uses.
1805 // Delete the original instructions.
1806 Changed |= HandleRefills();
1807
1808 // Handle non-saturating instructions by inserting convert(s) from sf to qf.
1809 Changed |= HandleNonSatInstr();
1810 QFNonSatMIs.clear();
1811 // Cleanup
1812 for (auto It : QFUsesMap)
1813 It.first->eraseFromParent();
1814 QFUsesMap.clear();
1815 IgnoreInsertConvList.clear();
1816
1817 // Option if enabled, checks for qf use-def mismatches
1818 if (EnablePostRAXqfCompliance) {
1819 dbgs() << "\nChecking for ABI compliance for XQF post register \
1820allocation for function: "
1821 << MF.getName() << "\n";
1822 DataFlowGraph DFG(MF, *HII, *HRI, *MDT, MDF);
1823 DFG.build();
1824 Liveness LV(*MRI, DFG);
1825 LV.computeLiveIns();
1826 XqfPostRADiagnosis VDiag(DFG, LV, HII);
1827 VDiag.runCompliance();
1828 }
1829 return Changed;
1830}
1831
1832//===----------------------------------------------------------------------===//
1833// Public Constructor Functions
1834//===----------------------------------------------------------------------===//
1835INITIALIZE_PASS_BEGIN(HexagonPostRAHandleQFP, "handle-qfp-spills-refills",
1836 "Hexagon Post RA Handle QFloat", false, false)
1837INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
1838INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontierWrapperPass)
1839INITIALIZE_PASS_END(HexagonPostRAHandleQFP, "handle-qfp-spills-refills",
1840 "Hexagon PostRA Handle QFloat", false, false)
1841
1842FunctionPass *llvm::createHexagonPostRAHandleQFP() {
1843 return new HexagonPostRAHandleQFP();
1844}
1845