1//===-------------------- HexagonXQFloatGenerator.cpp --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass enables generation of XQFloat instructions. XQF instructions
10// are more efficient, but can be less precise in comparison to IEEE ones.
11// Based on the accuracy preservation of the generated code, we enabled four
12// modes - Strict IEEE-754 compliant, IEEE-754 compliant, Lossy subnormals and
13// legacy mode.
14//
15// Strict IEEE mode adheres to similar accuracy and precision as of IEEE-754.
16//
17// IEEE-754 compliant mode excludes IEEE-754 overflows and lower precision
18// subnormals due to larger dynamic range than IEEE-754.
19// All subnormals have extra precision.
20//
21// Lossy subnormals mode without normalization result in a loss of accuracy.
22// This provides greater precision than a clamp of subnormals to 0.
23// If dataset excludes subnormals, it behavas as IEEE-754 compliant mode.
24//
25// The direct mode has a loss of 1 bit of accuracy compared to IEEE-754.
26//
27// V79 replaces the prior internal HVX floating point format for floating-point
28// arithmetic. The new internal HVX floating-point format yields results
29// identical to IEEE-754 round-to-even mode. The new format contains more bits
30// than IEEE-754, which optionally produces results with greater range and
31// accuracy. Only the HVX vector registers use the HVX floating-point format.
32// Memory maintains all floating-point data in IEEE-754 format,
33// and all loads/stores use the IEEE-754 format. A subset of HVX floating-point
34// operations transform IEEE-754 floating-point data to HVX floating-point data.
35// Subsequent HVX floating-point instructions may consume operands in the HVX
36// floating-point without conversion to IEEE-754, which allows for performant
37// & energy efficient code. The program does not need to switch between formats
38// continuously. The program must convert the HVX floating-point results to
39// IEEE-754 prior to storing to memory.
40
41// HVX floating-point achieves IEEE-754 compliance through normalization.
42// The program may skip normalization when faster calculation is desired, and
43// IEEE-754 compliance isn’t required. HVX floating-point contains two input
44// types: qf32, single precision floating point, and qf16, half precision
45// floating point. In Hexagon, IEEE-754 contains two input types: sf, single
46// precision floating point, and hf, half precision floating point.
47//
48// Only HVX floating-point source and destination instructions use HVX
49// floating-point values. Instructions specify the HVX floating-point format
50// with the qf16 and qf32 identifier. A source vector register will drop the
51// extended state of a HVX floating-point value when an instruction reads the
52// source vector register without the qf16 or qf32 identifier. A destination
53// vector register will reset its extended state when an instruction writes to
54// a vector register without the qf16 or qf32 identifier. When dropping the
55// extended state, the floating-point value loses accuracy. The program may
56// preserve the floating-point value by converting HVX floating-point values
57// to IEEE-754 values. Compiler must convert HVX floating-point values to
58// IEEE-754 values before using as an input to stores, permutes, shifts, and
59// any other operations that do not source the HVX floating-point format.
60//
61// Depending on the desired results, HVX floating-point operations may have
62// some requirements on the input sources. The HVX floating-point values
63// require normalization to achieve IEEE-754 compliance, while faster operations
64// may skip normalization. The program normalizes HVX floating-point values
65// before subsequent HVX floating-point operations, so the floating-point value
66// does not lose precision. The program also obtains results identical to
67// IEEE-754 by converting all HVX floating-point results to IEEE-754 format
68// before consumed in any subsequent operation. There are however cases where
69// this conversion is redundant, or the differences between IEEE-754 and HVX
70// floating-point may not be a concern.
71//
72// The conversion logic can be understood by the table below:
73//
74// ================================================================================================================================================
75// | | | |
76// | Inputs to add/subtarct | Inputs to
77// multiplication instuctions | Non-HVX floating
78// point | | instructions | | instruction
79// | | | | |
80// ===============================================================================================================================================|
81// Sources | IEEE- | HVX | HVX | sf | qf32 | qf32 | hf
82// | qf16 | qf16 | IEE-754 | HVX | HVX |
83// | 754 | floating | floating | | from | from | |
84// from | from | | floating | floating | | |
85// point | point | | mult | adder | | mult
86// | adder | | point | point | | | from |
87// from | | | | | | | |
88// from | from | | | multi | adder | |
89// | | | | | | mult |
90// adder | | | | | | | | | | |
91// | | |
92// ===============================================================================================================================================|
93// Strict | Direct | Convert | Convert | Normalize | Convert | Convert
94// | widening | Convert | Convert | Direct | Convert | Convert | IEEE-754
95// | Use | to | to | | to IEEE | to IEEE | multiply
96// | to IEEE, | to IEEE, | use | to | to | compliance | |
97// IEEE | IEEE | | then | then | then | widening
98// | widening | | IEEE | IEEE |
99// | | | | | normalize | normalize
100// | convert | multiply,| multiply,| | | |
101// | | | | | | | to IEEE
102// | convert | convert | | | | | |
103// | | | | | | to
104// IEEE | to IEEE | | | |
105// -----------------------------------------------------------------------------------------------------------------------------------------------|
106// IEEE-754 | Direct | Direct | Direct | Normalize | Direct | Normalize
107// | Widening | Direct | Widening | Direct | Convert | Convert | compliance
108// | Use | Use | Use | | use | | multiply
109// | use | multiply | use | to IEEE | to IEEE |
110// -----------------------------------------------------------------------------------------------------------------------------------------------|
111// Lossy | Direct | Direct | Direct | Direct | Direct | Normalize
112// | Direct | Direct | Widening | Direct | Convert | Convert | Subnormals
113// | Use | Use | Use | Use | use | | use |
114// use | multiply | use | to IEEE | to IEEE |
115// -----------------------------------------------------------------------------------------------------------------------------------------------|
116// Direct | Direct | Direct | Direct | Direct | Direct | Direct |
117// Direct | Direct | Direct | Direct | Direct | Direct | Lossy |
118// Use | Use | Use | Use | use | use | use |
119// use | use | use | use | use |
120// -----------------------------------------------------------------------------------------------------------------------------------------------|
121//
122// For v81, the normalization sequence changes. Instead of multiplying 0
123// and -0, a simple copy operation normalizes the unnormal value. Both
124// qf and IEEE-754 value can be unnormal.
125// Additionally for v81, we have two new vsub instructions which are handled.
126
127#define HEXAGON_XQFLOAT_GENERATOR "XQFloat Generator pass"
128
129#include "Hexagon.h"
130#include "HexagonInstrInfo.h"
131#include "HexagonSubtarget.h"
132#include "HexagonTargetMachine.h"
133#include "llvm/ADT/SmallPtrSet.h"
134#include "llvm/ADT/SmallVector.h"
135#include "llvm/ADT/Statistic.h"
136#include "llvm/CodeGen/MachineBasicBlock.h"
137#include "llvm/CodeGen/MachineFunction.h"
138#include "llvm/CodeGen/MachineFunctionPass.h"
139#include "llvm/CodeGen/MachineInstr.h"
140#include "llvm/CodeGen/MachineOperand.h"
141#include "llvm/CodeGen/Passes.h"
142#include "llvm/IR/DebugLoc.h"
143#include "llvm/Pass.h"
144#include "llvm/Support/CommandLine.h"
145#include "llvm/Support/Debug.h"
146#include "llvm/Support/raw_ostream.h"
147#include <vector>
148
149#define DEBUG_TYPE "hexagon-xqf-gen"
150
151using namespace llvm;
152
153extern cl::opt<QFloatMode> QFloatModeValue;
154
155// Master flag to enable XQF generations
156cl::opt<bool> EnableHVXXQFloat("enable-xqf-gen", cl::init(Val: false),
157 cl::desc("Enable XQFloat generations"));
158// Master flag to remove extraneous qf to sf/hf conversions
159cl::opt<bool>
160 EnableConversionsRemoval("enable-rem-conv", cl::init(Val: false),
161 cl::desc("Enable extraneous conversions removal"));
162
163// Diagnostic flags
164cl::opt<bool> PrintDebug("debug-print", cl::init(Val: false),
165 cl::desc("Print function mir after transformation"));
166// This vector contains the opcodes which generate qf32 from add/subtract
167static constexpr unsigned XQFPAdd32[] = {
168 // vector add instructions
169 Hexagon::V6_vadd_sf, Hexagon::V6_vadd_qf32, Hexagon::V6_vadd_qf32_mix,
170
171 // vector subtract instructions
172 Hexagon::V6_vsub_qf32, Hexagon::V6_vsub_qf32_mix, Hexagon::V6_vsub_sf,
173 Hexagon::V6_vsub_sf_mix};
174
175// This vector contains the opcodes which generate qf16 from add/subtract
176static constexpr unsigned XQFPAdd16[] = {
177 // vector add instructions
178 Hexagon::V6_vadd_hf, Hexagon::V6_vadd_qf16, Hexagon::V6_vadd_qf16_mix,
179
180 // vector subtract intrutions
181 Hexagon::V6_vsub_hf, Hexagon::V6_vsub_qf16, Hexagon::V6_vsub_qf16_mix,
182 Hexagon::V6_vsub_hf_mix};
183
184// This vector contains the opcodes which generate qf32 from multiplication
185static constexpr unsigned XQFPMult32[] = {
186 Hexagon::V6_vmpy_qf32, Hexagon::V6_vmpy_qf32_qf16, Hexagon::V6_vmpy_qf32_hf,
187 Hexagon::V6_vmpy_qf32_sf, Hexagon::V6_vmpy_qf32_mix_hf};
188// This vector contains the opcodes which generate qf16 from multiplication
189static constexpr unsigned XQFPMult16[] = {Hexagon::V6_vmpy_qf16,
190 Hexagon::V6_vmpy_qf16_hf,
191 Hexagon::V6_vmpy_qf16_mix_hf};
192
193namespace llvm {
194FunctionPass *createHexagonXQFloatGenerator();
195void initializeHexagonXQFloatGeneratorPass(PassRegistry &);
196} // namespace llvm
197
198namespace {
199
200struct HexagonXQFloatGenerator : public MachineFunctionPass {
201public:
202 static char ID;
203 HexagonXQFloatGenerator() : MachineFunctionPass(ID) {}
204
205 bool runOnMachineFunction(MachineFunction &MF) override;
206
207 StringRef getPassName() const override { return HEXAGON_XQFLOAT_GENERATOR; }
208
209 void getAnalysisUsage(AnalysisUsage &AU) const override {
210 MachineFunctionPass::getAnalysisUsage(AU);
211 }
212
213private:
214 // Handle each XQF optimization level
215 bool HandleStrictIEEE(MachineFunction &);
216 bool HandleCompliantIEEE(MachineFunction &);
217 bool HandleLossySubnormals(MachineFunction &);
218 bool HandleLossyLegacy(MachineFunction &);
219
220 // Checkers functions for input operands
221 bool checkIfInputFromAdder32(Register Reg);
222 bool checkIfInputFromAdder16(Register Reg);
223 bool checkIfInputFromMult32(Register Reg);
224 bool checkIfInputFromMult16(Register Reg);
225 bool deleteList();
226
227 // Helper functions for conversion/normalization/widening
228 bool widenMultiplicationInputF16(MachineInstr &, Register &, Register &,
229 Register &, bool);
230 bool widenMultiplicationInputF16Rt(MachineInstr &, Register &, Register &,
231 Register &);
232 void widenMultiplyInputHF(MachineInstr &, Register &, Register &, Register &);
233 bool normalizeMultiplicationInputF32(MachineInstr &, Register &, Register &,
234 Register &, Register &, bool &);
235 void normalizeMultiplicationInputSF(MachineInstr &, Register &, Register &,
236 Register &, Register &, bool &);
237 bool convertNormalizeMultOp32(MachineInstr &, Register &, Register &,
238 Register &, Register &, bool &);
239 bool convertWidenMultOp16(MachineInstr &, Register &, Register &, Register &,
240 bool);
241 bool convertWidenMultOp32(MachineInstr &, Register &, Register &, Register &,
242 bool);
243 void createPrologInstructions(MachineInstr &, Register &);
244 bool convertAddOpToIEEE16(MachineInstr &, Register &, Register &, Register &,
245 bool, bool, bool);
246 bool convertAddOpToIEEE32(MachineInstr &, Register &, Register &, Register &,
247 bool, bool, bool);
248 void generateQF16FromQF32(MachineInstr &, Register &, Register &);
249 bool convertIfInputToNonHVX(MachineInstr &, bool);
250 void createConvertInstr(MachineInstr *, Register &, Register &, bool);
251
252 // V81 specific normalization function
253 bool V81normalizeMultF32(MachineInstr &, Register &, Register &, Register &,
254 bool, bool, bool);
255
256 const HexagonSubtarget *HST = nullptr;
257 const HexagonInstrInfo *HII = nullptr;
258 MachineRegisterInfo *MRI = nullptr;
259
260 SmallVector<MachineInstr *, 16>
261 OriginalMI; // Hold the instructions to be deleted
262};
263
264// Print machine function
265static void debug_print([[maybe_unused]] MachineFunction &MF) {
266 dbgs() << "\n=== Printing function ===\n";
267#ifndef NDEBUG
268 for (MachineBasicBlock &MBB : MF)
269 MBB.dump();
270#endif // NDEBUG
271}
272
273// This class removes redundant vector convert instructions from qf to hf/sf.
274// Additionally, it relaces use of sf/hf registers with qf types.
275// The resulting code is complete without dangling instructions.
276// FIXME: Liveness is not preserved.
277class VectorConvertRemove {
278
279public:
280 VectorConvertRemove(MachineFunction &_MF, MachineRegisterInfo *_MRI,
281 const HexagonSubtarget *_HST)
282 : MF(_MF), MRI(_MRI), HST(_HST) {
283 HII = HST->getInstrInfo();
284 }
285
286 void run();
287
288private:
289 MachineFunction &MF;
290 MachineRegisterInfo *MRI;
291 const HexagonSubtarget *HST;
292 const HexagonInstrInfo *HII;
293
294 enum Operation { Add16, Add32, Sub16, Sub32, Mul16, Mul32 };
295 // Helper functions
296 void handle_addsub_sf_sf(MachineInstr &, Register &, Register &, Register &,
297 bool);
298 void handle_addsub_qf_sf(MachineInstr &, Register &, Register &, Register &,
299 bool);
300 void handle_addsubmul_hf_hf(MachineInstr &, Register &, Register &,
301 Register &, Operation);
302 void handle_addsubmul_qf_hf(MachineInstr &, Register &, Register &,
303 Register &, Operation);
304 void handle_qf32_mul_sf_sf(MachineInstr &, Register &, Register &,
305 Register &);
306 bool checkHVXUses32(MachineInstr *, MachineInstr *);
307 bool checkHVXUses16(MachineInstr *, MachineInstr *);
308 unsigned getOperation(Operation, bool, bool);
309
310 // List which holds conversion instructions
311 SmallPtrSet<MachineInstr *, 16> ConvInstrList;
312 // List which holds qf handling instructions
313 std::vector<MachineInstr *> SfHfInstrList;
314};
315
316// both : both operands are replaced
317unsigned VectorConvertRemove::getOperation(Operation Op, bool firstOpQf,
318 bool secOpQf) {
319 if (firstOpQf && secOpQf) {
320 switch (Op) {
321 case Add16:
322 return Hexagon::V6_vadd_qf16;
323 case Add32:
324 return Hexagon::V6_vadd_qf32;
325 case Sub16:
326 return Hexagon::V6_vsub_qf16;
327 case Sub32:
328 return Hexagon::V6_vsub_qf32;
329 case Mul16:
330 return Hexagon::V6_vmpy_qf16;
331 case Mul32:
332 return Hexagon::V6_vmpy_qf32_qf16;
333 }
334 } else if (firstOpQf) {
335 switch (Op) {
336 case Add16:
337 return Hexagon::V6_vadd_qf16_mix;
338 case Add32:
339 return Hexagon::V6_vadd_qf32_mix;
340 case Sub16:
341 return Hexagon::V6_vsub_qf16_mix;
342 case Sub32:
343 return Hexagon::V6_vsub_qf32_mix;
344 case Mul16:
345 return Hexagon::V6_vmpy_qf16_mix_hf;
346 case Mul32:
347 return Hexagon::V6_vmpy_qf32_mix_hf;
348 }
349 } else if (secOpQf) {
350 switch (Op) {
351 case Sub16:
352 return Hexagon::V6_vsub_hf_mix;
353 case Sub32:
354 return Hexagon::V6_vsub_sf_mix;
355 default:
356 break;
357 }
358 } else {
359 }
360 llvm_unreachable("Unknown opcode and operand combination!");
361}
362
363// Return false if there are multiple instructions where the qf32 is used
364// other than the instruction for which it is called
365bool VectorConvertRemove::checkHVXUses32(MachineInstr *MI,
366 MachineInstr *UseMI) {
367 Register convReg = MI->getOperand(i: 0).getReg();
368 // Iterate over all uses of the Def we are analyzing
369 for (auto &MO : make_range(x: MRI->use_begin(RegNo: convReg), y: MRI->use_end())) {
370 MachineInstr *UMI = MO.getParent();
371 if (UMI == UseMI)
372 continue;
373 // Since the convert cannot be deleted, we set the operand as NOT kill
374 MI->getOperand(i: 1).setIsKill(false);
375 return false;
376 }
377 return true;
378}
379
380// Return false if there are multiple instructions where the qf16 is used
381// other than the instruction for which it is called
382bool VectorConvertRemove::checkHVXUses16(MachineInstr *MI,
383 MachineInstr *UseMI) {
384 Register convReg = MI->getOperand(i: 0).getReg();
385 // Iterate over all uses of the Def we are analyzing
386 for (auto &MO : make_range(x: MRI->use_begin(RegNo: convReg), y: MRI->use_end())) {
387 MachineInstr *UMI = MO.getParent();
388 if (UMI == UseMI)
389 continue;
390 // Since the convert cannot be deleted, we set the operand as NOT kill
391 MI->getOperand(i: 1).setIsKill(false);
392 return false;
393 }
394 return true;
395}
396
397// Removes converts feeding to op(sf,sf), and replaces its sf operands with qf
398void VectorConvertRemove::handle_addsub_sf_sf(MachineInstr &MI, Register &Reg1,
399 Register &Reg2, Register &Dest,
400 bool isAdd) {
401
402 MachineBasicBlock &MBB = *MI.getParent();
403 const DebugLoc &DL = MI.getDebugLoc();
404
405 bool firstConv = false, secConv = false;
406 bool DefOp1_del = false, DefOp2_del = false;
407 Register Src1, Src2;
408
409 MachineInstr *DefOp1 = MRI->getVRegDef(Reg: Reg1);
410 MachineInstr *DefOp2 = MRI->getVRegDef(Reg: Reg2);
411 // check if the first operand is from a convert operation
412 if (DefOp1->getOpcode() == Hexagon::V6_vconv_sf_qf32) {
413 if (checkHVXUses32(MI: DefOp1, UseMI: &MI))
414 DefOp1_del = true;
415 Src1 = DefOp1->getOperand(i: 1).getReg();
416 firstConv = true;
417 }
418
419 // check if the second operand is from a convert operation
420 if (DefOp2->getOpcode() == Hexagon::V6_vconv_sf_qf32) {
421 if (checkHVXUses32(MI: DefOp2, UseMI: &MI))
422 DefOp2_del = true;
423 Src2 = DefOp2->getOperand(i: 1).getReg();
424 secConv = true;
425 }
426
427 if (firstConv && secConv) {
428 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
429 MCID: HII->get(Opcode: getOperation(Op: isAdd ? Operation::Add32 : Operation::Sub32,
430 firstOpQf: true, secOpQf: true)),
431 DestReg: Dest)
432 .addReg(RegNo: Src1)
433 .addReg(RegNo: Src2);
434 SfHfInstrList.push_back(x: &MI);
435 } else if (firstConv) {
436 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
437 MCID: HII->get(Opcode: getOperation(Op: isAdd ? Operation::Add32 : Operation::Sub32,
438 firstOpQf: true, secOpQf: false)),
439 DestReg: Dest)
440 .addReg(RegNo: Src1)
441 .addReg(RegNo: Reg2);
442 SfHfInstrList.push_back(x: &MI);
443 } else if (secConv) {
444 // For v79, there is no provision for 2nd op being qf for add/sub
445 if (HST->useHVXV81Ops()) {
446 if (isAdd)
447 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: Dest)
448 .addReg(RegNo: Src2)
449 .addReg(RegNo: Reg1);
450 else
451 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_sf_mix), DestReg: Dest)
452 .addReg(RegNo: Reg1)
453 .addReg(RegNo: Src2);
454 SfHfInstrList.push_back(x: &MI);
455 // For v79, there is no provision for 2nd op being qf for add/sub. Since
456 // add is commutative, the ops can be rotated.
457 } else if (HST->useHVXV79Ops()) {
458 // for vadd we interchange the ops, for vsub we ignore
459 if (isAdd) {
460 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: Dest)
461 .addReg(RegNo: Src2)
462 .addReg(RegNo: Reg1);
463 SfHfInstrList.push_back(x: &MI);
464 } else // don't delete the convert instruction for vsub
465 DefOp2_del = false;
466 }
467 }
468
469 if (DefOp1_del)
470 ConvInstrList.insert(Ptr: DefOp1);
471 if (DefOp2_del)
472 ConvInstrList.insert(Ptr: DefOp2);
473}
474
475// Removes converts feeding to op(hf,hf), and replaces its hf operands with qf
476void VectorConvertRemove::handle_addsubmul_hf_hf(MachineInstr &MI,
477 Register &Reg1, Register &Reg2,
478 Register &Dest, Operation Op) {
479
480 MachineBasicBlock &MBB = *MI.getParent();
481 const DebugLoc &DL = MI.getDebugLoc();
482
483 bool firstConv = false, secConv = false;
484 bool DefOp1_del = false, DefOp2_del = false;
485 bool isSub = Op == Operation::Sub16;
486 Register Src1, Src2;
487
488 MachineInstr *DefOp1 = MRI->getVRegDef(Reg: Reg1);
489 MachineInstr *DefOp2 = MRI->getVRegDef(Reg: Reg2);
490 // check if the first operand is from a convert operation
491 if (DefOp1->getOpcode() == Hexagon::V6_vconv_hf_qf16) {
492 if (checkHVXUses16(MI: DefOp1, UseMI: &MI))
493 DefOp1_del = true;
494 Src1 = DefOp1->getOperand(i: 1).getReg();
495 firstConv = true;
496 }
497
498 // check if the second operand is from a convert operation
499 if (DefOp2->getOpcode() == Hexagon::V6_vconv_hf_qf16) {
500 if (checkHVXUses16(MI: DefOp2, UseMI: &MI))
501 DefOp2_del = true;
502 Src2 = DefOp2->getOperand(i: 1).getReg();
503 secConv = true;
504 }
505
506 if (firstConv && secConv) {
507 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: true, secOpQf: true)), DestReg: Dest)
508 .addReg(RegNo: Src1)
509 .addReg(RegNo: Src2);
510 SfHfInstrList.push_back(x: &MI);
511 } else if (firstConv) {
512 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: true, secOpQf: false)), DestReg: Dest)
513 .addReg(RegNo: Src1)
514 .addReg(RegNo: Reg2);
515 SfHfInstrList.push_back(x: &MI);
516 } else if (secConv) {
517 // For v81, we interchange the ops for vadd/vmul
518 // for vsub we use qf as second operand
519 if (HST->useHVXV81Ops()) {
520 if (!isSub)
521 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: true, secOpQf: false)), DestReg: Dest)
522 .addReg(RegNo: Src2)
523 .addReg(RegNo: Reg1);
524 else
525 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: false, secOpQf: true)), DestReg: Dest)
526 .addReg(RegNo: Reg1)
527 .addReg(RegNo: Src2);
528 SfHfInstrList.push_back(x: &MI);
529 } else if (HST->useHVXV79Ops()) {
530 // for vadd/vmul we interchange the ops, for vsub we ignore
531 if (!isSub) {
532 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: true, secOpQf: false)), DestReg: Dest)
533 .addReg(RegNo: Src2)
534 .addReg(RegNo: Reg1);
535 SfHfInstrList.push_back(x: &MI);
536 } else // don't delete the convert instruction for vsub
537 DefOp2_del = false;
538 }
539 }
540
541 if (DefOp1_del)
542 ConvInstrList.insert(Ptr: DefOp1);
543 if (DefOp2_del)
544 ConvInstrList.insert(Ptr: DefOp2);
545}
546
547// Removes converts feeding to op(qf,sf), and replaces its sf operands with qf
548void VectorConvertRemove::handle_addsub_qf_sf(MachineInstr &MI, Register &Reg1,
549 Register &Reg2, Register &Dest,
550 bool isAdd) {
551 MachineBasicBlock &MBB = *MI.getParent();
552 const DebugLoc &DL = MI.getDebugLoc();
553 Register Src;
554 bool conv = false;
555
556 MachineInstr *DefOp = MRI->getVRegDef(Reg: Reg2);
557 // check if the second operand is from a convert operation
558 if (DefOp->getOpcode() == Hexagon::V6_vconv_sf_qf32) {
559 if (checkHVXUses32(MI: DefOp, UseMI: &MI))
560 ConvInstrList.insert(Ptr: DefOp);
561 Src = DefOp->getOperand(i: 1).getReg();
562 conv = true;
563 }
564
565 if (conv) {
566 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
567 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_qf32 : Hexagon::V6_vsub_qf32),
568 DestReg: Dest)
569 .addReg(RegNo: Reg1)
570 .addReg(RegNo: Src);
571 SfHfInstrList.push_back(x: &MI);
572 }
573}
574
575// Removes converts feeding to op(qf,hf), and replaces its hf operands with qf
576void VectorConvertRemove::handle_addsubmul_qf_hf(MachineInstr &MI,
577 Register &Reg1, Register &Reg2,
578 Register &Dest, Operation Op) {
579 MachineBasicBlock &MBB = *MI.getParent();
580 const DebugLoc &DL = MI.getDebugLoc();
581 Register Src;
582 bool conv = false;
583
584 MachineInstr *DefOp = MRI->getVRegDef(Reg: Reg2);
585 // check if the second operand is from a convert operation
586 if (DefOp->getOpcode() == Hexagon::V6_vconv_hf_qf16) {
587 if (checkHVXUses16(MI: DefOp, UseMI: &MI))
588 ConvInstrList.insert(Ptr: DefOp);
589 Src = DefOp->getOperand(i: 1).getReg();
590 conv = true;
591 }
592
593 if (conv) {
594 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: getOperation(Op, firstOpQf: true, secOpQf: true)), DestReg: Dest)
595 .addReg(RegNo: Reg1)
596 .addReg(RegNo: Src);
597 SfHfInstrList.push_back(x: &MI);
598 }
599}
600
601// Removes converts feeding to op(sf,sf), and replaces its sf operands with qf
602void VectorConvertRemove::handle_qf32_mul_sf_sf(MachineInstr &MI,
603 Register &Reg1, Register &Reg2,
604 Register &Dest) {
605 MachineBasicBlock &MBB = *MI.getParent();
606 const DebugLoc &DL = MI.getDebugLoc();
607 Register Src1, Src2;
608 bool firstConv = false, secConv = false;
609
610 MachineInstr *DefOp1 = MRI->getVRegDef(Reg: Reg1);
611 MachineInstr *DefOp2 = MRI->getVRegDef(Reg: Reg2);
612
613 if (DefOp1->getOpcode() == Hexagon::V6_vconv_sf_qf32 &&
614 DefOp2->getOpcode() == Hexagon::V6_vconv_sf_qf32) {
615 // If yes, we can remove the convert
616 if (checkHVXUses32(MI: DefOp1, UseMI: &MI) && checkHVXUses32(MI: DefOp2, UseMI: &MI)) {
617 ConvInstrList.insert(Ptr: DefOp1);
618 ConvInstrList.insert(Ptr: DefOp2);
619 }
620 Src1 = DefOp1->getOperand(i: 1).getReg();
621 Src2 = DefOp2->getOperand(i: 1).getReg();
622 firstConv = true;
623 secConv = true;
624 }
625
626 // If both are true, then only replace with qf32 = vmpy(qf32, qf32)
627 if (firstConv && secConv) {
628 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
629 .addReg(RegNo: Src1)
630 .addReg(RegNo: Src2);
631 SfHfInstrList.push_back(x: &MI);
632 }
633}
634
635void VectorConvertRemove::run() {
636 for (auto &MBB : MF) {
637 for (auto &MI : MBB) {
638 // Skip if the instruction does not have two operands,
639 // or is a bundle instruction
640 // or is a debug instruction
641 if (MI.getNumOperands() != 3 || MI.isDebugInstr())
642 continue;
643
644 auto Op1 = MI.getOperand(i: 1);
645 if (!Op1.isReg())
646 continue;
647 auto Op2 = MI.getOperand(i: 2);
648 if (!Op2.isReg())
649 continue;
650 auto Op0 = MI.getOperand(i: 0);
651 if (!Op0.isReg())
652 continue;
653 Register Reg1 = Op1.getReg();
654 Register Reg2 = Op2.getReg();
655 Register Dest = Op0.getReg();
656
657 switch (MI.getOpcode()) {
658 // TODO Handle the new vsub instructions
659 // qf32 = vadd(sf, sf)
660 case Hexagon::V6_vadd_sf:
661 handle_addsub_sf_sf(MI, Reg1, Reg2, Dest, isAdd: true);
662 break;
663 // qf32 = vsub(sf, sf)
664 case Hexagon::V6_vsub_sf:
665 handle_addsub_sf_sf(MI, Reg1, Reg2, Dest, isAdd: false);
666 break;
667 // qf32 = vadd(qf32, sf)
668 case Hexagon::V6_vadd_qf32_mix:
669 handle_addsub_qf_sf(MI, Reg1, Reg2, Dest, isAdd: true);
670 break;
671 // qf32 = vsub(qf32, sf)
672 case Hexagon::V6_vsub_qf32_mix:
673 handle_addsub_qf_sf(MI, Reg1, Reg2, Dest, isAdd: false);
674 break;
675 // qf16 = vadd(hf, hf)
676 case Hexagon::V6_vadd_hf:
677 handle_addsubmul_hf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Add16);
678 break;
679 // qf16 = vsub(hf, hf)
680 case Hexagon::V6_vsub_hf:
681 handle_addsubmul_hf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Sub16);
682 break;
683 // qf16 = vadd(qf16, hf)
684 case Hexagon::V6_vadd_qf16_mix:
685 handle_addsubmul_qf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Add16);
686 break;
687 // qf16 = vsub(qf16, hf)
688 case Hexagon::V6_vsub_qf16_mix:
689 handle_addsubmul_qf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Sub16);
690 break;
691 // qf32 = vmpy(sf, sf)
692 case Hexagon::V6_vmpy_qf32_sf:
693 handle_qf32_mul_sf_sf(MI, Reg1, Reg2, Dest);
694 break;
695 // qf32 = vmpy(hf, hf)
696 case Hexagon::V6_vmpy_qf32_hf:
697 handle_addsubmul_hf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Mul32);
698 break;
699 // qf32 = vmpy(qf16, hf)
700 case Hexagon::V6_vmpy_qf32_mix_hf:
701 handle_addsubmul_qf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Mul32);
702 break;
703 // qf16 = vmpy(hf, hf)
704 case Hexagon::V6_vmpy_qf16_hf:
705 handle_addsubmul_hf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Mul16);
706 break;
707 // qf16 = vmpy(qf16, hf)
708 case Hexagon::V6_vmpy_qf16_mix_hf:
709 handle_addsubmul_qf_hf(MI, Reg1, Reg2, Dest, Op: Operation::Mul16);
710 break;
711 default:
712 break;
713 }
714 }
715 }
716
717 // Delete the vadd/vsub/vmpy instructions
718 for (MachineInstr *sfhfMI : SfHfInstrList) {
719 LLVM_DEBUG(dbgs() << "deleting sf/hf instruction ");
720 LLVM_DEBUG(sfhfMI->dump());
721 sfhfMI->eraseFromParent();
722 }
723 // Delete conversion instructions
724 for (MachineInstr *convMI : ConvInstrList) {
725 LLVM_DEBUG(dbgs() << "deleting conversion instruction");
726 LLVM_DEBUG(convMI->dump());
727 convMI->eraseFromParent();
728 }
729}
730
731char HexagonXQFloatGenerator::ID = 0;
732
733} // namespace
734
735INITIALIZE_PASS(HexagonXQFloatGenerator, "hexagon-xqfloat-generator",
736 HEXAGON_XQFLOAT_GENERATOR, false, false)
737
738FunctionPass *llvm::createHexagonXQFloatGenerator() {
739 return new HexagonXQFloatGenerator();
740}
741
742// Returns true if qf32 input is from an adder/subtract unit
743bool HexagonXQFloatGenerator::checkIfInputFromAdder32(Register Reg) {
744 MachineInstr *Def = MRI->getVRegDef(Reg);
745 if (!Def)
746 return false;
747
748 // If the definition is a copy, we need to analyze its def again
749 if (Def->getOpcode() == TargetOpcode::COPY) {
750 Register SrcReg = Def->getOperand(i: 1).getReg();
751 if (SrcReg.isValid())
752 return checkIfInputFromAdder32(Reg: SrcReg);
753 return false;
754 } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) {
755 Register SrcReg1 = Def->getOperand(i: 1).getReg();
756 Register SrcReg2 = Def->getOperand(i: 2).getReg();
757 bool isTrue = false;
758 if (SrcReg1.isValid())
759 isTrue = checkIfInputFromAdder32(Reg: SrcReg1);
760 if (SrcReg2.isValid())
761 isTrue |= checkIfInputFromAdder32(Reg: SrcReg2);
762 return isTrue;
763 } else
764 return llvm::is_contained(Range: XQFPAdd32, Element: Def->getOpcode());
765}
766
767// Returns true if qf16 input is from an adder/subtract unit
768bool HexagonXQFloatGenerator::checkIfInputFromAdder16(Register Reg) {
769 MachineInstr *Def = MRI->getVRegDef(Reg);
770 if (!Def)
771 return false;
772
773 // if the definition is a copy, we need to analyze its def again
774 if (Def->getOpcode() == TargetOpcode::COPY) {
775 Register SrcReg = Def->getOperand(i: 1).getReg();
776 if (SrcReg.isValid())
777 return checkIfInputFromAdder16(Reg: SrcReg);
778 return false;
779 } else
780 return llvm::is_contained(Range: XQFPAdd16, Element: Def->getOpcode());
781}
782
783// Returns true if qf32 input is from a multiplier unit
784bool HexagonXQFloatGenerator::checkIfInputFromMult32(Register Reg) {
785 MachineInstr *Def = MRI->getVRegDef(Reg);
786 if (!Def)
787 return false;
788
789 // if the definition is a copy, we need to analyze its def again
790 if (Def->getOpcode() == TargetOpcode::COPY) {
791 Register SrcReg = Def->getOperand(i: 1).getReg();
792 if (SrcReg.isValid())
793 return checkIfInputFromMult32(Reg: SrcReg);
794 return false;
795 } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) {
796 Register SrcReg1 = Def->getOperand(i: 1).getReg();
797 Register SrcReg2 = Def->getOperand(i: 2).getReg();
798 bool isTrue = false;
799 if (SrcReg1.isValid())
800 isTrue |= checkIfInputFromMult32(Reg: SrcReg1);
801 if (SrcReg2.isValid())
802 isTrue |= checkIfInputFromMult32(Reg: SrcReg2);
803 return isTrue;
804 } else
805 return llvm::is_contained(Range: XQFPMult32, Element: Def->getOpcode());
806}
807
808// Returns true if qf16 input is from a multiplier unit
809bool HexagonXQFloatGenerator::checkIfInputFromMult16(Register Reg) {
810 MachineInstr *Def = MRI->getVRegDef(Reg);
811 if (!Def)
812 return false;
813
814 // if the definition is a copy, we need to analyze its def again
815 if (Def->getOpcode() == TargetOpcode::COPY) {
816 Register SrcReg = Def->getOperand(i: 1).getReg();
817 if (SrcReg.isValid())
818 return checkIfInputFromMult16(Reg: SrcReg);
819 return false;
820 } else
821 return llvm::is_contained(Range: XQFPMult16, Element: Def->getOpcode());
822}
823
824// Generates sf = qf32 instruction or hf = qf16 intruction
825void HexagonXQFloatGenerator::createConvertInstr(MachineInstr *UseMI,
826 Register &NewR, Register &OldR,
827 bool is32bit) {
828 const DebugLoc &DL = UseMI->getDebugLoc();
829 MachineBasicBlock *MBB = UseMI->getParent();
830 NewR = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
831 if (is32bit)
832 BuildMI(BB&: *MBB, I&: *UseMI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: NewR)
833 .addReg(RegNo: OldR);
834 else
835 BuildMI(BB&: *MBB, I&: *UseMI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: NewR)
836 .addReg(RegNo: OldR);
837}
838
839// Generate HVX to IEEE conversion instruction for all non-HVX uses
840bool HexagonXQFloatGenerator::convertIfInputToNonHVX(MachineInstr &MI,
841 bool is32bit) {
842 Register NewR;
843 bool Changed = false;
844 ;
845 Register Dest = MI.getOperand(i: 0).getReg();
846
847 // Iterate over all uses of the Def we are analyzing
848 for (auto &MO : make_range(x: MRI->use_begin(RegNo: Dest), y: MRI->use_end())) {
849 MachineInstr *UseMI = MO.getParent();
850 // Omit if the use is a REG_SEQUENCE instruction, since the only
851 // use of REG_SEQUENCE in qf context is transforming to IEEE.
852 // Omit for use in DBG instructions.
853 // Omit for use in PHI instructions since PHI result can be used as a qf
854 // operand.
855 if (UseMI->getOpcode() == TargetOpcode::REG_SEQUENCE ||
856 UseMI->getOpcode() == TargetOpcode::DBG_VALUE ||
857 UseMI->getOpcode() == TargetOpcode::DBG_LABEL ||
858 UseMI->getOpcode() == TargetOpcode::PHI)
859 continue;
860
861 // If 32-bit operand
862 if (is32bit) {
863 // If it is a copy instruction, we need to analyze it uses
864 if (UseMI->getOpcode() == TargetOpcode::COPY)
865 return convertIfInputToNonHVX(MI&: *UseMI, /* 32 bit */ is32bit: true);
866 if (!HII->usesQFOperand(MI: UseMI)) {
867 createConvertInstr(UseMI, NewR, OldR&: Dest, /*32 bit*/ is32bit: true);
868 MO.setReg(NewR);
869 Changed = true;
870 }
871 // If 16-bit operand
872 } else {
873 // If it is a copy instruction, we need to analyze it uses
874 if (UseMI->getOpcode() == TargetOpcode::COPY)
875 return convertIfInputToNonHVX(MI&: *UseMI, /* 16 bit */ is32bit: false);
876 if (!HII->usesQFOperand(MI: UseMI)) {
877 createConvertInstr(UseMI, NewR, OldR&: Dest, /*16 bit*/ is32bit: false);
878 MO.setReg(NewR);
879 Changed = true;
880 }
881 }
882 }
883 return Changed;
884}
885
886// generate qf16 = qf32 via:
887// hf = qf32
888// V0 = #0
889// qf16 = vsub(hf,V0)
890void HexagonXQFloatGenerator::generateQF16FromQF32(MachineInstr &MI,
891 Register &Dest,
892 Register &SrcReg) {
893
894 MachineBasicBlock &MBB = *MI.getParent();
895 const DebugLoc &DL = MI.getDebugLoc();
896
897 Register convertReg = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
898 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf32), DestReg: convertReg)
899 .addReg(RegNo: SrcReg);
900 Register VR0 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
901 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vd0), DestReg: VR0);
902
903 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_hf), DestReg: Dest)
904 .addReg(RegNo: convertReg)
905 .addReg(RegNo: VR0);
906}
907
908// Widen qf16 = vmpy(hf, hf) result unconditionally
909void HexagonXQFloatGenerator::widenMultiplyInputHF(MachineInstr &MI,
910 Register &Reg1,
911 Register &Reg2,
912 Register &Dest) {
913 Register output_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
914 MachineBasicBlock &MBB = *MI.getParent();
915 const DebugLoc &DL = MI.getDebugLoc();
916
917 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: output_mpy)
918 .addReg(RegNo: Reg1)
919 .addReg(RegNo: Reg2);
920 generateQF16FromQF32(MI, Dest, SrcReg&: output_mpy);
921}
922
923// Widen vmpy(qf16, qf16/hf) result conditionally
924bool HexagonXQFloatGenerator::widenMultiplicationInputF16(MachineInstr &MI,
925 Register &Reg1,
926 Register &Reg2,
927 Register &Dest,
928 bool twoOps) {
929 bool firstconvert = false, secondconvert = false;
930 MachineBasicBlock &MBB = *MI.getParent();
931 const DebugLoc &DL = MI.getDebugLoc();
932
933 // We widen only that operand which comes from add/subtract unit.
934 if (checkIfInputFromAdder16(Reg: Reg1))
935 firstconvert = true;
936 // twoOps == true suggest 2nd operand is qf16, else it is hf
937 if (twoOps && checkIfInputFromAdder16(Reg: Reg2))
938 secondconvert = true;
939
940 Register widenReg;
941 // if either operands from add/subtract unit, we widen
942 if (twoOps) {
943 if (firstconvert || secondconvert) {
944 widenReg = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
945 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_qf16), DestReg: widenReg)
946 .addReg(RegNo: Reg1)
947 .addReg(RegNo: Reg2);
948 } else {
949 return false;
950 }
951 } else {
952 if (firstconvert) {
953 widenReg = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
954 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: widenReg)
955 .addReg(RegNo: Reg1)
956 .addReg(RegNo: Reg2);
957 } else {
958 return false;
959 }
960 }
961
962 // generate qf16 = qf32
963 generateQF16FromQF32(MI, Dest, SrcReg&: widenReg);
964
965 return true;
966}
967
968// Handle qf16 = vmpy(qf16, Rt)
969// For strict IEEE mode, convert the qf16 to IEEE before widening
970bool HexagonXQFloatGenerator::widenMultiplicationInputF16Rt(MachineInstr &MI,
971 Register &Reg1,
972 Register &Reg2,
973 Register &Dest) {
974 // If the first input is not from an adder, for strict-ieee check if
975 // input from mult, else return false.
976 if (!checkIfInputFromAdder16(Reg: Reg1)) {
977 if (QFloatModeValue == QFloatMode::StrictIEEE) {
978 if (!checkIfInputFromMult16(Reg: Reg1))
979 return false;
980 } else
981 return false;
982 }
983
984 MachineBasicBlock &MBB = *MI.getParent();
985 const DebugLoc &DL = MI.getDebugLoc();
986
987 Register VSplatReg = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
988 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_lvsplatw), DestReg: VSplatReg).addReg(RegNo: Reg2);
989
990 Register widenReg = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
991 if (QFloatModeValue == QFloatMode::StrictIEEE) {
992 Register VHf = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
993 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VHf).addReg(RegNo: Reg1);
994 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: widenReg)
995 .addReg(RegNo: VHf)
996 .addReg(RegNo: VSplatReg);
997 } else {
998 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: widenReg)
999 .addReg(RegNo: Reg1)
1000 .addReg(RegNo: VSplatReg);
1001 }
1002
1003 // generate qf16 = qf32
1004 generateQF16FromQF32(MI, Dest, SrcReg&: widenReg);
1005 return true;
1006}
1007
1008// Handle qf32 = vadd/vsub(qf32/sf, qf32/sf)
1009// Handle vadd/vsub instructions with qf32 operands conditionally
1010// isAdd: true if an add instruction is analyzed, false for subtract
1011// isFirstOpQf: true if 1st operand is qf32 type, false if sf type
1012// isSecOpQf: true if 2nd operand is qf32 type, false if sf type
1013bool HexagonXQFloatGenerator::convertAddOpToIEEE32(
1014 MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
1015 bool isAdd, bool isFirstOpQf, bool isSecOpQf) {
1016
1017 Register VR1;
1018 Register VR2;
1019 bool firstconvert = false, secondconvert = false;
1020 MachineBasicBlock &MBB = *MI.getParent();
1021 const DebugLoc &DL = MI.getDebugLoc();
1022
1023 // If the first operand is qf32 type
1024 if (isFirstOpQf) {
1025 // If the first operand is from add/sub/mul unit,
1026 // generate IEEE conversion instruction sf = qf32
1027 if (checkIfInputFromAdder32(Reg: Reg1) || checkIfInputFromMult32(Reg: Reg1)) {
1028 VR1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1029 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: VR1)
1030 .addReg(RegNo: Reg1);
1031 firstconvert = true;
1032 }
1033 }
1034
1035 // If 2nd operand is of qf32 type
1036 if (isSecOpQf) {
1037 // If the second operand is from add/sub/mul unit,
1038 // generate IEEE conversion instruction
1039 if (checkIfInputFromAdder32(Reg: Reg2) || checkIfInputFromMult32(Reg: Reg2)) {
1040 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1041 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: VR2)
1042 .addReg(RegNo: Reg2);
1043 secondconvert = true;
1044 }
1045 }
1046
1047 // If both operands are qf32 type, use V6_v[add/sub]_sf instruction
1048 // If one of them is of sf type, use V6_v[add/sub]_qf32_mix instruction
1049 // Output is qf32
1050 if (isFirstOpQf && isSecOpQf) {
1051 if (firstconvert && secondconvert) {
1052 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1053 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), DestReg: Dest)
1054 .addReg(RegNo: VR1)
1055 .addReg(RegNo: VR2);
1056 } else if (firstconvert) {
1057 if (isAdd)
1058 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: Dest)
1059 .addReg(RegNo: Reg2)
1060 .addReg(RegNo: VR1);
1061 // For vsub type, for v81 we use a different opcode,
1062 // for v79, we convert the 2nd op to IEEE too.
1063 else {
1064 if (HST->useHVXV81Ops())
1065 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_sf_mix), DestReg: Dest)
1066 .addReg(RegNo: VR1)
1067 .addReg(RegNo: Reg2);
1068 else {
1069 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1070 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: VR2)
1071 .addReg(RegNo: Reg2);
1072 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_sf), DestReg: Dest)
1073 .addReg(RegNo: VR1)
1074 .addReg(RegNo: VR2);
1075 }
1076 }
1077 } else if (secondconvert) {
1078 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1079 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_qf32_mix
1080 : Hexagon::V6_vsub_qf32_mix),
1081 DestReg: Dest)
1082 .addReg(RegNo: Reg1)
1083 .addReg(RegNo: VR2);
1084 } else { // none of the inputs is from an add/sub/mul unit
1085 return false;
1086 }
1087 // handle vadd/vsub when the 1st op of original instruction is qf type
1088 } else if (isFirstOpQf) {
1089 if (firstconvert)
1090 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1091 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), DestReg: Dest)
1092 .addReg(RegNo: VR1)
1093 .addReg(RegNo: Reg2);
1094 else
1095 return false;
1096 // handle vadd/vsub when the 2nd op of original instruction is qf type
1097 } else if (isSecOpQf) {
1098 if (secondconvert)
1099 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1100 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), DestReg: Dest)
1101 .addReg(RegNo: Reg1)
1102 .addReg(RegNo: VR2);
1103 else
1104 return false;
1105 } else
1106 return false;
1107 return true;
1108}
1109
1110// Handle qf16 = vadd/vsub(qf16, qf16/hf)
1111// Handle vadd/vsub instructions with qf16 operands conditionally
1112// isAdd: true if an add instruction is analyzed, false for subtract
1113// isFirstOpQf: true if 1st operand is qf16 type, false if hf type
1114// isSecOpQf: true if 2nd operand is qf16 type, false if hf type
1115bool HexagonXQFloatGenerator::convertAddOpToIEEE16(
1116 MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
1117 bool isAdd, bool isFirstOpQf, bool isSecOpQf) {
1118
1119 MachineBasicBlock &MBB = *MI.getParent();
1120 const DebugLoc &DL = MI.getDebugLoc();
1121 Register VR1;
1122 Register VR2;
1123 bool firstconvert = false, secondconvert = false;
1124
1125 // If the first qf16 operand is from add/sub/mul unit,
1126 // generate IEEE conversion instruction
1127 if (isFirstOpQf) {
1128 if (checkIfInputFromAdder16(Reg: Reg1) || checkIfInputFromMult16(Reg: Reg1)) {
1129 VR1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1130 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR1)
1131 .addReg(RegNo: Reg1);
1132 firstconvert = true;
1133 }
1134 }
1135 if (isSecOpQf) {
1136 // If the second operand is from add/sub/mul unit,
1137 // generate IEEE conversion instruction
1138 if (checkIfInputFromAdder16(Reg: Reg2) || checkIfInputFromMult16(Reg: Reg2)) {
1139 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1140 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR2)
1141 .addReg(RegNo: Reg2);
1142 secondconvert = true;
1143 }
1144 }
1145
1146 // If both operands are qf16 type, use V6_v[add/sub]_hf instruction
1147 // If one of them is of hf type, use V6_v[add/sub]_qf16_mix instruction
1148 // Output is qf16
1149 if (isFirstOpQf && isSecOpQf) {
1150 if (firstconvert && secondconvert) {
1151 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1152 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), DestReg: Dest)
1153 .addReg(RegNo: VR1)
1154 .addReg(RegNo: VR2);
1155 } else if (firstconvert) {
1156 if (isAdd)
1157 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf16_mix), DestReg: Dest)
1158 .addReg(RegNo: Reg2)
1159 .addReg(RegNo: VR1);
1160 // For vsub type, for v81 we use a different opcode,
1161 // for v79, we convert the 2nd op to IEEE too.
1162 else {
1163 if (HST->useHVXV81Ops())
1164 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_hf_mix), DestReg: Dest)
1165 .addReg(RegNo: VR1)
1166 .addReg(RegNo: Reg2);
1167 else {
1168 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1169 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR2)
1170 .addReg(RegNo: Reg2);
1171 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vsub_hf), DestReg: Dest)
1172 .addReg(RegNo: VR1)
1173 .addReg(RegNo: VR2);
1174 }
1175 }
1176 } else if (secondconvert) {
1177 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1178 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_qf16_mix
1179 : Hexagon::V6_vsub_qf16_mix),
1180 DestReg: Dest)
1181 .addReg(RegNo: Reg1)
1182 .addReg(RegNo: VR2);
1183 } else { // none of the inputs is from an add/sub/mul unit
1184 return false;
1185 }
1186 // handle vadd/vsub when the 1st op of original instruction is qf type
1187 } else if (isFirstOpQf) {
1188 if (firstconvert)
1189 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1190 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), DestReg: Dest)
1191 .addReg(RegNo: VR1)
1192 .addReg(RegNo: Reg2);
1193 else
1194 return false;
1195 // handle vadd/vsub when the 2nd op of original instruction is qf type
1196 } else if (isSecOpQf) {
1197 if (secondconvert)
1198 BuildMI(BB&: MBB, I&: MI, MIMD: DL,
1199 MCID: HII->get(Opcode: isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), DestReg: Dest)
1200 .addReg(RegNo: Reg1)
1201 .addReg(RegNo: VR2);
1202 else
1203 return false;
1204 } else
1205 return false;
1206 return true;
1207}
1208
1209// Create the prolog
1210// v0 = #0
1211// R1 = #0x80000000
1212// v1.sf = vsplat(R1)
1213// v2.sf = vmpy(v0.sf, v1.sf)
1214void HexagonXQFloatGenerator::createPrologInstructions(MachineInstr &MI,
1215 Register &R_mpy) {
1216
1217 MachineBasicBlock &MBB = *MI.getParent();
1218 const DebugLoc &DL = MI.getDebugLoc();
1219
1220 Register VR0 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1221 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vd0), DestReg: VR0);
1222
1223 Register R_0 = MRI->createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
1224 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::A2_tfrsi), DestReg: R_0).addImm(Val: 0x80000000);
1225
1226 Register VR_0 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1227 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_lvsplatw), DestReg: VR_0).addReg(RegNo: R_0);
1228
1229 R_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1230 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_sf), DestReg: R_mpy)
1231 .addReg(RegNo: VR0)
1232 .addReg(RegNo: VR_0);
1233}
1234
1235bool HexagonXQFloatGenerator::V81normalizeMultF32(
1236 MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
1237 bool firstconvert, bool secondconvert, bool strictieee) {
1238 MachineBasicBlock &MBB = *MI.getParent();
1239 const DebugLoc &DL = MI.getDebugLoc();
1240 Register input_mpy1, input_mpy2;
1241
1242 auto Op =
1243 strictieee ? Hexagon::V6_vconv_qf32_sf : Hexagon::V6_vconv_qf32_qf32;
1244
1245 // Normalize both input operands
1246 if (firstconvert && secondconvert) {
1247 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1248 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1249
1250 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: input_mpy1).addReg(RegNo: Reg1);
1251 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: input_mpy2).addReg(RegNo: Reg2);
1252 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1253 .addReg(RegNo: input_mpy1)
1254 .addReg(RegNo: input_mpy2);
1255 }
1256 // Normalize only first operand
1257 else if (firstconvert) {
1258 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1259 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: input_mpy1).addReg(RegNo: Reg1);
1260 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1261 .addReg(RegNo: input_mpy1)
1262 .addReg(RegNo: Reg2);
1263 }
1264 // Normalize only second operand
1265 else if (secondconvert) {
1266 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1267 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Op), DestReg: input_mpy2).addReg(RegNo: Reg2);
1268 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1269 .addReg(RegNo: Reg1)
1270 .addReg(RegNo: input_mpy2);
1271 } else
1272 // we do nothing if the inputs are not from adder/sub/mult unit
1273 return false;
1274
1275 return true;
1276}
1277
1278// Normalize qf32 = vmpy(sf, sf) instruction unconditionally
1279void HexagonXQFloatGenerator::normalizeMultiplicationInputSF(
1280 MachineInstr &MI, Register &Src1, Register &Src2, Register &Dest,
1281 Register &R_mpy, bool &PrologCreated) {
1282
1283 MachineBasicBlock &MBB = *MI.getParent();
1284 const DebugLoc &DL = MI.getDebugLoc();
1285
1286 if (HST->useHVXV81Ops()) {
1287 Register input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1288 Register input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1289
1290 // Normalize both inputs
1291 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_qf32_sf), DestReg: input_mpy1)
1292 .addReg(RegNo: Src1);
1293 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_qf32_sf), DestReg: input_mpy2)
1294 .addReg(RegNo: Src2);
1295 // Add the new vmpy
1296 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1297 .addReg(RegNo: input_mpy1)
1298 .addReg(RegNo: input_mpy2);
1299 return;
1300 }
1301
1302 if (!PrologCreated) {
1303 createPrologInstructions(MI, R_mpy);
1304 PrologCreated = true;
1305 }
1306
1307 Register input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1308 Register input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1309 // Normalize both inputs
1310 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy1)
1311 .addReg(RegNo: R_mpy)
1312 .addReg(RegNo: Src1);
1313 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy2)
1314 .addReg(RegNo: R_mpy)
1315 .addReg(RegNo: Src2);
1316 // Add the new vmpy
1317 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1318 .addReg(RegNo: input_mpy1)
1319 .addReg(RegNo: input_mpy2);
1320}
1321
1322// Convert and normalize qf32 = vmpy(qf32, qf32) instructions conditionally
1323bool HexagonXQFloatGenerator::convertNormalizeMultOp32(
1324 MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
1325 Register &R_mpy, bool &PrologCreated) {
1326
1327 Register VR1, VR2;
1328 bool firstconvert = false, secondconvert = false;
1329 MachineBasicBlock &MBB = *MI.getParent();
1330 const DebugLoc &DL = MI.getDebugLoc();
1331
1332 // If the first operand is from add/subtract/multiply unit, generate IEEE
1333 // conversion instruction
1334 if (checkIfInputFromAdder32(Reg: Reg1) || checkIfInputFromMult32(Reg: Reg1)) {
1335 VR1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1336 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: VR1).addReg(RegNo: Reg1);
1337 firstconvert = true;
1338 }
1339
1340 if (checkIfInputFromAdder32(Reg: Reg2) || checkIfInputFromMult32(Reg: Reg2)) {
1341 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1342 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_sf_qf32), DestReg: VR2).addReg(RegNo: Reg2);
1343 secondconvert = true;
1344 }
1345
1346 if (HST->useHVXV81Ops()) {
1347 if (firstconvert && secondconvert)
1348 return V81normalizeMultF32(MI, Reg1&: VR1, Reg2&: VR2, Dest, firstconvert: true, secondconvert: true, strictieee: true);
1349 else if (firstconvert)
1350 return V81normalizeMultF32(MI, Reg1&: VR1, Reg2, Dest, firstconvert: true, secondconvert: false, strictieee: true);
1351 else if (secondconvert)
1352 return V81normalizeMultF32(MI, Reg1, Reg2&: VR2, Dest, firstconvert: false, secondconvert: true, strictieee: true);
1353 else
1354 return false;
1355 }
1356
1357 // create prolog if not already created
1358 if (!PrologCreated && (firstconvert || secondconvert)) {
1359 createPrologInstructions(MI, R_mpy);
1360 PrologCreated = true;
1361 }
1362
1363 Register input_mpy1, input_mpy2;
1364
1365 // Normalize both IEEE converts
1366 if (firstconvert && secondconvert) {
1367 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1368 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1369
1370 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy1)
1371 .addReg(RegNo: R_mpy)
1372 .addReg(RegNo: VR1);
1373 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy2)
1374 .addReg(RegNo: R_mpy)
1375 .addReg(RegNo: VR2);
1376 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1377 .addReg(RegNo: input_mpy1)
1378 .addReg(RegNo: input_mpy2);
1379 // Normalize only first operand
1380 } else if (firstconvert) {
1381 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1382
1383 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy1)
1384 .addReg(RegNo: R_mpy)
1385 .addReg(RegNo: VR1);
1386 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1387 .addReg(RegNo: input_mpy1)
1388 .addReg(RegNo: Reg2);
1389 // Normalize only second operand
1390 } else if (secondconvert) {
1391 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1392
1393 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32_mix), DestReg: input_mpy2)
1394 .addReg(RegNo: R_mpy)
1395 .addReg(RegNo: VR2);
1396 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1397 .addReg(RegNo: Reg1)
1398 .addReg(RegNo: input_mpy2);
1399 } else {
1400 // we do nothing if the inputs are not fromadder/subtracter/multiplier unit
1401 return false;
1402 }
1403 return true;
1404}
1405
1406// Convert to IEEE and widen qf16 = vmpy(qf16/hf, qf16) conditionally
1407// Then convert qf32 to qf16
1408// twoOps: true if the first operand is qf type, false if hf type
1409bool HexagonXQFloatGenerator::convertWidenMultOp16(MachineInstr &MI,
1410 Register &Reg1,
1411 Register &Reg2,
1412 Register &Dest,
1413 bool twoOps) {
1414
1415 Register VR1, VR2, output_mpy;
1416 bool firstconvert = false,
1417 secondconvert = false; // normalize with hf or qf16 operands
1418 MachineBasicBlock &MBB = *MI.getParent();
1419 const DebugLoc &DL = MI.getDebugLoc();
1420
1421 // If the first operand is from add/sub/mul unit,
1422 // generate IEEE conversion instruction
1423 if (checkIfInputFromAdder16(Reg: Reg1) || checkIfInputFromMult16(Reg: Reg1)) {
1424 VR1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1425 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR1).addReg(RegNo: Reg1);
1426 firstconvert = true;
1427 }
1428
1429 if (twoOps) {
1430 if (checkIfInputFromAdder16(Reg: Reg2) || checkIfInputFromMult16(Reg: Reg2)) {
1431 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1432 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR2)
1433 .addReg(RegNo: Reg2);
1434 secondconvert = true;
1435 }
1436 }
1437
1438 if (twoOps) {
1439 // Both operands have been converted to IEEE
1440 if (firstconvert && secondconvert) {
1441 output_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
1442 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: output_mpy)
1443 .addReg(RegNo: VR1)
1444 .addReg(RegNo: VR2);
1445 // Only one operand has been converted to IEEE
1446 } else if (firstconvert) {
1447 output_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
1448 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: output_mpy)
1449 .addReg(RegNo: Reg2)
1450 .addReg(RegNo: VR1);
1451 } else if (secondconvert) {
1452 output_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
1453 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: output_mpy)
1454 .addReg(RegNo: Reg1)
1455 .addReg(RegNo: VR2);
1456 } else {
1457 // Neither have to be transformed
1458 return false;
1459 }
1460 } else {
1461 if (firstconvert) {
1462 output_mpy = MRI->createVirtualRegister(RegClass: &Hexagon::HvxWRRegClass);
1463 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: output_mpy)
1464 .addReg(RegNo: VR1)
1465 .addReg(RegNo: Reg2);
1466 } else
1467 return false;
1468 }
1469
1470 // convert qf32 to qf16
1471 generateQF16FromQF32(MI, Dest, SrcReg&: output_mpy);
1472
1473 return true;
1474}
1475
1476// Convert to IEEE and perform qf32 = vmpy(qf16/hf, qf16) conditionally
1477// Final output is qf32 type
1478bool HexagonXQFloatGenerator::convertWidenMultOp32(MachineInstr &MI,
1479 Register &Reg1,
1480 Register &Reg2,
1481 Register &Dest,
1482 bool twoOps) {
1483 Register VR1, VR2;
1484 bool firstconvert = false,
1485 secondconvert = false; // normalize with hf or qf16 operands
1486 MachineBasicBlock &MBB = *MI.getParent();
1487 const DebugLoc &DL = MI.getDebugLoc();
1488
1489 // If the first operand is from add/subtract/multiply unit, generate IEEE
1490 // conversion instruction
1491 if (checkIfInputFromAdder16(Reg: Reg1) || checkIfInputFromMult16(Reg: Reg1)) {
1492 VR1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1493 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR1).addReg(RegNo: Reg1);
1494 firstconvert = true;
1495 }
1496
1497 if (twoOps) {
1498 if (checkIfInputFromAdder16(Reg: Reg2) || checkIfInputFromMult16(Reg: Reg2)) {
1499 VR2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1500 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vconv_hf_qf16), DestReg: VR2)
1501 .addReg(RegNo: Reg2);
1502 secondconvert = true;
1503 }
1504 }
1505
1506 if (twoOps) {
1507 // Both operands have been converted to IEEE
1508 if (firstconvert && secondconvert) {
1509 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: Dest)
1510 .addReg(RegNo: VR1)
1511 .addReg(RegNo: VR2);
1512 // Only one operand has been converted to IEEE
1513 } else if (firstconvert) {
1514 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: Dest)
1515 .addReg(RegNo: Reg2)
1516 .addReg(RegNo: VR1);
1517 } else if (secondconvert) {
1518 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_mix_hf), DestReg: Dest)
1519 .addReg(RegNo: Reg1)
1520 .addReg(RegNo: VR2);
1521 } else
1522 // Neither have to be transformed
1523 return false;
1524 } else {
1525 if (firstconvert)
1526 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32_hf), DestReg: Dest)
1527 .addReg(RegNo: VR1)
1528 .addReg(RegNo: Reg2);
1529 else
1530 return false;
1531 }
1532
1533 return true;
1534}
1535
1536// Normalize instructions of type qf32 = vmpy(qf32, qf32)
1537bool HexagonXQFloatGenerator::normalizeMultiplicationInputF32(
1538 MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
1539 Register &R_mpy, bool &PrologCreated) {
1540 bool firstconvert = false, secondconvert = false;
1541 MachineBasicBlock &MBB = *MI.getParent();
1542 const DebugLoc &DL = MI.getDebugLoc();
1543
1544 // We normalize only that operand which comes from add/subtract unit.
1545 if (checkIfInputFromAdder32(Reg: Reg1))
1546 firstconvert = true;
1547 if (checkIfInputFromAdder32(Reg: Reg2))
1548 secondconvert = true;
1549
1550 // v81 normalization
1551 if (HST->useHVXV81Ops())
1552 return V81normalizeMultF32(MI, Reg1, Reg2, Dest, firstconvert,
1553 secondconvert, strictieee: false);
1554
1555 // create normalization operand conditionally for v79
1556 if ((!PrologCreated && (firstconvert || secondconvert))) {
1557 createPrologInstructions(MI, R_mpy);
1558 PrologCreated = true;
1559 }
1560
1561 Register input_mpy1, input_mpy2;
1562
1563 // Normalize both input operands
1564 if (firstconvert && secondconvert) {
1565 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1566 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1567
1568 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32), DestReg: input_mpy1)
1569 .addReg(RegNo: R_mpy)
1570 .addReg(RegNo: Reg1);
1571 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32), DestReg: input_mpy2)
1572 .addReg(RegNo: R_mpy)
1573 .addReg(RegNo: Reg2);
1574 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1575 .addReg(RegNo: input_mpy1)
1576 .addReg(RegNo: input_mpy2);
1577 // Normalize only first operand
1578 } else if (firstconvert) {
1579 input_mpy1 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1580
1581 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32), DestReg: input_mpy1)
1582 .addReg(RegNo: R_mpy)
1583 .addReg(RegNo: Reg1);
1584 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1585 .addReg(RegNo: input_mpy1)
1586 .addReg(RegNo: Reg2);
1587 // Normalize only second operand
1588 } else if (secondconvert) {
1589 input_mpy2 = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1590
1591 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vadd_qf32), DestReg: input_mpy2)
1592 .addReg(RegNo: R_mpy)
1593 .addReg(RegNo: Reg2);
1594 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: HII->get(Opcode: Hexagon::V6_vmpy_qf32), DestReg: Dest)
1595 .addReg(RegNo: input_mpy2)
1596 .addReg(RegNo: Reg1);
1597 } else {
1598 // we do nothing if the inputs are not from adder/subtracter/multiplier unit
1599 return false;
1600 }
1601
1602 return true;
1603}
1604
1605bool HexagonXQFloatGenerator::deleteList() {
1606 if (OriginalMI.empty())
1607 return false;
1608 bool Changed = false;
1609 for (MachineInstr *origMI : OriginalMI) {
1610 LLVM_DEBUG(dbgs() << "deleting redundant instruction");
1611 LLVM_DEBUG(origMI->dump());
1612 origMI->eraseFromParent();
1613 Changed = true;
1614 }
1615 OriginalMI.clear();
1616 return Changed;
1617}
1618
1619// Parent function to handle Loosy subnormal transformations
1620bool HexagonXQFloatGenerator::HandleLossySubnormals(MachineFunction &MF) {
1621 bool Changed = false;
1622 Register R_mpy;
1623 for (auto &MBB : MF) {
1624 bool PrologCreated = false;
1625 for (auto &MI : MBB) {
1626 Changed |= deleteList();
1627 // Skip if the instruction does not have two operands,
1628 // or is a bundle instruction
1629 // or is a debug instruction
1630 if (MI.getNumOperands() != 3 || MI.isDebugInstr())
1631 continue;
1632 auto Op1 = MI.getOperand(i: 1);
1633 if (!Op1.isReg())
1634 continue;
1635 auto Op2 = MI.getOperand(i: 2);
1636 if (!Op2.isReg())
1637 continue;
1638 auto Op0 = MI.getOperand(i: 0);
1639 if (!Op0.isReg())
1640 continue;
1641 Register Reg1 = Op1.getReg();
1642 Register Reg2 = Op2.getReg();
1643 Register Dest = Op0.getReg();
1644
1645 // FIXME Do not process physical registers as operands
1646 if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
1647 continue;
1648
1649 switch (MI.getOpcode()) {
1650 // qf32 = vmpy(qf32, qf32)
1651 // Normalize one or both input operands
1652 // if from add/sub unit
1653 case Hexagon::V6_vmpy_qf32:
1654 if (normalizeMultiplicationInputF32(MI, Reg1, Reg2, Dest, R_mpy,
1655 PrologCreated))
1656 OriginalMI.push_back(Elt: &MI);
1657 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1658 break;
1659
1660 // qf16 = vmpy(qf16, qf16)
1661 // Widening multiply to qf32 and convert back to qf16
1662 // if any of the operands are from add/sub unit
1663 case Hexagon::V6_vmpy_qf16:
1664 if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, twoOps: true))
1665 OriginalMI.push_back(Elt: &MI);
1666 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1667 break;
1668
1669 // qf16 = vmpy(qf16, Rt.hf)
1670 // Splat Rt to vector and then widening multiply
1671 // and then convert back to qf16
1672 // if first operand is from add/sub unit
1673 case Hexagon::V6_vmpy_rt_qf16:
1674 if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
1675 OriginalMI.push_back(Elt: &MI);
1676 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1677 break;
1678
1679 // qf16 = vmpy(qf16, hf)
1680 // Widening multiply to qf32 and convert back to qf16
1681 // if first operand is from add/sub unit
1682 case Hexagon::V6_vmpy_qf16_mix_hf:
1683 if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, twoOps: false))
1684 OriginalMI.push_back(Elt: &MI);
1685 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1686 break;
1687 // Check if use of qf32 generating add/sub/mul instructions
1688 // are used as non-HVX operands.
1689 // If yes, convert the use to IEEE
1690 case Hexagon::V6_vadd_sf:
1691 case Hexagon::V6_vadd_qf32:
1692 case Hexagon::V6_vadd_qf32_mix:
1693 case Hexagon::V6_vsub_sf:
1694 case Hexagon::V6_vsub_qf32:
1695 case Hexagon::V6_vsub_qf32_mix:
1696 case Hexagon::V6_vsub_sf_mix:
1697 case Hexagon::V6_vmpy_qf32_qf16:
1698 case Hexagon::V6_vmpy_qf32_hf:
1699 case Hexagon::V6_vmpy_qf32_mix_hf:
1700 case Hexagon::V6_vmpy_rt_sf:
1701 case Hexagon::V6_vmpy_qf32_sf:
1702 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1703 break;
1704 // Check if use of qf16 generating add/sub/mul instructions
1705 // are used as non-HVX operands.
1706 // If yes, convert the use to IEEE
1707 case Hexagon::V6_vadd_hf:
1708 case Hexagon::V6_vsub_hf:
1709 case Hexagon::V6_vadd_qf16:
1710 case Hexagon::V6_vsub_qf16:
1711 case Hexagon::V6_vadd_qf16_mix:
1712 case Hexagon::V6_vsub_qf16_mix:
1713 case Hexagon::V6_vsub_hf_mix:
1714 case Hexagon::V6_vmpy_qf16_hf:
1715 case Hexagon::V6_vmpy_rt_hf:
1716 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1717 break;
1718 default:
1719 break;
1720 }
1721 }
1722 }
1723 if (OriginalMI.empty() || !Changed)
1724 return false;
1725 return true;
1726}
1727
1728// Parent function to handle all IEEE-754 compliant transformations
1729bool HexagonXQFloatGenerator::HandleCompliantIEEE(MachineFunction &MF) {
1730 bool Changed = false;
1731 Register R_mpy;
1732 for (auto &MBB : MF) {
1733 bool PrologCreated = false;
1734 for (auto &MI : MBB) {
1735 Changed |= deleteList();
1736 // Skip if the instruction does not have two operands,
1737 // or is a bundle instruction
1738 // or is a debug instruction
1739 if (MI.getNumOperands() != 3 || MI.isDebugInstr())
1740 continue;
1741
1742 auto Op1 = MI.getOperand(i: 1);
1743 if (!Op1.isReg())
1744 continue;
1745 auto Op2 = MI.getOperand(i: 2);
1746 if (!Op2.isReg())
1747 continue;
1748 auto Op0 = MI.getOperand(i: 0);
1749 if (!Op0.isReg())
1750 continue;
1751 Register Reg1 = Op1.getReg();
1752 Register Reg2 = Op2.getReg();
1753 Register Dest = Op0.getReg();
1754 Register VRtSplat;
1755
1756 // FIXME Do not process physical registers as operands
1757 if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
1758 continue;
1759
1760 switch (MI.getOpcode()) {
1761
1762 // ==== Handle multiplication instructions ====
1763
1764 // qf32 = vmpy(sf, Rt.sf)
1765 // Splat Rt to a vector
1766 // Normalize both input operands unconditionally
1767 case Hexagon::V6_vmpy_rt_sf:
1768 VRtSplat = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1769 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: HII->get(Opcode: Hexagon::V6_lvsplatw),
1770 DestReg: VRtSplat)
1771 .addReg(RegNo: Reg2);
1772 normalizeMultiplicationInputSF(MI, Src1&: Reg1, Src2&: VRtSplat, Dest, R_mpy,
1773 PrologCreated);
1774 OriginalMI.push_back(Elt: &MI);
1775 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1776 break;
1777
1778 // qf32 = vmpy(sf, sf)
1779 // Normalize both operands unconditionally
1780 case Hexagon::V6_vmpy_qf32_sf:
1781 normalizeMultiplicationInputSF(MI, Src1&: Reg1, Src2&: Reg2, Dest, R_mpy,
1782 PrologCreated);
1783 OriginalMI.push_back(Elt: &MI);
1784 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1785 break;
1786
1787 // qf32 = vmpy(qf32, qf32)
1788 // Normalize one or both input operands
1789 // if from add/sub unit
1790 case Hexagon::V6_vmpy_qf32:
1791 if (normalizeMultiplicationInputF32(MI, Reg1, Reg2, Dest, R_mpy,
1792 PrologCreated))
1793 OriginalMI.push_back(Elt: &MI);
1794 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1795 break;
1796
1797 // qf16 = vmpy(hf, rt)
1798 // Splat Rt to vector and then widening multiply
1799 case Hexagon::V6_vmpy_rt_hf:
1800 VRtSplat = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1801 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: HII->get(Opcode: Hexagon::V6_lvsplatw),
1802 DestReg: VRtSplat)
1803 .addReg(RegNo: Reg2);
1804 widenMultiplyInputHF(MI, Reg1, Reg2&: VRtSplat, Dest);
1805 OriginalMI.push_back(Elt: &MI);
1806 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1807 break;
1808
1809 // Widening multiply
1810 // qf16 = vmpy(hf, hf)
1811 case Hexagon::V6_vmpy_qf16_hf:
1812 widenMultiplyInputHF(MI, Reg1, Reg2, Dest);
1813 OriginalMI.push_back(Elt: &MI);
1814 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1815 break;
1816
1817 // qf16 = vmpy(qf16, qf16)
1818 // Widening multiply to qf32 and convert back to qf16
1819 // if any of the operands are from add/sub unit
1820 case Hexagon::V6_vmpy_qf16:
1821 if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, twoOps: true))
1822 OriginalMI.push_back(Elt: &MI);
1823 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1824 break;
1825
1826 // qf16 = vmpy(qf16, Rt.hf)
1827 // Splat Rt to vector and then widening multiply
1828 // and then convert back to qf16
1829 // if first operand is from add/sub unit
1830 case Hexagon::V6_vmpy_rt_qf16:
1831 if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
1832 OriginalMI.push_back(Elt: &MI);
1833 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1834 break;
1835
1836 // qf16 = vmpy(qf16, hf)
1837 // Widening multiply to qf32 and convert back to qf16
1838 // if first operand is from add/sub unit
1839 case Hexagon::V6_vmpy_qf16_mix_hf:
1840 if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, twoOps: false))
1841 OriginalMI.push_back(Elt: &MI);
1842 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1843 break;
1844
1845 // Check if use of qf32/qf16 generating add/sub/mul
1846 // instructions are used as non-HVX operands.
1847 // If yes, convert the use to IEEE
1848 case Hexagon::V6_vadd_sf:
1849 case Hexagon::V6_vadd_qf32:
1850 case Hexagon::V6_vadd_qf32_mix:
1851 case Hexagon::V6_vsub_sf:
1852 case Hexagon::V6_vsub_qf32:
1853 case Hexagon::V6_vsub_qf32_mix:
1854 case Hexagon::V6_vsub_sf_mix:
1855 case Hexagon::V6_vmpy_qf32_qf16:
1856 case Hexagon::V6_vmpy_qf32_hf:
1857 case Hexagon::V6_vmpy_qf32_mix_hf:
1858 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1859 break;
1860 case Hexagon::V6_vadd_hf:
1861 case Hexagon::V6_vsub_hf:
1862 case Hexagon::V6_vadd_qf16:
1863 case Hexagon::V6_vsub_qf16:
1864 case Hexagon::V6_vadd_qf16_mix:
1865 case Hexagon::V6_vsub_qf16_mix:
1866 case Hexagon::V6_vsub_hf_mix:
1867 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1868 break;
1869 default:
1870 break;
1871 }
1872 }
1873 }
1874 if (OriginalMI.empty() || !Changed)
1875 return false;
1876 return true;
1877}
1878
1879// Parent function to do strict IEEE transformations
1880bool HexagonXQFloatGenerator::HandleStrictIEEE(MachineFunction &MF) {
1881
1882 bool Changed = false;
1883 Register R_mpy;
1884 for (auto &MBB : MF) {
1885 bool PrologCreated = false;
1886 for (auto &MI : MBB) {
1887 Changed |= deleteList();
1888 // Skip if the instruction does not have two operands,
1889 // or is a bundle instruction
1890 // or is a debug instruction
1891 if (MI.getNumOperands() != 3 || MI.isDebugInstr())
1892 continue;
1893
1894 auto Op1 = MI.getOperand(i: 1);
1895 if (!Op1.isReg())
1896 continue;
1897 auto Op2 = MI.getOperand(i: 2);
1898 if (!Op2.isReg())
1899 continue;
1900 auto Op0 = MI.getOperand(i: 0);
1901 if (!Op0.isReg())
1902 continue;
1903 Register Reg1 = Op1.getReg();
1904 Register Reg2 = Op2.getReg();
1905 Register Dest = Op0.getReg();
1906 Register VRtSplat;
1907
1908 // FIXME Do not process physical registers as operands
1909 if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
1910 continue;
1911
1912 switch (MI.getOpcode()) {
1913 // ==== Handle add/subtract instructions ====
1914 // Convert one or both the input operands to IEEE 32-bit
1915 // if from add/sub/mult unit(s)
1916 // qf32 = vadd(qf32, qf32)
1917 case Hexagon::V6_vadd_qf32:
1918 if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, isAdd: true, isFirstOpQf: true, isSecOpQf: true))
1919 OriginalMI.push_back(Elt: &MI);
1920 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1921 break;
1922 // qf32 = vsub(qf32, qf32)
1923 case Hexagon::V6_vsub_qf32:
1924 if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: true, isSecOpQf: true))
1925 OriginalMI.push_back(Elt: &MI);
1926 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1927 break;
1928 // Convert only the first input operand to IEEE 32-bit
1929 // if it is from add/sub/mult unit
1930 // qf32 = vadd(qf32, sf)
1931 case Hexagon::V6_vadd_qf32_mix:
1932 if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, isAdd: true, isFirstOpQf: true, isSecOpQf: false))
1933 OriginalMI.push_back(Elt: &MI);
1934 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1935 break;
1936 // qf32 = vsub(qf32, sf)
1937 case Hexagon::V6_vsub_qf32_mix:
1938 if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: true, isSecOpQf: false))
1939 OriginalMI.push_back(Elt: &MI);
1940 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1941 break;
1942 // qf32 = vsub(sf, qf32)
1943 case Hexagon::V6_vsub_sf_mix:
1944 if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: false, isSecOpQf: true))
1945 OriginalMI.push_back(Elt: &MI);
1946 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1947 break;
1948 break;
1949
1950 // Convert one or both the input operands to IEEE 16-bit
1951 // if from add/sub/mult unit(s)
1952 // qf16 = vadd(qf16, qf16)
1953 case Hexagon::V6_vadd_qf16:
1954 if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, isAdd: true, isFirstOpQf: true, isSecOpQf: true))
1955 OriginalMI.push_back(Elt: &MI);
1956 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1957 break;
1958 // qf16 = vsub(qf16, qf16)
1959 case Hexagon::V6_vsub_qf16:
1960 if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: true, isSecOpQf: true))
1961 OriginalMI.push_back(Elt: &MI);
1962 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1963 break;
1964 // Convert only the first input operand IEEE 16-bit
1965 // if it is from add/sub/mul unit
1966 // qf16 = vadd(qf16, hf)
1967 case Hexagon::V6_vadd_qf16_mix:
1968 if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, isAdd: true, isFirstOpQf: true, isSecOpQf: false))
1969 OriginalMI.push_back(Elt: &MI);
1970 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1971 break;
1972 // qf16 = vsub(qf16, hf)
1973 case Hexagon::V6_vsub_qf16_mix:
1974 if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: true, isSecOpQf: false))
1975 OriginalMI.push_back(Elt: &MI);
1976 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1977 break;
1978 // qf16 = vsub(hf, qf16)
1979 case Hexagon::V6_vsub_hf_mix:
1980 if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, isAdd: false, isFirstOpQf: false, isSecOpQf: true))
1981 OriginalMI.push_back(Elt: &MI);
1982 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
1983 break;
1984
1985 // ==== Handle multiplication instructions ====
1986
1987 // qf32 = vmpy(sf, Rt.sf)
1988 // Splat Rt to a vector
1989 // Normalize both input operands unconditionally
1990 case Hexagon::V6_vmpy_rt_sf:
1991 VRtSplat = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
1992 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: HII->get(Opcode: Hexagon::V6_lvsplatw),
1993 DestReg: VRtSplat)
1994 .addReg(RegNo: Reg2);
1995 normalizeMultiplicationInputSF(MI, Src1&: Reg1, Src2&: VRtSplat, Dest, R_mpy,
1996 PrologCreated);
1997 OriginalMI.push_back(Elt: &MI);
1998 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
1999 break;
2000
2001 // Normalize both operands unconditionally
2002 // qf32 = vmpy(sf, sf)
2003 case Hexagon::V6_vmpy_qf32_sf:
2004 normalizeMultiplicationInputSF(MI, Src1&: Reg1, Src2&: Reg2, Dest, R_mpy,
2005 PrologCreated);
2006 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
2007 OriginalMI.push_back(Elt: &MI);
2008 break;
2009
2010 // Convert one or both input operands to IEEE 32-bit
2011 // if from add/sub/mult unit and normalize
2012 // qf32 = vmpy(qf32, qf32)
2013 case Hexagon::V6_vmpy_qf32:
2014 if (convertNormalizeMultOp32(MI, Reg1, Reg2, Dest, R_mpy,
2015 PrologCreated))
2016 OriginalMI.push_back(Elt: &MI);
2017 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
2018 break;
2019
2020 // Convert one or both input operands to IEEE 16-bit
2021 // if from mul/add/sub unit;
2022 // then widening multiply to generate qf32
2023 // then convert to qf16
2024 // qf16 = vmpy(qf16, qf16)
2025 case Hexagon::V6_vmpy_qf16:
2026 if (convertWidenMultOp16(MI, Reg1, Reg2, Dest, twoOps: true))
2027 OriginalMI.push_back(Elt: &MI);
2028 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2029 break;
2030
2031 // Convert one or both input operands to IEEE 16-bit
2032 // if from mul/add/sub unit;
2033 // then widening multiply to generate qf32
2034 // qf32 = vmpy(qf16, qf16)
2035 case Hexagon::V6_vmpy_qf32_qf16:
2036 if (convertWidenMultOp32(MI, Reg1, Reg2, Dest, twoOps: true))
2037 OriginalMI.push_back(Elt: &MI);
2038 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
2039 break;
2040
2041 // qf16 = vmpy(hf, rt)
2042 // Splat Rt to vector and then widening multiply
2043 case Hexagon::V6_vmpy_rt_hf:
2044 VRtSplat = MRI->createVirtualRegister(RegClass: &Hexagon::HvxVRRegClass);
2045 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: HII->get(Opcode: Hexagon::V6_lvsplatw),
2046 DestReg: VRtSplat)
2047 .addReg(RegNo: Reg2);
2048 widenMultiplyInputHF(MI, Reg1, Reg2&: VRtSplat, Dest);
2049 OriginalMI.push_back(Elt: &MI);
2050 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2051 break;
2052
2053 // Widening multiply, then convert to IEEE
2054 // qf16 = vmpy(hf, hf)
2055 case Hexagon::V6_vmpy_qf16_hf:
2056 widenMultiplyInputHF(MI, Reg1, Reg2, Dest);
2057 OriginalMI.push_back(Elt: &MI);
2058 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2059 break;
2060
2061 // qf16 = vmpy(qf16, Rt.hf)
2062 // Splat Rt to vector and then widening multiply
2063 // and then convert back to qf16
2064 // if first operand is from add/sub unit
2065 case Hexagon::V6_vmpy_rt_qf16:
2066 if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
2067 OriginalMI.push_back(Elt: &MI);
2068 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2069 break;
2070
2071 // qf16 = vmpy(qf16, hf)
2072 // Convert only the first input operans to IEEE 16-bit
2073 // if from mul/add/sub unit;
2074 // then widening multiply to generate qf32
2075 // then convert back to qf16
2076 case Hexagon::V6_vmpy_qf16_mix_hf:
2077 if (convertWidenMultOp16(MI, Reg1, Reg2, Dest, twoOps: false))
2078 OriginalMI.push_back(Elt: &MI);
2079 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2080 break;
2081
2082 // qf32 = vmpy(qf16, hf)
2083 // Convert only the first input operans to IEEE 16-bit
2084 // if from mul/add/sub unit;
2085 // then widening multiply to generate qf32
2086 case Hexagon::V6_vmpy_qf32_mix_hf:
2087 if (convertWidenMultOp32(MI, Reg1, Reg2, Dest, twoOps: false))
2088 OriginalMI.push_back(Elt: &MI);
2089 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
2090 break;
2091 // Check if use of qf32/qf16 generating add/sub/mul
2092 // instructions are used as non-HVX operands.
2093 // If yes, convert the use to IEEE
2094 case Hexagon::V6_vadd_sf:
2095 case Hexagon::V6_vsub_sf:
2096 Changed |= convertIfInputToNonHVX(MI, is32bit: true);
2097 break;
2098 case Hexagon::V6_vadd_hf:
2099 case Hexagon::V6_vsub_hf:
2100 Changed |= convertIfInputToNonHVX(MI, is32bit: false);
2101 break;
2102 default:
2103 break;
2104 }
2105 }
2106 }
2107 if (OriginalMI.empty() || !Changed)
2108 return false;
2109 return true;
2110}
2111
2112// There is no conversions in lossy mode
2113bool HexagonXQFloatGenerator::HandleLossyLegacy(MachineFunction &MF) {
2114 return false;
2115}
2116
2117bool HexagonXQFloatGenerator::runOnMachineFunction(MachineFunction &MF) {
2118 if (!EnableHVXXQFloat || (QFloatModeValue == QFloatMode::Legacy))
2119 return false;
2120
2121 bool Changed = false;
2122 HST = &MF.getSubtarget<HexagonSubtarget>();
2123 HII = HST->getInstrInfo();
2124 MRI = &MF.getRegInfo();
2125
2126 if (EnableConversionsRemoval &&
2127 !(QFloatModeValue == QFloatMode::StrictIEEE)) {
2128 VectorConvertRemove VCR(MF, MRI, HST);
2129 VCR.run();
2130 LLVM_DEBUG(dbgs() << "\nExtraneous conversion instructions removed for "
2131 << MF.getName());
2132 if (PrintDebug)
2133 debug_print(MF);
2134 }
2135
2136 switch (QFloatModeValue) {
2137 case QFloatMode::StrictIEEE:
2138 LLVM_DEBUG(dbgs() << "\nGenerating code for STRICT-IEEE mode.\n");
2139 Changed = HandleStrictIEEE(MF);
2140 break;
2141 case QFloatMode::IEEE:
2142 LLVM_DEBUG(dbgs() << "\nGenerating code for IEEE mode.\n");
2143 Changed = HandleCompliantIEEE(MF);
2144 break;
2145 case QFloatMode::Lossy:
2146 LLVM_DEBUG(dbgs() << "\nGenerating code for LOSSY mode.\n");
2147 Changed = HandleLossySubnormals(MF);
2148 break;
2149 case QFloatMode::Legacy:
2150 LLVM_DEBUG(dbgs() << "\nGenerating code for LEGACY mode.\n");
2151 Changed = HandleLossyLegacy(MF);
2152 break;
2153 }
2154 LLVM_DEBUG(dbgs() << "...fine");
2155
2156 // Delete the original instructions
2157 for (MachineInstr *origMI : OriginalMI) {
2158 LLVM_DEBUG(origMI->dump());
2159 origMI->eraseFromParent();
2160 }
2161 OriginalMI.clear();
2162
2163 return Changed;
2164}
2165