1//===- OpenACCUtilsReduction.cpp - OpenACC reduction utilities ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/OpenACC/OpenACCUtilsReduction.h"
10#include "mlir/Dialect/Arith/Utils/Utils.h"
11#include "mlir/Dialect/Complex/IR/Complex.h"
12#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
13#include "llvm/Support/raw_ostream.h"
14
15using namespace mlir;
16
17namespace mlir {
18namespace acc {
19
20static bool isFloatOrComplexType(Type ty) {
21 return isa<FloatType, ComplexType>(Val: ty);
22}
23
24SmallVector<GPUParallelDimAttr>
25getReductionCombineParDims(ReductionCombineOp reductionCombineOp) {
26 if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: reductionCombineOp))
27 return SmallVector<GPUParallelDimAttr>(parDimsAttr.getArray());
28 llvm_unreachable(
29 "expected parallel dimensions attribute for reduction combine op");
30}
31
32SmallVector<GPUParallelDimAttr>
33getReductionCombineParDims(ReductionCombineRegionOp combineRegionOp) {
34 for (Operation *user : combineRegionOp.getSrcVar().getUsers()) {
35 if (auto accumulateOp = dyn_cast<ReductionAccumulateOp>(Val: user))
36 return SmallVector<GPUParallelDimAttr>(
37 accumulateOp.getParDims().getArray());
38 }
39 if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: combineRegionOp))
40 return SmallVector<GPUParallelDimAttr>(parDimsAttr.getArray());
41 return {};
42}
43
44ReductionOperator translateAtomicRMWKind(arith::AtomicRMWKind kind) {
45 switch (kind) {
46 case arith::AtomicRMWKind::addf:
47 case arith::AtomicRMWKind::addi:
48 return ReductionOperator::AccAdd;
49 case arith::AtomicRMWKind::mulf:
50 case arith::AtomicRMWKind::muli:
51 return ReductionOperator::AccMul;
52 case arith::AtomicRMWKind::maxs:
53 case arith::AtomicRMWKind::maxu:
54 case arith::AtomicRMWKind::maximumf:
55 case arith::AtomicRMWKind::maxnumf:
56 return ReductionOperator::AccMax;
57 case arith::AtomicRMWKind::minu:
58 case arith::AtomicRMWKind::mins:
59 case arith::AtomicRMWKind::minimumf:
60 case arith::AtomicRMWKind::minnumf:
61 return ReductionOperator::AccMin;
62 case arith::AtomicRMWKind::andi:
63 return ReductionOperator::AccIand;
64 case arith::AtomicRMWKind::ori:
65 return ReductionOperator::AccIor;
66 case arith::AtomicRMWKind::xori:
67 return ReductionOperator::AccXor;
68 case arith::AtomicRMWKind::assign:
69 break;
70 }
71 llvm_unreachable("unsupported atomic kind");
72}
73
74std::optional<arith::AtomicRMWKind>
75translateACCReductionOperator(ReductionOperator redOp, Type type) {
76 if (type.isInteger() && type.isUnsignedInteger())
77 return std::nullopt;
78
79 if (auto reducible = dyn_cast<ReducibleType>(Val&: type)) {
80 if (std::optional<arith::AtomicRMWKind> kind =
81 reducible.getAtomicRMWKind(redOp))
82 return kind;
83 return std::nullopt;
84 }
85
86 switch (redOp) {
87 case ReductionOperator::AccAdd:
88 if (type.isInteger())
89 return arith::AtomicRMWKind::addi;
90 if (isFloatOrComplexType(ty: type))
91 return arith::AtomicRMWKind::addf;
92 break;
93 case ReductionOperator::AccMul:
94 if (type.isInteger())
95 return arith::AtomicRMWKind::muli;
96 if (isFloatOrComplexType(ty: type))
97 return arith::AtomicRMWKind::mulf;
98 break;
99 case ReductionOperator::AccMax:
100 if (type.isInteger())
101 return arith::AtomicRMWKind::maxs;
102 if (type.isFloat())
103 return arith::AtomicRMWKind::maxnumf;
104 break;
105 case ReductionOperator::AccMaximumf:
106 return arith::AtomicRMWKind::maximumf;
107 case ReductionOperator::AccMaxnumf:
108 return arith::AtomicRMWKind::maxnumf;
109 case ReductionOperator::AccMin:
110 if (type.isInteger())
111 return arith::AtomicRMWKind::mins;
112 if (type.isFloat())
113 return arith::AtomicRMWKind::minnumf;
114 break;
115 case ReductionOperator::AccMinimumf:
116 return arith::AtomicRMWKind::minimumf;
117 case ReductionOperator::AccMinnumf:
118 return arith::AtomicRMWKind::minnumf;
119 case ReductionOperator::AccIand:
120 case ReductionOperator::AccLand:
121 if (type.isInteger())
122 return arith::AtomicRMWKind::andi;
123 break;
124 case ReductionOperator::AccIor:
125 case ReductionOperator::AccLor:
126 if (type.isInteger())
127 return arith::AtomicRMWKind::ori;
128 break;
129 case ReductionOperator::AccXor:
130 case ReductionOperator::AccNeqv:
131 if (type.isInteger())
132 return arith::AtomicRMWKind::xori;
133 break;
134 case ReductionOperator::AccEqv:
135 case ReductionOperator::AccNone:
136 break;
137 }
138 return std::nullopt;
139}
140
141static TypedAttr getReductionIdentityValueAttr(arith::AtomicRMWKind kind,
142 Type type, OpBuilder &builder,
143 Location loc,
144 bool useOnlyFiniteValue) {
145 if (type.isIntOrIndexOrFloat()) {
146 TypedAttr attr = arith::getIdentityValueAttr(kind, resultType: type, builder, loc,
147 useOnlyFiniteValue);
148 if (!attr)
149 emitError(loc) << "reduction identity: operator not supported " << kind;
150 return attr;
151 }
152 if (auto complexTy = dyn_cast<ComplexType>(Val&: type)) {
153 auto eltTy = dyn_cast<FloatType>(Val: complexTy.getElementType());
154 if (!eltTy) {
155 emitError(loc) << "reduction identity: complex with non-floating "
156 "element type";
157 return nullptr;
158 }
159 switch (kind) {
160 case arith::AtomicRMWKind::addf: {
161 TypedAttr scalarAttr = arith::getIdentityValueAttr(
162 kind, resultType: eltTy, builder, loc, useOnlyFiniteValue);
163 assert(scalarAttr && "expected scalar identity for complex reduction");
164 double d = cast<FloatAttr>(Val&: scalarAttr).getValue().convertToDouble();
165 return complex::NumberAttr::get(type: complexTy, real: d, imag: d);
166 }
167 case arith::AtomicRMWKind::mulf: {
168 TypedAttr scalarAttr = arith::getIdentityValueAttr(
169 kind, resultType: eltTy, builder, loc, useOnlyFiniteValue);
170 assert(scalarAttr &&
171 "expected scalar identity for complex mulf reduction");
172 auto realPart = cast<FloatAttr>(Val&: scalarAttr).getValue();
173 return complex::NumberAttr::get(type: complexTy, real: realPart.convertToDouble(),
174 imag: 0.0);
175 }
176 default:
177 emitError(loc)
178 << "reduction identity: operator not supported for complex " << kind;
179 return nullptr;
180 }
181 }
182 emitError(loc) << "reduction identity: type not supported " << type;
183 return nullptr;
184}
185
186Value createIdentityValue(OpBuilder &b, Location loc, Type type,
187 arith::AtomicRMWKind kind, bool useOnlyFiniteValue) {
188 TypedAttr typedAttr =
189 getReductionIdentityValueAttr(kind, type, builder&: b, loc, useOnlyFiniteValue);
190 assert(typedAttr && "expected identity attribute");
191 if (auto numAttr = dyn_cast<complex::NumberAttr>(Val&: typedAttr)) {
192 auto complexTy = cast<ComplexType>(Val: numAttr.getType());
193 auto floatElt = cast<FloatType>(Val: complexTy.getElementType());
194 Value realVal = arith::ConstantOp::create(
195 builder&: b, location: loc, value: b.getFloatAttr(type: floatElt, value: numAttr.getReal()));
196 Value imagVal = arith::ConstantOp::create(
197 builder&: b, location: loc, value: b.getFloatAttr(type: floatElt, value: numAttr.getImag()));
198 return complex::CreateOp::create(builder&: b, location: loc, complex: complexTy, real: realVal, imaginary: imagVal);
199 }
200 return arith::ConstantOp::create(builder&: b, location: loc, value: typedAttr);
201}
202
203Value generateReductionOp(OpBuilder &b, Location loc, Value lhs, Value rhs,
204 arith::AtomicRMWKind kind) {
205 assert(lhs.getType() == rhs.getType() &&
206 "expected same type for lhs and rhs");
207 if (isa<ComplexType>(Val: lhs.getType())) {
208 switch (kind) {
209 case arith::AtomicRMWKind::addf:
210 return complex::AddOp::create(builder&: b, location: loc, lhs, rhs);
211 case arith::AtomicRMWKind::mulf:
212 return complex::MulOp::create(builder&: b, location: loc, lhs, rhs);
213 default:
214 llvm_unreachable("unsupported complex atomic reduction kind");
215 }
216 }
217 return arith::getReductionOp(op: kind, builder&: b, loc, lhs, rhs);
218}
219
220} // namespace acc
221} // namespace mlir
222