1//===- QuantUtils.cpp -----------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains TOSA numerical support functions and quantization
10// attribute builders.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
15
16using namespace mlir;
17using namespace mlir::tosa;
18
19/// From a scale value, generates multiplier and shift values where
20/// mantissa is in [-1.0,-0.5] or [0.5, 1.0] such that
21/// multiplier = mantissa*2^shift for 16-bit scaling.
22static void computeMultiplierAndShiftTosaScale16(double scale,
23 int32_t &multiplier,
24 int32_t &shift) {
25
26 const double mantissa = std::frexp(x: scale, exponent: &shift);
27 auto shiftedM = std::round(x: mantissa * (int64_t(1) << 15));
28
29 // Can't be greater than 1.0.
30 assert(shiftedM <= (int64_t(1) << 15) &&
31 "Shifted mantissa exceeds 16 signed bits");
32
33 if (shiftedM == (int64_t(1) << 15)) {
34 shiftedM /= 2;
35 shift++;
36 }
37
38 // TOSA expects right shift to be positive and embed (1 << 15) into right
39 // shift bits.
40 shift = (-shift) + 15;
41
42 assert(shiftedM <= std::numeric_limits<int32_t>::max() &&
43 "Shifted mantissa exceeds 32-bit signed output type");
44
45 multiplier = static_cast<int32_t>(shiftedM);
46
47 // Shifting tops out at 62 bits. Right shift to make 62 bits the max.
48 // The limit of 62 on shift allows the shift to be decomposed as
49 // two right shifts of 31.
50 if (shift > 62) {
51 // Shifting the multiplier by more than 31-bits is unnecessary.
52 multiplier = multiplier >> std::min<int32_t>(a: 31, b: shift - 62);
53 shift = 62;
54 }
55}
56
57/// From a scale value, generates multiplier and shift values where
58/// mantissa is in [-1.0,-0.5] or [0.5, 1.0] such that
59/// multiplier = mantissa*2^shift for 32-bit scaling.
60static void computeMultiplierAndShiftTosaScale32(double scale,
61 int32_t &multiplier,
62 int32_t &shift) {
63
64 const double mantissa = std::frexp(x: scale, exponent: &shift);
65 auto shiftedM = std::round(x: mantissa * (int64_t(1) << 31));
66
67 // Can't be greater than 1.0.
68 assert(shiftedM <= (int64_t(1) << 31) &&
69 "Shifted mantissa exceeds 32 signed bits");
70 if (shiftedM == (int64_t(1) << 31)) {
71 shiftedM /= 2;
72 shift++;
73 }
74
75 // TOSA expects right shift to be positive, and embed (1 << 31) into right
76 // shift bits.
77 shift = (-shift) + 31;
78
79 assert(shiftedM <= std::numeric_limits<int32_t>::max() &&
80 "Shifted mantissa exceeds 32-bit signed output type");
81
82 multiplier = static_cast<int32_t>(shiftedM);
83
84 // Shifting tops out at 62 bits. Right shift to make 62 bits the max.
85 // The limit of 62 on shift allows the shift to be decomposed as
86 // two right shifts of 31.
87 if (shift > 62) {
88 // Shifting the multiplier by more than 32-bits is unnecessary.
89 multiplier = multiplier >> std::min<int32_t>(a: 31, b: shift - 62);
90 shift = 62;
91 }
92}
93
94/// Generates a quantized multiplier/shift from double.
95bool mlir::tosa::computeMultiplierAndShift(double scale, int32_t &multiplier,
96 int32_t &shift, int32_t scaleWidth) {
97
98 switch (scaleWidth) {
99 case 16:
100 computeMultiplierAndShiftTosaScale16(scale, multiplier, shift);
101
102 // In some cases computeMultiplierAndShiftTosaScale16 can return
103 // a value less then 2, which is not valid in the TOSA spec.
104 return (!(shift < 2));
105 case 32:
106 computeMultiplierAndShiftTosaScale32(scale, multiplier, shift);
107
108 // In some cases computeMultiplierAndShiftTosaScale32 can return
109 // a value less then 2, which is not valid in the TOSA spec.
110 return (!(shift < 2));
111 default:
112 assert(0 && "Unsupported Tosa quantized_scale regime specified!");
113 return false;
114 }
115}
116
117#define GET_UQTYPE(inputType) \
118 (llvm::dyn_cast<quant::UniformQuantizedType>((inputType).getElementType()))
119#define GET_QTYPE(inputType) \
120 (llvm::dyn_cast<quant::QuantizedType>((inputType).getElementType()))
121
122static std::optional<std::pair<std::int64_t, std::int64_t>>
123getConvZeroPoints(Value input, Value weight) {
124
125 auto inputType = dyn_cast<ShapedType>(Val: input.getType());
126 auto weightType = dyn_cast<ShapedType>(Val: weight.getType());
127
128 if (!inputType || !weightType)
129 return std::nullopt;
130
131 auto inputQType = GET_UQTYPE(inputType);
132 auto weightPerTensorQType = GET_UQTYPE(weightType);
133 auto weightPerAxisQType =
134 dyn_cast<quant::UniformQuantizedPerAxisType>(Val: weightType.getElementType());
135
136 // Weights must be either per-tensor quantized or per-axis quantized.
137 assert(!((bool)weightPerTensorQType && (bool)weightPerAxisQType) &&
138 "Weights must be either per-tensor or per-axis quantized");
139
140 // Either all quantized or all not quantized.
141 assert(!((bool)inputQType ^
142 ((bool)weightPerTensorQType || (bool)weightPerAxisQType)) &&
143 "Inputs and weights must be all quantized or all not quantized");
144
145 if (inputQType) {
146 int64_t inputZp = inputQType.getZeroPoint();
147 int64_t weightZp = 0;
148
149 if (weightPerTensorQType) {
150 weightZp = weightPerTensorQType.getZeroPoint();
151 } else if (weightPerAxisQType) {
152 weightZp = weightPerAxisQType.getZeroPoints().front();
153 }
154
155 return std::make_pair(x&: inputZp, y&: weightZp);
156 }
157
158 return std::nullopt;
159}
160
161std::pair<Value, Value>
162mlir::tosa::createZPsAsConst(OpBuilder &builder, Value input, Value weight) {
163 std::int64_t inputZp, weightZp;
164
165 Type inputZpType = getElementTypeOrSelf(type: input.getType());
166 if (isa<BlockScaledType>(Val: inputZpType))
167 inputZpType = builder.getF32Type();
168 Type weightZpType = getElementTypeOrSelf(type: weight.getType());
169 if (isa<BlockScaledType>(Val: weightZpType))
170 weightZpType = builder.getF32Type();
171
172 if (mlir::isa<FloatType>(Val: inputZpType) && mlir::isa<FloatType>(Val: weightZpType)) {
173 inputZp = 0;
174 weightZp = 0;
175 } else {
176 auto maybeZps = getConvZeroPoints(input, weight);
177 if (!maybeZps.has_value())
178 return {};
179
180 inputZp = maybeZps->first;
181 weightZp = maybeZps->second;
182 }
183
184 auto maybeInputZpValue =
185 createZeroPointTensor(builder, loc: input.getLoc(), srcElemType: inputZpType, zp: inputZp);
186 if (!maybeInputZpValue.has_value())
187 return {};
188
189 auto maybeWeightZpValue =
190 createZeroPointTensor(builder, loc: weight.getLoc(), srcElemType: weightZpType, zp: weightZp);
191 if (!maybeWeightZpValue.has_value())
192 return {};
193
194 return std::make_pair(x&: *maybeInputZpValue, y&: *maybeWeightZpValue);
195}
196
197/// Method to build ConvOpQuantizationAttr, called from
198/// ConvOpQuantInfoBuilder/TransConvOpQuantInfoBuilder:
199/// input_zp: input zeropoint
200/// weight_zp: weight zeropoint.
201ConvOpQuantizationAttr
202mlir::tosa::buildConvOpQuantizationAttr(OpBuilder &builder, Value input,
203 Value weight) {
204
205 auto maybeZps = getConvZeroPoints(input, weight);
206 if (!maybeZps.has_value())
207 return nullptr;
208
209 return builder.getAttr<tosa::ConvOpQuantizationAttr>(args&: maybeZps->first,
210 args&: maybeZps->second);
211}
212
213/// Builds MatMulOpQuantizationAttr, called from
214/// MatMulOpQuantInfoBuilder:
215/// aZp: input a zeropoint
216/// bZp: input b zeropoint.
217MatMulOpQuantizationAttr
218mlir::tosa::buildMatMulOpQuantizationAttr(OpBuilder &builder, Value a,
219 Value b) {
220
221 auto aType = dyn_cast<ShapedType>(Val: a.getType());
222 auto bType = dyn_cast<ShapedType>(Val: b.getType());
223
224 if (!aType || !bType)
225 return nullptr;
226
227 auto aQType = GET_UQTYPE(aType);
228 auto bQType = GET_UQTYPE(bType);
229
230 // A and B are either all quantized or all not quantized.
231 assert(!((bool)aQType ^ (bool)bQType) &&
232 "Matmul operands must be all quantized or all not quantized");
233
234 if (aQType) {
235 return builder.getAttr<tosa::MatMulOpQuantizationAttr>(
236 args: aQType.getZeroPoint(), args: bQType.getZeroPoint());
237 }
238
239 return nullptr;
240}
241
242/// Builds UnaryOpQuantizationAttr
243/// UnaryOpQuantInfoBuilder:
244/// inputZp: input zeropoint
245/// outputZp: output zeropoint.
246UnaryOpQuantizationAttr
247mlir::tosa::buildUnaryOpQuantizationAttr(OpBuilder &builder, Value input,
248 Type outputRawType) {
249
250 auto inputType = dyn_cast<ShapedType>(Val: input.getType());
251 auto outputType = dyn_cast<ShapedType>(Val&: outputRawType);
252
253 if (!inputType || !outputType)
254 return nullptr;
255
256 auto inputQType = GET_UQTYPE(inputType);
257 auto outputQType = GET_UQTYPE(outputType);
258
259 // Either all quantized or all not quantized.
260 assert(!((bool)inputQType ^ (bool)outputQType) &&
261 "Unary inputs/outputs must be all quantized or all not quantized");
262
263 if (inputQType) {
264 return builder.getAttr<UnaryOpQuantizationAttr>(args: inputQType.getZeroPoint(),
265 args: outputQType.getZeroPoint());
266 }
267
268 return nullptr;
269}
270
271/// Builds PadOpQuantizationAttr, called from PadOpQuantInfoBuilder:
272/// inputZp: input zeropoint.
273PadOpQuantizationAttr mlir::tosa::buildPadOpQuantizationAttr(OpBuilder &builder,
274 Value input) {
275
276 auto inputType = dyn_cast<ShapedType>(Val: input.getType());
277
278 if (!inputType)
279 return nullptr;
280
281 auto inputQType = GET_UQTYPE(inputType);
282
283 if (inputQType) {
284 return builder.getAttr<tosa::PadOpQuantizationAttr>(
285 args: inputQType.getZeroPoint());
286 }
287
288 return nullptr;
289}
290
291/// Builds output type for a quantized ConvOp with the right bitwidth.
292/// This is called by the builder when dealing with quantized content.
293Type mlir::tosa::buildConvOpResultTypeInfo(OpBuilder &builder, Type outputType,
294 Value input, Value weight) {
295
296 auto inputType = dyn_cast<ShapedType>(Val: input.getType());
297 auto weightType = dyn_cast<ShapedType>(Val: weight.getType());
298
299 assert(inputType && weightType &&
300 "Could not extract input or weight tensors from Conv op");
301
302 auto inputQType = GET_QTYPE(inputType);
303 auto weightQType = GET_QTYPE(weightType);
304
305 assert(inputQType && weightQType &&
306 "Could not extract input or weight tensor types from Conv op");
307
308 unsigned inputBits = inputQType.getStorageTypeIntegralWidth();
309 unsigned weightBits = weightQType.getStorageTypeIntegralWidth();
310
311 auto outputShapedType = dyn_cast<ShapedType>(Val&: outputType);
312 assert(outputShapedType &&
313 "Could not extract output shape type from Conv op");
314
315 IntegerType accElementType;
316 if (inputBits == 16 && weightBits == 8)
317 accElementType = builder.getIntegerType(width: 48);
318 else
319 accElementType = builder.getI32Type();
320 auto accType = outputShapedType.clone(elementType: accElementType);
321 return accType;
322}
323
324/// Builds Tosa quantization attributes from min/max values.
325Type mlir::tosa::buildQTypeFromMinMax(OpBuilder builder, Type inputDType,
326 Attribute minAttr, Attribute maxAttr,
327 IntegerAttr quantBits, int filterQuantDim,
328 bool isSigned, BoolAttr narrowRange) {
329
330 quant::QuantizedType retType;
331
332 auto convfunc =
333 quant::ExpressedToQuantizedConverter::forInputType(inputType: inputDType);
334
335 auto minElems = dyn_cast<DenseFPElementsAttr>(Val&: minAttr);
336 auto maxElems = dyn_cast<DenseFPElementsAttr>(Val&: maxAttr);
337
338 SmallVector<double, 2> min, max;
339
340 // At least one is per-axis quantized elementsattr.
341 if (minElems || maxElems) {
342 // Must have the same number of elements.
343 if (minElems.getNumElements() != maxElems.getNumElements())
344 return {};
345 min.reserve(N: minElems.getNumElements());
346 max.reserve(N: maxElems.getNumElements());
347 for (auto i : minElems)
348 min.push_back(Elt: FloatAttr::getValueAsDouble(val: i));
349 for (auto i : maxElems)
350 max.push_back(Elt: FloatAttr::getValueAsDouble(val: i));
351 } else { // Just a single FP value.
352 auto minVal = dyn_cast<FloatAttr>(Val&: minAttr);
353 if (minVal)
354 min.push_back(Elt: minVal.getValueAsDouble());
355 else
356 return {};
357 auto maxVal = dyn_cast<FloatAttr>(Val&: maxAttr);
358 if (maxVal)
359 max.push_back(Elt: maxVal.getValueAsDouble());
360 else
361 return {};
362 }
363
364 if (min.size() == max.size()) {
365 if (min.size() == 1) { // Per-tensor quantization with one min/max pair.
366 retType = quant::fakeQuantAttrsToType(
367 loc: builder.getUnknownLoc(), numBits: quantBits.getInt(), rmin: min[0], rmax: max[0],
368 narrowRange: narrowRange.getValue(), expressedType: convfunc.expressedType, isSigned);
369 } else if (min.size() > 1) { // Per-axis quant on filterQuantDim.
370 auto shape = dyn_cast<ShapedType>(Val&: inputDType);
371 if (!shape)
372 return {};
373 if ((filterQuantDim) >= 0 && (shape.getRank() > filterQuantDim)) {
374 retType = quant::fakeQuantAttrsToType(
375 loc: builder.getUnknownLoc(), numBits: quantBits.getInt(), quantizedDimension: filterQuantDim, rmins: min[0],
376 rmax: max[0], narrowRange: narrowRange.getValue(), expressedType: convfunc.expressedType, isSigned);
377 }
378 } else {
379 return {};
380 }
381 } else {
382 return {};
383 }
384
385 if (!retType)
386 return {};
387
388 return convfunc.convert(elementalType: retType);
389}
390
391/// Builds Tosa quantization attributes from min/max values.
392TypeAttr
393mlir::tosa::buildQTypeAttrFromMinMax(OpBuilder builder, Type inputDtype,
394 Attribute minAttr, Attribute maxAttr,
395 IntegerAttr quantBits, int filterQuantDim,
396 bool isSigned, BoolAttr narrowRange) {
397
398 return TypeAttr::get(type: buildQTypeFromMinMax(builder, inputDType: inputDtype, minAttr,
399 maxAttr, quantBits, filterQuantDim,
400 isSigned, narrowRange));
401}
402
403Type mlir::tosa::getStorageElementTypeFromQuantized(
404 quant::QuantizedType quantType) {
405 auto quantEty = quantType.getStorageType();
406 // StorageType doesn't capture the sign information
407 // Explicitly create unsigned type if needed
408 if (!quantType.isSigned()) {
409 quantEty = IntegerType::get(context: quantEty.getContext(),
410 width: quantEty.getIntOrFloatBitWidth(),
411 signedness: IntegerType::Unsigned);
412 }
413 return quantEty;
414}
415