1//===- ComplexToSPIRV.cpp - Complex to SPIR-V Patterns --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns to convert Complex dialect to SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRV.h"
14#include "mlir/Dialect/Complex/IR/Complex.h"
15#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
16#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
17#include "mlir/Transforms/DialectConversion.h"
18
19#define DEBUG_TYPE "complex-to-spirv-pattern"
20
21using namespace mlir;
22
23//===----------------------------------------------------------------------===//
24// Operation conversion
25//===----------------------------------------------------------------------===//
26
27namespace {
28/// Creates a scalar floating-point constant of the given value.
29Value createFPConstant(OpBuilder &builder, Location loc, Type type,
30 double value) {
31 return spirv::ConstantOp::create(
32 builder, location: loc, constant: type, value: builder.getFloatAttr(type: cast<FloatType>(Val&: type), value));
33}
34
35/// Computes abs(re + i*im) as max * sqrt(1 + (min / max)^2) to avoid overflow
36/// in squaring. The result is NaN when both components are zero.
37template <typename SqrtOp, typename FAbsOp, typename FMaxOp, typename FMinOp>
38Value createComplexAbs(OpBuilder &builder, Location loc, Value re, Value im) {
39 Value absRe = FAbsOp::create(builder, loc, re);
40 Value absIm = FAbsOp::create(builder, loc, im);
41 Value max = FMaxOp::create(builder, loc, absRe, absIm);
42 Value min = FMinOp::create(builder, loc, absRe, absIm);
43 Value ratio = spirv::FDivOp::create(builder, location: loc, operand1: min, operand2: max);
44 Value ratioSq = spirv::FMulOp::create(builder, location: loc, operand1: ratio, operand2: ratio);
45 Value one = createFPConstant(builder, loc, type: re.getType(), value: 1.0);
46 Value ratioSqPlusOne = spirv::FAddOp::create(builder, location: loc, operand1: ratioSq, operand2: one);
47 Value sqrt = SqrtOp::create(builder, loc, ratioSqPlusOne);
48 return spirv::FMulOp::create(builder, location: loc, operand1: max, operand2: sqrt);
49}
50
51/// Returns true when both components are zero.
52Value createIsZero(OpBuilder &builder, Location loc, Value re, Value im) {
53 Value zero = createFPConstant(builder, loc, type: re.getType(), value: 0.0);
54 Value reIsZero = spirv::FOrdEqualOp::create(builder, location: loc, operand1: re, operand2: zero);
55 Value imIsZero = spirv::FOrdEqualOp::create(builder, location: loc, operand1: im, operand2: zero);
56 return spirv::LogicalAndOp::create(builder, location: loc, operand1: reIsZero, operand2: imIsZero);
57}
58
59/// Computes FinalOp(abs) * (cos(angle * angleScale) + i*sin(angle *
60/// angleScale)) where abs and angle are the modulus and argument of re + i*im.
61/// Zero input is not handled.
62template <typename SqrtOp, typename FAbsOp, typename FMaxOp, typename FMinOp,
63 typename SinOp, typename CosOp, typename Atan2Op, typename FinalOp>
64std::pair<Value, Value> createComplexSqrtOrRsqrt(OpBuilder &builder,
65 Location loc, Value re,
66 Value im, double angleScale) {
67 Value abs =
68 createComplexAbs<SqrtOp, FAbsOp, FMaxOp, FMinOp>(builder, loc, re, im);
69 Value finalAbs = FinalOp::create(builder, loc, abs);
70 Value angle = Atan2Op::create(builder, loc, im, re);
71 Value scale = createFPConstant(builder, loc, type: re.getType(), value: angleScale);
72 Value scaledAngle = spirv::FMulOp::create(builder, location: loc, operand1: angle, operand2: scale);
73 Value cos = CosOp::create(builder, loc, scaledAngle);
74 Value sin = SinOp::create(builder, loc, scaledAngle);
75 Value resultRe = spirv::FMulOp::create(builder, location: loc, operand1: finalAbs, operand2: cos);
76 Value resultIm = spirv::FMulOp::create(builder, location: loc, operand1: finalAbs, operand2: sin);
77 return {resultRe, resultIm};
78}
79
80struct ConstantOpPattern final : OpConversionPattern<complex::ConstantOp> {
81 using Base::Base;
82
83 LogicalResult
84 matchAndRewrite(complex::ConstantOp constOp, OpAdaptor adaptor,
85 ConversionPatternRewriter &rewriter) const override {
86 auto spirvType =
87 getTypeConverter()->convertType<ShapedType>(t: constOp.getType());
88 if (!spirvType)
89 return rewriter.notifyMatchFailure(arg&: constOp,
90 msg: "unable to convert result type");
91
92 rewriter.replaceOpWithNewOp<spirv::ConstantOp>(
93 op: constOp, args&: spirvType,
94 args: DenseElementsAttr::get(type: spirvType, values: constOp.getValue().getValue()));
95 return success();
96 }
97};
98
99struct CreateOpPattern final : OpConversionPattern<complex::CreateOp> {
100 using Base::Base;
101
102 LogicalResult
103 matchAndRewrite(complex::CreateOp createOp, OpAdaptor adaptor,
104 ConversionPatternRewriter &rewriter) const override {
105 Type spirvType = getTypeConverter()->convertType(t: createOp.getType());
106 if (!spirvType)
107 return rewriter.notifyMatchFailure(arg&: createOp,
108 msg: "unable to convert result type");
109
110 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
111 op: createOp, args&: spirvType, args: adaptor.getOperands());
112 return success();
113 }
114};
115
116struct ReOpPattern final : OpConversionPattern<complex::ReOp> {
117 using Base::Base;
118
119 LogicalResult
120 matchAndRewrite(complex::ReOp reOp, OpAdaptor adaptor,
121 ConversionPatternRewriter &rewriter) const override {
122 Type spirvType = getTypeConverter()->convertType(t: reOp.getType());
123 if (!spirvType)
124 return rewriter.notifyMatchFailure(arg&: reOp, msg: "unable to convert result type");
125
126 rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
127 op: reOp, args: adaptor.getComplex(), args: llvm::ArrayRef(0));
128 return success();
129 }
130};
131
132struct ImOpPattern final : OpConversionPattern<complex::ImOp> {
133 using Base::Base;
134
135 LogicalResult
136 matchAndRewrite(complex::ImOp imOp, OpAdaptor adaptor,
137 ConversionPatternRewriter &rewriter) const override {
138 Type spirvType = getTypeConverter()->convertType(t: imOp.getType());
139 if (!spirvType)
140 return rewriter.notifyMatchFailure(arg&: imOp, msg: "unable to convert result type");
141
142 rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
143 op: imOp, args: adaptor.getComplex(), args: llvm::ArrayRef(1));
144 return success();
145 }
146};
147
148template <typename ComplexOp, typename SPIRVOp>
149struct ElementwiseBinaryOpPattern final : OpConversionPattern<ComplexOp> {
150 using OpConversionPattern<ComplexOp>::OpConversionPattern;
151 using OpAdaptor = typename ComplexOp::Adaptor;
152
153 LogicalResult
154 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
155 ConversionPatternRewriter &rewriter) const override {
156 Type spirvType =
157 this->getTypeConverter()->convertType(op.getResult().getType());
158 if (!spirvType)
159 return rewriter.notifyMatchFailure(op, "unable to convert result type");
160
161 Location loc = op.getLoc();
162 Value lhs = adaptor.getLhs();
163 Value rhs = adaptor.getRhs();
164
165 Value lhsRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {0});
166 Value lhsIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {1});
167 Value rhsRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {0});
168 Value rhsIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {1});
169
170 Value resultRe = SPIRVOp::create(rewriter, loc, lhsRe, rhsRe);
171 Value resultIm = SPIRVOp::create(rewriter, loc, lhsIm, rhsIm);
172
173 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
174 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
175 return success();
176 }
177};
178
179template <typename ComplexOp, typename SPIRVCompareOp, typename SPIRVCombinerOp>
180struct ComparisonOpPattern final : OpConversionPattern<ComplexOp> {
181 using OpConversionPattern<ComplexOp>::OpConversionPattern;
182 using OpAdaptor = typename ComplexOp::Adaptor;
183
184 LogicalResult
185 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
186 ConversionPatternRewriter &rewriter) const override {
187 Type spirvType =
188 this->getTypeConverter()->convertType(op.getResult().getType());
189 if (!spirvType)
190 return rewriter.notifyMatchFailure(op, "unable to convert result type");
191
192 Location loc = op.getLoc();
193 Value lhs = adaptor.getLhs();
194 Value rhs = adaptor.getRhs();
195
196 Value lhsRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {0});
197 Value lhsIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {1});
198 Value rhsRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {0});
199 Value rhsIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {1});
200
201 Value cmpRe = SPIRVCompareOp::create(rewriter, loc, lhsRe, rhsRe);
202 Value cmpIm = SPIRVCompareOp::create(rewriter, loc, lhsIm, rhsIm);
203
204 rewriter.replaceOpWithNewOp<SPIRVCombinerOp>(op, spirvType, cmpRe, cmpIm);
205 return success();
206 }
207};
208
209struct MulOpPattern final : OpConversionPattern<complex::MulOp> {
210 using Base::Base;
211
212 LogicalResult
213 matchAndRewrite(complex::MulOp op, OpAdaptor adaptor,
214 ConversionPatternRewriter &rewriter) const override {
215 Type spirvType = getTypeConverter()->convertType(t: op.getResult().getType());
216 if (!spirvType)
217 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
218
219 Location loc = op.getLoc();
220 Value lhs = adaptor.getLhs();
221 Value rhs = adaptor.getRhs();
222
223 Value a = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {0});
224 Value b = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {1});
225 Value c = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {0});
226 Value d = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {1});
227
228 Value ac = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: a, operand2: c);
229 Value bd = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: b, operand2: d);
230 Value ad = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: a, operand2: d);
231 Value bc = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: b, operand2: c);
232 Value resultRe = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: ac, operand2: bd);
233 Value resultIm = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: ad, operand2: bc);
234
235 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
236 op, args&: spirvType, args: llvm::ArrayRef<Value>{resultRe, resultIm});
237 return success();
238 }
239};
240
241template <typename SqrtOp>
242struct AbsOpPattern final : OpConversionPattern<complex::AbsOp> {
243 using OpConversionPattern<complex::AbsOp>::OpConversionPattern;
244
245 LogicalResult
246 matchAndRewrite(complex::AbsOp op, OpAdaptor adaptor,
247 ConversionPatternRewriter &rewriter) const override {
248 Type spirvType =
249 this->getTypeConverter()->convertType(op.getResult().getType());
250 if (!spirvType)
251 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
252
253 Location loc = op.getLoc();
254 Value complexVal = adaptor.getComplex();
255
256 Value re =
257 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
258 Value im =
259 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
260
261 Value reSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: re, operand2: re);
262 Value imSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: im, operand2: im);
263 Value sum = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: reSq, operand2: imSq);
264
265 rewriter.replaceOpWithNewOp<SqrtOp>(op, sum);
266 return success();
267 }
268};
269
270template <typename ComplexOp, bool NegateReal>
271struct NegationOpPattern final : OpConversionPattern<ComplexOp> {
272 using OpConversionPattern<ComplexOp>::OpConversionPattern;
273 using OpAdaptor = typename ComplexOp::Adaptor;
274
275 LogicalResult
276 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
277 ConversionPatternRewriter &rewriter) const override {
278 Type spirvType =
279 this->getTypeConverter()->convertType(op.getResult().getType());
280 if (!spirvType)
281 return rewriter.notifyMatchFailure(op, "unable to convert result type");
282
283 Location loc = op.getLoc();
284 Value complexVal = adaptor.getComplex();
285
286 Value re =
287 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
288 Value im =
289 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
290
291 Value resultRe =
292 NegateReal ? spirv::FNegateOp::create(builder&: rewriter, location: loc, operand: re) : re;
293 Value resultIm = spirv::FNegateOp::create(builder&: rewriter, location: loc, operand: im);
294
295 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
296 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
297 return success();
298 }
299};
300
301struct DivOpPattern final : OpConversionPattern<complex::DivOp> {
302 using Base::Base;
303
304 LogicalResult
305 matchAndRewrite(complex::DivOp op, OpAdaptor adaptor,
306 ConversionPatternRewriter &rewriter) const override {
307 Type spirvType = getTypeConverter()->convertType(t: op.getResult().getType());
308 if (!spirvType)
309 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
310
311 Location loc = op.getLoc();
312 Value lhs = adaptor.getLhs();
313 Value rhs = adaptor.getRhs();
314
315 Value a = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {0});
316 Value b = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {1});
317 Value c = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {0});
318 Value d = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {1});
319
320 Value ac = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: a, operand2: c);
321 Value bd = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: b, operand2: d);
322 Value bc = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: b, operand2: c);
323 Value ad = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: a, operand2: d);
324 Value cc = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: c, operand2: c);
325 Value dd = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: d, operand2: d);
326 Value denom = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: cc, operand2: dd);
327 Value numRe = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: ac, operand2: bd);
328 Value numIm = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: bc, operand2: ad);
329 Value resultRe = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: numRe, operand2: denom);
330 Value resultIm = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: numIm, operand2: denom);
331
332 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
333 op, args&: spirvType, args: llvm::ArrayRef<Value>{resultRe, resultIm});
334 return success();
335 }
336};
337
338template <typename Atan2Op>
339struct AngleOpPattern final : OpConversionPattern<complex::AngleOp> {
340 using OpConversionPattern<complex::AngleOp>::OpConversionPattern;
341
342 LogicalResult
343 matchAndRewrite(complex::AngleOp op, OpAdaptor adaptor,
344 ConversionPatternRewriter &rewriter) const override {
345 Type spirvType =
346 this->getTypeConverter()->convertType(op.getResult().getType());
347 if (!spirvType)
348 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
349
350 Location loc = op.getLoc();
351 Value complexVal = adaptor.getComplex();
352
353 Value re =
354 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
355 Value im =
356 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
357
358 rewriter.replaceOpWithNewOp<Atan2Op>(op, im, re);
359 return success();
360 }
361};
362
363/// Computes complex.sign as z / abs(z), selecting a zero result when z is
364/// zero to avoid a division by zero. abs(z) is computed as
365/// max * sqrt(1 + (min / max)^2) to avoid overflow in squaring.
366template <typename SqrtOp, typename FAbsOp, typename FMaxOp, typename FMinOp>
367struct SignOpPattern final : OpConversionPattern<complex::SignOp> {
368 using OpConversionPattern<complex::SignOp>::OpConversionPattern;
369
370 LogicalResult
371 matchAndRewrite(complex::SignOp op, OpAdaptor adaptor,
372 ConversionPatternRewriter &rewriter) const override {
373 Type spirvType =
374 this->getTypeConverter()->convertType(op.getResult().getType());
375 if (!spirvType)
376 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
377
378 Location loc = op.getLoc();
379 Value complexVal = adaptor.getComplex();
380
381 Value re =
382 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
383 Value im =
384 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
385
386 Value abs =
387 createComplexAbs<SqrtOp, FAbsOp, FMaxOp, FMinOp>(rewriter, loc, re, im);
388
389 Value signRe = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: re, operand2: abs);
390 Value signIm = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: im, operand2: abs);
391
392 Value isZero = createIsZero(builder&: rewriter, loc, re, im);
393
394 // Select per component: spirv.Select with a scalar condition and a
395 // composite result requires SPIR-V 1.4, so operate on scalars here and
396 // assemble the result afterwards.
397 Value resultRe = spirv::SelectOp::create(builder&: rewriter, location: loc, condition: isZero, true_value: re, false_value: signRe);
398 Value resultIm = spirv::SelectOp::create(builder&: rewriter, location: loc, condition: isZero, true_value: im, false_value: signIm);
399
400 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
401 op, args&: spirvType, args: llvm::ArrayRef<Value>{resultRe, resultIm});
402 return success();
403 }
404};
405
406enum class SinCos { Sin, Cos };
407
408/// Lowers complex.sin/cos following the identities
409/// sin(x + iy) = sin(x) * cosh(y) + i * cos(x) * sinh(y)
410/// cos(x + iy) = cos(x) * cosh(y) - i * sin(x) * sinh(y)
411template <SinCos Kind, typename SinOp, typename CosOp, typename SinhOp,
412 typename CoshOp,
413 typename ComplexOp = std::conditional_t<
414 Kind == SinCos::Sin, complex::SinOp, complex::CosOp>>
415struct TrigonometricOpPattern final : OpConversionPattern<ComplexOp> {
416 using OpConversionPattern<ComplexOp>::OpConversionPattern;
417 using OpAdaptor = typename ComplexOp::Adaptor;
418
419 LogicalResult
420 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
421 ConversionPatternRewriter &rewriter) const override {
422 Type spirvType =
423 this->getTypeConverter()->convertType(op.getResult().getType());
424 if (!spirvType)
425 return rewriter.notifyMatchFailure(op, "unable to convert result type");
426
427 Location loc = op.getLoc();
428 Value complexVal = adaptor.getComplex();
429
430 Value re =
431 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
432 Value im =
433 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
434
435 Value sin = SinOp::create(rewriter, loc, re);
436 Value cos = CosOp::create(rewriter, loc, re);
437 Value sinh = SinhOp::create(rewriter, loc, im);
438 Value cosh = CoshOp::create(rewriter, loc, im);
439
440 bool isSin = Kind == SinCos::Sin;
441 Value resultRe =
442 spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: isSin ? sin : cos, operand2: cosh);
443 Value resultIm =
444 spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: isSin ? cos : sin, operand2: sinh);
445 if (!isSin)
446 resultIm = spirv::FNegateOp::create(builder&: rewriter, location: loc, operand: resultIm);
447
448 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
449 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
450 return success();
451 }
452};
453
454/// Lowers complex.exp/expm1 following the identities
455/// exp(x + iy) = exp(x) * cos(y) + i * exp(x) * sin(y)
456/// expm1(x + iy) = exp(x) * cos(y) - 1 + i * exp(x) * sin(y)
457template <bool IsExpm1, typename ExpOp, typename SinOp, typename CosOp,
458 typename ComplexOp =
459 std::conditional_t<IsExpm1, complex::Expm1Op, complex::ExpOp>>
460struct ExpOpPattern final : OpConversionPattern<ComplexOp> {
461 using OpConversionPattern<ComplexOp>::OpConversionPattern;
462 using OpAdaptor = typename ComplexOp::Adaptor;
463
464 LogicalResult
465 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
466 ConversionPatternRewriter &rewriter) const override {
467 Type spirvType =
468 this->getTypeConverter()->convertType(op.getResult().getType());
469 if (!spirvType)
470 return rewriter.notifyMatchFailure(op, "unable to convert result type");
471
472 Location loc = op.getLoc();
473 Value complexVal = adaptor.getComplex();
474
475 Value re =
476 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
477 Value im =
478 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
479
480 Value expRe = ExpOp::create(rewriter, loc, re);
481 Value sinIm = SinOp::create(rewriter, loc, im);
482 Value cosIm = CosOp::create(rewriter, loc, im);
483 Value resultRe = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: expRe, operand2: cosIm);
484 if (IsExpm1) {
485 Value one = createFPConstant(builder&: rewriter, loc, type: re.getType(), value: 1.0);
486 resultRe = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: resultRe, operand2: one);
487 }
488 Value resultIm = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: expRe, operand2: sinIm);
489
490 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
491 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
492 return success();
493 }
494};
495
496enum class SqrtKind { Sqrt, Rsqrt };
497
498/// Lowers complex.sqrt/rsqrt following the trigonometric form
499/// sqrt(z) = sqrt(abs) * (cos(angle/2) + i*sin(angle/2))
500/// rsqrt(z) = rsqrt(abs) * (cos(-angle/2) + i*sin(-angle/2))
501/// where angle = atan2(im, re). Atan2 is undefined for zero input, so zero
502/// maps to 0 for sqrt and to inf + i*nan for rsqrt.
503template <SqrtKind Kind, typename SqrtOp, typename RsqrtOp, typename FAbsOp,
504 typename FMaxOp, typename FMinOp, typename SinOp, typename CosOp,
505 typename Atan2Op,
506 typename ComplexOp = std::conditional_t<
507 Kind == SqrtKind::Sqrt, complex::SqrtOp, complex::RsqrtOp>>
508struct SqrtOpPattern final : OpConversionPattern<ComplexOp> {
509 using OpConversionPattern<ComplexOp>::OpConversionPattern;
510 using OpAdaptor = typename ComplexOp::Adaptor;
511
512 LogicalResult
513 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
514 ConversionPatternRewriter &rewriter) const override {
515 Type spirvType =
516 this->getTypeConverter()->convertType(op.getResult().getType());
517 if (!spirvType)
518 return rewriter.notifyMatchFailure(op, "unable to convert result type");
519
520 Location loc = op.getLoc();
521 Value complexVal = adaptor.getComplex();
522
523 Value re =
524 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
525 Value im =
526 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
527 Type elemType = re.getType();
528
529 bool isSqrt = Kind == SqrtKind::Sqrt;
530 using FinalOp = std::conditional_t<Kind == SqrtKind::Sqrt, SqrtOp, RsqrtOp>;
531 auto [resultRe, resultIm] =
532 createComplexSqrtOrRsqrt<SqrtOp, FAbsOp, FMaxOp, FMinOp, SinOp, CosOp,
533 Atan2Op, FinalOp>(rewriter, loc, re, im,
534 isSqrt ? 0.5 : -0.5);
535
536 Value isZero = createIsZero(builder&: rewriter, loc, re, im);
537 Value zeroRe = createFPConstant(
538 builder&: rewriter, loc, type: elemType,
539 value: isSqrt ? 0.0 : std::numeric_limits<double>::infinity());
540 Value zeroIm = createFPConstant(
541 builder&: rewriter, loc, type: elemType,
542 value: isSqrt ? 0.0 : std::numeric_limits<double>::quiet_NaN());
543 resultRe = spirv::SelectOp::create(rewriter, loc, isZero, zeroRe, resultRe);
544 resultIm = spirv::SelectOp::create(rewriter, loc, isZero, zeroIm, resultIm);
545
546 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
547 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
548 return success();
549 }
550};
551
552/// Lowers complex.log/log1p following the identities
553/// log(x + iy) = log(abs(x + iy)) + i * atan2(y, x)
554/// log1p(x + iy) = log(abs(1 + x + iy)) + i * atan2(y, 1 + x)
555template <bool IsLog1p, typename SqrtOp, typename LogOp, typename Atan2Op,
556 typename ComplexOp =
557 std::conditional_t<IsLog1p, complex::Log1pOp, complex::LogOp>>
558struct LogOpPattern final : OpConversionPattern<ComplexOp> {
559 using OpConversionPattern<ComplexOp>::OpConversionPattern;
560 using OpAdaptor = typename ComplexOp::Adaptor;
561
562 LogicalResult
563 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
564 ConversionPatternRewriter &rewriter) const override {
565 Type spirvType =
566 this->getTypeConverter()->convertType(op.getResult().getType());
567 if (!spirvType)
568 return rewriter.notifyMatchFailure(op, "unable to convert result type");
569
570 Location loc = op.getLoc();
571 Value complexVal = adaptor.getComplex();
572
573 Value re =
574 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
575 Value im =
576 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
577
578 if (IsLog1p) {
579 Value one = createFPConstant(builder&: rewriter, loc, type: re.getType(), value: 1.0);
580 re = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: one, operand2: re);
581 }
582
583 Value reSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: re, operand2: re);
584 Value imSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: im, operand2: im);
585 Value sum = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: reSq, operand2: imSq);
586 Value abs = SqrtOp::create(rewriter, loc, sum);
587
588 Value resultRe = LogOp::create(rewriter, loc, abs);
589 Value resultIm = Atan2Op::create(rewriter, loc, im, re);
590
591 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
592 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
593 return success();
594 }
595};
596
597/// Lowers complex.tan/tanh following the identities
598/// tan(x + iy) = (sin(2x) + i * sinh(2y)) / (cos(2x) + cosh(2y))
599/// tanh(x + iy) = (sinh(2x) + i * sin(2y)) / (cosh(2x) + cos(2y))
600template <bool IsTanh, typename SinOp, typename CosOp, typename SinhOp,
601 typename CoshOp,
602 typename ComplexOp =
603 std::conditional_t<IsTanh, complex::TanhOp, complex::TanOp>>
604struct TanTanhOpPattern final : OpConversionPattern<ComplexOp> {
605 using OpConversionPattern<ComplexOp>::OpConversionPattern;
606 using OpAdaptor = typename ComplexOp::Adaptor;
607
608 LogicalResult
609 matchAndRewrite(ComplexOp op, OpAdaptor adaptor,
610 ConversionPatternRewriter &rewriter) const override {
611 Type spirvType =
612 this->getTypeConverter()->convertType(op.getResult().getType());
613 if (!spirvType)
614 return rewriter.notifyMatchFailure(op, "unable to convert result type");
615
616 Location loc = op.getLoc();
617 Value complexVal = adaptor.getComplex();
618
619 Value re =
620 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {0});
621 Value im =
622 spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: complexVal, indices: {1});
623
624 Value two = createFPConstant(builder&: rewriter, loc, type: re.getType(), value: 2.0);
625 Value twoRe = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: re, operand2: two);
626 Value twoIm = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: im, operand2: two);
627
628 Value trigArg = IsTanh ? twoIm : twoRe;
629 Value hypArg = IsTanh ? twoRe : twoIm;
630 Value sin = SinOp::create(rewriter, loc, trigArg);
631 Value cos = CosOp::create(rewriter, loc, trigArg);
632 Value sinh = SinhOp::create(rewriter, loc, hypArg);
633 Value cosh = CoshOp::create(rewriter, loc, hypArg);
634
635 Value denom = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: cos, operand2: cosh);
636 Value sinQuot = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: sin, operand2: denom);
637 Value sinhQuot = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: sinh, operand2: denom);
638 Value resultRe = IsTanh ? sinhQuot : sinQuot;
639 Value resultIm = IsTanh ? sinQuot : sinhQuot;
640
641 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
642 op, spirvType, llvm::ArrayRef<Value>{resultRe, resultIm});
643 return success();
644 }
645};
646
647/// Computes complex.atan2(lhs, rhs) using the complex-logarithm identity
648/// atan2(y, x) = -i * log((x + i*y) / sqrt(x**2 + y**2)), expressed directly
649/// in terms of the real/imaginary components of x and y.
650template <typename SqrtOp, typename FAbsOp, typename FMaxOp, typename FMinOp,
651 typename SinOp, typename CosOp, typename Atan2Op, typename LogOp>
652struct Atan2OpPattern final : OpConversionPattern<complex::Atan2Op> {
653 using OpConversionPattern<complex::Atan2Op>::OpConversionPattern;
654
655 LogicalResult
656 matchAndRewrite(complex::Atan2Op op, OpAdaptor adaptor,
657 ConversionPatternRewriter &rewriter) const override {
658 Type spirvType = getTypeConverter()->convertType(op.getResult().getType());
659 if (!spirvType)
660 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
661
662 Location loc = op.getLoc();
663 Value y = adaptor.getLhs();
664 Value x = adaptor.getRhs();
665
666 Value yRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: y, indices: {0});
667 Value yIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: y, indices: {1});
668 Value xRe = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: x, indices: {0});
669 Value xIm = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: x, indices: {1});
670
671 // w = x**2 + y**2.
672 Value xReSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: xRe, operand2: xRe);
673 Value xImSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: xIm, operand2: xIm);
674 Value xSqRe = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: xReSq, operand2: xImSq);
675 Value xReXIm = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: xRe, operand2: xIm);
676 Value xSqIm = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: xReXIm, operand2: xReXIm);
677
678 Value yReSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: yRe, operand2: yRe);
679 Value yImSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: yIm, operand2: yIm);
680 Value ySqRe = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: yReSq, operand2: yImSq);
681 Value yReYIm = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: yRe, operand2: yIm);
682 Value ySqIm = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: yReYIm, operand2: yReYIm);
683
684 Value wRe = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: xSqRe, operand2: ySqRe);
685 Value wIm = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: xSqIm, operand2: ySqIm);
686
687 // sqrtW = sqrt(w), computed via the trigonometric form.
688 auto [sqrtWRe, sqrtWIm] =
689 createComplexSqrtOrRsqrt<SqrtOp, FAbsOp, FMaxOp, FMinOp, SinOp, CosOp,
690 Atan2Op, SqrtOp>(rewriter, loc, wRe, wIm,
691 /*angleScale=*/0.5);
692
693 // num = x + i*y.
694 Value numRe = spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: xRe, operand2: yIm);
695 Value numIm = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: xIm, operand2: yRe);
696
697 // quotient = num / sqrtW.
698 Value denomRe = spirv::FMulOp::create(rewriter, loc, sqrtWRe, sqrtWRe);
699 Value denomIm = spirv::FMulOp::create(rewriter, loc, sqrtWIm, sqrtWIm);
700 Value denom = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: denomRe, operand2: denomIm);
701
702 Value numReSqrtWRe = spirv::FMulOp::create(rewriter, loc, numRe, sqrtWRe);
703 Value numImSqrtWIm = spirv::FMulOp::create(rewriter, loc, numIm, sqrtWIm);
704 Value quotNumRe =
705 spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: numReSqrtWRe, operand2: numImSqrtWIm);
706 Value numImSqrtWRe = spirv::FMulOp::create(rewriter, loc, numIm, sqrtWRe);
707 Value numReSqrtWIm = spirv::FMulOp::create(rewriter, loc, numRe, sqrtWIm);
708 Value quotNumIm =
709 spirv::FSubOp::create(builder&: rewriter, location: loc, operand1: numImSqrtWRe, operand2: numReSqrtWIm);
710
711 Value quotRe = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: quotNumRe, operand2: denom);
712 Value quotIm = spirv::FDivOp::create(builder&: rewriter, location: loc, operand1: quotNumIm, operand2: denom);
713
714 // log(quotient) = log(abs(quotient)) + i*atan2(quotIm, quotRe).
715 Value quotReSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: quotRe, operand2: quotRe);
716 Value quotImSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: quotIm, operand2: quotIm);
717 Value quotAbsSq = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: quotReSq, operand2: quotImSq);
718 Value quotAbs = SqrtOp::create(rewriter, loc, quotAbsSq);
719 Value logRe = LogOp::create(rewriter, loc, quotAbs);
720 Value logIm = Atan2Op::create(rewriter, loc, quotIm, quotRe);
721
722 // result = -i * log(quotient) = logIm - i*logRe.
723 Value resultIm = spirv::FNegateOp::create(builder&: rewriter, location: loc, operand: logRe);
724
725 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
726 op, args&: spirvType, args: llvm::ArrayRef<Value>{logIm, resultIm});
727 return success();
728 }
729};
730
731/// Lowers complex.pow following the identity
732/// (a + ib)^(c + id) = abs^c * exp(-d * arg) * (cos(q) + i * sin(q))
733/// where abs = |a + ib|, arg = atan2(b, a), q = c * arg + d * log(abs)
734template <typename SqrtOp, typename LogOp, typename ExpOp, typename SinOp,
735 typename CosOp, typename Atan2Op, typename PowOp>
736struct PowOpPattern final : OpConversionPattern<complex::PowOp> {
737 using OpConversionPattern<complex::PowOp>::OpConversionPattern;
738
739 LogicalResult
740 matchAndRewrite(complex::PowOp op, OpAdaptor adaptor,
741 ConversionPatternRewriter &rewriter) const override {
742 Type spirvType = getTypeConverter()->convertType(op.getResult().getType());
743 if (!spirvType)
744 return rewriter.notifyMatchFailure(arg&: op, msg: "unable to convert result type");
745
746 Location loc = op.getLoc();
747 Value lhs = adaptor.getLhs();
748 Value rhs = adaptor.getRhs();
749
750 Value a = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {0});
751 Value b = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: lhs, indices: {1});
752 Value c = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {0});
753 Value d = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: rhs, indices: {1});
754
755 Value aSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: a, operand2: a);
756 Value bSq = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: b, operand2: b);
757 Value absSq = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: aSq, operand2: bSq);
758 Value abs = SqrtOp::create(rewriter, loc, absSq);
759 Value absToC = PowOp::create(rewriter, loc, abs, c);
760
761 Value argLhs = Atan2Op::create(rewriter, loc, b, a);
762 Value negD = spirv::FNegateOp::create(builder&: rewriter, location: loc, operand: d);
763 Value negDArgLhs = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: negD, operand2: argLhs);
764 Value expNegDArgLhs = ExpOp::create(rewriter, loc, negDArgLhs);
765 Value coeff = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: absToC, operand2: expNegDArgLhs);
766
767 Value lnAbs = LogOp::create(rewriter, loc, abs);
768 Value cArgLhs = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: c, operand2: argLhs);
769 Value dLnAbs = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: d, operand2: lnAbs);
770 Value q = spirv::FAddOp::create(builder&: rewriter, location: loc, operand1: cArgLhs, operand2: dLnAbs);
771 Value cosQ = CosOp::create(rewriter, loc, q);
772 Value sinQ = SinOp::create(rewriter, loc, q);
773
774 Value resultRe = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: coeff, operand2: cosQ);
775 Value resultIm = spirv::FMulOp::create(builder&: rewriter, location: loc, operand1: coeff, operand2: sinQ);
776
777 rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
778 op, args&: spirvType, args: llvm::ArrayRef<Value>{resultRe, resultIm});
779 return success();
780 }
781};
782} // namespace
783
784//===----------------------------------------------------------------------===//
785// Pattern population
786//===----------------------------------------------------------------------===//
787
788void mlir::populateComplexToSPIRVPatterns(
789 const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) {
790 MLIRContext *context = patterns.getContext();
791
792 patterns
793 .add<ConstantOpPattern, CreateOpPattern, ReOpPattern, ImOpPattern,
794 ElementwiseBinaryOpPattern<complex::AddOp, spirv::FAddOp>,
795 ElementwiseBinaryOpPattern<complex::SubOp, spirv::FSubOp>,
796 ComparisonOpPattern<complex::EqualOp, spirv::FOrdEqualOp,
797 spirv::LogicalAndOp>,
798 ComparisonOpPattern<complex::NotEqualOp, spirv::FUnordNotEqualOp,
799 spirv::LogicalOrOp>,
800 MulOpPattern, DivOpPattern,
801 NegationOpPattern<complex::NegOp, /*NegateReal=*/true>,
802 NegationOpPattern<complex::ConjOp, /*NegateReal=*/false>,
803 AbsOpPattern<spirv::GLSqrtOp>, AbsOpPattern<spirv::CLSqrtOp>,
804 AngleOpPattern<spirv::GLAtan2Op>, AngleOpPattern<spirv::CLAtan2Op>,
805 SignOpPattern<spirv::GLSqrtOp, spirv::GLFAbsOp, spirv::GLFMaxOp,
806 spirv::GLFMinOp>,
807 SignOpPattern<spirv::CLSqrtOp, spirv::CLFAbsOp, spirv::CLFMaxOp,
808 spirv::CLFMinOp>,
809 TrigonometricOpPattern<SinCos::Sin, spirv::GLSinOp, spirv::GLCosOp,
810 spirv::GLSinhOp, spirv::GLCoshOp>,
811 TrigonometricOpPattern<SinCos::Sin, spirv::CLSinOp, spirv::CLCosOp,
812 spirv::CLSinhOp, spirv::CLCoshOp>,
813 TrigonometricOpPattern<SinCos::Cos, spirv::GLSinOp, spirv::GLCosOp,
814 spirv::GLSinhOp, spirv::GLCoshOp>,
815 TrigonometricOpPattern<SinCos::Cos, spirv::CLSinOp, spirv::CLCosOp,
816 spirv::CLSinhOp, spirv::CLCoshOp>,
817 ExpOpPattern</*IsExpm1=*/false, spirv::GLExpOp, spirv::GLSinOp,
818 spirv::GLCosOp>,
819 ExpOpPattern</*IsExpm1=*/false, spirv::CLExpOp, spirv::CLSinOp,
820 spirv::CLCosOp>,
821 ExpOpPattern</*IsExpm1=*/true, spirv::GLExpOp, spirv::GLSinOp,
822 spirv::GLCosOp>,
823 ExpOpPattern</*IsExpm1=*/true, spirv::CLExpOp, spirv::CLSinOp,
824 spirv::CLCosOp>,
825 LogOpPattern</*IsLog1p=*/false, spirv::GLSqrtOp, spirv::GLLogOp,
826 spirv::GLAtan2Op>,
827 LogOpPattern</*IsLog1p=*/false, spirv::CLSqrtOp, spirv::CLLogOp,
828 spirv::CLAtan2Op>,
829 LogOpPattern</*IsLog1p=*/true, spirv::GLSqrtOp, spirv::GLLogOp,
830 spirv::GLAtan2Op>,
831 LogOpPattern</*IsLog1p=*/true, spirv::CLSqrtOp, spirv::CLLogOp,
832 spirv::CLAtan2Op>>(arg: typeConverter, args&: context);
833 patterns.add<
834 SqrtOpPattern<SqrtKind::Sqrt, spirv::GLSqrtOp, spirv::GLInverseSqrtOp,
835 spirv::GLFAbsOp, spirv::GLFMaxOp, spirv::GLFMinOp,
836 spirv::GLSinOp, spirv::GLCosOp, spirv::GLAtan2Op>,
837 SqrtOpPattern<SqrtKind::Sqrt, spirv::CLSqrtOp, spirv::CLRsqrtOp,
838 spirv::CLFAbsOp, spirv::CLFMaxOp, spirv::CLFMinOp,
839 spirv::CLSinOp, spirv::CLCosOp, spirv::CLAtan2Op>,
840 SqrtOpPattern<SqrtKind::Rsqrt, spirv::GLSqrtOp, spirv::GLInverseSqrtOp,
841 spirv::GLFAbsOp, spirv::GLFMaxOp, spirv::GLFMinOp,
842 spirv::GLSinOp, spirv::GLCosOp, spirv::GLAtan2Op>,
843 SqrtOpPattern<SqrtKind::Rsqrt, spirv::CLSqrtOp, spirv::CLRsqrtOp,
844 spirv::CLFAbsOp, spirv::CLFMaxOp, spirv::CLFMinOp,
845 spirv::CLSinOp, spirv::CLCosOp, spirv::CLAtan2Op>,
846 TanTanhOpPattern</*IsTanh=*/false, spirv::GLSinOp, spirv::GLCosOp,
847 spirv::GLSinhOp, spirv::GLCoshOp>,
848 TanTanhOpPattern</*IsTanh=*/false, spirv::CLSinOp, spirv::CLCosOp,
849 spirv::CLSinhOp, spirv::CLCoshOp>,
850 TanTanhOpPattern</*IsTanh=*/true, spirv::GLSinOp, spirv::GLCosOp,
851 spirv::GLSinhOp, spirv::GLCoshOp>,
852 TanTanhOpPattern</*IsTanh=*/true, spirv::CLSinOp, spirv::CLCosOp,
853 spirv::CLSinhOp, spirv::CLCoshOp>,
854 Atan2OpPattern<spirv::GLSqrtOp, spirv::GLFAbsOp, spirv::GLFMaxOp,
855 spirv::GLFMinOp, spirv::GLSinOp, spirv::GLCosOp,
856 spirv::GLAtan2Op, spirv::GLLogOp>,
857 Atan2OpPattern<spirv::CLSqrtOp, spirv::CLFAbsOp, spirv::CLFMaxOp,
858 spirv::CLFMinOp, spirv::CLSinOp, spirv::CLCosOp,
859 spirv::CLAtan2Op, spirv::CLLogOp>,
860 PowOpPattern<spirv::GLSqrtOp, spirv::GLLogOp, spirv::GLExpOp,
861 spirv::GLSinOp, spirv::GLCosOp, spirv::GLAtan2Op,
862 spirv::GLPowOp>,
863 PowOpPattern<spirv::CLSqrtOp, spirv::CLLogOp, spirv::CLExpOp,
864 spirv::CLSinOp, spirv::CLCosOp, spirv::CLAtan2Op,
865 spirv::CLPowOp>>(arg: typeConverter, args&: context);
866}
867