1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains descriptions of the various Vector Predication intrinsics.
10// This is used as a central place for enumerating the different instructions
11// and should eventually be the place to put comments about the instructions.
12//
13//===----------------------------------------------------------------------===//
14
15// NOTE: NO INCLUDE GUARD DESIRED!
16
17// Provide definitions of macros so that users of this file do not have to
18// define everything to use it...
19//
20// Register a VP intrinsic and begin its property scope.
21// All VP intrinsic scopes are top level, ie it is illegal to place a
22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope.
23// \p VPID The VP intrinsic id.
24// \p MASKPOS The mask operand position.
25// \p EVLPOS The explicit vector length operand position.
26#ifndef BEGIN_REGISTER_VP_INTRINSIC
27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS)
28#endif
29
30// End the property scope of a VP intrinsic.
31#ifndef END_REGISTER_VP_INTRINSIC
32#define END_REGISTER_VP_INTRINSIC(VPID)
33#endif
34
35// Register a new VP SDNode and begin its property scope.
36// When the SDNode scope is nested within a VP intrinsic scope, it is
37// implicitly registered as the canonical SDNode for this VP intrinsic. There
38// is one VP intrinsic that maps directly to one SDNode that goes by the
39// same name. Since the operands are also the same, we open the property
40// scopes for both the VPIntrinsic and the SDNode at once.
41// \p VPSD The SelectionDAG Node id (eg VP_ADD).
42// \p LEGALPOS The operand position of the SDNode that is used for legalizing.
43// If LEGALPOS < 0, then the return type given by
44// TheNode->getValueType(-1-LEGALPOS) is used.
45// \p TDNAME The name of the TableGen definition of this SDNode.
46// \p MASKPOS The mask operand position.
47// \p EVLPOS The explicit vector length operand position.
48#ifndef BEGIN_REGISTER_VP_SDNODE
49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS)
50#endif
51
52// End the property scope of a new VP SDNode.
53#ifndef END_REGISTER_VP_SDNODE
54#define END_REGISTER_VP_SDNODE(VPSD)
55#endif
56
57// Helper macro to set up the mapping from VP intrinsic to ISD opcode.
58// Note: More than one VP intrinsic may map to one ISD opcode.
59#ifndef HELPER_MAP_VPID_TO_VPSD
60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
61#endif
62
63// Helper macros for the common "1:1 - Intrinsic : SDNode" case.
64//
65// There is one VP intrinsic that maps directly to one SDNode that goes by the
66// same name. Since the operands are also the same, we open the property
67// scopes for both the VPIntrinsic and the SDNode at once.
68//
69// \p VPID The canonical name (eg `vp_add`, which at the same time is the
70// name of the intrinsic and the TableGen def of the SDNode).
71// \p MASKPOS The mask operand position.
72// \p EVLPOS The explicit vector length operand position.
73// \p VPSD The SelectionDAG Node id (eg VP_ADD).
74// \p LEGALPOS The operand position of the SDNode that is used for legalizing
75// this SDNode. This can be `-1`, in which case the return type of
76// the SDNode is used.
77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \
78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \
79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \
80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
81
82#define END_REGISTER_VP(VPID, VPSD) \
83 END_REGISTER_VP_INTRINSIC(VPID) \
84 END_REGISTER_VP_SDNODE(VPSD)
85
86// The following macros attach properties to the scope they are placed in. This
87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the
88// scope.
89//
90// Property Macros {
91
92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode.
93// \p OPC The opcode of the instruction with the same function.
94#ifndef VP_PROPERTY_FUNCTIONAL_OPC
95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC)
96#endif
97
98// If operation can have rounding or fp exceptions, maps to corresponding
99// constrained fp intrinsic.
100#ifndef VP_PROPERTY_CONSTRAINEDFP
101#define VP_PROPERTY_CONSTRAINEDFP(INTRINID)
102#endif
103
104// The intrinsic and/or SDNode has the same function as this ISD Opcode.
105// \p SDOPC The opcode of the instruction with the same function.
106#ifndef VP_PROPERTY_FUNCTIONAL_SDOPC
107#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)
108#endif
109
110// Map this VP intrinsic to its canonical functional intrinsic.
111// \p INTRIN The non-VP intrinsics with the same function.
112#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC
113#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN)
114#endif
115
116// This VP Intrinsic has no functionally-equivalent non-VP opcode or intrinsic.
117#ifndef VP_PROPERTY_NO_FUNCTIONAL
118#define VP_PROPERTY_NO_FUNCTIONAL
119#endif
120
121// A property to infer VP binary-op SDNode opcodes automatically.
122#ifndef VP_PROPERTY_BINARYOP
123#define VP_PROPERTY_BINARYOP
124#endif
125
126/// } Property Macros
127
128///// Integer Arithmetic {
129
130// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl).
131#ifdef HELPER_REGISTER_BINARY_INT_VP
132#error \
133 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!"
134#endif
135#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC) \
136 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \
137 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
138 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
139 VP_PROPERTY_BINARYOP \
140 END_REGISTER_VP(VPID, VPSD)
141
142// llvm.vp.add(x,y,mask,vlen)
143HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add, ADD)
144
145// llvm.vp.and(x,y,mask,vlen)
146HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And, AND)
147
148// llvm.vp.ashr(x,y,mask,vlen)
149HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_SRA, AShr, SRA)
150
151// llvm.vp.lshr(x,y,mask,vlen)
152HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_SRL, LShr, SRL)
153
154// llvm.vp.mul(x,y,mask,vlen)
155HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul, MUL)
156
157// llvm.vp.or(x,y,mask,vlen)
158HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or, OR)
159
160// llvm.vp.sdiv(x,y,mask,vlen)
161HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV)
162
163// llvm.vp.shl(x,y,mask,vlen)
164HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl, SHL)
165
166// llvm.vp.srem(x,y,mask,vlen)
167HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM)
168
169// llvm.vp.sub(x,y,mask,vlen)
170HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub, SUB)
171
172// llvm.vp.udiv(x,y,mask,vlen)
173HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV)
174
175// llvm.vp.urem(x,y,mask,vlen)
176HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM)
177
178// llvm.vp.xor(x,y,mask,vlen)
179HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor, XOR)
180
181#undef HELPER_REGISTER_BINARY_INT_VP
182
183// llvm.vp.smin(x,y,mask,vlen)
184BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1)
185VP_PROPERTY_BINARYOP
186VP_PROPERTY_FUNCTIONAL_SDOPC(SMIN)
187VP_PROPERTY_FUNCTIONAL_INTRINSIC(smin)
188END_REGISTER_VP(vp_smin, VP_SMIN)
189
190// llvm.vp.smax(x,y,mask,vlen)
191BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1)
192VP_PROPERTY_BINARYOP
193VP_PROPERTY_FUNCTIONAL_SDOPC(SMAX)
194VP_PROPERTY_FUNCTIONAL_INTRINSIC(smax)
195END_REGISTER_VP(vp_smax, VP_SMAX)
196
197// llvm.vp.umin(x,y,mask,vlen)
198BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1)
199VP_PROPERTY_BINARYOP
200VP_PROPERTY_FUNCTIONAL_SDOPC(UMIN)
201VP_PROPERTY_FUNCTIONAL_INTRINSIC(umin)
202END_REGISTER_VP(vp_umin, VP_UMIN)
203
204// llvm.vp.umax(x,y,mask,vlen)
205BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1)
206VP_PROPERTY_BINARYOP
207VP_PROPERTY_FUNCTIONAL_SDOPC(UMAX)
208VP_PROPERTY_FUNCTIONAL_INTRINSIC(umax)
209END_REGISTER_VP(vp_umax, VP_UMAX)
210
211// llvm.vp.abs(x,is_int_min_poison,mask,vlen)
212BEGIN_REGISTER_VP_INTRINSIC(vp_abs, 2, 3)
213BEGIN_REGISTER_VP_SDNODE(VP_ABS, -1, vp_abs, 1, 2)
214HELPER_MAP_VPID_TO_VPSD(vp_abs, VP_ABS)
215VP_PROPERTY_FUNCTIONAL_INTRINSIC(abs)
216VP_PROPERTY_FUNCTIONAL_SDOPC(ABS)
217END_REGISTER_VP(vp_abs, VP_ABS)
218
219// llvm.vp.bswap(x,mask,vlen)
220BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1)
221VP_PROPERTY_FUNCTIONAL_INTRINSIC(bswap)
222VP_PROPERTY_FUNCTIONAL_SDOPC(BSWAP)
223END_REGISTER_VP(vp_bswap, VP_BSWAP)
224
225// llvm.vp.bitreverse(x,mask,vlen)
226BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1)
227VP_PROPERTY_FUNCTIONAL_INTRINSIC(bitreverse)
228VP_PROPERTY_FUNCTIONAL_SDOPC(BITREVERSE)
229END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE)
230
231// llvm.vp.ctpop(x,mask,vlen)
232BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1)
233VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctpop)
234VP_PROPERTY_FUNCTIONAL_SDOPC(CTPOP)
235END_REGISTER_VP(vp_ctpop, VP_CTPOP)
236
237// llvm.vp.ctlz(x,is_zero_poison,mask,vlen)
238BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 2, 3)
239BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2)
240VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctlz)
241VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ)
242END_REGISTER_VP_SDNODE(VP_CTLZ)
243BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2)
244VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ_ZERO_UNDEF)
245END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF)
246END_REGISTER_VP_INTRINSIC(vp_ctlz)
247
248// llvm.vp.cttz(x,is_zero_poison,mask,vlen)
249BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 2, 3)
250BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2)
251VP_PROPERTY_FUNCTIONAL_INTRINSIC(cttz)
252VP_PROPERTY_FUNCTIONAL_SDOPC(CTTZ)
253END_REGISTER_VP_SDNODE(VP_CTTZ)
254BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2)
255END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF)
256END_REGISTER_VP_INTRINSIC(vp_cttz)
257
258// llvm.vp.cttz.elts(x,is_zero_poison,mask,vl)
259BEGIN_REGISTER_VP_INTRINSIC(vp_cttz_elts, 2, 3)
260VP_PROPERTY_NO_FUNCTIONAL
261BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS, 0, vp_cttz_elts, 1, 2)
262END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS)
263BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_UNDEF, 0, vp_cttz_elts_zero_undef, 1, 2)
264END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_UNDEF)
265END_REGISTER_VP_INTRINSIC(vp_cttz_elts)
266
267// llvm.vp.fshl(x,y,z,mask,vlen)
268BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1)
269VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshl)
270VP_PROPERTY_FUNCTIONAL_SDOPC(FSHL)
271END_REGISTER_VP(vp_fshl, VP_FSHL)
272
273// llvm.vp.fshr(x,y,z,mask,vlen)
274BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1)
275VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshr)
276VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR)
277END_REGISTER_VP(vp_fshr, VP_FSHR)
278
279// llvm.vp.sadd.sat(x,y,mask,vlen)
280BEGIN_REGISTER_VP(vp_sadd_sat, 2, 3, VP_SADDSAT, -1)
281VP_PROPERTY_BINARYOP
282VP_PROPERTY_FUNCTIONAL_INTRINSIC(sadd_sat)
283VP_PROPERTY_FUNCTIONAL_SDOPC(SADDSAT)
284END_REGISTER_VP(vp_sadd_sat, VP_SADDSAT)
285
286// llvm.vp.uadd.sat(x,y,mask,vlen)
287BEGIN_REGISTER_VP(vp_uadd_sat, 2, 3, VP_UADDSAT, -1)
288VP_PROPERTY_BINARYOP
289VP_PROPERTY_FUNCTIONAL_INTRINSIC(uadd_sat)
290VP_PROPERTY_FUNCTIONAL_SDOPC(UADDSAT)
291END_REGISTER_VP(vp_uadd_sat, VP_UADDSAT)
292
293// llvm.vp.ssub.sat(x,y,mask,vlen)
294BEGIN_REGISTER_VP(vp_ssub_sat, 2, 3, VP_SSUBSAT, -1)
295VP_PROPERTY_BINARYOP
296VP_PROPERTY_FUNCTIONAL_INTRINSIC(ssub_sat)
297VP_PROPERTY_FUNCTIONAL_SDOPC(SSUBSAT)
298END_REGISTER_VP(vp_ssub_sat, VP_SSUBSAT)
299
300// llvm.vp.usub.sat(x,y,mask,vlen)
301BEGIN_REGISTER_VP(vp_usub_sat, 2, 3, VP_USUBSAT, -1)
302VP_PROPERTY_BINARYOP
303VP_PROPERTY_FUNCTIONAL_INTRINSIC(usub_sat)
304VP_PROPERTY_FUNCTIONAL_SDOPC(USUBSAT)
305END_REGISTER_VP(vp_usub_sat, VP_USUBSAT)
306///// } Integer Arithmetic
307
308///// Floating-Point Arithmetic {
309
310// Specialized helper macro for floating-point binary operators
311// <operation>(%x, %y, %mask, %evl).
312#ifdef HELPER_REGISTER_BINARY_FP_VP
313#error \
314 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!"
315#endif
316#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \
317 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \
318 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
319 VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_##OPSUFFIX) \
320 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
321 VP_PROPERTY_BINARYOP \
322 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
323
324// llvm.vp.fadd(x,y,mask,vlen)
325HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd, FADD)
326
327// llvm.vp.fsub(x,y,mask,vlen)
328HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub, FSUB)
329
330// llvm.vp.fmul(x,y,mask,vlen)
331HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul, FMUL)
332
333// llvm.vp.fdiv(x,y,mask,vlen)
334HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv, FDIV)
335
336// llvm.vp.frem(x,y,mask,vlen)
337HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem, FREM)
338
339#undef HELPER_REGISTER_BINARY_FP_VP
340
341// llvm.vp.fneg(x,mask,vlen)
342BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1)
343VP_PROPERTY_FUNCTIONAL_OPC(FNeg)
344VP_PROPERTY_FUNCTIONAL_SDOPC(FNEG)
345END_REGISTER_VP(vp_fneg, VP_FNEG)
346
347// llvm.vp.fabs(x,mask,vlen)
348BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1)
349VP_PROPERTY_FUNCTIONAL_INTRINSIC(fabs)
350VP_PROPERTY_FUNCTIONAL_SDOPC(FABS)
351END_REGISTER_VP(vp_fabs, VP_FABS)
352
353// llvm.vp.sqrt(x,mask,vlen)
354BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1)
355VP_PROPERTY_FUNCTIONAL_INTRINSIC(sqrt)
356VP_PROPERTY_FUNCTIONAL_SDOPC(FSQRT)
357END_REGISTER_VP(vp_sqrt, VP_SQRT)
358
359// llvm.vp.fma(x,y,z,mask,vlen)
360BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1)
361VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fma)
362VP_PROPERTY_FUNCTIONAL_INTRINSIC(fma)
363VP_PROPERTY_FUNCTIONAL_SDOPC(FMA)
364END_REGISTER_VP(vp_fma, VP_FMA)
365
366// llvm.vp.fmuladd(x,y,z,mask,vlen)
367BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1)
368VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fmuladd)
369VP_PROPERTY_FUNCTIONAL_INTRINSIC(fmuladd)
370VP_PROPERTY_FUNCTIONAL_SDOPC(FMAD)
371END_REGISTER_VP(vp_fmuladd, VP_FMULADD)
372
373// llvm.vp.copysign(x,y,mask,vlen)
374BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1)
375VP_PROPERTY_BINARYOP
376VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
377VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign)
378END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
379
380// llvm.vp.minnum(x,y,mask,vlen)
381BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
382VP_PROPERTY_BINARYOP
383VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
384VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum)
385END_REGISTER_VP(vp_minnum, VP_FMINNUM)
386
387// llvm.vp.maxnum(x,y,mask,vlen)
388BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
389VP_PROPERTY_BINARYOP
390VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
391VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum)
392END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
393
394// llvm.vp.minimum(x,y,mask,vlen)
395BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1)
396VP_PROPERTY_BINARYOP
397VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM)
398VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum)
399END_REGISTER_VP(vp_minimum, VP_FMINIMUM)
400
401// llvm.vp.maximum(x,y,mask,vlen)
402BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1)
403VP_PROPERTY_BINARYOP
404VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
405VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
406END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
407
408// llvm.vp.ceil(x,mask,vlen)
409BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
410VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)
411VP_PROPERTY_FUNCTIONAL_SDOPC(FCEIL)
412END_REGISTER_VP(vp_ceil, VP_FCEIL)
413
414// llvm.vp.floor(x,mask,vlen)
415BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1)
416VP_PROPERTY_FUNCTIONAL_INTRINSIC(floor)
417VP_PROPERTY_FUNCTIONAL_SDOPC(FFLOOR)
418END_REGISTER_VP(vp_floor, VP_FFLOOR)
419
420// llvm.vp.round(x,mask,vlen)
421BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1)
422VP_PROPERTY_FUNCTIONAL_INTRINSIC(round)
423VP_PROPERTY_FUNCTIONAL_SDOPC(FROUND)
424END_REGISTER_VP(vp_round, VP_FROUND)
425
426// llvm.vp.roundeven(x,mask,vlen)
427BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1)
428VP_PROPERTY_FUNCTIONAL_INTRINSIC(roundeven)
429VP_PROPERTY_FUNCTIONAL_SDOPC(FROUNDEVEN)
430END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN)
431
432// llvm.vp.roundtozero(x,mask,vlen)
433BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1)
434VP_PROPERTY_FUNCTIONAL_INTRINSIC(trunc)
435VP_PROPERTY_FUNCTIONAL_SDOPC(FTRUNC)
436END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO)
437
438// llvm.vp.rint(x,mask,vlen)
439BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1)
440VP_PROPERTY_FUNCTIONAL_INTRINSIC(rint)
441VP_PROPERTY_FUNCTIONAL_SDOPC(FRINT)
442END_REGISTER_VP(vp_rint, VP_FRINT)
443
444// llvm.vp.nearbyint(x,mask,vlen)
445BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1)
446VP_PROPERTY_FUNCTIONAL_INTRINSIC(nearbyint)
447VP_PROPERTY_FUNCTIONAL_SDOPC(FNEARBYINT)
448END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT)
449
450// llvm.vp.lrint(x,mask,vlen)
451BEGIN_REGISTER_VP(vp_lrint, 1, 2, VP_LRINT, 0)
452VP_PROPERTY_FUNCTIONAL_INTRINSIC(lrint)
453VP_PROPERTY_FUNCTIONAL_SDOPC(LRINT)
454END_REGISTER_VP(vp_lrint, VP_LRINT)
455
456// llvm.vp.llrint(x,mask,vlen)
457BEGIN_REGISTER_VP(vp_llrint, 1, 2, VP_LLRINT, 0)
458VP_PROPERTY_FUNCTIONAL_INTRINSIC(llrint)
459VP_PROPERTY_FUNCTIONAL_SDOPC(LLRINT)
460END_REGISTER_VP(vp_llrint, VP_LLRINT)
461
462///// } Floating-Point Arithmetic
463
464///// Type Casts {
465// Specialized helper macro for type conversions.
466// <operation>(%x, %mask, %evl).
467#ifdef HELPER_REGISTER_FP_CAST_VP
468#error \
469 "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!"
470#endif
471#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \
472 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \
473 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
474 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
475 VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_##OPSUFFIX) \
476 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
477
478// llvm.vp.fptoui(x,mask,vlen)
479HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, FP_TO_UINT)
480
481// llvm.vp.fptosi(x,mask,vlen)
482HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, FP_TO_SINT)
483
484// llvm.vp.uitofp(x,mask,vlen)
485HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, UINT_TO_FP)
486
487// llvm.vp.sitofp(x,mask,vlen)
488HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, SINT_TO_FP)
489
490// llvm.vp.fptrunc(x,mask,vlen)
491HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, FP_ROUND)
492
493// llvm.vp.fpext(x,mask,vlen)
494HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, FP_EXTEND)
495
496#undef HELPER_REGISTER_FP_CAST_VP
497
498// Specialized helper macro for integer type conversions.
499// <operation>(%x, %mask, %evl).
500#ifdef HELPER_REGISTER_INT_CAST_VP
501#error \
502 "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!"
503#endif
504#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \
505 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \
506 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
507 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
508 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
509
510// llvm.vp.trunc(x,mask,vlen)
511HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc, TRUNCATE)
512
513// llvm.vp.zext(x,mask,vlen)
514HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt, ZERO_EXTEND)
515
516// llvm.vp.sext(x,mask,vlen)
517HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt, SIGN_EXTEND)
518
519// llvm.vp.ptrtoint(x,mask,vlen)
520BEGIN_REGISTER_VP(vp_ptrtoint, 1, 2, VP_PTRTOINT, -1)
521VP_PROPERTY_FUNCTIONAL_OPC(PtrToInt)
522END_REGISTER_VP(vp_ptrtoint, VP_PTRTOINT)
523
524// llvm.vp.inttoptr(x,mask,vlen)
525BEGIN_REGISTER_VP(vp_inttoptr, 1, 2, VP_INTTOPTR, -1)
526VP_PROPERTY_FUNCTIONAL_OPC(IntToPtr)
527END_REGISTER_VP(vp_inttoptr, VP_INTTOPTR)
528
529#undef HELPER_REGISTER_INT_CAST_VP
530
531///// } Type Casts
532
533///// Comparisons {
534
535// VP_SETCC (ISel only)
536BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4)
537END_REGISTER_VP_SDNODE(VP_SETCC)
538
539// llvm.vp.fcmp(x,y,cc,mask,vlen)
540BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4)
541HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC)
542VP_PROPERTY_FUNCTIONAL_OPC(FCmp)
543VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fcmp)
544END_REGISTER_VP_INTRINSIC(vp_fcmp)
545
546// llvm.vp.icmp(x,y,cc,mask,vlen)
547BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4)
548HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC)
549VP_PROPERTY_FUNCTIONAL_OPC(ICmp)
550END_REGISTER_VP_INTRINSIC(vp_icmp)
551
552///// } Comparisons
553
554// llvm.vp.is.fpclass(on_true,on_false,mask,vlen)
555BEGIN_REGISTER_VP(vp_is_fpclass, 2, 3, VP_IS_FPCLASS, 0)
556VP_PROPERTY_FUNCTIONAL_INTRINSIC(is_fpclass)
557END_REGISTER_VP(vp_is_fpclass, VP_IS_FPCLASS)
558
559///// Memory Operations {
560// llvm.vp.store(val,ptr,mask,vlen)
561BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3)
562// chain = VP_STORE chain,val,base,offset,mask,evl
563BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5)
564HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE)
565VP_PROPERTY_FUNCTIONAL_OPC(Store)
566VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store)
567END_REGISTER_VP(vp_store, VP_STORE)
568
569// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen)
570BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4)
571// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl
572VP_PROPERTY_NO_FUNCTIONAL
573BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6)
574HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
575END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
576
577// llvm.vp.scatter(ptr,val,mask,vlen)
578BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3)
579// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl
580BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6)
581HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER)
582VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter)
583END_REGISTER_VP(vp_scatter, VP_SCATTER)
584
585// llvm.vp.load(ptr,mask,vlen)
586BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2)
587// val,chain = VP_LOAD chain,base,offset,mask,evl
588BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4)
589HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD)
590VP_PROPERTY_FUNCTIONAL_OPC(Load)
591VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load)
592END_REGISTER_VP(vp_load, VP_LOAD)
593
594BEGIN_REGISTER_VP_INTRINSIC(vp_load_ff, 1, 2)
595// val,chain = VP_LOAD_FF chain,base,mask,evl
596BEGIN_REGISTER_VP_SDNODE(VP_LOAD_FF, -1, vp_load_ff, 2, 3)
597HELPER_MAP_VPID_TO_VPSD(vp_load_ff, VP_LOAD_FF)
598VP_PROPERTY_NO_FUNCTIONAL
599END_REGISTER_VP(vp_load_ff, VP_LOAD_FF)
600// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen)
601BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3)
602// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl
603VP_PROPERTY_NO_FUNCTIONAL
604BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5)
605HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
606END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
607
608// llvm.vp.gather(ptr,mask,vlen)
609BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2)
610// val,chain = VP_GATHER chain,base,indices,scale,mask,evl
611BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5)
612HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER)
613VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather)
614END_REGISTER_VP(vp_gather, VP_GATHER)
615
616///// } Memory Operations
617
618///// Reductions {
619
620// Specialized helper macro for VP reductions (%start, %x, %mask, %evl).
621#ifdef HELPER_REGISTER_REDUCTION_VP
622#error \
623 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
624#endif
625#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN, SDOPC) \
626 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \
627 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
628 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
629 END_REGISTER_VP(VPID, VPSD)
630
631// llvm.vp.reduce.add(start,x,mask,vlen)
632HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
633 vector_reduce_add, VECREDUCE_ADD)
634
635// llvm.vp.reduce.mul(start,x,mask,vlen)
636HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
637 vector_reduce_mul, VECREDUCE_MUL)
638
639// llvm.vp.reduce.and(start,x,mask,vlen)
640HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
641 vector_reduce_and, VECREDUCE_AND)
642
643// llvm.vp.reduce.or(start,x,mask,vlen)
644HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
645 vector_reduce_or, VECREDUCE_OR)
646
647// llvm.vp.reduce.xor(start,x,mask,vlen)
648HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
649 vector_reduce_xor, VECREDUCE_XOR)
650
651// llvm.vp.reduce.smax(start,x,mask,vlen)
652HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
653 vector_reduce_smax, VECREDUCE_SMAX)
654
655// llvm.vp.reduce.smin(start,x,mask,vlen)
656HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
657 vector_reduce_smin, VECREDUCE_SMIN)
658
659// llvm.vp.reduce.umax(start,x,mask,vlen)
660HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
661 vector_reduce_umax, VECREDUCE_UMAX)
662
663// llvm.vp.reduce.umin(start,x,mask,vlen)
664HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
665 vector_reduce_umin, VECREDUCE_UMIN)
666
667// llvm.vp.reduce.fmax(start,x,mask,vlen)
668HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
669 vector_reduce_fmax, VECREDUCE_FMAX)
670
671// llvm.vp.reduce.fmin(start,x,mask,vlen)
672HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
673 vector_reduce_fmin, VECREDUCE_FMIN)
674
675// llvm.vp.reduce.fmaximum(start,x,mask,vlen)
676HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
677 vector_reduce_fmaximum, VECREDUCE_FMAXIMUM)
678
679// llvm.vp.reduce.fminimum(start,x,mask,vlen)
680HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
681 vector_reduce_fminimum, VECREDUCE_FMINIMUM)
682
683#undef HELPER_REGISTER_REDUCTION_VP
684
685// Specialized helper macro for VP reductions as above but with two forms:
686// sequential and reassociative. These manifest as the presence of 'reassoc'
687// fast-math flags in the IR and as two distinct ISD opcodes in the
688// SelectionDAG.
689// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which
690// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set.
691#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP
692#error \
693 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!"
694#endif
695#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, SDOPC, SEQ_SDOPC, INTRIN) \
696 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \
697 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \
698 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
699 END_REGISTER_VP_SDNODE(VPSD) \
700 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \
701 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \
702 VP_PROPERTY_FUNCTIONAL_SDOPC(SEQ_SDOPC) \
703 END_REGISTER_VP_SDNODE(SEQ_VPSD) \
704 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
705 END_REGISTER_VP_INTRINSIC(VPID)
706
707// llvm.vp.reduce.fadd(start,x,mask,vlen)
708HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD,
709 VP_REDUCE_SEQ_FADD, VECREDUCE_FADD,
710 VECREDUCE_SEQ_FADD, vector_reduce_fadd)
711
712// llvm.vp.reduce.fmul(start,x,mask,vlen)
713HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
714 VP_REDUCE_SEQ_FMUL, VECREDUCE_FMUL,
715 VECREDUCE_SEQ_FMUL, vector_reduce_fmul)
716
717#undef HELPER_REGISTER_REDUCTION_SEQ_VP
718
719///// } Reduction
720
721///// Shuffles {
722
723// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported
724// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other
725// VP intrinsics, these two have a defined result on lanes where the mask is
726// false.
727//
728// llvm.vp.select(cond,on_true,on_false,vlen)
729BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1)
730VP_PROPERTY_FUNCTIONAL_OPC(Select)
731VP_PROPERTY_FUNCTIONAL_SDOPC(VSELECT)
732END_REGISTER_VP(vp_select, VP_SELECT)
733
734// llvm.vp.merge(cond,on_true,on_false,pivot)
735BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1)
736VP_PROPERTY_NO_FUNCTIONAL
737END_REGISTER_VP(vp_merge, VP_MERGE)
738
739BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1)
740VP_PROPERTY_NO_FUNCTIONAL
741END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE)
742
743// llvm.experimental.vp.reverse(x,mask,vlen)
744BEGIN_REGISTER_VP(experimental_vp_reverse, 1, 2,
745 EXPERIMENTAL_VP_REVERSE, -1)
746VP_PROPERTY_NO_FUNCTIONAL
747END_REGISTER_VP(experimental_vp_reverse, EXPERIMENTAL_VP_REVERSE)
748
749///// } Shuffles
750
751#undef BEGIN_REGISTER_VP
752#undef BEGIN_REGISTER_VP_INTRINSIC
753#undef BEGIN_REGISTER_VP_SDNODE
754#undef END_REGISTER_VP
755#undef END_REGISTER_VP_INTRINSIC
756#undef END_REGISTER_VP_SDNODE
757#undef HELPER_MAP_VPID_TO_VPSD
758#undef VP_PROPERTY_BINARYOP
759#undef VP_PROPERTY_CONSTRAINEDFP
760#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC
761#undef VP_PROPERTY_FUNCTIONAL_OPC
762#undef VP_PROPERTY_FUNCTIONAL_SDOPC
763#undef VP_PROPERTY_NO_FUNCTIONAL
764