1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains descriptions of the various Vector Predication intrinsics.
10// This is used as a central place for enumerating the different instructions
11// and should eventually be the place to put comments about the instructions.
12//
13//===----------------------------------------------------------------------===//
14
15// NOTE: NO INCLUDE GUARD DESIRED!
16
17// Provide definitions of macros so that users of this file do not have to
18// define everything to use it...
19//
20// Register a VP intrinsic and begin its property scope.
21// All VP intrinsic scopes are top level, ie it is illegal to place a
22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope.
23// \p VPID The VP intrinsic id.
24// \p MASKPOS The mask operand position.
25// \p EVLPOS The explicit vector length operand position.
26#ifndef BEGIN_REGISTER_VP_INTRINSIC
27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS)
28#endif
29
30// End the property scope of a VP intrinsic.
31#ifndef END_REGISTER_VP_INTRINSIC
32#define END_REGISTER_VP_INTRINSIC(VPID)
33#endif
34
35// Register a new VP SDNode and begin its property scope.
36// When the SDNode scope is nested within a VP intrinsic scope, it is
37// implicitly registered as the canonical SDNode for this VP intrinsic. There
38// is one VP intrinsic that maps directly to one SDNode that goes by the
39// same name. Since the operands are also the same, we open the property
40// scopes for both the VPIntrinsic and the SDNode at once.
41// \p VPSD The SelectionDAG Node id (eg VP_ADD).
42// \p LEGALPOS The operand position of the SDNode that is used for legalizing.
43// If LEGALPOS < 0, then the return type given by
44// TheNode->getValueType(-1-LEGALPOS) is used.
45// \p TDNAME The name of the TableGen definition of this SDNode.
46// \p MASKPOS The mask operand position.
47// \p EVLPOS The explicit vector length operand position.
48#ifndef BEGIN_REGISTER_VP_SDNODE
49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS)
50#endif
51
52// End the property scope of a new VP SDNode.
53#ifndef END_REGISTER_VP_SDNODE
54#define END_REGISTER_VP_SDNODE(VPSD)
55#endif
56
57// Helper macro to set up the mapping from VP intrinsic to ISD opcode.
58// Note: More than one VP intrinsic may map to one ISD opcode.
59#ifndef HELPER_MAP_VPID_TO_VPSD
60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
61#endif
62
63// Helper macros for the common "1:1 - Intrinsic : SDNode" case.
64//
65// There is one VP intrinsic that maps directly to one SDNode that goes by the
66// same name. Since the operands are also the same, we open the property
67// scopes for both the VPIntrinsic and the SDNode at once.
68//
69// \p VPID The canonical name (eg `vp_add`, which at the same time is the
70// name of the intrinsic and the TableGen def of the SDNode).
71// \p MASKPOS The mask operand position.
72// \p EVLPOS The explicit vector length operand position.
73// \p VPSD The SelectionDAG Node id (eg VP_ADD).
74// \p LEGALPOS The operand position of the SDNode that is used for legalizing
75// this SDNode. This can be `-1`, in which case the return type of
76// the SDNode is used.
77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \
78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \
79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \
80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
81
82#define END_REGISTER_VP(VPID, VPSD) \
83 END_REGISTER_VP_INTRINSIC(VPID) \
84 END_REGISTER_VP_SDNODE(VPSD)
85
86// The following macros attach properties to the scope they are placed in. This
87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the
88// scope.
89//
90// Property Macros {
91
92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode.
93// \p OPC The opcode of the instruction with the same function.
94#ifndef VP_PROPERTY_FUNCTIONAL_OPC
95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC)
96#endif
97
98// If operation can have rounding or fp exceptions, maps to corresponding
99// constrained fp intrinsic.
100#ifndef VP_PROPERTY_CONSTRAINEDFP
101#define VP_PROPERTY_CONSTRAINEDFP(INTRINID)
102#endif
103
104// The intrinsic and/or SDNode has the same function as this ISD Opcode.
105// \p SDOPC The opcode of the instruction with the same function.
106#ifndef VP_PROPERTY_FUNCTIONAL_SDOPC
107#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)
108#endif
109
110// Map this VP intrinsic to its canonical functional intrinsic.
111// \p INTRIN The non-VP intrinsics with the same function.
112#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC
113#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN)
114#endif
115
116// This VP Intrinsic has no functionally-equivalent non-VP opcode or intrinsic.
117#ifndef VP_PROPERTY_NO_FUNCTIONAL
118#define VP_PROPERTY_NO_FUNCTIONAL
119#endif
120
121// A property to infer VP binary-op SDNode opcodes automatically.
122#ifndef VP_PROPERTY_BINARYOP
123#define VP_PROPERTY_BINARYOP
124#endif
125
126/// } Property Macros
127
128///// Integer Arithmetic {
129
130// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl).
131#ifdef HELPER_REGISTER_BINARY_INT_VP
132#error \
133 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!"
134#endif
135#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC) \
136 BEGIN_REGISTER_VP_SDNODE(VPSD, -1, VPID, 2, 3) \
137 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
138 VP_PROPERTY_BINARYOP \
139 END_REGISTER_VP_SDNODE(VPSD)
140
141// llvm.vp.sdiv(x,y,mask,vlen)
142BEGIN_REGISTER_VP_INTRINSIC(vp_sdiv, 2, 3)
143HELPER_MAP_VPID_TO_VPSD(vp_sdiv, VP_SDIV)
144VP_PROPERTY_FUNCTIONAL_OPC(SDiv)
145HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV)
146END_REGISTER_VP_INTRINSIC(vp_sdiv)
147
148// llvm.vp.srem(x,y,mask,vlen)
149BEGIN_REGISTER_VP_INTRINSIC(vp_srem, 2, 3)
150HELPER_MAP_VPID_TO_VPSD(vp_srem, VP_SREM)
151VP_PROPERTY_FUNCTIONAL_OPC(SRem)
152HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM)
153END_REGISTER_VP_INTRINSIC(vp_srem)
154
155// llvm.vp.udiv(x,y,mask,vlen)
156BEGIN_REGISTER_VP_INTRINSIC(vp_udiv, 2, 3)
157HELPER_MAP_VPID_TO_VPSD(vp_udiv, VP_UDIV)
158VP_PROPERTY_FUNCTIONAL_OPC(UDiv)
159HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV)
160END_REGISTER_VP_INTRINSIC(vp_udiv)
161
162// llvm.vp.urem(x,y,mask,vlen)
163BEGIN_REGISTER_VP_INTRINSIC(vp_urem, 2, 3)
164HELPER_MAP_VPID_TO_VPSD(vp_urem, VP_UREM)
165VP_PROPERTY_FUNCTIONAL_OPC(URem)
166HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM)
167END_REGISTER_VP_INTRINSIC(vp_urem)
168
169#undef HELPER_REGISTER_BINARY_INT_VP
170
171// llvm.vp.cttz.elts(x,is_zero_poison,mask,vl)
172BEGIN_REGISTER_VP_INTRINSIC(vp_cttz_elts, 2, 3)
173VP_PROPERTY_NO_FUNCTIONAL
174BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS, 0, vp_cttz_elts, 1, 2)
175END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS)
176BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_POISON, 0, vp_cttz_elts_zero_poison, 1, 2)
177END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_POISON)
178END_REGISTER_VP_INTRINSIC(vp_cttz_elts)
179
180///// } Integer Arithmetic
181
182///// Memory Operations {
183// llvm.vp.store(val,ptr,mask,vlen)
184BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3)
185// chain = VP_STORE chain,val,base,offset,mask,evl
186BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5)
187HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE)
188VP_PROPERTY_FUNCTIONAL_OPC(Store)
189VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store)
190END_REGISTER_VP(vp_store, VP_STORE)
191
192// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen)
193BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4)
194// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl
195VP_PROPERTY_NO_FUNCTIONAL
196BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6)
197HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
198END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
199
200// llvm.vp.scatter(ptr,val,mask,vlen)
201BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3)
202// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl
203BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6)
204HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER)
205VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter)
206END_REGISTER_VP(vp_scatter, VP_SCATTER)
207
208// llvm.vp.load(ptr,mask,vlen)
209BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2)
210// val,chain = VP_LOAD chain,base,offset,mask,evl
211BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4)
212HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD)
213VP_PROPERTY_FUNCTIONAL_OPC(Load)
214VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load)
215END_REGISTER_VP(vp_load, VP_LOAD)
216
217BEGIN_REGISTER_VP_INTRINSIC(vp_load_ff, 1, 2)
218// val,chain = VP_LOAD_FF chain,base,mask,evl
219BEGIN_REGISTER_VP_SDNODE(VP_LOAD_FF, -1, vp_load_ff, 2, 3)
220HELPER_MAP_VPID_TO_VPSD(vp_load_ff, VP_LOAD_FF)
221VP_PROPERTY_NO_FUNCTIONAL
222END_REGISTER_VP(vp_load_ff, VP_LOAD_FF)
223// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen)
224BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3)
225// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl
226VP_PROPERTY_NO_FUNCTIONAL
227BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5)
228HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
229END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
230
231// llvm.vp.gather(ptr,mask,vlen)
232BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2)
233// val,chain = VP_GATHER chain,base,indices,scale,mask,evl
234BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5)
235HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER)
236VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather)
237END_REGISTER_VP(vp_gather, VP_GATHER)
238
239///// } Memory Operations
240
241///// Reductions {
242
243// Specialized helper macro for VP reductions (%start, %x, %mask, %evl).
244#ifdef HELPER_REGISTER_REDUCTION_VP
245#error \
246 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
247#endif
248#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN, SDOPC) \
249 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \
250 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
251 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
252 END_REGISTER_VP(VPID, VPSD)
253
254// llvm.vp.reduce.add(start,x,mask,vlen)
255HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
256 vector_reduce_add, VECREDUCE_ADD)
257
258// llvm.vp.reduce.mul(start,x,mask,vlen)
259HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
260 vector_reduce_mul, VECREDUCE_MUL)
261
262// llvm.vp.reduce.and(start,x,mask,vlen)
263HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
264 vector_reduce_and, VECREDUCE_AND)
265
266// llvm.vp.reduce.or(start,x,mask,vlen)
267HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
268 vector_reduce_or, VECREDUCE_OR)
269
270// llvm.vp.reduce.xor(start,x,mask,vlen)
271HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
272 vector_reduce_xor, VECREDUCE_XOR)
273
274// llvm.vp.reduce.smax(start,x,mask,vlen)
275HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
276 vector_reduce_smax, VECREDUCE_SMAX)
277
278// llvm.vp.reduce.smin(start,x,mask,vlen)
279HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
280 vector_reduce_smin, VECREDUCE_SMIN)
281
282// llvm.vp.reduce.umax(start,x,mask,vlen)
283HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
284 vector_reduce_umax, VECREDUCE_UMAX)
285
286// llvm.vp.reduce.umin(start,x,mask,vlen)
287HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
288 vector_reduce_umin, VECREDUCE_UMIN)
289
290// llvm.vp.reduce.fmax(start,x,mask,vlen)
291HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
292 vector_reduce_fmax, VECREDUCE_FMAX)
293
294// llvm.vp.reduce.fmin(start,x,mask,vlen)
295HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
296 vector_reduce_fmin, VECREDUCE_FMIN)
297
298// llvm.vp.reduce.fmaximum(start,x,mask,vlen)
299HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
300 vector_reduce_fmaximum, VECREDUCE_FMAXIMUM)
301
302// llvm.vp.reduce.fminimum(start,x,mask,vlen)
303HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
304 vector_reduce_fminimum, VECREDUCE_FMINIMUM)
305
306#undef HELPER_REGISTER_REDUCTION_VP
307
308// Specialized helper macro for VP reductions as above but with two forms:
309// sequential and reassociative. These manifest as the presence of 'reassoc'
310// fast-math flags in the IR and as two distinct ISD opcodes in the
311// SelectionDAG.
312// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which
313// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set.
314#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP
315#error \
316 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!"
317#endif
318#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, SDOPC, SEQ_SDOPC, INTRIN) \
319 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \
320 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \
321 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
322 END_REGISTER_VP_SDNODE(VPSD) \
323 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \
324 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \
325 VP_PROPERTY_FUNCTIONAL_SDOPC(SEQ_SDOPC) \
326 END_REGISTER_VP_SDNODE(SEQ_VPSD) \
327 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
328 END_REGISTER_VP_INTRINSIC(VPID)
329
330// llvm.vp.reduce.fadd(start,x,mask,vlen)
331HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD,
332 VP_REDUCE_SEQ_FADD, VECREDUCE_FADD,
333 VECREDUCE_SEQ_FADD, vector_reduce_fadd)
334
335// llvm.vp.reduce.fmul(start,x,mask,vlen)
336HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
337 VP_REDUCE_SEQ_FMUL, VECREDUCE_FMUL,
338 VECREDUCE_SEQ_FMUL, vector_reduce_fmul)
339
340#undef HELPER_REGISTER_REDUCTION_SEQ_VP
341
342///// } Reduction
343
344///// Shuffles {
345
346// The mask 'cond' operand of llvm.vp.merge is not reported
347// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other
348// VP intrinsics, vp.merge has a defined result on lanes where the mask is
349// false.
350
351// llvm.vp.merge(cond,on_true,on_false,pivot)
352BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1)
353VP_PROPERTY_NO_FUNCTIONAL
354END_REGISTER_VP(vp_merge, VP_MERGE)
355
356BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1)
357VP_PROPERTY_NO_FUNCTIONAL
358END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE)
359
360// llvm.experimental.vp.reverse(x,mask,vlen)
361BEGIN_REGISTER_VP(experimental_vp_reverse, 1, 2,
362 EXPERIMENTAL_VP_REVERSE, -1)
363VP_PROPERTY_NO_FUNCTIONAL
364END_REGISTER_VP(experimental_vp_reverse, EXPERIMENTAL_VP_REVERSE)
365
366///// } Shuffles
367
368#undef BEGIN_REGISTER_VP
369#undef BEGIN_REGISTER_VP_INTRINSIC
370#undef BEGIN_REGISTER_VP_SDNODE
371#undef END_REGISTER_VP
372#undef END_REGISTER_VP_INTRINSIC
373#undef END_REGISTER_VP_SDNODE
374#undef HELPER_MAP_VPID_TO_VPSD
375#undef VP_PROPERTY_BINARYOP
376#undef VP_PROPERTY_CONSTRAINEDFP
377#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC
378#undef VP_PROPERTY_FUNCTIONAL_OPC
379#undef VP_PROPERTY_FUNCTIONAL_SDOPC
380#undef VP_PROPERTY_NO_FUNCTIONAL
381