1#ifndef BLAKE3_IMPL_H
2#define BLAKE3_IMPL_H
3
4#include <assert.h>
5#include <stdbool.h>
6#include <stddef.h>
7#include <stdint.h>
8#include <string.h>
9
10#include "llvm-c/blake3.h"
11// For \p LLVM_LIBRARY_VISIBILITY
12#include "llvm/Support/Compiler.h"
13
14#include "llvm_blake3_prefix.h"
15
16// internal flags
17enum blake3_flags {
18 CHUNK_START = 1 << 0,
19 CHUNK_END = 1 << 1,
20 PARENT = 1 << 2,
21 ROOT = 1 << 3,
22 KEYED_HASH = 1 << 4,
23 DERIVE_KEY_CONTEXT = 1 << 5,
24 DERIVE_KEY_MATERIAL = 1 << 6,
25};
26
27// This C implementation tries to support recent versions of GCC, Clang, and
28// MSVC.
29#if defined(_MSC_VER)
30#define INLINE static __forceinline
31#else
32#define INLINE static inline __attribute__((always_inline))
33#endif
34
35#if defined(__x86_64__) || defined(_M_X64)
36#define IS_X86
37#define IS_X86_64
38#endif
39
40#if defined(__i386__) || defined(_M_IX86)
41#define IS_X86
42#define IS_X86_32
43#endif
44
45#if defined(__aarch64__) || defined(_M_ARM64)
46#define IS_AARCH64
47#endif
48
49#if defined(IS_X86)
50#if defined(_MSC_VER)
51#include <intrin.h>
52#endif
53#include <immintrin.h>
54#endif
55
56#if !defined(BLAKE3_USE_NEON)
57 // If BLAKE3_USE_NEON not manually set, autodetect based on
58 // AArch64ness and endianness.
59 #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)
60 #define BLAKE3_USE_NEON 1
61 #else
62 #define BLAKE3_USE_NEON 0
63 #endif
64#endif
65
66#if defined(IS_X86)
67#define MAX_SIMD_DEGREE 16
68#elif BLAKE3_USE_NEON == 1
69#define MAX_SIMD_DEGREE 4
70#else
71#define MAX_SIMD_DEGREE 1
72#endif
73
74// There are some places where we want a static size that's equal to the
75// MAX_SIMD_DEGREE, but also at least 2.
76#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
77
78static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
79 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
80 0x1F83D9ABUL, 0x5BE0CD19UL};
81
82static const uint8_t MSG_SCHEDULE[7][16] = {
83 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
84 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
85 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
86 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
87 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
88 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
89 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
90};
91
92/* Find index of the highest set bit */
93/* x is assumed to be nonzero. */
94static unsigned int highest_one(uint64_t x) {
95#if defined(__GNUC__) || defined(__clang__)
96 return 63 ^ __builtin_clzll(x);
97#elif defined(_MSC_VER) && defined(IS_X86_64)
98 unsigned long index;
99 _BitScanReverse64(&index, x);
100 return index;
101#elif defined(_MSC_VER) && defined(IS_X86_32)
102 if(x >> 32) {
103 unsigned long index;
104 _BitScanReverse(&index, (unsigned long)(x >> 32));
105 return 32 + index;
106 } else {
107 unsigned long index;
108 _BitScanReverse(&index, (unsigned long)x);
109 return index;
110 }
111#else
112 unsigned int c = 0;
113 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
114 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
115 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
116 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
117 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
118 if(x & 0x0000000000000002ULL) { c += 1; }
119 return c;
120#endif
121}
122
123// Count the number of 1 bits.
124INLINE unsigned int popcnt(uint64_t x) {
125#if defined(__GNUC__) || defined(__clang__)
126 return __builtin_popcountll(x);
127#else
128 unsigned int count = 0;
129 while (x != 0) {
130 count += 1;
131 x &= x - 1;
132 }
133 return count;
134#endif
135}
136
137// Largest power of two less than or equal to x. As a special case, returns 1
138// when x is 0.
139INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
140 return 1ULL << highest_one(x: x | 1);
141}
142
143INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
144
145INLINE uint32_t counter_high(uint64_t counter) {
146 return (uint32_t)(counter >> 32);
147}
148
149INLINE uint32_t load32(const void *src) {
150 const uint8_t *p = (const uint8_t *)src;
151 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
152 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
153}
154
155INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
156 uint32_t key_words[8]) {
157 key_words[0] = load32(src: &key[0 * 4]);
158 key_words[1] = load32(src: &key[1 * 4]);
159 key_words[2] = load32(src: &key[2 * 4]);
160 key_words[3] = load32(src: &key[3 * 4]);
161 key_words[4] = load32(src: &key[4 * 4]);
162 key_words[5] = load32(src: &key[5 * 4]);
163 key_words[6] = load32(src: &key[6 * 4]);
164 key_words[7] = load32(src: &key[7 * 4]);
165}
166
167INLINE void store32(void *dst, uint32_t w) {
168 uint8_t *p = (uint8_t *)dst;
169 p[0] = (uint8_t)(w >> 0);
170 p[1] = (uint8_t)(w >> 8);
171 p[2] = (uint8_t)(w >> 16);
172 p[3] = (uint8_t)(w >> 24);
173}
174
175INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
176 store32(dst: &bytes_out[0 * 4], w: cv_words[0]);
177 store32(dst: &bytes_out[1 * 4], w: cv_words[1]);
178 store32(dst: &bytes_out[2 * 4], w: cv_words[2]);
179 store32(dst: &bytes_out[3 * 4], w: cv_words[3]);
180 store32(dst: &bytes_out[4 * 4], w: cv_words[4]);
181 store32(dst: &bytes_out[5 * 4], w: cv_words[5]);
182 store32(dst: &bytes_out[6 * 4], w: cv_words[6]);
183 store32(dst: &bytes_out[7 * 4], w: cv_words[7]);
184}
185
186LLVM_LIBRARY_VISIBILITY
187void blake3_compress_in_place(uint32_t cv[8],
188 const uint8_t block[BLAKE3_BLOCK_LEN],
189 uint8_t block_len, uint64_t counter,
190 uint8_t flags);
191
192LLVM_LIBRARY_VISIBILITY
193void blake3_compress_xof(const uint32_t cv[8],
194 const uint8_t block[BLAKE3_BLOCK_LEN],
195 uint8_t block_len, uint64_t counter, uint8_t flags,
196 uint8_t out[64]);
197
198LLVM_LIBRARY_VISIBILITY
199void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
200 size_t blocks, const uint32_t key[8], uint64_t counter,
201 bool increment_counter, uint8_t flags,
202 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
203
204LLVM_LIBRARY_VISIBILITY
205size_t blake3_simd_degree(void);
206
207
208// Declarations for implementation-specific functions.
209LLVM_LIBRARY_VISIBILITY
210void blake3_compress_in_place_portable(uint32_t cv[8],
211 const uint8_t block[BLAKE3_BLOCK_LEN],
212 uint8_t block_len, uint64_t counter,
213 uint8_t flags);
214
215LLVM_LIBRARY_VISIBILITY
216void blake3_compress_xof_portable(const uint32_t cv[8],
217 const uint8_t block[BLAKE3_BLOCK_LEN],
218 uint8_t block_len, uint64_t counter,
219 uint8_t flags, uint8_t out[64]);
220
221LLVM_LIBRARY_VISIBILITY
222void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
223 size_t blocks, const uint32_t key[8],
224 uint64_t counter, bool increment_counter,
225 uint8_t flags, uint8_t flags_start,
226 uint8_t flags_end, uint8_t *out);
227
228#if defined(IS_X86)
229#if !defined(BLAKE3_NO_SSE2)
230LLVM_LIBRARY_VISIBILITY
231void blake3_compress_in_place_sse2(uint32_t cv[8],
232 const uint8_t block[BLAKE3_BLOCK_LEN],
233 uint8_t block_len, uint64_t counter,
234 uint8_t flags);
235LLVM_LIBRARY_VISIBILITY
236void blake3_compress_xof_sse2(const uint32_t cv[8],
237 const uint8_t block[BLAKE3_BLOCK_LEN],
238 uint8_t block_len, uint64_t counter,
239 uint8_t flags, uint8_t out[64]);
240LLVM_LIBRARY_VISIBILITY
241void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
242 size_t blocks, const uint32_t key[8],
243 uint64_t counter, bool increment_counter,
244 uint8_t flags, uint8_t flags_start,
245 uint8_t flags_end, uint8_t *out);
246#endif
247#if !defined(BLAKE3_NO_SSE41)
248LLVM_LIBRARY_VISIBILITY
249void blake3_compress_in_place_sse41(uint32_t cv[8],
250 const uint8_t block[BLAKE3_BLOCK_LEN],
251 uint8_t block_len, uint64_t counter,
252 uint8_t flags);
253LLVM_LIBRARY_VISIBILITY
254void blake3_compress_xof_sse41(const uint32_t cv[8],
255 const uint8_t block[BLAKE3_BLOCK_LEN],
256 uint8_t block_len, uint64_t counter,
257 uint8_t flags, uint8_t out[64]);
258LLVM_LIBRARY_VISIBILITY
259void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
260 size_t blocks, const uint32_t key[8],
261 uint64_t counter, bool increment_counter,
262 uint8_t flags, uint8_t flags_start,
263 uint8_t flags_end, uint8_t *out);
264#endif
265#if !defined(BLAKE3_NO_AVX2)
266LLVM_LIBRARY_VISIBILITY
267void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
268 size_t blocks, const uint32_t key[8],
269 uint64_t counter, bool increment_counter,
270 uint8_t flags, uint8_t flags_start,
271 uint8_t flags_end, uint8_t *out);
272#endif
273#if !defined(BLAKE3_NO_AVX512)
274LLVM_LIBRARY_VISIBILITY
275void blake3_compress_in_place_avx512(uint32_t cv[8],
276 const uint8_t block[BLAKE3_BLOCK_LEN],
277 uint8_t block_len, uint64_t counter,
278 uint8_t flags);
279
280LLVM_LIBRARY_VISIBILITY
281void blake3_compress_xof_avx512(const uint32_t cv[8],
282 const uint8_t block[BLAKE3_BLOCK_LEN],
283 uint8_t block_len, uint64_t counter,
284 uint8_t flags, uint8_t out[64]);
285
286LLVM_LIBRARY_VISIBILITY
287void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
288 size_t blocks, const uint32_t key[8],
289 uint64_t counter, bool increment_counter,
290 uint8_t flags, uint8_t flags_start,
291 uint8_t flags_end, uint8_t *out);
292#endif
293#endif
294
295#if BLAKE3_USE_NEON == 1
296LLVM_LIBRARY_VISIBILITY
297void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
298 size_t blocks, const uint32_t key[8],
299 uint64_t counter, bool increment_counter,
300 uint8_t flags, uint8_t flags_start,
301 uint8_t flags_end, uint8_t *out);
302#endif
303
304
305#endif /* BLAKE3_IMPL_H */
306