1//===-- sse2 floating point env manipulation utilities ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENV_MXCSR_UTILS_H
10#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENV_MXCSR_UTILS_H
11
12#include "hdr/stdint_proxy.h"
13#include "hdr/types/fenv_t.h"
14#include "src/__support/CPP/bit.h"
15#include "src/__support/FPUtil/x86_64/fenv_x86_common.h"
16#include "src/__support/macros/attributes.h" // LIBC_INLINE
17#include "src/__support/macros/config.h"
18#include "src/__support/macros/properties/architectures.h"
19#include "src/__support/macros/properties/compiler.h"
20#include "src/__support/macros/sanitizer.h"
21
22#include <immintrin.h>
23
24namespace LIBC_NAMESPACE_DECL {
25namespace fputil {
26
27namespace sse {
28
29using internal::ExceptionFlags;
30using internal::RoundingControl;
31
32// SSE FPU environment from Intel 64 and IA-32 Architectures Software Developer
33// Manuals - Chapter 10
34// https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html
35//
36// The SSE floating point environment will be save/load with LDMXCSR/STMXCSR
37// instructions, which will return the following 4-byte structure in 32-bit
38// mode (see section 10.2.3, figure 10-3 in the manual linked above).
39
40// SSE MXCSR register (32-bit) structure: (section 10.2.3 in the manual)
41// - Bit 0: Invalid Exception
42// - Bit 1: Denormal Exception
43// - Bit 2: Division-by-zero Exception
44// - Bit 3: Overflow Exception
45// - Bit 4: Underflow Exception
46// - Bit 5: Inexact Exception
47// - Bit 6: Denormal Are Zeros (DAZ)
48// - Bit 7: Invalid Exception Mask
49// - Bit 8: Denormal Exception Mask
50// - Bit 9: Division-by-zero Exception Mask
51// - Bit 10: Overflow Exception Mask
52// - Bit 11: Underflow Exception Mask
53// - Bit 12: Inexact Exception Mask
54// - Bit 13-14: Rounding Control
55// - Bit 15: Flush Denormal To Zero (FTZ)
56// - Bit 16-31: Reserved, will raise general-protection exception if set to
57// non-zero.
58
59LIBC_INLINE static uint32_t get_mxcsr() { return _mm_getcsr(); }
60
61LIBC_INLINE static void write_mxcsr(uint32_t w) { _mm_setcsr(i: w); }
62
63LIBC_INLINE static void clear_except(uint16_t excepts) {
64 uint32_t mxcsr = get_mxcsr();
65 mxcsr &= ~static_cast<uint32_t>(excepts);
66 write_mxcsr(w: mxcsr);
67}
68
69LIBC_INLINE static uint16_t test_except(uint16_t excepts) {
70 uint32_t mxcsr = get_mxcsr();
71 return static_cast<uint16_t>(excepts & ExceptionFlags::ALL_F & mxcsr);
72}
73
74LIBC_INLINE static uint16_t get_except() {
75 uint32_t mxcsr = ~get_mxcsr();
76 return static_cast<uint16_t>(
77 (mxcsr >> ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION) &
78 ExceptionFlags::ALL_F);
79}
80
81LIBC_INLINE static void set_except(uint16_t excepts) {
82 _MM_SET_EXCEPTION_STATE(excepts);
83}
84
85LIBC_INLINE static void raise_except(uint16_t excepts) {
86 uint32_t mxcsr = get_mxcsr();
87 mxcsr |= excepts & ExceptionFlags::ALL_F;
88 write_mxcsr(w: mxcsr);
89#ifdef LIBC_TRAP_ON_RAISE_FP_EXCEPT
90 // We will try to trigger the SIGFPE if floating point exceptions are not
91 // masked. Since we already set all the floating point exception flags, we
92 // only need to trigger the trap on one of them.
93 static constexpr float EXCEPTION_INPUTS[6][2] = {
94 // FE_INVALID: 0.0 * inf
95 {0.0f, cpp::bit_cast<float>(0x7f80'0000U)},
96 // FE_DENORM: 1.0 * 0x1.0p-128
97 {1.0f, 0x1.0p-128f},
98 // FE_DIVBYZERO: 1.0 / 0.0
99 {1.0f, 0.0f},
100 // FE_OVERFLOW: 0x1.0p127 * 0x1.0p127
101 {0x1.0p127f, 0x1.0p127f},
102 // FE_UNDERFLOW: 0x1.0p-126 * 0x1.0p-126
103 {0x1.0p-126f, 0x1.0p-126f},
104 // FE_INEXACT: (1 + 2^-12) * (1 + 2^-12)
105 {0x1.001p0f, 0x1.001p0f}};
106
107 uint32_t except_masks =
108 (~(get_mxcsr() >> ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION)) &
109 excepts;
110 if (except_masks) {
111 int idx = cpp::countr_zero(except_masks);
112 if (idx == 2) {
113 // FE_DIVBYZERO, we need floating point division operations.
114 [[maybe_unused]] volatile float z = EXCEPTION_INPUTS[idx][0];
115 z /= EXCEPTION_INPUTS[idx][1];
116 } else {
117 // For the remaining exceptions, we use floating point multiplications.
118 [[maybe_unused]] volatile float z = EXCEPTION_INPUTS[idx][0];
119 z *= EXCEPTION_INPUTS[idx][1];
120 }
121 }
122#endif // LIBC_TRAP_ON_RAISE_FP_EXCEPT
123}
124
125LIBC_INLINE static uint16_t enable_except(uint16_t excepts) {
126 uint32_t mxcsr = get_mxcsr();
127 uint16_t old_excepts =
128 (mxcsr >> ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION) &
129 ExceptionFlags::ALL_F;
130 mxcsr &= ~(static_cast<uint32_t>(excepts)
131 << ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION);
132 write_mxcsr(w: mxcsr);
133 return old_excepts;
134}
135
136LIBC_INLINE static uint16_t disable_except(uint16_t excepts) {
137 uint32_t mxcsr = get_mxcsr();
138 uint16_t old_excepts =
139 (mxcsr >> ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION) &
140 ExceptionFlags::ALL_F;
141 mxcsr |= (static_cast<uint32_t>(excepts)
142 << ExceptionFlags::MXCSR_EXCEPTION_MASK_BIT_POSITION);
143 write_mxcsr(w: mxcsr);
144 return old_excepts;
145}
146
147LIBC_INLINE static uint16_t get_round() {
148 uint32_t mxcsr = get_mxcsr();
149 return static_cast<uint16_t>(mxcsr >> RoundingControl::MXCSR_BIT_POSITION) &
150 RoundingControl::ROUNDING_MASK;
151}
152
153LIBC_INLINE static void set_round(uint16_t rounding_mode) {
154 uint32_t mxcsr = get_mxcsr();
155 rounding_mode <<= RoundingControl::MXCSR_BIT_POSITION;
156 // Clear rounding bits.
157 mxcsr &= (~RoundingControl::MXCSR_ROUNDING_MASK);
158 write_mxcsr(w: mxcsr | rounding_mode);
159}
160
161} // namespace sse
162
163} // namespace fputil
164} // namespace LIBC_NAMESPACE_DECL
165
166#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENV_MXCSR_UTILS_H
167