utils/
multiversion.rs

1//! Implementation of [`multiversion!`](crate::multiversion!) macro.
2
3use std::error::Error;
4use std::fmt::{self, Display, Formatter};
5use std::str::FromStr;
6use std::sync::{LazyLock, OnceLock};
7
8/// Macro to generate multiversioned functions.
9///
10/// This macro generates multiple versions of the provided functions, each optimized for different
11/// hardware, as well as dynamic dispatch functions which select the best version to use at runtime
12/// based on feature support.
13///
14/// This allows using instruction set extensions such as
15/// [AVX2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2)
16/// if the binary is run on a compatible processor, without enabling it at compile time, which would
17/// create a binary which can't be run on processors without AVX2 support.
18///
19/// The first three rules are supported:
20///
21/// 1. The first rule matches a dynamic dispatch function, optionally followed by extra private
22///    helper functions. Dynamic dispatch is controlled at runtime by supplying a path to a
23///    [`LazyLock`] containing a [`Version`] value in the `dyn_dispatch` attribute.
24///
25/// 2. The second rule matches library functions without a dynamic dispatch function.
26///
27/// 3. The third rule takes a name of a library function generated using the second rule and
28///    expands to a [`LazyLock`] evaluating to the fastest version, unless a global override is set
29///    using [`Version::set_override`]. The return value is intended to be stored in a `static` item
30///    for use in a `dyn_dispatch` attribute.
31///
32/// The first two rules start with a use statement which can be used for wildcard imports of
33/// multiversioned libraries. Each path will be expanded to include the version's name - e.g. the
34/// `avx2` version will expand `use {utils::simd::*};` into `use {utils::simd::avx2::*};`. Other
35/// imports should be outside the macro invocation.
36///
37/// Supported function syntax is limited as this is a declarative macro. Dynamic dispatch functions
38/// are the most limited, supporting only basic arguments and optionally a return type. Library
39/// functions may have complex definitions including generics, up to the maximum supported number of
40/// token trees per function definition.
41///
42/// Versions enabling additional features require the `unsafe` crate feature to be enabled, as
43/// features are enabled using the
44/// [`target_feature`](https://doc.rust-lang.org/reference/attributes/codegen.html#the-target_feature-attribute)
45/// attribute, which requires functions to be marked as unsafe. A safe fallback version will always
46/// be generated.
47///
48/// See [`crate::md5`] as an example. [`multiversion_test!`](crate::multiversion_test!) can be used
49/// for testing multiversioned libraries.
50#[macro_export]
51macro_rules! multiversion {
52    // One dynamic dispatch function, optionally with extra helper functions.
53    (
54        use {$($($path:ident::)+*),*};
55
56        #[dyn_dispatch = $dispatch:path]
57        $(#[$m:meta])* $v:vis fn $name:ident($($arg_name:ident: $arg_type:ty),*$(,)?) $(-> $ret:ty)? $body:block
58
59        $($tail:tt)*
60    ) => {
61        /// [`multiversion!`] dynamic dispatch implementations.
62        mod $name {
63            #[allow(clippy::allow_attributes, unused_imports)]
64            use {super::*, $crate::multiversion}; // multiversion import needed for rustdoc links
65
66            $crate::multiversion!{
67                use {$($($path::)+*),*};
68                $(#[$m])* pub fn $name($($arg_name: $arg_type),*) $(-> $ret)? $body
69                $($tail)*
70            }
71        }
72
73        /// [`multiversion!`] dynamic dispatch function.
74        #[inline]
75        $v fn $name($($arg_name: $arg_type),*) $(-> $ret)? {
76            use $crate::multiversion::Version::*;
77
78            match *$dispatch {
79                Scalar => $name::scalar::$name($($arg_name),*),
80                Array128 => $name::array128::$name($($arg_name),*),
81                Array256 => $name::array256::$name($($arg_name),*),
82                #[cfg(feature = "all-simd")]
83                Array4096 => $name::array4096::$name($($arg_name),*),
84                #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
85                AVX2 => unsafe { $name::avx2::$name($($arg_name),*) },
86                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
87                AVX2x2 => unsafe { $name::avx2x2::$name($($arg_name),*) },
88                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
89                AVX2x4 => unsafe { $name::avx2x4::$name($($arg_name),*) },
90                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
91                AVX2x8 => unsafe { $name::avx2x8::$name($($arg_name),*) },
92                #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
93                Neon => $name::neon::$name($($arg_name),*),
94                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
95                Neonx2 => $name::neonx2::$name($($arg_name),*),
96                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
97                Neonx4 => $name::neonx4::$name($($arg_name),*),
98                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
99                Neonx8 => $name::neonx8::$name($($arg_name),*),
100            }
101        }
102    };
103
104    // Library-only definition, without a dynamic dispatch function.
105    (
106        use {$($($path:ident::)+*),*};
107
108        $($tail:tt)+
109    ) => {
110        /// [`multiversion!`] scalar implementation.
111        pub mod scalar {
112            #![allow(
113                clippy::reversed_empty_ranges,
114                clippy::range_plus_one,
115                clippy::modulo_one,
116                clippy::trivially_copy_pass_by_ref
117            )]
118
119            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
120            use {super::*, $($($path::)+scalar::*),*};
121
122            $($tail)*
123        }
124
125        /// [`multiversion!`] array128 implementation.
126        pub mod array128 {
127            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
128            use {super::*, $($($path::)+array128::*),*};
129
130            $($tail)*
131        }
132
133        /// [`multiversion!`] array256 implementation.
134        pub mod array256 {
135            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
136            use {super::*, $($($path::)+array256::*),*};
137
138            $($tail)*
139        }
140
141        /// [`multiversion!`] array4096 implementation.
142        #[cfg(feature="all-simd")]
143        pub mod array4096 {
144            #![allow(clippy::large_types_passed_by_value)]
145
146            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
147            use {super::*, $($($path::)+array4096::*),*};
148
149            $($tail)*
150        }
151
152        /// [`multiversion!`] avx2 implementation.
153        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
154        pub mod avx2 {
155            #![allow(clippy::missing_safety_doc)]
156
157            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
158            use {super::*, $($($path::)+avx2::*),*};
159
160            $crate::multiversion!{@helper target_feature(enable = "avx2") $($tail)*}
161        }
162
163        /// [`multiversion!`] avx2x2 implementation.
164        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
165        pub mod avx2x2 {
166            #![allow(clippy::missing_safety_doc)]
167
168            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
169            use {super::*, $($($path::)+avx2x2::*),*};
170
171            $crate::multiversion!{@helper target_feature(enable = "avx2") $($tail)*}
172        }
173
174        /// [`multiversion!`] avx2x4 implementation.
175        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
176        pub mod avx2x4 {
177            #![allow(clippy::missing_safety_doc)]
178
179            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
180            use {super::*, $($($path::)+avx2x4::*),*};
181
182            $crate::multiversion!{@helper target_feature(enable = "avx2") $($tail)*}
183        }
184
185        /// [`multiversion!`] avx2x8 implementation.
186        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
187        pub mod avx2x8 {
188            #![allow(clippy::missing_safety_doc)]
189
190            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
191            use {super::*, $($($path::)+avx2x8::*),*};
192
193            $crate::multiversion!{@helper target_feature(enable = "avx2") $($tail)*}
194        }
195
196        /// [`multiversion!`] neon implementation.
197        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
198        pub mod neon {
199            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
200            use {super::*, $($($path::)+neon::*),*};
201
202            $($tail)*
203        }
204
205        /// [`multiversion!`] neonx2 implementation.
206        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
207        pub mod neonx2 {
208            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
209            use {super::*, $($($path::)+neonx2::*),*};
210
211            $($tail)*
212        }
213
214        /// [`multiversion!`] neonx4 implementation.
215        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
216        pub mod neonx4 {
217            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
218            use {super::*, $($($path::)+neonx4::*),*};
219
220            $($tail)*
221        }
222
223        /// [`multiversion!`] neonx8 implementation.
224        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
225        pub mod neonx8 {
226            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
227            use {super::*, $($($path::)+neonx8::*),*};
228
229            $($tail)*
230        }
231    };
232
233    // Microbenchmark for dynamic dispatch
234    (fastest($name:ident())) => {
235        ::std::sync::LazyLock::new(#[cfg_attr(target_family = "wasm", expect(unreachable_code))] || {
236            use $crate::multiversion::Version::*;
237
238            // Instant::now() isn't implemented in WebAssembly, so hardcode implementations
239            #[cfg(all(target_family = "wasm", target_feature = "simd128"))]
240            return Array256;
241            #[cfg(all(target_family = "wasm"))]
242            return Array128;
243
244            if let Some(version) = $crate::multiversion::Version::get_override() {
245                return version;
246            }
247
248            $crate::multiversion::VERSIONS
249                .iter()
250                .map(|&x| {
251                    let start = ::std::time::Instant::now();
252                    ::std::hint::black_box(match x {
253                        Scalar => scalar::$name(),
254                        Array128 => array128::$name(),
255                        Array256 => array256::$name(),
256                        #[cfg(feature = "all-simd")]
257                        Array4096 => array4096::$name(),
258                        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
259                        AVX2 => unsafe { avx2::$name() },
260                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
261                        AVX2x2 => unsafe { avx2x2::$name() },
262                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
263                        AVX2x4 => unsafe { avx2x4::$name() },
264                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
265                        AVX2x8 => unsafe { avx2x8::$name() },
266                        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
267                        Neon => neon::$name(),
268                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
269                        Neonx2 => neonx2::$name(),
270                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
271                        Neonx4 => neonx4::$name(),
272                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
273                        Neonx8 => neonx8::$name(),
274                    });
275                    (start.elapsed(), x)
276                })
277                // .inspect(|x| { dbg!(x); })
278                .min_by_key(|x| x.0)
279                .unwrap()
280                .1
281        })
282    };
283
284    // Allow helper functions to have complex definitions up to the max number of supported token
285    // trees. This structure is needed as `$($t:tt)+ $b:block` is ambiguous
286    (@helper $t:meta) => {};
287    // Replace #[inline(always)] which is incompatible with target_feature with normal #[inline]
288    (@helper $t:meta #[inline(always)] $($tail:tt)*) => {$crate::multiversion!{@helper $t #[inline] $($tail)*}};
289    (@helper $t:meta $(#[$m:meta])* $v:vis const $n:ident: $ty:ty = $e:expr; $($tail:tt)*) => {$(#[$m])* $v const $n: $ty = $e; $crate::multiversion!{@helper $t $($tail)*}};
290    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
291    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
292    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
293    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
294    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
295    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
296    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
297    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
298    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
299    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
300    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
301    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
302    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
303    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
304    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
305    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
306    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
307    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
308    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
309    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
310    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
311    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
312    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
313    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
314    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
315    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
316    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
317    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $t27:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 $t27 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
318    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $t27:tt $t28:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 $t27 $t28 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
319    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $t27:tt $t28:tt $t29:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 $t27 $t28 $t29 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
320    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $t27:tt $t28:tt $t29:tt $t30:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 $t27 $t28 $t29 $t30 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
321    (@helper $t:meta $(#[$m:meta])* $v:vis fn $n:ident $t0:tt $t1:tt $t2:tt $t3:tt $t4:tt $t5:tt $t6:tt $t7:tt $t8:tt $t9:tt $t10:tt $t11:tt $t12:tt $t13:tt $t14:tt $t15:tt $t16:tt $t17:tt $t18:tt $t19:tt $t20:tt $t21:tt $t22:tt $t23:tt $t24:tt $t25:tt $t26:tt $t27:tt $t28:tt $t29:tt $t30:tt $t31:tt $b:block $($tail:tt)*) => {$(#[$m])* #[$t] $v unsafe fn $n $t0 $t1 $t2 $t3 $t4 $t5 $t6 $t7 $t8 $t9 $t10 $t11 $t12 $t13 $t14 $t15 $t16 $t17 $t18 $t19 $t20 $t21 $t22 $t23 $t24 $t25 $t26 $t27 $t28 $t29 $t30 $t31 { #[allow(clippy::allow_attributes, clippy::macro_metavars_in_unsafe, unused_unsafe)] unsafe { $b } } $crate::multiversion!{@helper $t $($tail)*}};
322}
323
324/// Helper for testing and benchmarking [`multiversion!`] library functions.
325///
326/// The first rule is for testing and creates individual
327/// [`#[test]`](https://doc.rust-lang.org/reference/attributes/testing.html) functions for each
328/// implementation.
329///
330/// The second rule is more general, duplicating the same expression for each implementation and
331/// is useful for benchmarking.
332///
333/// `#[target_feature(...)]` isn't applied to the test code as the feature-specific code should
334/// be elsewhere, inside a [`multiversion!`] macro.
335#[macro_export]
336macro_rules! multiversion_test {
337    (
338        use {$($($path:ident::)+*),*};
339
340        #[test]
341        $(#[$m:meta])* $v:vis fn multiversion() $body:block
342    ) => {
343        #[test]
344        $(#[$m])*
345        fn scalar() {
346            #![allow(
347                clippy::reversed_empty_ranges,
348                clippy::range_plus_one,
349                clippy::modulo_one,
350                clippy::trivially_copy_pass_by_ref
351            )]
352
353            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
354            use {$($($path::)+scalar::*),*};
355
356            $body
357        }
358
359        #[test]
360        $(#[$m])*
361        fn array128() {
362            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
363            use {$($($path::)+array128::*),*};
364
365            $body
366        }
367
368        #[test]
369        $(#[$m])*
370        fn array256() {
371            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
372            use {$($($path::)+array256::*),*};
373
374            $body
375        }
376
377        #[test]
378        #[cfg(feature = "all-simd")]
379        $(#[$m])*
380        fn array4096() {
381            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
382            use {$($($path::)+array4096::*),*};
383
384            $body
385        }
386
387        #[test]
388        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
389        $(#[$m])*
390        fn avx2() {
391            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
392            use {$($($path::)+avx2::*),*};
393
394            if !$crate::multiversion::Version::AVX2.supported() {
395                use std::io::{stdout, Write};
396                let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2 due to missing avx2 support", module_path!());
397                return;
398            }
399
400            unsafe { $body }
401        }
402
403        #[test]
404        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
405        $(#[$m])*
406        fn avx2x2() {
407            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
408            use {$($($path::)+avx2x2::*),*};
409
410            if !$crate::multiversion::Version::AVX2x2.supported() {
411                use std::io::{stdout, Write};
412                let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x2 due to missing avx2 support", module_path!());
413                return;
414            }
415
416            unsafe { $body }
417        }
418
419        #[test]
420        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
421        $(#[$m])*
422        fn avx2x4() {
423            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
424            use {$($($path::)+avx2x4::*),*};
425
426            if !$crate::multiversion::Version::AVX2x4.supported() {
427                use std::io::{stdout, Write};
428                let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x4 due to missing avx2 support", module_path!());
429                return;
430            }
431
432            unsafe { $body }
433        }
434
435        #[test]
436        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
437        $(#[$m])*
438        fn avx2x8() {
439            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
440            use {$($($path::)+avx2x8::*),*};
441
442            if !$crate::multiversion::Version::AVX2x8.supported() {
443                use std::io::{stdout, Write};
444                let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x8 due to missing avx2 support", module_path!());
445                return;
446            }
447
448            unsafe { $body }
449        }
450
451        #[test]
452        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
453        $(#[$m])*
454        fn neon() {
455            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
456            use {$($($path::)+neon::*),*};
457
458            $body
459        }
460
461        #[test]
462        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
463        $(#[$m])*
464        fn neonx2() {
465            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
466            use {$($($path::)+neonx2::*),*};
467
468            $body
469        }
470
471        #[test]
472        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
473        $(#[$m])*
474        fn neonx4() {
475            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
476            use {$($($path::)+neonx4::*),*};
477
478            $body
479        }
480
481        #[test]
482        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
483        $(#[$m])*
484        fn neonx8() {
485            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
486            use {$($($path::)+neonx8::*),*};
487
488            $body
489        }
490    };
491
492    (
493        use {$($($path:ident::)+*),*};
494
495        { $($tail:tt)+ }
496    ) => {
497        #[allow(
498            clippy::reversed_empty_ranges,
499            clippy::range_plus_one,
500            clippy::modulo_one,
501            clippy::trivially_copy_pass_by_ref
502        )]
503        {
504            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
505            use {$($($path::)+scalar::*),*};
506
507            $crate::multiversion_test!(@expr { $($tail)+ });
508        }
509
510        {
511            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
512            use {$($($path::)+array128::*),*};
513
514            $crate::multiversion_test!(@expr { $($tail)+ });
515        }
516
517        {
518            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
519            use {$($($path::)+array256::*),*};
520
521            $crate::multiversion_test!(@expr { $($tail)+ });
522        }
523
524        #[cfg(feature = "all-simd")]
525        #[allow(clippy::large_types_passed_by_value)]
526        {
527            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
528            use {$($($path::)+array4096::*),*};
529
530            $crate::multiversion_test!(@expr { $($tail)+ });
531        }
532
533        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
534        if $crate::multiversion::Version::AVX2.supported() {
535            unsafe {
536                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
537                use {$($($path::)+avx2::*),*};
538
539                $crate::multiversion_test!(@expr { $($tail)+ });
540            }
541
542            #[cfg(feature = "all-simd")]
543            unsafe {
544                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
545                use {$($($path::)+avx2x2::*),*};
546
547                $crate::multiversion_test!(@expr { $($tail)+ });
548            }
549
550            #[cfg(feature = "all-simd")]
551            unsafe {
552                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
553                use {$($($path::)+avx2x4::*),*};
554
555                $crate::multiversion_test!(@expr { $($tail)+ });
556            }
557
558            #[cfg(feature = "all-simd")]
559            unsafe {
560                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
561                use {$($($path::)+avx2x8::*),*};
562
563                $crate::multiversion_test!(@expr { $($tail)+ });
564            }
565        }
566
567        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
568        {
569            {
570                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
571                use {$($($path::)+neon::*),*};
572
573                $crate::multiversion_test!(@expr { $($tail)+ });
574            }
575
576            #[cfg(feature = "all-simd")]
577            {
578                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
579                use {$($($path::)+neonx2::*),*};
580
581                $crate::multiversion_test!(@expr { $($tail)+ });
582            }
583
584            #[cfg(feature = "all-simd")]
585            {
586                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
587                use {$($($path::)+neonx4::*),*};
588
589                $crate::multiversion_test!(@expr { $($tail)+ });
590            }
591
592            #[cfg(feature = "all-simd")]
593            {
594                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
595                use {$($($path::)+neonx8::*),*};
596
597                $crate::multiversion_test!(@expr { $($tail)+ });
598            }
599        }
600    };
601    (@expr $e:expr) => { $e }
602}
603
604macro_rules! versions_impl {
605    ($($
606        (#[$m:meta])*
607        $name:ident $(if $supported:expr)?,
608    )+) => {
609        /// Versions generated by [`multiversion!`](crate::multiversion!) on this platform.
610        #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default)]
611        pub enum Version {
612            #[default] // First item is the default
613            $(
614                $(#[$m])*
615                $(#[doc = concat!("Requires `", stringify!($supported), "`.")])?
616                $name,
617            )+
618        }
619
620        impl Version {
621            /// Check if this version is supported at runtime.
622            #[must_use]
623            pub fn supported(self) -> bool {
624                match self {
625                    $($(#[$m])* Version::$name => true $(&& $supported)?,)+
626                }
627            }
628        }
629
630        impl FromStr for Version {
631            type Err = UnknownVersion;
632
633            /// Implementation is case-insensitive.
634            fn from_str(x: &str) -> Result<Self, Self::Err> {
635                $(
636                    $(#[$m])*
637                    if stringify!($name).eq_ignore_ascii_case(x) { return Ok(Version::$name); }
638                )+
639                Err(UnknownVersion(x.to_string()))
640            }
641        }
642
643        /// Runtime generated list of supported versions.
644        pub static VERSIONS: LazyLock<Vec<Version>> = LazyLock::new(|| {
645            let mut vec = vec![$($(#[$m])* Version::$name,)+];
646            vec.retain(|i| i.supported());
647            vec
648        });
649    };
650}
651versions_impl! {
652    Scalar,
653    Array128,
654    Array256,
655    #[cfg(feature = "all-simd")]
656    Array4096,
657    #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
658    AVX2 if std::arch::is_x86_feature_detected!("avx2"),
659    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
660    AVX2x2 if std::arch::is_x86_feature_detected!("avx2"),
661    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
662    AVX2x4 if std::arch::is_x86_feature_detected!("avx2"),
663    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
664    AVX2x8 if std::arch::is_x86_feature_detected!("avx2"),
665    #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
666    Neon,
667    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
668    Neonx2,
669    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
670    Neonx4,
671    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
672    Neonx8,
673}
674
675static OVERRIDE: OnceLock<Option<Version>> = OnceLock::new();
676
677impl Version {
678    /// Get the global version override, if any.
679    ///
680    /// This function will return [`Some`] containing the version provided to
681    /// [`Version::set_override`] if it was previously called, or [`None`] otherwise.
682    ///
683    /// All dynamic dispatch implementations should respect this value.
684    pub fn get_override() -> Option<Version> {
685        *OVERRIDE.get_or_init(|| None)
686    }
687
688    /// Set the global version override.
689    ///
690    /// # Panics
691    ///
692    /// This function will panic if any of the following conditions are met:
693    ///
694    /// - The provided version is not [supported](Self::supported).
695    ///
696    /// - This function is called more than once.
697    ///
698    /// - This function is called after calling [`Version::get_override`] (or any multiversioned
699    ///   dynamic dispatch function that respects the global override).
700    pub fn set_override(version: Version) {
701        assert!(version.supported(), "{version:?} is not supported!");
702
703        if OVERRIDE.set(Some(version)).is_err() {
704            // Value returned in Err() is the value passed to set, not the existing value
705            if Self::get_override().is_none() {
706                panic!("Version::set_override must be called before get_override");
707            } else {
708                panic!("Version::set_override called more than once");
709            }
710        }
711    }
712}
713
714/// Error type returned when trying to convert an invalid string to a [`Version`].
715#[derive(Debug)]
716pub struct UnknownVersion(String);
717
718impl Display for UnknownVersion {
719    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
720        write!(f, "unknown function version: {:#}", self.0)
721    }
722}
723
724impl Error for UnknownVersion {}
utils/multiversion.rs

utils/
multiversion.rs
