Skip to main content

utils/
multiversion.rs

1//! Implementation of [`multiversion!`](crate::multiversion!) macro.
2
3use std::error::Error;
4use std::fmt::{self, Display, Formatter};
5use std::str::FromStr;
6use std::sync::OnceLock;
7
8/// Macro to generate multiversioned functions.
9///
10/// This macro generates multiple versions of the provided functions, each optimized for different
11/// hardware, as well as dynamic dispatch functions which select the best version to use at runtime
12/// based on feature support.
13///
14/// This allows using instruction set extensions such as
15/// [AVX2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2)
16/// if the binary is run on a compatible processor, without enabling it at compile time, which would
17/// create a binary which can't be run on processors without AVX2 support.
18///
19/// The first three rules are supported:
20///
21/// 1. The first rule matches a dynamic dispatch function, optionally followed by extra private
22///    helper functions. Dynamic dispatch is controlled at runtime by supplying a path to a
23///    [`LazyLock`](std::sync::LazyLock) containing a [`Version`] value in the `dyn_dispatch`
24///    attribute.
25///
26/// 2. The second rule matches library functions without a dynamic dispatch function.
27///
28/// 3. The third rule takes a name of a library function generated using the second rule and
29///    expands to a [`LazyLock`](std::sync::LazyLock) evaluating to the fastest version, unless a
30///    global override is set using [`Version::set_override`]. The return value is intended to be
31///    stored in a `static` item for use in a `dyn_dispatch` attribute.
32///
33/// The first two rules start with a use statement which can be used for wildcard imports of
34/// multiversioned libraries. Each path will be expanded to include the version's name - e.g. the
35/// `avx2` version will expand `use {utils::simd::*};` into `use {utils::simd::avx2::*};`. Other
36/// imports should be outside the macro invocation.
37///
38/// Supported function syntax is limited as this is a declarative macro. Dynamic dispatch functions
39/// are the most limited, supporting only basic arguments and optionally a return type.
40///
41/// Versions enabling additional features require the `unsafe` crate feature to be enabled, as
42/// features are enabled using the
43/// [`target_feature`](https://doc.rust-lang.org/reference/attributes/codegen.html#the-target_feature-attribute)
44/// attribute, which requires callers to use `unsafe` blocks. A safe fallback version will always
45/// be generated.
46///
47/// See [`crate::md5`] as an example. [`multiversion_test!`](crate::multiversion_test!) can be used
48/// for testing multiversioned libraries.
49#[macro_export]
50macro_rules! multiversion {
51    // One dynamic dispatch function, optionally with extra helper functions.
52    (
53        use {$($($path:ident::)+*),*};
54
55        #[dyn_dispatch = $dispatch:path]
56        $(#[$m:meta])* $v:vis fn $name:ident($($arg_name:ident: $arg_type:ty),*$(,)?) $(-> $ret:ty)? $body:block
57
58        $($tail:tt)*
59    ) => {
60        /// [`multiversion!`] dynamic dispatch implementations.
61        mod $name {
62            #[allow(clippy::allow_attributes, unused_imports)]
63            use {super::*, $crate::multiversion}; // multiversion import needed for rustdoc links
64
65            $crate::multiversion!{
66                use {$($($path::)+*),*};
67                $(#[$m])* pub fn $name($($arg_name: $arg_type),*) $(-> $ret)? $body
68                $($tail)*
69            }
70        }
71
72        /// [`multiversion!`] dynamic dispatch function.
73        #[inline]
74        $v fn $name($($arg_name: $arg_type),*) $(-> $ret)? {
75            use $crate::multiversion::Version::*;
76
77            match *$dispatch {
78                Scalar => $name::scalar::$name($($arg_name),*),
79                Array128 => $name::array128::$name($($arg_name),*),
80                Array256 => $name::array256::$name($($arg_name),*),
81                #[cfg(feature = "all-simd")]
82                Array4096 => $name::array4096::$name($($arg_name),*),
83                #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
84                AVX2 => unsafe { $name::avx2::$name($($arg_name),*) },
85                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
86                AVX2x2 => unsafe { $name::avx2x2::$name($($arg_name),*) },
87                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
88                AVX2x4 => unsafe { $name::avx2x4::$name($($arg_name),*) },
89                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
90                AVX2x8 => unsafe { $name::avx2x8::$name($($arg_name),*) },
91                #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
92                AVX512 => unsafe { $name::avx512::$name($($arg_name),*) },
93                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
94                AVX512x2 => unsafe { $name::avx512x2::$name($($arg_name),*) },
95                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
96                AVX512x4 => unsafe { $name::avx512x4::$name($($arg_name),*) },
97                #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
98                AVX512x8 => unsafe { $name::avx512x8::$name($($arg_name),*) },
99                #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
100                Neon => unsafe { $name::neon::$name($($arg_name),*) },
101                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
102                Neonx2 => unsafe { $name::neonx2::$name($($arg_name),*) },
103                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
104                Neonx4 => unsafe { $name::neonx4::$name($($arg_name),*) },
105                #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
106                Neonx8 => unsafe { $name::neonx8::$name($($arg_name),*) },
107            }
108        }
109    };
110
111    // Library-only definition, without a dynamic dispatch function.
112    (
113        use {$($($path:ident::)+*),*};
114
115        $($tail:tt)+
116    ) => {
117        /// [`multiversion!`] scalar implementation.
118        pub mod scalar {
119            #![allow(
120                clippy::reversed_empty_ranges,
121                clippy::range_plus_one,
122                clippy::modulo_one,
123                clippy::trivially_copy_pass_by_ref
124            )]
125
126            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
127            use {super::*, $($($path::)+scalar::*),*};
128
129            $($tail)*
130        }
131
132        /// [`multiversion!`] array128 implementation.
133        pub mod array128 {
134            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
135            use {super::*, $($($path::)+array128::*),*};
136
137            $($tail)*
138        }
139
140        /// [`multiversion!`] array256 implementation.
141        pub mod array256 {
142            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
143            use {super::*, $($($path::)+array256::*),*};
144
145            $($tail)*
146        }
147
148        /// [`multiversion!`] array4096 implementation.
149        #[cfg(feature="all-simd")]
150        pub mod array4096 {
151            #![allow(clippy::large_types_passed_by_value)]
152
153            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
154            use {super::*, $($($path::)+array4096::*),*};
155
156            $($tail)*
157        }
158
159        /// [`multiversion!`] avx2 implementation.
160        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
161        pub mod avx2 {
162            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
163            use {super::*, $($($path::)+avx2::*),*};
164
165            $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
166        }
167
168        /// [`multiversion!`] avx2x2 implementation.
169        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
170        pub mod avx2x2 {
171            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
172            use {super::*, $($($path::)+avx2x2::*),*};
173
174            $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
175        }
176
177        /// [`multiversion!`] avx2x4 implementation.
178        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
179        pub mod avx2x4 {
180            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
181            use {super::*, $($($path::)+avx2x4::*),*};
182
183            $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
184        }
185
186        /// [`multiversion!`] avx2x8 implementation.
187        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
188        pub mod avx2x8 {
189            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
190            use {super::*, $($($path::)+avx2x8::*),*};
191
192            $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
193        }
194
195        /// [`multiversion!`] avx512 implementation.
196        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
197        pub mod avx512 {
198            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
199            use {super::*, $($($path::)+avx512::*),*};
200
201            $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
202        }
203
204        /// [`multiversion!`] avx512x2 implementation.
205        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
206        pub mod avx512x2 {
207            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
208            use {super::*, $($($path::)+avx512x2::*),*};
209
210            $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
211        }
212
213        /// [`multiversion!`] avx512x4 implementation.
214        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
215        pub mod avx512x4 {
216            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
217            use {super::*, $($($path::)+avx512x4::*),*};
218
219            $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
220        }
221
222        /// [`multiversion!`] avx512x8 implementation.
223        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
224        pub mod avx512x8 {
225            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
226            use {super::*, $($($path::)+avx512x8::*),*};
227
228            $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
229        }
230
231        /// [`multiversion!`] neon implementation.
232        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
233        pub mod neon {
234            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
235            use {super::*, $($($path::)+neon::*),*};
236
237            $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
238        }
239
240        /// [`multiversion!`] neonx2 implementation.
241        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
242        pub mod neonx2 {
243            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
244            use {super::*, $($($path::)+neonx2::*),*};
245
246            $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
247        }
248
249        /// [`multiversion!`] neonx4 implementation.
250        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
251        pub mod neonx4 {
252            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
253            use {super::*, $($($path::)+neonx4::*),*};
254
255            $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
256        }
257
258        /// [`multiversion!`] neonx8 implementation.
259        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
260        pub mod neonx8 {
261            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
262            use {super::*, $($($path::)+neonx8::*),*};
263
264            $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
265        }
266    };
267
268    // Microbenchmark for dynamic dispatch
269    //
270    // The first time each version runs it is slower than normal (page faults, cold caches), which
271    // is often enough to select the wrong version. Each version is timed up to three times, keeping
272    // the best time seen so far:
273    //  1. All versions once, dropping only those over 3x slower than the fastest, so one slightly
274    //     slower first run can't eliminate the eventual winner.
275    //  2. The remaining versions again, keeping the fastest half (at least two).
276    //  3. The remaining versions again, then the fastest is selected.
277    // The scalar version is skipped when a SIMD version is supported to speed up the process.
278    (fastest($name:ident())) => {
279        ::std::sync::LazyLock::new(#[cfg_attr(target_family = "wasm", expect(unreachable_code))] || {
280            use $crate::multiversion::Version::*;
281
282            // Instant::now() isn't implemented in WebAssembly, so hardcode implementations
283            #[cfg(all(target_family = "wasm", target_feature = "simd128"))]
284            return Array256;
285            #[cfg(all(target_family = "wasm"))]
286            return Array128;
287
288            if let Some(version) = $crate::multiversion::Version::get_override() {
289                return version;
290            }
291
292            let time = |x: $crate::multiversion::Version| {
293                let start = ::std::time::Instant::now();
294                ::std::hint::black_box(match x {
295                        Scalar => scalar::$name(),
296                        Array128 => array128::$name(),
297                        Array256 => array256::$name(),
298                        #[cfg(feature = "all-simd")]
299                        Array4096 => array4096::$name(),
300                        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
301                        AVX2 => unsafe { avx2::$name() },
302                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
303                        AVX2x2 => unsafe { avx2x2::$name() },
304                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
305                        AVX2x4 => unsafe { avx2x4::$name() },
306                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
307                        AVX2x8 => unsafe { avx2x8::$name() },
308                        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
309                        AVX512 => unsafe { avx512::$name() },
310                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
311                        AVX512x2 => unsafe { avx512x2::$name() },
312                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
313                        AVX512x4 => unsafe { avx512x4::$name() },
314                        #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
315                        AVX512x8 => unsafe { avx512x8::$name() },
316                        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
317                        Neon => unsafe { neon::$name() },
318                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
319                        Neonx2 => unsafe { neonx2::$name() },
320                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
321                        Neonx4 => unsafe { neonx4::$name() },
322                        #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
323                        Neonx8 => unsafe { neonx8::$name() },
324                });
325                start.elapsed()
326            };
327
328            let is_scalar_or_array = |x: $crate::multiversion::Version| match x {
329                Scalar | Array128 | Array256 => true,
330                #[cfg(feature = "all-simd")]
331                Array4096 => true,
332                #[allow(clippy::allow_attributes, unreachable_patterns)]
333                _ => false,
334            };
335            let has_simd = $crate::multiversion::VERSIONS.iter().any(|&x| !is_scalar_or_array(x));
336
337            // Round 1: all versions, dropping only those far behind the fastest
338            let mut candidates: ::std::vec::Vec<($crate::multiversion::Version, ::std::time::Duration)> =
339                $crate::multiversion::VERSIONS
340                    .iter()
341                    .copied()
342                    .filter(|&x| !(has_simd && x == Scalar))
343                    .map(|x| (x, time(x)))
344                    .collect();
345            let leader = candidates.iter().map(|&(_, t)| t).min().unwrap();
346            candidates.retain(|&(_, t)| t <= leader * 3);
347
348            if candidates.len() > 1 {
349                // Round 2: the remaining versions again, keeping the fastest half
350                for (x, best) in &mut candidates {
351                    *best = (*best).min(time(*x));
352                }
353                candidates.sort_by_key(|&(_, t)| t);
354                if candidates.len() > 2 {
355                    candidates.truncate((candidates.len() / 2).max(2));
356                }
357
358                // Round 3: the remaining versions again
359                for (x, best) in &mut candidates {
360                    *best = (*best).min(time(*x));
361                }
362            }
363
364            candidates.into_iter().min_by_key(|&(_, t)| t).unwrap().0
365        })
366    };
367
368    // Helper rules to add target_feature to helper functions
369    (@enable $t:meta) => {};
370    (@enable $t:meta
371        $(#[$m:meta])* $v:vis const $n:ident: $ty:ty = $e:expr; $($tail:tt)*
372    ) => {
373        $(#[$m])* $v const $n: $ty = $e;
374        $crate::multiversion!{@enable $t $($tail)*}
375    };
376    // Replace #[inline(always)] which is incompatible with target_feature with normal #[inline]
377    (@enable $t:meta
378        #[inline(always)] $($tail:tt)*
379    ) => {
380        $crate::multiversion!{@enable $t
381            #[inline] $($tail)*
382        }
383    };
384    (@enable $t:meta
385        $(#[$m:meta])* $v:vis fn $($tail:tt)*
386    ) => {
387        $crate::multiversion!{@one_item $t
388            $(#[$m])*
389            #[$t]
390            #[allow(clippy::allow_attributes, clippy::missing_safety_doc)]
391            $v fn $($tail)*
392        }
393    };
394
395    // Helper rule to pop the first item out of the tt sequence
396    (@one_item $t:meta $item:item $($tail:tt)*) => {
397        $item
398        $crate::multiversion!{@enable $t $($tail)*}
399    };
400}
401
402/// Helper for testing and benchmarking [`multiversion!`] library functions.
403///
404/// The first rule is for testing and creates individual
405/// [`#[test]`](https://doc.rust-lang.org/reference/attributes/testing.html) functions for each
406/// implementation.
407///
408/// The second rule is more general, duplicating the same expression for each implementation and
409/// is useful for benchmarking.
410///
411/// `#[target_feature(...)]` isn't applied to the test code as the feature-specific code should
412/// be elsewhere, inside a [`multiversion!`] macro.
413#[macro_export]
414macro_rules! multiversion_test {
415    (
416        use {$($($path:ident::)+*),*};
417
418        #[test]
419        $(#[$m:meta])* $v:vis fn $name:ident() $body:block
420    ) => {
421        mod $name {
422            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
423            use super::*;
424
425            #[test]
426            $(#[$m])*
427            fn scalar() {
428                #![allow(
429                    clippy::reversed_empty_ranges,
430                    clippy::range_plus_one,
431                    clippy::modulo_one,
432                    clippy::trivially_copy_pass_by_ref
433                )]
434
435                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
436                use {$($($path::)+scalar::*),*};
437
438                $body
439            }
440
441            #[test]
442            $(#[$m])*
443            fn array128() {
444                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
445                use {$($($path::)+array128::*),*};
446
447                $body
448            }
449
450            #[test]
451            $(#[$m])*
452            fn array256() {
453                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
454                use {$($($path::)+array256::*),*};
455
456                $body
457            }
458
459            #[test]
460            #[cfg(feature = "all-simd")]
461            $(#[$m])*
462            fn array4096() {
463                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
464                use {$($($path::)+array4096::*),*};
465
466                $body
467            }
468
469            #[test]
470            #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
471            $(#[$m])*
472            fn avx2() {
473                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
474                use {$($($path::)+avx2::*),*};
475
476                if !$crate::multiversion::Version::AVX2.supported() {
477                    use std::io::{stdout, Write};
478                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2 due to missing avx2 support", module_path!());
479                    return;
480                }
481
482                unsafe { $body }
483            }
484
485            #[test]
486            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
487            $(#[$m])*
488            fn avx2x2() {
489                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
490                use {$($($path::)+avx2x2::*),*};
491
492                if !$crate::multiversion::Version::AVX2x2.supported() {
493                    use std::io::{stdout, Write};
494                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x2 due to missing avx2 support", module_path!());
495                    return;
496                }
497
498                unsafe { $body }
499            }
500
501            #[test]
502            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
503            $(#[$m])*
504            fn avx2x4() {
505                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
506                use {$($($path::)+avx2x4::*),*};
507
508                if !$crate::multiversion::Version::AVX2x4.supported() {
509                    use std::io::{stdout, Write};
510                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x4 due to missing avx2 support", module_path!());
511                    return;
512                }
513
514                unsafe { $body }
515            }
516
517            #[test]
518            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
519            $(#[$m])*
520            fn avx2x8() {
521                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
522                use {$($($path::)+avx2x8::*),*};
523
524                if !$crate::multiversion::Version::AVX2x8.supported() {
525                    use std::io::{stdout, Write};
526                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x8 due to missing avx2 support", module_path!());
527                    return;
528                }
529
530                unsafe { $body }
531            }
532
533            #[test]
534            #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
535            $(#[$m])*
536            fn avx512() {
537                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
538                use {$($($path::)+avx512::*),*};
539
540                if !$crate::multiversion::Version::AVX512.supported() {
541                    use std::io::{stdout, Write};
542                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512 due to missing avx512 support", module_path!());
543                    return;
544                }
545
546                unsafe { $body }
547            }
548
549            #[test]
550            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
551            $(#[$m])*
552            fn avx512x2() {
553                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
554                use {$($($path::)+avx512x2::*),*};
555
556                if !$crate::multiversion::Version::AVX512x2.supported() {
557                    use std::io::{stdout, Write};
558                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x2 due to missing avx512 support", module_path!());
559                    return;
560                }
561
562                unsafe { $body }
563            }
564
565            #[test]
566            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
567            $(#[$m])*
568            fn avx512x4() {
569                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
570                use {$($($path::)+avx512x4::*),*};
571
572                if !$crate::multiversion::Version::AVX512x4.supported() {
573                    use std::io::{stdout, Write};
574                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x4 due to missing avx512 support", module_path!());
575                    return;
576                }
577
578                unsafe { $body }
579            }
580
581            #[test]
582            #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
583            $(#[$m])*
584            fn avx512x8() {
585                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
586                use {$($($path::)+avx512x8::*),*};
587
588                if !$crate::multiversion::Version::AVX512x8.supported() {
589                    use std::io::{stdout, Write};
590                    let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x8 due to missing avx512 support", module_path!());
591                    return;
592                }
593
594                unsafe { $body }
595            }
596
597            #[test]
598            #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
599            $(#[$m])*
600            fn neon() {
601                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
602                use {$($($path::)+neon::*),*};
603
604                unsafe { $body }
605            }
606
607            #[test]
608            #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
609            $(#[$m])*
610            fn neonx2() {
611                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
612                use {$($($path::)+neonx2::*),*};
613
614                unsafe { $body }
615            }
616
617            #[test]
618            #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
619            $(#[$m])*
620            fn neonx4() {
621                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
622                use {$($($path::)+neonx4::*),*};
623
624                unsafe { $body }
625            }
626
627            #[test]
628            #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
629            $(#[$m])*
630            fn neonx8() {
631                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
632                use {$($($path::)+neonx8::*),*};
633
634                unsafe { $body }
635            }
636        }
637    };
638
639    (
640        use {$($($path:ident::)+*),*};
641
642        { $($tail:tt)+ }
643    ) => {
644        #[allow(
645            clippy::reversed_empty_ranges,
646            clippy::range_plus_one,
647            clippy::modulo_one,
648            clippy::trivially_copy_pass_by_ref
649        )]
650        {
651            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
652            use {$($($path::)+scalar::*),*};
653
654            $crate::multiversion_test!(@expr { $($tail)+ });
655        }
656
657        {
658            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
659            use {$($($path::)+array128::*),*};
660
661            $crate::multiversion_test!(@expr { $($tail)+ });
662        }
663
664        {
665            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
666            use {$($($path::)+array256::*),*};
667
668            $crate::multiversion_test!(@expr { $($tail)+ });
669        }
670
671        #[cfg(feature = "all-simd")]
672        #[allow(clippy::large_types_passed_by_value)]
673        {
674            #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
675            use {$($($path::)+array4096::*),*};
676
677            $crate::multiversion_test!(@expr { $($tail)+ });
678        }
679
680        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
681        if $crate::multiversion::Version::AVX2.supported() {
682            unsafe {
683                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
684                use {$($($path::)+avx2::*),*};
685
686                $crate::multiversion_test!(@expr { $($tail)+ });
687            }
688
689            #[cfg(feature = "all-simd")]
690            unsafe {
691                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
692                use {$($($path::)+avx2x2::*),*};
693
694                $crate::multiversion_test!(@expr { $($tail)+ });
695            }
696
697            #[cfg(feature = "all-simd")]
698            unsafe {
699                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
700                use {$($($path::)+avx2x4::*),*};
701
702                $crate::multiversion_test!(@expr { $($tail)+ });
703            }
704
705            #[cfg(feature = "all-simd")]
706            unsafe {
707                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
708                use {$($($path::)+avx2x8::*),*};
709
710                $crate::multiversion_test!(@expr { $($tail)+ });
711            }
712        }
713
714        #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
715        if $crate::multiversion::Version::AVX512.supported() {
716            unsafe {
717                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
718                use {$($($path::)+avx512::*),*};
719
720                $crate::multiversion_test!(@expr { $($tail)+ });
721            }
722
723            #[cfg(feature = "all-simd")]
724            unsafe {
725                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
726                use {$($($path::)+avx512x2::*),*};
727
728                $crate::multiversion_test!(@expr { $($tail)+ });
729            }
730
731            #[cfg(feature = "all-simd")]
732            unsafe {
733                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
734                use {$($($path::)+avx512x4::*),*};
735
736                $crate::multiversion_test!(@expr { $($tail)+ });
737            }
738
739            #[cfg(feature = "all-simd")]
740            unsafe {
741                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
742                use {$($($path::)+avx512x8::*),*};
743
744                $crate::multiversion_test!(@expr { $($tail)+ });
745            }
746        }
747
748        #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
749        {
750            unsafe {
751                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
752                use {$($($path::)+neon::*),*};
753
754                $crate::multiversion_test!(@expr { $($tail)+ });
755            }
756
757            #[cfg(feature = "all-simd")]
758            unsafe {
759                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
760                use {$($($path::)+neonx2::*),*};
761
762                $crate::multiversion_test!(@expr { $($tail)+ });
763            }
764
765            #[cfg(feature = "all-simd")]
766            unsafe {
767                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
768                use {$($($path::)+neonx4::*),*};
769
770                $crate::multiversion_test!(@expr { $($tail)+ });
771            }
772
773            #[cfg(feature = "all-simd")]
774            unsafe {
775                #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
776                use {$($($path::)+neonx8::*),*};
777
778                $crate::multiversion_test!(@expr { $($tail)+ });
779            }
780        }
781    };
782    (@expr $e:expr) => { $e }
783}
784
785macro_rules! versions_impl {
786    ($($
787        (#[$m:meta])*
788        $name:ident $(if $supported:expr)?,
789    )+) => {
790        /// Versions generated by [`multiversion!`](crate::multiversion!) on this platform.
791        #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default)]
792        pub enum Version {
793            #[default] // First item is the default
794            $(
795                $(#[$m])*
796                $(#[doc = concat!("Requires `", stringify!($supported), "`.")])?
797                $name,
798            )+
799        }
800
801        impl Version {
802            /// All versions on this platform, including any unsupported by the current processor.
803            #[must_use]
804            pub const fn all() -> &'static [Self] {
805                &[$($(#[$m])* Self::$name,)+]
806            }
807
808            /// Check if this version is supported at runtime.
809            #[must_use]
810            pub fn supported(self) -> bool {
811                match self {
812                    $($(#[$m])* Version::$name => true $(&& $supported)?,)+
813                }
814            }
815        }
816
817        impl FromStr for Version {
818            type Err = UnknownVersion;
819
820            /// Implementation is case-insensitive.
821            fn from_str(x: &str) -> Result<Self, Self::Err> {
822                $(
823                    $(#[$m])*
824                    if stringify!($name).eq_ignore_ascii_case(x) { return Ok(Version::$name); }
825                )+
826                Err(UnknownVersion(x.to_string()))
827            }
828        }
829
830        /// Runtime generated list of supported versions.
831        pub static VERSIONS: std::sync::LazyLock<Vec<Version>> = std::sync::LazyLock::new(|| {
832            let mut vec = Version::all().to_vec();
833            vec.retain(|i| i.supported());
834            vec
835        });
836    };
837}
838versions_impl! {
839    Scalar,
840    Array128,
841    Array256,
842    #[cfg(feature = "all-simd")]
843    Array4096,
844    #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
845    AVX2 if std::arch::is_x86_feature_detected!("avx2"),
846    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
847    AVX2x2 if std::arch::is_x86_feature_detected!("avx2"),
848    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
849    AVX2x4 if std::arch::is_x86_feature_detected!("avx2"),
850    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
851    AVX2x8 if std::arch::is_x86_feature_detected!("avx2"),
852    #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
853    AVX512 if std::arch::is_x86_feature_detected!("avx512f"),
854    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
855    AVX512x2 if std::arch::is_x86_feature_detected!("avx512f"),
856    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
857    AVX512x4 if std::arch::is_x86_feature_detected!("avx512f"),
858    #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
859    AVX512x8 if std::arch::is_x86_feature_detected!("avx512f"),
860    #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
861    Neon,
862    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
863    Neonx2,
864    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
865    Neonx4,
866    #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
867    Neonx8,
868}
869
870static OVERRIDE: OnceLock<Option<Version>> = OnceLock::new();
871
872impl Version {
873    /// Get the global version override, if any.
874    ///
875    /// This function will return [`Some`] containing the version provided to
876    /// [`Version::set_override`] if it was previously called, or [`None`] otherwise.
877    ///
878    /// All dynamic dispatch implementations should respect this value.
879    pub fn get_override() -> Option<Version> {
880        *OVERRIDE.get_or_init(|| None)
881    }
882
883    /// Set the global version override.
884    ///
885    /// # Panics
886    ///
887    /// This function will panic if any of the following conditions are met:
888    ///
889    /// - The provided version is not [supported](Self::supported).
890    ///
891    /// - This function is called more than once.
892    ///
893    /// - This function is called after calling [`Version::get_override`] (or any multiversioned
894    ///   dynamic dispatch function that respects the global override).
895    pub fn set_override(version: Version) {
896        assert!(version.supported(), "{version:?} is not supported!");
897
898        if OVERRIDE.set(Some(version)).is_err() {
899            // Value returned in Err() is the value passed to set, not the existing value
900            assert!(
901                Self::get_override().is_some(),
902                "Version::set_override must be called before get_override"
903            );
904            panic!("Version::set_override called more than once");
905        }
906    }
907}
908
909/// Error type returned when trying to convert an invalid string to a [`Version`].
910#[derive(Debug)]
911pub struct UnknownVersion(String);
912
913impl Display for UnknownVersion {
914    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
915        write!(f, "unknown function version: {:#}", self.0)
916    }
917}
918
919impl Error for UnknownVersion {}