1use std::error::Error;
4use std::fmt::{self, Display, Formatter};
5use std::str::FromStr;
6use std::sync::OnceLock;
7
8#[macro_export]
50macro_rules! multiversion {
51 (
53 use {$($($path:ident::)+*),*};
54
55 #[dyn_dispatch = $dispatch:path]
56 $(#[$m:meta])* $v:vis fn $name:ident($($arg_name:ident: $arg_type:ty),*$(,)?) $(-> $ret:ty)? $body:block
57
58 $($tail:tt)*
59 ) => {
60 mod $name {
62 #[allow(clippy::allow_attributes, unused_imports)]
63 use {super::*, $crate::multiversion}; $crate::multiversion!{
66 use {$($($path::)+*),*};
67 $(#[$m])* pub fn $name($($arg_name: $arg_type),*) $(-> $ret)? $body
68 $($tail)*
69 }
70 }
71
72 #[inline]
74 $v fn $name($($arg_name: $arg_type),*) $(-> $ret)? {
75 use $crate::multiversion::Version::*;
76
77 match *$dispatch {
78 Scalar => $name::scalar::$name($($arg_name),*),
79 Array128 => $name::array128::$name($($arg_name),*),
80 Array256 => $name::array256::$name($($arg_name),*),
81 #[cfg(feature = "all-simd")]
82 Array4096 => $name::array4096::$name($($arg_name),*),
83 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
84 AVX2 => unsafe { $name::avx2::$name($($arg_name),*) },
85 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
86 AVX2x2 => unsafe { $name::avx2x2::$name($($arg_name),*) },
87 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
88 AVX2x4 => unsafe { $name::avx2x4::$name($($arg_name),*) },
89 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
90 AVX2x8 => unsafe { $name::avx2x8::$name($($arg_name),*) },
91 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
92 AVX512 => unsafe { $name::avx512::$name($($arg_name),*) },
93 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
94 AVX512x2 => unsafe { $name::avx512x2::$name($($arg_name),*) },
95 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
96 AVX512x4 => unsafe { $name::avx512x4::$name($($arg_name),*) },
97 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
98 AVX512x8 => unsafe { $name::avx512x8::$name($($arg_name),*) },
99 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
100 Neon => unsafe { $name::neon::$name($($arg_name),*) },
101 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
102 Neonx2 => unsafe { $name::neonx2::$name($($arg_name),*) },
103 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
104 Neonx4 => unsafe { $name::neonx4::$name($($arg_name),*) },
105 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
106 Neonx8 => unsafe { $name::neonx8::$name($($arg_name),*) },
107 }
108 }
109 };
110
111 (
113 use {$($($path:ident::)+*),*};
114
115 $($tail:tt)+
116 ) => {
117 pub mod scalar {
119 #![allow(
120 clippy::reversed_empty_ranges,
121 clippy::range_plus_one,
122 clippy::modulo_one,
123 clippy::trivially_copy_pass_by_ref
124 )]
125
126 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
127 use {super::*, $($($path::)+scalar::*),*};
128
129 $($tail)*
130 }
131
132 pub mod array128 {
134 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
135 use {super::*, $($($path::)+array128::*),*};
136
137 $($tail)*
138 }
139
140 pub mod array256 {
142 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
143 use {super::*, $($($path::)+array256::*),*};
144
145 $($tail)*
146 }
147
148 #[cfg(feature="all-simd")]
150 pub mod array4096 {
151 #![allow(clippy::large_types_passed_by_value)]
152
153 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
154 use {super::*, $($($path::)+array4096::*),*};
155
156 $($tail)*
157 }
158
159 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
161 pub mod avx2 {
162 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
163 use {super::*, $($($path::)+avx2::*),*};
164
165 $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
166 }
167
168 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
170 pub mod avx2x2 {
171 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
172 use {super::*, $($($path::)+avx2x2::*),*};
173
174 $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
175 }
176
177 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
179 pub mod avx2x4 {
180 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
181 use {super::*, $($($path::)+avx2x4::*),*};
182
183 $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
184 }
185
186 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
188 pub mod avx2x8 {
189 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
190 use {super::*, $($($path::)+avx2x8::*),*};
191
192 $crate::multiversion!{@enable target_feature(enable = "avx2") $($tail)*}
193 }
194
195 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
197 pub mod avx512 {
198 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
199 use {super::*, $($($path::)+avx512::*),*};
200
201 $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
202 }
203
204 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
206 pub mod avx512x2 {
207 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
208 use {super::*, $($($path::)+avx512x2::*),*};
209
210 $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
211 }
212
213 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
215 pub mod avx512x4 {
216 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
217 use {super::*, $($($path::)+avx512x4::*),*};
218
219 $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
220 }
221
222 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
224 pub mod avx512x8 {
225 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
226 use {super::*, $($($path::)+avx512x8::*),*};
227
228 $crate::multiversion!{@enable target_feature(enable = "avx512f") $($tail)*}
229 }
230
231 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
233 pub mod neon {
234 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
235 use {super::*, $($($path::)+neon::*),*};
236
237 $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
238 }
239
240 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
242 pub mod neonx2 {
243 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
244 use {super::*, $($($path::)+neonx2::*),*};
245
246 $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
247 }
248
249 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
251 pub mod neonx4 {
252 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
253 use {super::*, $($($path::)+neonx4::*),*};
254
255 $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
256 }
257
258 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
260 pub mod neonx8 {
261 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
262 use {super::*, $($($path::)+neonx8::*),*};
263
264 $crate::multiversion!{@enable target_feature(enable = "neon") $($tail)*}
265 }
266 };
267
268 (fastest($name:ident())) => {
279 ::std::sync::LazyLock::new(#[cfg_attr(target_family = "wasm", expect(unreachable_code))] || {
280 use $crate::multiversion::Version::*;
281
282 #[cfg(all(target_family = "wasm", target_feature = "simd128"))]
284 return Array256;
285 #[cfg(all(target_family = "wasm"))]
286 return Array128;
287
288 if let Some(version) = $crate::multiversion::Version::get_override() {
289 return version;
290 }
291
292 let time = |x: $crate::multiversion::Version| {
293 let start = ::std::time::Instant::now();
294 ::std::hint::black_box(match x {
295 Scalar => scalar::$name(),
296 Array128 => array128::$name(),
297 Array256 => array256::$name(),
298 #[cfg(feature = "all-simd")]
299 Array4096 => array4096::$name(),
300 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
301 AVX2 => unsafe { avx2::$name() },
302 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
303 AVX2x2 => unsafe { avx2x2::$name() },
304 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
305 AVX2x4 => unsafe { avx2x4::$name() },
306 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
307 AVX2x8 => unsafe { avx2x8::$name() },
308 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
309 AVX512 => unsafe { avx512::$name() },
310 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
311 AVX512x2 => unsafe { avx512x2::$name() },
312 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
313 AVX512x4 => unsafe { avx512x4::$name() },
314 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
315 AVX512x8 => unsafe { avx512x8::$name() },
316 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
317 Neon => unsafe { neon::$name() },
318 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
319 Neonx2 => unsafe { neonx2::$name() },
320 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
321 Neonx4 => unsafe { neonx4::$name() },
322 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
323 Neonx8 => unsafe { neonx8::$name() },
324 });
325 start.elapsed()
326 };
327
328 let is_scalar_or_array = |x: $crate::multiversion::Version| match x {
329 Scalar | Array128 | Array256 => true,
330 #[cfg(feature = "all-simd")]
331 Array4096 => true,
332 #[allow(clippy::allow_attributes, unreachable_patterns)]
333 _ => false,
334 };
335 let has_simd = $crate::multiversion::VERSIONS.iter().any(|&x| !is_scalar_or_array(x));
336
337 let mut candidates: ::std::vec::Vec<($crate::multiversion::Version, ::std::time::Duration)> =
339 $crate::multiversion::VERSIONS
340 .iter()
341 .copied()
342 .filter(|&x| !(has_simd && x == Scalar))
343 .map(|x| (x, time(x)))
344 .collect();
345 let leader = candidates.iter().map(|&(_, t)| t).min().unwrap();
346 candidates.retain(|&(_, t)| t <= leader * 3);
347
348 if candidates.len() > 1 {
349 for (x, best) in &mut candidates {
351 *best = (*best).min(time(*x));
352 }
353 candidates.sort_by_key(|&(_, t)| t);
354 if candidates.len() > 2 {
355 candidates.truncate((candidates.len() / 2).max(2));
356 }
357
358 for (x, best) in &mut candidates {
360 *best = (*best).min(time(*x));
361 }
362 }
363
364 candidates.into_iter().min_by_key(|&(_, t)| t).unwrap().0
365 })
366 };
367
368 (@enable $t:meta) => {};
370 (@enable $t:meta
371 $(#[$m:meta])* $v:vis const $n:ident: $ty:ty = $e:expr; $($tail:tt)*
372 ) => {
373 $(#[$m])* $v const $n: $ty = $e;
374 $crate::multiversion!{@enable $t $($tail)*}
375 };
376 (@enable $t:meta
378 #[inline(always)] $($tail:tt)*
379 ) => {
380 $crate::multiversion!{@enable $t
381 #[inline] $($tail)*
382 }
383 };
384 (@enable $t:meta
385 $(#[$m:meta])* $v:vis fn $($tail:tt)*
386 ) => {
387 $crate::multiversion!{@one_item $t
388 $(#[$m])*
389 #[$t]
390 #[allow(clippy::allow_attributes, clippy::missing_safety_doc)]
391 $v fn $($tail)*
392 }
393 };
394
395 (@one_item $t:meta $item:item $($tail:tt)*) => {
397 $item
398 $crate::multiversion!{@enable $t $($tail)*}
399 };
400}
401
402#[macro_export]
414macro_rules! multiversion_test {
415 (
416 use {$($($path:ident::)+*),*};
417
418 #[test]
419 $(#[$m:meta])* $v:vis fn $name:ident() $body:block
420 ) => {
421 mod $name {
422 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
423 use super::*;
424
425 #[test]
426 $(#[$m])*
427 fn scalar() {
428 #![allow(
429 clippy::reversed_empty_ranges,
430 clippy::range_plus_one,
431 clippy::modulo_one,
432 clippy::trivially_copy_pass_by_ref
433 )]
434
435 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
436 use {$($($path::)+scalar::*),*};
437
438 $body
439 }
440
441 #[test]
442 $(#[$m])*
443 fn array128() {
444 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
445 use {$($($path::)+array128::*),*};
446
447 $body
448 }
449
450 #[test]
451 $(#[$m])*
452 fn array256() {
453 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
454 use {$($($path::)+array256::*),*};
455
456 $body
457 }
458
459 #[test]
460 #[cfg(feature = "all-simd")]
461 $(#[$m])*
462 fn array4096() {
463 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
464 use {$($($path::)+array4096::*),*};
465
466 $body
467 }
468
469 #[test]
470 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
471 $(#[$m])*
472 fn avx2() {
473 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
474 use {$($($path::)+avx2::*),*};
475
476 if !$crate::multiversion::Version::AVX2.supported() {
477 use std::io::{stdout, Write};
478 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2 due to missing avx2 support", module_path!());
479 return;
480 }
481
482 unsafe { $body }
483 }
484
485 #[test]
486 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
487 $(#[$m])*
488 fn avx2x2() {
489 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
490 use {$($($path::)+avx2x2::*),*};
491
492 if !$crate::multiversion::Version::AVX2x2.supported() {
493 use std::io::{stdout, Write};
494 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x2 due to missing avx2 support", module_path!());
495 return;
496 }
497
498 unsafe { $body }
499 }
500
501 #[test]
502 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
503 $(#[$m])*
504 fn avx2x4() {
505 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
506 use {$($($path::)+avx2x4::*),*};
507
508 if !$crate::multiversion::Version::AVX2x4.supported() {
509 use std::io::{stdout, Write};
510 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x4 due to missing avx2 support", module_path!());
511 return;
512 }
513
514 unsafe { $body }
515 }
516
517 #[test]
518 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
519 $(#[$m])*
520 fn avx2x8() {
521 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
522 use {$($($path::)+avx2x8::*),*};
523
524 if !$crate::multiversion::Version::AVX2x8.supported() {
525 use std::io::{stdout, Write};
526 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx2x8 due to missing avx2 support", module_path!());
527 return;
528 }
529
530 unsafe { $body }
531 }
532
533 #[test]
534 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
535 $(#[$m])*
536 fn avx512() {
537 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
538 use {$($($path::)+avx512::*),*};
539
540 if !$crate::multiversion::Version::AVX512.supported() {
541 use std::io::{stdout, Write};
542 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512 due to missing avx512 support", module_path!());
543 return;
544 }
545
546 unsafe { $body }
547 }
548
549 #[test]
550 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
551 $(#[$m])*
552 fn avx512x2() {
553 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
554 use {$($($path::)+avx512x2::*),*};
555
556 if !$crate::multiversion::Version::AVX512x2.supported() {
557 use std::io::{stdout, Write};
558 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x2 due to missing avx512 support", module_path!());
559 return;
560 }
561
562 unsafe { $body }
563 }
564
565 #[test]
566 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
567 $(#[$m])*
568 fn avx512x4() {
569 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
570 use {$($($path::)+avx512x4::*),*};
571
572 if !$crate::multiversion::Version::AVX512x4.supported() {
573 use std::io::{stdout, Write};
574 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x4 due to missing avx512 support", module_path!());
575 return;
576 }
577
578 unsafe { $body }
579 }
580
581 #[test]
582 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
583 $(#[$m])*
584 fn avx512x8() {
585 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
586 use {$($($path::)+avx512x8::*),*};
587
588 if !$crate::multiversion::Version::AVX512x8.supported() {
589 use std::io::{stdout, Write};
590 let _ = writeln!(&mut stdout(), "warning: skipping test in {}::avx512x8 due to missing avx512 support", module_path!());
591 return;
592 }
593
594 unsafe { $body }
595 }
596
597 #[test]
598 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
599 $(#[$m])*
600 fn neon() {
601 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
602 use {$($($path::)+neon::*),*};
603
604 unsafe { $body }
605 }
606
607 #[test]
608 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
609 $(#[$m])*
610 fn neonx2() {
611 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
612 use {$($($path::)+neonx2::*),*};
613
614 unsafe { $body }
615 }
616
617 #[test]
618 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
619 $(#[$m])*
620 fn neonx4() {
621 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
622 use {$($($path::)+neonx4::*),*};
623
624 unsafe { $body }
625 }
626
627 #[test]
628 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
629 $(#[$m])*
630 fn neonx8() {
631 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
632 use {$($($path::)+neonx8::*),*};
633
634 unsafe { $body }
635 }
636 }
637 };
638
639 (
640 use {$($($path:ident::)+*),*};
641
642 { $($tail:tt)+ }
643 ) => {
644 #[allow(
645 clippy::reversed_empty_ranges,
646 clippy::range_plus_one,
647 clippy::modulo_one,
648 clippy::trivially_copy_pass_by_ref
649 )]
650 {
651 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
652 use {$($($path::)+scalar::*),*};
653
654 $crate::multiversion_test!(@expr { $($tail)+ });
655 }
656
657 {
658 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
659 use {$($($path::)+array128::*),*};
660
661 $crate::multiversion_test!(@expr { $($tail)+ });
662 }
663
664 {
665 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
666 use {$($($path::)+array256::*),*};
667
668 $crate::multiversion_test!(@expr { $($tail)+ });
669 }
670
671 #[cfg(feature = "all-simd")]
672 #[allow(clippy::large_types_passed_by_value)]
673 {
674 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
675 use {$($($path::)+array4096::*),*};
676
677 $crate::multiversion_test!(@expr { $($tail)+ });
678 }
679
680 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
681 if $crate::multiversion::Version::AVX2.supported() {
682 unsafe {
683 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
684 use {$($($path::)+avx2::*),*};
685
686 $crate::multiversion_test!(@expr { $($tail)+ });
687 }
688
689 #[cfg(feature = "all-simd")]
690 unsafe {
691 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
692 use {$($($path::)+avx2x2::*),*};
693
694 $crate::multiversion_test!(@expr { $($tail)+ });
695 }
696
697 #[cfg(feature = "all-simd")]
698 unsafe {
699 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
700 use {$($($path::)+avx2x4::*),*};
701
702 $crate::multiversion_test!(@expr { $($tail)+ });
703 }
704
705 #[cfg(feature = "all-simd")]
706 unsafe {
707 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
708 use {$($($path::)+avx2x8::*),*};
709
710 $crate::multiversion_test!(@expr { $($tail)+ });
711 }
712 }
713
714 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
715 if $crate::multiversion::Version::AVX512.supported() {
716 unsafe {
717 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
718 use {$($($path::)+avx512::*),*};
719
720 $crate::multiversion_test!(@expr { $($tail)+ });
721 }
722
723 #[cfg(feature = "all-simd")]
724 unsafe {
725 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
726 use {$($($path::)+avx512x2::*),*};
727
728 $crate::multiversion_test!(@expr { $($tail)+ });
729 }
730
731 #[cfg(feature = "all-simd")]
732 unsafe {
733 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
734 use {$($($path::)+avx512x4::*),*};
735
736 $crate::multiversion_test!(@expr { $($tail)+ });
737 }
738
739 #[cfg(feature = "all-simd")]
740 unsafe {
741 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
742 use {$($($path::)+avx512x8::*),*};
743
744 $crate::multiversion_test!(@expr { $($tail)+ });
745 }
746 }
747
748 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
749 {
750 unsafe {
751 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
752 use {$($($path::)+neon::*),*};
753
754 $crate::multiversion_test!(@expr { $($tail)+ });
755 }
756
757 #[cfg(feature = "all-simd")]
758 unsafe {
759 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
760 use {$($($path::)+neonx2::*),*};
761
762 $crate::multiversion_test!(@expr { $($tail)+ });
763 }
764
765 #[cfg(feature = "all-simd")]
766 unsafe {
767 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
768 use {$($($path::)+neonx4::*),*};
769
770 $crate::multiversion_test!(@expr { $($tail)+ });
771 }
772
773 #[cfg(feature = "all-simd")]
774 unsafe {
775 #[allow(clippy::allow_attributes, unused_imports, clippy::wildcard_imports)]
776 use {$($($path::)+neonx8::*),*};
777
778 $crate::multiversion_test!(@expr { $($tail)+ });
779 }
780 }
781 };
782 (@expr $e:expr) => { $e }
783}
784
785macro_rules! versions_impl {
786 ($($
787 (#[$m:meta])*
788 $name:ident $(if $supported:expr)?,
789 )+) => {
790 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default)]
792 pub enum Version {
793 #[default] $(
795 $(#[$m])*
796 $(#[doc = concat!("Requires `", stringify!($supported), "`.")])?
797 $name,
798 )+
799 }
800
801 impl Version {
802 #[must_use]
804 pub const fn all() -> &'static [Self] {
805 &[$($(#[$m])* Self::$name,)+]
806 }
807
808 #[must_use]
810 pub fn supported(self) -> bool {
811 match self {
812 $($(#[$m])* Version::$name => true $(&& $supported)?,)+
813 }
814 }
815 }
816
817 impl FromStr for Version {
818 type Err = UnknownVersion;
819
820 fn from_str(x: &str) -> Result<Self, Self::Err> {
822 $(
823 $(#[$m])*
824 if stringify!($name).eq_ignore_ascii_case(x) { return Ok(Version::$name); }
825 )+
826 Err(UnknownVersion(x.to_string()))
827 }
828 }
829
830 pub static VERSIONS: std::sync::LazyLock<Vec<Version>> = std::sync::LazyLock::new(|| {
832 let mut vec = Version::all().to_vec();
833 vec.retain(|i| i.supported());
834 vec
835 });
836 };
837}
838versions_impl! {
839 Scalar,
840 Array128,
841 Array256,
842 #[cfg(feature = "all-simd")]
843 Array4096,
844 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
845 AVX2 if std::arch::is_x86_feature_detected!("avx2"),
846 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
847 AVX2x2 if std::arch::is_x86_feature_detected!("avx2"),
848 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
849 AVX2x4 if std::arch::is_x86_feature_detected!("avx2"),
850 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
851 AVX2x8 if std::arch::is_x86_feature_detected!("avx2"),
852 #[cfg(all(feature = "unsafe", any(target_arch = "x86", target_arch = "x86_64")))]
853 AVX512 if std::arch::is_x86_feature_detected!("avx512f"),
854 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
855 AVX512x2 if std::arch::is_x86_feature_detected!("avx512f"),
856 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
857 AVX512x4 if std::arch::is_x86_feature_detected!("avx512f"),
858 #[cfg(all(feature = "unsafe", feature = "all-simd", any(target_arch = "x86", target_arch = "x86_64")))]
859 AVX512x8 if std::arch::is_x86_feature_detected!("avx512f"),
860 #[cfg(all(feature = "unsafe", target_arch = "aarch64"))]
861 Neon,
862 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
863 Neonx2,
864 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
865 Neonx4,
866 #[cfg(all(feature = "unsafe", feature = "all-simd", target_arch = "aarch64"))]
867 Neonx8,
868}
869
870static OVERRIDE: OnceLock<Option<Version>> = OnceLock::new();
871
872impl Version {
873 pub fn get_override() -> Option<Version> {
880 *OVERRIDE.get_or_init(|| None)
881 }
882
883 pub fn set_override(version: Version) {
896 assert!(version.supported(), "{version:?} is not supported!");
897
898 if OVERRIDE.set(Some(version)).is_err() {
899 assert!(
901 Self::get_override().is_some(),
902 "Version::set_override must be called before get_override"
903 );
904 panic!("Version::set_override called more than once");
905 }
906 }
907}
908
909#[derive(Debug)]
911pub struct UnknownVersion(String);
912
913impl Display for UnknownVersion {
914 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
915 write!(f, "unknown function version: {:#}", self.0)
916 }
917}
918
919impl Error for UnknownVersion {}