This is the mail archive of the
libstdc++@gcc.gnu.org
mailing list for the libstdc++ project.
Re: [PATCH] Implementation of std::experimental::simd of TS 19570 ed.2
- From: Matthias Kretz <m dot kretz at gsi dot de>
- To: <gcc-patches at gcc dot gnu dot org>
- Cc: <libstdc++ at gcc dot gnu dot org>
- Date: Mon, 11 Mar 2019 17:51:10 +0100
- Subject: Re: [PATCH] Implementation of std::experimental::simd of TS 19570 ed.2
- References: <2667705.X7s29ds1zI@mars>
Hi,
attached is a major update to the previous patch that compiles and passes on
skylake, skylake-avx512, aarch64 armv8 (as far as I could test) and seems to
compile fine (without explicit SIMD support) for POWER9.
I reorganized, underscored, and reformatted the code a lot. And there's still
a lot more to do for full coding style conformance.
The license headers call out the copyright of GSI (my employer) with a BSD-3
license. We still have not completed the process of Copyright Assignment. I
hope BSD-3 is good enough for an initial experimental release.
Cheers,
Matthias
On Freitag, 9. November 2018 01:45:45 CET Matthias Kretz wrote:
> Hi,
>
> the attached patch implements section 9 of the (about to be published) C++
> Parallelism TS ed.2 (latest WD at
> http://www.open-std.org/jtc1/sc22/wg21/docs/ papers/2018/n4773.pdf)
>
> The code has been extensively tested for x86_64 and needs some more work to
> work (not necessarily at full efficiency) for other targets.
>
> The integration of the code into the libstdc++ tree is untested. However, it
> is header-only at this point, so it might just work (it never does, though,
> does it?).
> Especially the full integration of my tests is a challenge, but I guess we
> can start small and expand the number of tests.
>
> I also will have to work some more on naming convention.
>
> Copyright assignment forms are in progress (I sent my part, but my employers
> part is still under way).
>
> Cheers,
> Matthias
--
──────────────────────────────────────────────────────────────────────────
Dr. Matthias Kretz https://kretzfamily.de
GSI Helmholtzzentrum für Schwerionenforschung https://gsi.de
SIMD easy and portable https://github.com/VcDevel/Vc
──────────────────────────────────────────────────────────────────────────
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index cf12d748371..4fbf748378b 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -2043,6 +2043,15 @@ Target supports compiling @code{avx512f} instructions.
@item avx512f_runtime
Target supports the execution of @code{avx512f} instructions.
+@item avx512vl_runtime
+Target supports the execution of @code{avx512vl} instructions.
+
+@item avx512dq_runtime
+Target supports the execution of @code{avx512dq} instructions.
+
+@item avx512bw_runtime
+Target supports the execution of @code{avx512bw} instructions.
+
@item cell_hw
Test system can execute AltiVec and Cell PPU instructions.
@@ -2082,6 +2091,15 @@ Target supports compiling @code{sse2} instructions.
@item sse2_runtime
Target supports the execution of @code{sse2} instructions.
+@item sse3_runtime
+Target supports the execution of @code{sse3} instructions.
+
+@item ssse3_runtime
+Target supports the execution of @code{ssse3} instructions.
+
+@item sse4_runtime
+Target supports the execution of @code{sse4} instructions.
+
@item sync_char_short
Target supports atomic operations on @code{char} and @code{short}.
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index d6e7a354e37..87cf959e517 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -1703,6 +1703,54 @@ proc check_sse2_hw_available { } {
}]
}
+# Return 1 if the target supports executing SSE2 instructions, 0
+# otherwise. Cache the result.
+
+proc check_sse3_hw_available { } {
+ return [check_cached_effective_target sse3_hw_available {
+ # If this is not the right target then we can skip the test.
+ if { !([istarget i?86-*-*] || [istarget x86_64-*-*]) } {
+ expr 0
+ } else {
+ check_runtime_nocache sse3_hw_available {
+ #include "cpuid.h"
+ int main ()
+ {
+ unsigned int eax, ebx, ecx, edx;
+ if (!__get_cpuid (1, &eax, &ebx, &ecx, &edx))
+ return 1;
+
+ return !(ecx & bit_SSE3);
+ }
+ } ""
+ }
+ }]
+}
+
+# Return 1 if the target supports executing SSE2 instructions, 0
+# otherwise. Cache the result.
+
+proc check_ssse3_hw_available { } {
+ return [check_cached_effective_target ssse3_hw_available {
+ # If this is not the right target then we can skip the test.
+ if { !([istarget i?86-*-*] || [istarget x86_64-*-*]) } {
+ expr 0
+ } else {
+ check_runtime_nocache ssse3_hw_available {
+ #include "cpuid.h"
+ int main ()
+ {
+ unsigned int eax, ebx, ecx, edx;
+ if (!__get_cpuid (1, &eax, &ebx, &ecx, &edx))
+ return 1;
+
+ return !(ecx & bit_SSSE3);
+ }
+ } ""
+ }
+ }]
+}
+
# Return 1 if the target supports executing SSE4 instructions, 0
# otherwise. Cache the result.
@@ -1818,6 +1866,90 @@ proc check_avx512f_hw_available { } {
}]
}
+# Return 1 if the target supports executing AVX512 vector length extensions
+# instructions, 0 otherwise. Cache the result.
+
+proc check_avx512vl_hw_available { } {
+ return [check_cached_effective_target avx512vl_hw_available {
+ # If this is not the right target then we can skip the test.
+ if { !([istarget x86_64-*-*] || [istarget i?86-*-*]) } {
+ expr 0
+ } else {
+ check_runtime_nocache avx512vl_hw_available {
+ #include <stddef.h>
+ #include "cpuid.h"
+ int main ()
+ {
+ unsigned int eax, ebx, ecx, edx;
+ if (__get_cpuid_max (0, NULL) < 7)
+ return 1;
+ __cpuid (1, eax, ebx, ecx, edx);
+ if (!(ecx & bit_OSXSAVE))
+ return 1;
+ __cpuid_count (7, 0, eax, ebx, ecx, edx);
+ return !(ebx & bit_AVX512VL);
+ }
+ } ""
+ }
+ }]
+}
+
+# Return 1 if the target supports executing AVX512 doubleword and quadword
+# instructions, 0 otherwise. Cache the result.
+
+proc check_avx512dq_hw_available { } {
+ return [check_cached_effective_target avx512dq_hw_available {
+ # If this is not the right target then we can skip the test.
+ if { !([istarget x86_64-*-*] || [istarget i?86-*-*]) } {
+ expr 0
+ } else {
+ check_runtime_nocache avx512dq_hw_available {
+ #include <stddef.h>
+ #include "cpuid.h"
+ int main ()
+ {
+ unsigned int eax, ebx, ecx, edx;
+ if (__get_cpuid_max (0, NULL) < 7)
+ return 1;
+ __cpuid (1, eax, ebx, ecx, edx);
+ if (!(ecx & bit_OSXSAVE))
+ return 1;
+ __cpuid_count (7, 0, eax, ebx, ecx, edx);
+ return !(ebx & bit_AVX512DQ);
+ }
+ } ""
+ }
+ }]
+}
+
+# Return 1 if the target supports executing AVX512 byte and word instructions, 0
+# otherwise. Cache the result.
+
+proc check_avx512bw_hw_available { } {
+ return [check_cached_effective_target avx512bw_hw_available {
+ # If this is not the right target then we can skip the test.
+ if { !([istarget x86_64-*-*] || [istarget i?86-*-*]) } {
+ expr 0
+ } else {
+ check_runtime_nocache avx512bw_hw_available {
+ #include <stddef.h>
+ #include "cpuid.h"
+ int main ()
+ {
+ unsigned int eax, ebx, ecx, edx;
+ if (__get_cpuid_max (0, NULL) < 7)
+ return 1;
+ __cpuid (1, eax, ebx, ecx, edx);
+ if (!(ecx & bit_OSXSAVE))
+ return 1;
+ __cpuid_count (7, 0, eax, ebx, ecx, edx);
+ return !(ebx & bit_AVX512BW);
+ }
+ } ""
+ }
+ }]
+}
+
# Return 1 if the target supports running SSE executables, 0 otherwise.
proc check_effective_target_sse_runtime { } {
@@ -1840,6 +1972,28 @@ proc check_effective_target_sse2_runtime { } {
return 0
}
+# Return 1 if the target supports running SSE3 executables, 0 otherwise.
+
+proc check_effective_target_sse3_runtime { } {
+ if { [check_effective_target_sse3]
+ && [check_sse3_hw_available]
+ && [check_sse_os_support_available] } {
+ return 1
+ }
+ return 0
+}
+
+# Return 1 if the target supports running SSSE3 executables, 0 otherwise.
+
+proc check_effective_target_ssse3_runtime { } {
+ if { [check_effective_target_ssse3]
+ && [check_ssse3_hw_available]
+ && [check_sse_os_support_available] } {
+ return 1
+ }
+ return 0
+}
+
# Return 1 if the target supports running SSE4 executables, 0 otherwise.
proc check_effective_target_sse4_runtime { } {
@@ -1884,6 +2038,39 @@ proc check_effective_target_avx512f_runtime { } {
return 0
}
+# Return 1 if the target supports running AVX512vl executables, 0 otherwise.
+
+proc check_effective_target_avx512vl_runtime { } {
+ if { [check_effective_target_avx512vl]
+ && [check_avx512vl_hw_available]
+ && [check_avx512_os_support_available] } {
+ return 1
+ }
+ return 0
+}
+
+# Return 1 if the target supports running AVX512dq executables, 0 otherwise.
+
+proc check_effective_target_avx512dq_runtime { } {
+ if { [check_effective_target_avx512dq]
+ && [check_avx512dq_hw_available]
+ && [check_avx512_os_support_available] } {
+ return 1
+ }
+ return 0
+}
+
+# Return 1 if the target supports running AVX512bw executables, 0 otherwise.
+
+proc check_effective_target_avx512bw_runtime { } {
+ if { [check_effective_target_avx512bw]
+ && [check_avx512bw_hw_available]
+ && [check_avx512_os_support_available] } {
+ return 1
+ }
+ return 0
+}
+
# Return 1 if bmi2 instructions can be compiled.
proc check_effective_target_bmi2 { } {
if { !([istarget i?86-*-*] || [istarget x86_64-*-*]) } {
diff --git a/libstdc++-v3/doc/xml/manual/status_cxx2017.xml b/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
index bb82e34bba7..5c65429f5be 100644
--- a/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
+++ b/libstdc++-v3/doc/xml/manual/status_cxx2017.xml
@@ -1003,6 +1003,17 @@ and test for <code>__STDCPP_MATH_SPEC_FUNCS__ >= 201003L</code>.
<entry>Library Fundamentals 2 TS</entry>
</row>
+ <row>
+ <entry>
+ <link xmlns:xlink="http://www.w3.org/1999/xlink"; xlink:href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0214r9.pdf";>
+ P0214R9
+ </link>
+ </entry>
+ <entry>Data-Parallel Types</entry>
+ <entry>in progress</entry>
+ <entry>Parallelism 2 TS</entry>
+ </row>
+
</tbody>
</tgroup>
</table>
@@ -1144,6 +1155,208 @@ and test for <code>__STDCPP_MATH_SPEC_FUNCS__ >= 201003L</code>.
If <code>!is_regular_file(p)</code>, an error is reported.
</para>
+ <section xml:id="iso.2017.par2ts" xreflabel="Implementation Specific Behavior of the Parallelism 2 TS"><info><title>Parallelism 2 TS</title></info>
+
+ <para>
+ <emphasis>9.2.1 [parallel.simd.abi]</emphasis>
+ <code>max_fixed_size<T></code> is 32, independent of <code>T</code>.
+ </para>
+
+ <para>
+ When targeting 32-bit x86,
+ <classname>simd_abi::compatible<T></classname> is an alias for
+ <classname>simd_abi::scalar</classname>. When targeting 64-bit x86
+ (including x32), <classname>simd_abi::compatible<T></classname> is
+ an alias for <classname>simd_abi::__sse</classname>, unless
+ <code>T</code> is <code>long double</code>, in which case it is an alias
+ for <classname>simd_abi::scalar</classname>.
+ </para>
+
+ <para>
+ When targeting x86,
+ <classname>simd_abi::native<T></classname> is an alias for one of
+ <classname>simd_abi::__sse</classname>,
+ <classname>simd_abi::__avx</classname>, or
+ <classname>simd_abi::__avx512</classname>, depending on the machine
+ options the compiler was invoked with.
+ </para>
+
+ <para>
+ For any other targeted machine
+ <classname>simd_abi::compatible<T></classname> and
+ <classname>simd_abi::native<T></classname> are aliases for
+ <classname>simd_abi::scalar</classname>. (subject to change)
+ </para>
+
+ <para>
+ The extended ABI tag types defined in the
+ <code>std::experimental::parallelism_v2::simd_abi</code> namespace are:
+ <classname>simd_abi::__sse_x<N></classname>,
+ <classname>simd_abi::__avx_x<N></classname>,
+ <classname>simd_abi::__avx512_x<N></classname>,
+ <classname>simd_abi::__neon_x<N></classname>,
+ <classname>simd_abi::__sse_n<T, N></classname>,
+ <classname>simd_abi::__avx_n<T, N></classname>,
+ <classname>simd_abi::__avx512_n<T, N></classname>,
+ <classname>simd_abi::__neon_n<T, N></classname>,
+ <classname>simd_abi::__sse</classname>,
+ <classname>simd_abi::__avx</classname>,
+ <classname>simd_abi::__avx512</classname>,
+ <classname>simd_abi::__neon</classname>,
+ <classname>simd_abi::__neon64</classname>, and
+ <classname>simd_abi::__neon128</classname>.
+ </para>
+
+ <para>
+ <classname>simd_abi::deduce<T, N, Abis...>::type</classname>,
+ with <code>N > 1</code> is an alias for an extended ABI tag, if a
+ supported extended ABI tag exists. Otherwise it is an alias for
+ <classname>simd_abi::fixed_size<N></classname>. If multiple
+ supported extended ABI tags satisfy the constraint, the widest register
+ type is chosen.
+ </para>
+
+ <para>
+ <emphasis>9.2.2 [parallel.simd.traits]</emphasis>
+ <classname>memory_alignment<T, U>::value</classname> is
+ <code>sizeof(U) * T::size()</code> rounded up to the next power-of-two
+ value.
+ </para>
+
+ <para>
+ <emphasis>9.3.1 [parallel.simd.overview]</emphasis>
+ Given an extended ABI tag <code>Abi</code>,
+ <classname>simd<T, Abi></classname> is supported according to the following table:
+ <table frame="all" xml:id="table.par2ts_simd_support">
+ <title>Support for Extended ABI Tags</title>
+
+ <tgroup cols="3" align="left" colsep="0" rowsep="1">
+ <colspec colname="c1"/>
+ <colspec colname="c2"/>
+ <colspec colname="c3"/>
+ <thead>
+ <row>
+ <entry>ABI tag</entry>
+ <entry>value type <code>T</code></entry>
+ <entry>required machine option</entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry>
+ <classname>__sse</classname>
+ </entry>
+ <entry morerows="2"><code>float</code></entry>
+ <entry morerows="2">"-msse"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__sse_n<float, N></classname> for <code>N</code> one of 2, 3, 4, 8, 12, or 16
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__sse_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ <row>
+ <entry>
+ <classname>__sse</classname>
+ </entry>
+ <entry morerows="2">any vectorizable type, except <code>long double</code> and <code>float</code></entry>
+ <entry morerows="2">"-msse2"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__sse_n<T, N></classname> for either <code>N > 1 && N * sizeof(T) <= 16</code> or <code>N * sizeof(T)</code> equal to 32, 48, or 64
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__sse_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ <row>
+ <entry>
+ <classname>__avx</classname>
+ </entry>
+ <entry morerows="2"><code>double</code> and <code>float</code></entry>
+ <entry morerows="2">"-mavx"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx_n<T, N></classname> for either <code>N * sizeof(T) > 16 && N * sizeof(T) <= 32</code> or <code>N * sizeof(T)</code> equal to 64, 96, or 128
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ <row>
+ <entry>
+ <classname>__avx</classname>
+ </entry>
+ <entry morerows="2">any integral vectorizable type</entry>
+ <entry morerows="2">"-mavx2"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx_n<T, N></classname> for either <code>N * sizeof(T) > 16 && N * sizeof(T) <= 32</code> or <code>N * sizeof(T)</code> equal to 64, 96, or 128
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ <row>
+ <entry>
+ <classname>__avx512</classname>
+ </entry>
+ <entry morerows="2"><code>float</code>, <code>double</code>, and any integral vectorizable type with <code>sizeof(T) >= 4</code></entry>
+ <entry morerows="2">"-mavx512f"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx512_n<T, N></classname> for either <code>N * sizeof(T) > 16 && N * sizeof(T) <= 32</code> or <code>N * sizeof(T)</code> equal to 64, 96, or 128
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx512_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ <row>
+ <entry>
+ <classname>__avx512</classname>
+ </entry>
+ <entry morerows="2">any integral vectorizable type with <code>sizeof(T) < 4</code></entry>
+ <entry morerows="2">"-mavx512bw"</entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx512_n<T, N></classname> for either <code>N * sizeof(T) > 16 && N * sizeof(T) <= 32</code> or <code>N * sizeof(T)</code> equal to 64, 96, or 128
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <classname>__avx512_x<N></classname> for <code>N</code> one of 1, 2, 3, or 4
+ </entry>
+ </row>
+
+ </tbody>
+ </tgroup>
+ </table>
+ </para>
+
+ </section>
</section>
diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am
index 6541caccc6d..a187b5675cb 100644
--- a/libstdc++-v3/include/Makefile.am
+++ b/libstdc++-v3/include/Makefile.am
@@ -694,6 +694,7 @@ experimental_headers = \
${experimental_srcdir}/ratio \
${experimental_srcdir}/regex \
${experimental_srcdir}/set \
+ ${experimental_srcdir}/simd \
${experimental_srcdir}/socket \
${experimental_srcdir}/source_location \
${experimental_srcdir}/string \
@@ -715,6 +716,13 @@ experimental_bits_headers = \
${experimental_bits_srcdir}/net.h \
${experimental_bits_srcdir}/shared_ptr.h \
${experimental_bits_srcdir}/string_view.tcc \
+ ${experimental_bits_srcdir}/simd.h \
+ ${experimental_bits_srcdir}/simd_abis.h \
+ ${experimental_bits_srcdir}/simd_debug.h \
+ ${experimental_bits_srcdir}/simd_detail.h \
+ ${experimental_bits_srcdir}/simd_math.h \
+ ${experimental_bits_srcdir}/simd_workarounds.h \
+ ${experimental_bits_srcdir}/simd_x86_conversions.h \
${experimental_bits_filesystem_headers}
if ENABLE_FILESYSTEM_TS
diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in
index bafe69cd374..c88d5ddba7a 100644
--- a/libstdc++-v3/include/Makefile.in
+++ b/libstdc++-v3/include/Makefile.in
@@ -1037,6 +1037,7 @@ experimental_headers = \
${experimental_srcdir}/ratio \
${experimental_srcdir}/regex \
${experimental_srcdir}/set \
+ ${experimental_srcdir}/simd \
${experimental_srcdir}/socket \
${experimental_srcdir}/source_location \
${experimental_srcdir}/string \
@@ -1058,6 +1059,13 @@ experimental_bits_headers = \
${experimental_bits_srcdir}/net.h \
${experimental_bits_srcdir}/shared_ptr.h \
${experimental_bits_srcdir}/string_view.tcc \
+ ${experimental_bits_srcdir}/simd.h \
+ ${experimental_bits_srcdir}/simd_abis.h \
+ ${experimental_bits_srcdir}/simd_debug.h \
+ ${experimental_bits_srcdir}/simd_detail.h \
+ ${experimental_bits_srcdir}/simd_math.h \
+ ${experimental_bits_srcdir}/simd_workarounds.h \
+ ${experimental_bits_srcdir}/simd_x86_conversions.h \
${experimental_bits_filesystem_headers}
@ENABLE_FILESYSTEM_TS_FALSE@experimental_filesystem_headers =
diff --git a/libstdc++-v3/include/experimental/bits/simd.h b/libstdc++-v3/include/experimental/bits/simd.h
new file mode 100644
index 00000000000..a95084aef1b
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd.h
@@ -0,0 +1,6117 @@
+// Definition of the public simd interfaces -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
+#define _GLIBCXX_EXPERIMENTAL_SIMD_H
+
+#if __cplusplus >= 201703L
+
+#include "simd_detail.h"
+#include <bitset>
+#include <climits>
+#include <cstring>
+#include <functional>
+#include <iosfwd>
+#include <limits>
+#include <utility>
+
+#if _GLIBCXX_SIMD_X86INTRIN
+#include <x86intrin.h>
+#elif _GLIBCXX_SIMD_HAVE_NEON
+#include <arm_neon.h>
+#endif
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+
+#if !_GLIBCXX_SIMD_X86INTRIN
+using __m128 [[__gnu__::__vector_size__(16)]] = float;
+using __m128d [[__gnu__::__vector_size__(16)]] = double;
+using __m128i [[__gnu__::__vector_size__(16)]] = long long;
+using __m256 [[__gnu__::__vector_size__(32)]] = float;
+using __m256d [[__gnu__::__vector_size__(32)]] = double;
+using __m256i [[__gnu__::__vector_size__(32)]] = long long;
+using __m512 [[__gnu__::__vector_size__(64)]] = float;
+using __m512d [[__gnu__::__vector_size__(64)]] = double;
+using __m512i [[__gnu__::__vector_size__(64)]] = long long;
+#endif
+
+// load/store flags {{{
+struct element_aligned_tag {};
+struct vector_aligned_tag {};
+template <size_t _N>
+struct overaligned_tag
+{
+ static constexpr size_t _S_alignment = _N;
+};
+inline constexpr element_aligned_tag element_aligned = {};
+inline constexpr vector_aligned_tag vector_aligned = {};
+template <size_t _N>
+inline constexpr overaligned_tag<_N> overaligned = {};
+// }}}
+
+// vvv ---- type traits ---- vvv
+// integer type aliases{{{
+using _UChar = unsigned char;
+using _SChar = signed char;
+using _UShort = unsigned short;
+using _UInt = unsigned int;
+using _ULong = unsigned long;
+using _ULLong = unsigned long long;
+using _LLong = long long;
+//}}}
+// __is_equal {{{
+template <typename _Tp, _Tp __a, _Tp __b>
+struct __is_equal : public false_type
+{
+};
+template <typename _Tp, _Tp __a>
+struct __is_equal<_Tp, __a, __a> : public true_type
+{
+};
+
+// }}}
+// __identity/__id{{{
+template <typename _Tp>
+struct __identity
+{
+ using type = _Tp;
+};
+template <typename _Tp>
+using __id = typename __identity<_Tp>::type;
+
+// }}}
+// __first_of_pack{{{
+template <typename _T0, typename...>
+struct __first_of_pack
+{
+ using type = _T0;
+};
+template <typename... _Ts>
+using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
+
+//}}}
+// __value_type_or_identity_t {{{
+template <typename _Tp>
+typename _Tp::value_type __value_type_or_identity_impl(int);
+template <typename _Tp>
+_Tp __value_type_or_identity_impl(float);
+template <typename _Tp>
+using __value_type_or_identity_t =
+ decltype(__value_type_or_identity_impl<_Tp>(int()));
+
+// }}}
+// __is_vectorizable {{{
+template <typename _Tp>
+struct __is_vectorizable : public std::is_arithmetic<_Tp>
+{
+};
+template <>
+struct __is_vectorizable<bool> : public false_type
+{
+};
+template <typename _Tp>
+inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
+// Deduces to a vectorizable type
+template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
+using _Vectorizable = _Tp;
+
+// }}}
+// _LoadStorePtr / __is_possible_loadstore_conversion {{{
+template <typename _Ptr, typename _ValueType>
+struct __is_possible_loadstore_conversion
+ : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {
+};
+template <> struct __is_possible_loadstore_conversion<bool, bool> : true_type {
+};
+// Deduces to a type allowed for load/store with the given value type.
+template <typename _Ptr, typename _ValueType,
+ typename = enable_if_t<__is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
+using _LoadStorePtr = _Ptr;
+
+// }}}
+// _SizeConstant{{{
+template <size_t _X> using _SizeConstant = integral_constant<size_t, _X>;
+// }}}
+// __is_bitmask{{{
+template <typename _Tp, typename = std::void_t<>>
+struct __is_bitmask : false_type
+{
+ constexpr __is_bitmask(const _Tp&) noexcept {}
+};
+template <typename _Tp>
+inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
+
+// the __mmaskXX case:
+template <typename _Tp>
+struct __is_bitmask<
+ _Tp,
+ std::void_t<decltype(std::declval<unsigned&>() = std::declval<_Tp>() & 1u)>>
+: true_type
+{
+ constexpr __is_bitmask(const _Tp&) noexcept {}
+};
+
+// }}}
+// __int_for_sizeof{{{
+template <size_t> struct __int_for_sizeof;
+template <> struct __int_for_sizeof<1> { using type = signed char; };
+template <> struct __int_for_sizeof<2> { using type = signed short; };
+template <> struct __int_for_sizeof<4> { using type = signed int; };
+template <> struct __int_for_sizeof<8> { using type = signed long long; };
+#ifdef __SIZEOF_INT128__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+template <> struct __int_for_sizeof<16> { using type = __int128; };
+#pragma GCC diagnostic pop
+#endif // __SIZEOF_INT128__
+template <typename _Tp>
+using __int_for_sizeof_t = typename __int_for_sizeof<sizeof(_Tp)>::type;
+template <size_t _N>
+using __int_with_sizeof_t = typename __int_for_sizeof<_N>::type;
+
+// }}}
+// __is_fixed_size_abi{{{
+template <typename _Tp>
+struct __is_fixed_size_abi : false_type
+{
+};
+template <int _N>
+struct __is_fixed_size_abi<simd_abi::fixed_size<_N>> : true_type
+{
+};
+
+template <typename _Tp>
+inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
+
+// }}}
+// constexpr feature detection{{{
+constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
+constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
+constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
+constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
+constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
+constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
+constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
+constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
+constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
+constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
+constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
+constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
+constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
+constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
+constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
+constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
+constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
+constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
+constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
+constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
+constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
+constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
+constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
+constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
+
+constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
+// }}}
+// __is_abi {{{
+template <template <int> class _Abi, int _Bytes>
+constexpr int __abi_bytes_impl(_Abi<_Bytes>*)
+{
+ return _Bytes;
+}
+template <typename _Tp>
+constexpr int __abi_bytes_impl(_Tp*)
+{
+ return -1;
+}
+template <typename _Abi>
+inline constexpr int
+ __abi_bytes_v = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
+
+template <typename _Abi0, typename _Abi1>
+constexpr bool __is_abi()
+{
+ return std::is_same_v<_Abi0, _Abi1>;
+}
+template <template <int> class _Abi0, typename _Abi1>
+constexpr bool __is_abi()
+{
+ return std::is_same_v<_Abi0<__abi_bytes_v<_Abi1>>, _Abi1>;
+}
+template <typename _Abi0, template <int> class _Abi1>
+constexpr bool __is_abi()
+{
+ return std::is_same_v<_Abi1<__abi_bytes_v<_Abi0>>, _Abi0>;
+}
+template <template <int> class _Abi0, template <int> class _Abi1>
+constexpr bool __is_abi()
+{
+ return std::is_same_v<_Abi0<0>, _Abi1<0>>;
+}
+
+// }}}
+// __is_combined_abi{{{
+template <template <int, typename> class _Combine, int _N, typename _Abi>
+constexpr bool __is_combined_abi(_Combine<_N, _Abi>*)
+{
+ return std::is_same_v<_Combine<_N, _Abi>, simd_abi::__combine<_N, _Abi>>;
+}
+template <typename _Abi>
+constexpr bool __is_combined_abi(_Abi*)
+{
+ return false;
+}
+
+template <typename _Abi>
+constexpr bool __is_combined_abi()
+{
+ return __is_combined_abi(static_cast<_Abi*>(nullptr));
+}
+
+// }}}
+// ^^^ ---- type traits ---- ^^^
+
+// __unused{{{
+template <typename _Tp>
+static constexpr void __unused(_Tp&&)
+{
+}
+
+// }}}
+// __assert_unreachable{{{
+template <typename _Tp>
+struct __assert_unreachable
+{
+ static_assert(!std::is_same_v<_Tp, _Tp>, "this should be unreachable");
+};
+
+// }}}
+// __size_or_zero_v {{{
+template <typename _Tp, typename _A, size_t _N = simd_size<_Tp, _A>::value>
+constexpr size_t __size_or_zero_dispatch(int)
+{
+ return _N;
+}
+template <typename _Tp, typename _A>
+constexpr size_t __size_or_zero_dispatch(float)
+{
+ return 0;
+}
+template <typename _Tp, typename _A>
+inline constexpr size_t __size_or_zero_v = __size_or_zero_dispatch<_Tp, _A>(0);
+
+// }}}
+// __promote_preserving_unsigned{{{
+// work around crazy semantics of unsigned integers of lower rank than int:
+// Before applying an operator the operands are promoted to int. In which case over- or
+// underflow is UB, even though the operand types were unsigned.
+template <typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr const _Tp&
+ __promote_preserving_unsigned(const _Tp& __x)
+{
+ return __x;
+}
+_GLIBCXX_SIMD_INTRINSIC constexpr unsigned int
+ __promote_preserving_unsigned(const unsigned char& __x)
+{
+ return __x;
+}
+_GLIBCXX_SIMD_INTRINSIC constexpr unsigned int
+ __promote_preserving_unsigned(const unsigned short& __x)
+{
+ return __x;
+}
+
+// }}}
+// _ExactBool{{{
+class _ExactBool
+{
+ const bool _M_data;
+
+public:
+ _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b)
+ : _M_data(__b)
+ {
+ }
+ _ExactBool(int) = delete;
+ _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
+};
+
+// }}}
+// __execute_on_index_sequence(_with_return){{{
+template <typename _F, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __execute_on_index_sequence(_F&& __f, std::index_sequence<_I...>)
+{
+ auto&& __x = {(__f(_SizeConstant<_I>()), 0)...};
+ __unused(__x);
+}
+
+template <typename _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __execute_on_index_sequence(_F&&, std::index_sequence<>)
+{
+}
+
+template <typename _R, typename _F, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R
+ __execute_on_index_sequence_with_return(_F&& __f, std::index_sequence<_I...>)
+{
+ return _R{__f(_SizeConstant<_I>())...};
+}
+
+// }}}
+// __execute_n_times{{{
+template <size_t _N, typename _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void __execute_n_times(_F&& __f)
+{
+ __execute_on_index_sequence(std::forward<_F>(__f),
+ std::make_index_sequence<_N>{});
+}
+
+// }}}
+// __generate_from_n_evaluations{{{
+template <size_t _N, typename _R, typename _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __generate_from_n_evaluations(_F&& __f)
+{
+ return __execute_on_index_sequence_with_return<_R>(
+ std::forward<_F>(__f), std::make_index_sequence<_N>{});
+}
+
+// }}}
+// __call_with_subscripts{{{
+template <size_t... _It, typename _Tp, typename _F>
+_GLIBCXX_SIMD_INTRINSIC auto
+ __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _F&& __fun)
+{
+ return __fun(__x[_It]...);
+}
+
+// }}}
+// __may_alias{{{
+/**\internal
+ * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an aliasing pointer. This
+ * adds the __may_alias attribute to _Tp (with compilers that support it).
+ */
+template <typename _Tp> using __may_alias [[__gnu__::__may_alias__]] = _Tp;
+
+// }}}
+// _UnsupportedBase {{{
+// simd and simd_mask base for unsupported <_Tp, _Abi>
+struct _UnsupportedBase
+{
+ _UnsupportedBase() = delete;
+ _UnsupportedBase(const _UnsupportedBase&) = delete;
+ _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
+ ~_UnsupportedBase() = delete;
+};
+
+// }}}
+// _InvalidTraits {{{
+/**
+ * \internal
+ * Defines the implementation of __a given <_Tp, _Abi>.
+ *
+ * Implementations must ensure that only valid <_Tp, _Abi> instantiations are possible.
+ * Static assertions in the type definition do not suffice. It is important that
+ * SFINAE works.
+ */
+struct _InvalidTraits
+{
+ using _IsValid = false_type;
+ using _Simd_base = _UnsupportedBase;
+ using _Mask_base = _UnsupportedBase;
+
+ static constexpr size_t _S_simd_align = 1;
+ struct _SimdImpl;
+ struct _SimdMember {};
+ struct _SimdCastType;
+
+ static constexpr size_t _S_mask_align = 1;
+ struct _MaskImpl;
+ struct _MaskMember {};
+ struct _MaskCastType;
+};
+// }}}
+// _SimdTraits {{{
+template <typename _Tp, typename _Abi, typename = std::void_t<>>
+struct _SimdTraits : _InvalidTraits
+{
+};
+
+// }}}
+// __get_impl_t/traits_t{{{
+template <typename _Tp>
+struct __get_impl;
+template <typename _Tp>
+using __get_impl_t = typename __get_impl<__remove_cvref_t<_Tp>>::_Impl;
+template <typename _Tp>
+using __get_traits_t = typename __get_impl<__remove_cvref_t<_Tp>>::_Traits;
+
+// }}}
+// __next_power_of_2{{{
+/**
+ * \internal
+ * Returns the next power of 2 larger than or equal to \p __x.
+ */
+constexpr std::size_t __next_power_of_2(std::size_t __x)
+{
+ return (__x & (__x - 1)) == 0 ? __x
+ : __next_power_of_2((__x | (__x >> 1)) + 1);
+}
+
+// }}}
+// __private_init, __bitset_init{{{
+/**
+ * \internal
+ * Tag used for private init constructor of simd and simd_mask
+ */
+inline constexpr struct _PrivateInit {} __private_init = {};
+inline constexpr struct _BitsetInit {} __bitset_init = {};
+
+// }}}
+// __is_narrowing_conversion<_From, _To>{{{
+template <typename _From,
+ typename _To,
+ bool = std::is_arithmetic<_From>::value,
+ bool = std::is_arithmetic<_To>::value>
+struct __is_narrowing_conversion;
+
+// ignore "warning C4018: '<': signed/unsigned mismatch" in the following trait. The implicit
+// conversions will do the right thing here.
+template <typename _From, typename _To>
+struct __is_narrowing_conversion<_From, _To, true, true>
+: public __bool_constant<(
+ std::numeric_limits<_From>::digits > std::numeric_limits<_To>::digits ||
+ std::numeric_limits<_From>::max() > std::numeric_limits<_To>::max() ||
+ std::numeric_limits<_From>::lowest() < std::numeric_limits<_To>::lowest() ||
+ (std::is_signed<_From>::value && std::is_unsigned<_To>::value))>
+{
+};
+
+template <typename _Tp>
+struct __is_narrowing_conversion<bool, _Tp, true, true> : public true_type
+{
+};
+template <>
+struct __is_narrowing_conversion<bool, bool, true, true> : public false_type
+{
+};
+template <typename _Tp>
+struct __is_narrowing_conversion<_Tp, _Tp, true, true> : public false_type
+{
+};
+
+template <typename _From, typename _To>
+struct __is_narrowing_conversion<_From, _To, false, true>
+: public negation<std::is_convertible<_From, _To>>
+{
+};
+
+// }}}
+// __converts_to_higher_integer_rank{{{
+template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
+struct __converts_to_higher_integer_rank : public true_type
+{
+};
+template <typename _From, typename _To>
+struct __converts_to_higher_integer_rank<_From, _To, false>
+: public std::is_same<decltype(std::declval<_From>() + std::declval<_To>()),
+ _To>
+{
+};
+
+// }}}
+// __is_aligned(_v){{{
+template <typename _Flag, size_t _Alignment>
+struct __is_aligned;
+template <size_t _Alignment>
+struct __is_aligned<vector_aligned_tag, _Alignment> : public true_type
+{
+};
+template <size_t _Alignment>
+struct __is_aligned<element_aligned_tag, _Alignment> : public false_type
+{
+};
+template <size_t _GivenAlignment, size_t _Alignment>
+struct __is_aligned<overaligned_tag<_GivenAlignment>, _Alignment>
+: public std::integral_constant<bool, (_GivenAlignment >= _Alignment)>
+{
+};
+template <typename _Flag, size_t _Alignment>
+inline constexpr bool __is_aligned_v = __is_aligned<_Flag, _Alignment>::value;
+
+// }}}
+// __data(simd/simd_mask) {{{
+template <typename _Tp, typename _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr const auto& __data(const simd<_Tp, _A>& __x);
+template <typename _Tp, typename _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto& __data(simd<_Tp, _A>& __x);
+
+template <typename _Tp, typename _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr const auto&
+ __data(const simd_mask<_Tp, _A>& __x);
+template <typename _Tp, typename _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto& __data(simd_mask<_Tp, _A>& __x);
+
+// }}}
+// _SimdConverter {{{
+template <typename _FromT, typename _FromA, typename _ToT, typename _ToA>
+struct _SimdConverter;
+
+template <typename _Tp, typename _A>
+struct _SimdConverter<_Tp, _A, _Tp, _A>
+{
+ template <typename _U>
+ _GLIBCXX_SIMD_INTRINSIC const _U& operator()(const _U& __x)
+ {
+ return __x;
+ }
+};
+
+// }}}
+// __to_value_type_or_member_type {{{
+template <typename _V>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto
+ __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
+{
+ return __data(__x);
+}
+
+template <typename _V>
+_GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
+ __to_value_type_or_member_type(const typename _V::value_type& __x)
+{
+ return __x;
+}
+
+// }}}
+// __bool_storage_member_type{{{
+template <size_t _Size>
+struct __bool_storage_member_type;
+
+template <size_t _Size>
+using __bool_storage_member_type_t =
+ typename __bool_storage_member_type<_Size>::type;
+
+// }}}
+// _SimdTuple {{{
+// why not std::tuple?
+// 1. std::tuple gives no guarantee about the storage order, but I require storage
+// equivalent to std::array<_Tp, _N>
+// 2. direct access to the element type (first template argument)
+// 3. enforces equal element type, only different _Abi types are allowed
+template <typename _Tp, typename... _Abis> struct _SimdTuple;
+
+//}}}
+// __fixed_size_storage_t {{{
+template <typename _Tp, int _N>
+struct __fixed_size_storage;
+
+template <typename _Tp, int _N>
+using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _N>::type;
+
+// }}}
+// _SimdWrapper fwd decl{{{
+template <typename _Tp, size_t _Size, typename = std::void_t<>>
+struct _SimdWrapper;
+
+template <typename _Tp>
+using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
+template <typename _Tp>
+using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
+template <typename _Tp>
+using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
+template <typename _Tp>
+using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
+
+// }}}
+// __bit_iteration{{{
+constexpr _UInt __popcount(_UInt __x) { return __builtin_popcount(__x); }
+constexpr _ULong __popcount(_ULong __x) { return __builtin_popcountl(__x); }
+constexpr _ULLong __popcount(_ULLong __x) { return __builtin_popcountll(__x); }
+
+constexpr _UInt __ctz(_UInt __x) { return __builtin_ctz(__x); }
+constexpr _ULong __ctz(_ULong __x) { return __builtin_ctzl(__x); }
+constexpr _ULLong __ctz(_ULLong __x) { return __builtin_ctzll(__x); }
+constexpr _UInt __clz(_UInt __x) { return __builtin_clz(__x); }
+constexpr _ULong __clz(_ULong __x) { return __builtin_clzl(__x); }
+constexpr _ULLong __clz(_ULLong __x) { return __builtin_clzll(__x); }
+
+template <typename _Tp, typename _F>
+void __bit_iteration(_Tp __mask, _F&& __f)
+{
+ static_assert(sizeof(_ULLong) >= sizeof(_Tp));
+ std::conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
+ if constexpr (std::is_convertible_v<_Tp, decltype(__k)>) {
+ __k = __mask;
+ } else {
+ __k = __mask.to_ullong();
+ }
+ switch (__popcount(__k)) {
+ default:
+ do {
+ __f(__ctz(__k));
+ __k &= (__k - 1);
+ } while (__k);
+ break;
+ /*case 3:
+ __f(__ctz(__k));
+ __k &= (__k - 1);
+ [[fallthrough]];*/
+ case 2:
+ __f(__ctz(__k));
+ [[fallthrough]];
+ case 1:
+ __f(__popcount(~decltype(__k)()) - 1 - __clz(__k));
+ [[fallthrough]];
+ case 0:
+ break;
+ }
+}
+
+//}}}
+// __firstbit{{{
+template <typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST auto __firstbit(_Tp __bits)
+{
+ static_assert(std::is_integral_v<_Tp>,
+ "__firstbit requires an integral argument");
+ if constexpr (sizeof(_Tp) <= sizeof(int))
+ {
+ return __builtin_ctz(__bits);
+ }
+ else if constexpr (alignof(_ULLong) == 8)
+ {
+ return __builtin_ctzll(__bits);
+ }
+ else
+ {
+ _UInt __lo = __bits;
+ return __lo == 0 ? 32 + __builtin_ctz(__bits >> 32) : __builtin_ctz(__lo);
+ }
+}
+
+// }}}
+// __lastbit{{{
+template <typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST auto __lastbit(_Tp __bits)
+{
+ static_assert(std::is_integral_v<_Tp>,
+ "__firstbit requires an integral argument");
+ if constexpr (sizeof(_Tp) <= sizeof(int))
+ {
+ return 31 - __builtin_clz(__bits);
+ }
+ else if constexpr (alignof(_ULLong) == 8)
+ {
+ return 63 - __builtin_clzll(__bits);
+ }
+ else
+ {
+ _UInt __lo = __bits;
+ _UInt __hi = __bits >> 32u;
+ return __hi == 0 ? 31 - __builtin_clz(__lo) : 63 - __builtin_clz(__hi);
+ }
+}
+
+// }}}
+// __convert_mask declaration {{{
+template <typename _To, typename _From>
+inline _To __convert_mask(_From __k);
+
+// }}}
+// __shift_left, __shift_right, __increment, __decrement {{{
+template <typename _Tp = void>
+struct __shift_left
+{
+ constexpr _Tp operator()(const _Tp& __a, const _Tp& __b) const
+ {
+ return __a << __b;
+ }
+};
+template <>
+struct __shift_left<void>
+{
+ template <typename _L, typename _R>
+ constexpr auto operator()(_L&& __a, _R&& __b) const
+ {
+ return std::forward<_L>(__a) << std::forward<_R>(__b);
+ }
+};
+template <typename _Tp = void>
+struct __shift_right
+{
+ constexpr _Tp operator()(const _Tp& __a, const _Tp& __b) const
+ {
+ return __a >> __b;
+ }
+};
+template <>
+struct __shift_right<void>
+{
+ template <typename _L, typename _R>
+ constexpr auto operator()(_L&& __a, _R&& __b) const
+ {
+ return std::forward<_L>(__a) >> std::forward<_R>(__b);
+ }
+};
+template <typename _Tp = void>
+struct __increment
+{
+ constexpr _Tp operator()(_Tp __a) const { return ++__a; }
+};
+template <>
+struct __increment<void>
+{
+ template <typename _Tp>
+ constexpr _Tp operator()(_Tp __a) const
+ {
+ return ++__a;
+ }
+};
+template <typename _Tp = void>
+struct __decrement
+{
+ constexpr _Tp operator()(_Tp __a) const { return --__a; }
+};
+template <>
+struct __decrement<void>
+{
+ template <typename _Tp>
+ constexpr _Tp operator()(_Tp __a) const
+ {
+ return --__a;
+ }
+};
+
+// }}}
+// _ValuePreserving(OrInt) {{{
+template <typename _From,
+ typename _To,
+ typename = enable_if_t<negation<
+ __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
+using _ValuePreserving = _From;
+
+template <typename _From,
+ typename _To,
+ typename _DecayedFrom = __remove_cvref_t<_From>,
+ typename = enable_if_t<conjunction<
+ is_convertible<_From, _To>,
+ disjunction<
+ is_same<_DecayedFrom, _To>,
+ is_same<_DecayedFrom, int>,
+ conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
+ negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
+using _ValuePreservingOrInt = _From;
+
+// }}}
+// __intrinsic_type {{{
+template <typename _Tp, size_t _Bytes, typename = std::void_t<>> struct __intrinsic_type;
+template <typename _Tp, size_t _Size>
+using __intrinsic_type_t = typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
+template <typename _Tp> using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
+template <typename _Tp> using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
+template <typename _Tp> using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
+template <typename _Tp> using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
+template <typename _Tp> using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
+template <typename _Tp> using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
+template <typename _Tp> using __intrinsic_type128_t = typename __intrinsic_type<_Tp, 128>::type;
+
+// }}}
+
+// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
+// __min_vector_size {{{
+static inline constexpr int __min_vector_size =
+#if _GLIBCXX_SIMD_HAVE_NEON
+ 8
+#else
+ 16
+#endif
+ ;
+
+// }}}
+// __vector_type {{{
+template <typename _Tp, size_t _N, typename = void> struct __vector_type_n {};
+
+// special case 1-element to be _Tp itself
+template <typename _Tp>
+struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>> {
+ using type = _Tp;
+};
+
+// else, use GNU-style builtin vector types
+template <typename _Tp, size_t _N>
+struct __vector_type_n<_Tp, _N, enable_if_t<__is_vectorizable_v<_Tp>>> {
+ static constexpr size_t _Bytes = __next_power_of_2(_N * sizeof(_Tp));
+ using type [[__gnu__::__vector_size__(_Bytes)]] = _Tp;
+};
+
+template <typename _Tp, size_t _Bytes>
+struct __vector_type : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {
+ static_assert(_Bytes % sizeof(_Tp) == 0);
+};
+
+template <typename _Tp, size_t _Size>
+using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
+template <typename _Tp> using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
+template <typename _Tp> using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
+template <typename _Tp> using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
+template <typename _Tp> using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
+template <typename _Tp> using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
+template <typename _Tp> using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
+template <typename _Tp> using __vector_type128_t = typename __vector_type<_Tp, 128>::type;
+
+// }}}
+// __is_vector_type {{{
+template <typename _Tp, typename = std::void_t<>> struct __is_vector_type : false_type {};
+template <typename _Tp>
+struct __is_vector_type<
+ _Tp,
+ std::void_t<typename __vector_type<decltype(std::declval<_Tp>()[0]), sizeof(_Tp)>::type>>
+ : std::is_same<
+ _Tp, typename __vector_type<decltype(std::declval<_Tp>()[0]), sizeof(_Tp)>::type> {
+};
+
+template <typename _Tp>
+inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
+
+// }}}
+// _VectorTraits{{{
+template <typename _Tp, typename = std::void_t<>>
+struct _VectorTraits;
+template <typename _Tp>
+struct _VectorTraits<_Tp, std::void_t<enable_if_t<__is_vector_type_v<_Tp>>>>
+{
+ using type = _Tp;
+ using value_type = decltype(std::declval<_Tp>()[0]);
+ static constexpr int _S_width = sizeof(_Tp) / sizeof(value_type);
+ template <typename _U, int _W = _S_width>
+ static constexpr bool __is = std::is_same_v<value_type, _U>&& _W == _S_width;
+};
+template <typename _Tp, size_t _N>
+struct _VectorTraits<_SimdWrapper<_Tp, _N>,
+ std::void_t<__vector_type_t<_Tp, _N>>>
+{
+ using type = __vector_type_t<_Tp, _N>;
+ using value_type = _Tp;
+ static constexpr int _S_width = _N;
+ template <typename _U, int _W = _S_width>
+ static constexpr bool __is = std::is_same_v<value_type, _U>&& _W == _S_width;
+};
+
+// }}}
+// __vector_bitcast{{{
+template <typename _To, typename _From, typename _FromVT = _VectorTraits<_From>>
+_GLIBCXX_SIMD_INTRINSIC constexpr
+ typename __vector_type<_To, sizeof(_From)>::type
+ __vector_bitcast(_From __x)
+{
+ return reinterpret_cast<typename __vector_type<_To, sizeof(_From)>::type>(
+ __x);
+}
+template <typename _To, typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr
+ typename __vector_type<_To, sizeof(_SimdWrapper<_Tp, _N>)>::type
+ __vector_bitcast(const _SimdWrapper<_Tp, _N>& __x)
+{
+ return reinterpret_cast<
+ typename __vector_type<_To, sizeof(_SimdWrapper<_Tp, _N>)>::type>(
+ __x._M_data);
+}
+
+// }}}
+// __convert_x86 declarations {{{
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
+template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_To __convert_x86(_Tp);
+
+template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_To __convert_x86(_Tp, _Tp);
+
+template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_To __convert_x86(_Tp, _Tp, _Tp, _Tp);
+
+template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
+#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
+
+//}}}
+// __vector_convert {{{
+// implementation requires an index sequence
+template <typename _To, typename _From, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, index_sequence<_I...>)
+{
+ using _Tp = typename _VectorTraits<_To>::value_type;
+ return _To{static_cast<_Tp>(__a[_I])...};
+}
+
+template <typename _To, typename _From, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b,
+ index_sequence<_I...>)
+{
+ using _Tp = typename _VectorTraits<_To>::value_type;
+ return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...};
+}
+
+template <typename _To, typename _From, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c,
+ index_sequence<_I...>)
+{
+ using _Tp = typename _VectorTraits<_To>::value_type;
+ return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
+ static_cast<_Tp>(__c[_I])...};
+}
+
+template <typename _To, typename _From, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c,
+ _From __d, index_sequence<_I...>)
+{
+ using _Tp = typename _VectorTraits<_To>::value_type;
+ return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
+ static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...};
+}
+
+template <typename _To, typename _From, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From __a, _From __b, _From __c,
+ _From __d, _From __e, _From __f,
+ _From __g, _From __h,
+ index_sequence<_I...>)
+{
+ using _Tp = typename _VectorTraits<_To>::value_type;
+ return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
+ static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
+ static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
+ static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...};
+}
+
+// Defer actual conversion to the overload that takes an index sequence. Note that this
+// function adds zeros or drops values off the end if you don't ensure matching width.
+template <typename _To, typename... _From, typename _ToT = _VectorTraits<_To>,
+ typename _FromT = _VectorTraits<__first_of_pack_t<_From...>>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From... __xs)
+{
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
+ return __convert_x86<_To>(__xs...);
+#else
+ return __vector_convert<_To>(__xs...,
+ make_index_sequence<std::min(_ToT::_S_width, _FromT::_S_width)>());
+#endif
+}
+
+// This overload takes a vectorizable type _To and produces a return type that matches the
+// width.
+template <typename _To, typename... _From, typename = enable_if_t<__is_vectorizable_v<_To>>,
+ typename _FromT = _VectorTraits<__first_of_pack_t<_From...>>, typename = int>
+_GLIBCXX_SIMD_INTRINSIC constexpr _To __vector_convert(_From... __xs)
+{
+ return __vector_convert<__vector_type_t<_To, _FromT::_S_width>>(__xs...);
+}
+
+// }}}
+// __to_intrin {{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
+ typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_width>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __to_intrin(_Tp __x)
+{
+ return reinterpret_cast<_R>(__x);
+}
+template <typename _Tp, size_t _N, typename _R = __intrinsic_type_t<_Tp, _N>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __to_intrin(_SimdWrapper<_Tp, _N> __x)
+{
+ return reinterpret_cast<_R>(__x._M_data);
+}
+
+// }}}
+// __make_vector{{{
+template <typename _Tp, typename... _Args>
+_GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
+ __make_vector(_Args&&... args)
+{
+ return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(args)...};
+}
+
+// }}}
+// __vector_broadcast{{{
+template <size_t _N, typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _N> __vector_broadcast(_Tp __x)
+{
+ if constexpr (_N == 2) {
+ return __vector_type_t<_Tp, 2>{__x, __x};
+ } else if constexpr (_N == 4) {
+ return __vector_type_t<_Tp, 4>{__x, __x, __x, __x};
+ } else if constexpr (_N == 8) {
+ return __vector_type_t<_Tp, 8>{__x, __x, __x, __x, __x, __x, __x, __x};
+ } else if constexpr (_N == 16) {
+ return __vector_type_t<_Tp, 16>{__x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x};
+ } else if constexpr (_N == 32) {
+ return __vector_type_t<_Tp, 32>{__x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x};
+ } else if constexpr (_N == 64) {
+ return __vector_type_t<_Tp, 64>{
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x};
+ } else if constexpr (_N == 128) {
+ return __vector_type_t<_Tp, 128>{
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x, __x,
+ __x, __x, __x, __x, __x, __x, __x, __x};
+ }
+}
+
+// }}}
+// __generate_vector{{{
+template <typename _Tp, size_t _N, typename _G, size_t... _I>
+_GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _N>
+ __generate_vector_impl(_G&& __gen, std::index_sequence<_I...>)
+{
+ return __vector_type_t<_Tp, _N>{
+ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
+}
+
+template <typename _V, typename _VVT = _VectorTraits<_V>, typename _G>
+_GLIBCXX_SIMD_INTRINSIC constexpr _V __generate_vector(_G&& __gen)
+{
+ return __generate_vector_impl<typename _VVT::value_type, _VVT::_S_width>(
+ std::forward<_G>(__gen), std::make_index_sequence<_VVT::_S_width>());
+}
+
+template <typename _Tp, size_t _N, typename _G>
+_GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _N>
+ __generate_vector(_G&& __gen)
+{
+ return __generate_vector_impl<_Tp, _N>(std::forward<_G>(__gen),
+ std::make_index_sequence<_N>());
+}
+
+// }}}
+// __vector_load{{{
+template <typename _Tp, size_t _N, size_t _M = _N * sizeof(_Tp), typename _F>
+__vector_type_t<_Tp, _N> __vector_load(const void* __p, _F)
+{
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ using _U = std::conditional_t<
+ (std::is_integral_v<_Tp> || _M < 4), long long,
+ std::conditional_t<(std::is_same_v<_Tp, double> || _M < 8), float, _Tp>>;
+ using _V = __vector_type_t<_U, _N * sizeof(_Tp) / sizeof(_U)>;
+#else // _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ using _V = __vector_type_t<_Tp, _N>;
+#endif // _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ _V __r;
+ static_assert(_M <= sizeof(_V));
+ if constexpr (std::is_same_v<_F, element_aligned_tag>) {}
+ else if constexpr (std::is_same_v<_F, vector_aligned_tag>)
+ __p = __builtin_assume_aligned(__p, alignof(__vector_type_t<_Tp, _N>));
+ else
+ __p = __builtin_assume_aligned(__p, _F::_S_alignment);
+ std::memcpy(&__r, __p, _M);
+ return reinterpret_cast<__vector_type_t<_Tp, _N>>(__r);
+}
+
+// }}}
+// __vector_load16 {{{
+template <typename _Tp, size_t _M = 16, typename _F>
+__vector_type16_t<_Tp> __vector_load16(const void* __p, _F __f)
+{
+ return __vector_load<_Tp, 16 / sizeof(_Tp), _M>(__p, __f);
+}
+
+// }}}
+// __vector_store{{{
+template <size_t _M = 0,
+ typename _B,
+ typename _BVT = _VectorTraits<_B>,
+ typename _F>
+void __vector_store(const _B __v, void* __p, _F)
+{
+ using _Tp = typename _BVT::value_type;
+ constexpr size_t _N = _BVT::_S_width;
+ constexpr size_t _Bytes = _M == 0 ? _N * sizeof(_Tp) : _M;
+ static_assert(_Bytes <= sizeof(__v));
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ using _U = std::conditional_t<
+ (std::is_integral_v<_Tp> || _Bytes < 4), long long,
+ std::conditional_t<(std::is_same_v<_Tp, double> || _Bytes < 8), float,
+ _Tp>>;
+ const auto __vv = __vector_bitcast<_U>(__v);
+#else // _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ const __vector_type_t<_Tp, _N> __vv = __v;
+#endif // _GLIBCXX_SIMD_WORKAROUND_XXX_2
+ if constexpr (std::is_same_v<_F, vector_aligned_tag>)
+ __p = __builtin_assume_aligned(__p, alignof(__vector_type_t<_Tp, _N>));
+ else if constexpr (!std::is_same_v<_F, element_aligned_tag>)
+ __p = __builtin_assume_aligned(__p, _F::_S_alignment);
+ if constexpr ((_Bytes & (_Bytes - 1)) != 0)
+ {
+ constexpr size_t _MoreBytes = __next_power_of_2(_Bytes);
+ alignas(_MoreBytes) char __tmp[_MoreBytes];
+ std::memcpy(__tmp, &__vv, _MoreBytes);
+ std::memcpy(__p, __tmp, _Bytes);
+ }
+ else
+ std::memcpy(__p, &__vv, _Bytes);
+}
+
+// }}}
+// __allbits{{{
+template <typename _V>
+inline constexpr _V __allbits = reinterpret_cast<_V>(
+ ~__intrinsic_type_t<_LLong, sizeof(_V) / sizeof(_LLong)>());
+
+// }}}
+// __xor{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __xor(_Tp __a, typename _TVT::type __b) noexcept
+{
+#if _GLIBCXX_SIMD_X86INTRIN
+ if constexpr (_TVT::template __is<float, 4> && __have_sse)
+ return _mm_xor_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 2> && __have_sse2)
+ return _mm_xor_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 8> && __have_avx)
+ return _mm256_xor_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 4> && __have_avx)
+ return _mm256_xor_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 16> && __have_avx512dq)
+ return _mm512_xor_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 8> && __have_avx512dq)
+ return _mm512_xor_pd(__a, __b);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ return reinterpret_cast<typename _TVT::type>(
+ __vector_bitcast<unsigned>(__a) ^ __vector_bitcast<unsigned>(__b));
+}
+
+// }}}
+// __or{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __or(_Tp __a, typename _TVT::type __b) noexcept
+{
+#if _GLIBCXX_SIMD_X86INTRIN
+ if constexpr (_TVT::template __is<float, 4> && __have_sse)
+ return _mm_or_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 2> && __have_sse2)
+ return _mm_or_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 8> && __have_avx)
+ return _mm256_or_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 4> && __have_avx)
+ return _mm256_or_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 16> && __have_avx512dq)
+ return _mm512_or_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 8> && __have_avx512dq)
+ return _mm512_or_pd(__a, __b);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ return reinterpret_cast<typename _TVT::type>(
+ __vector_bitcast<unsigned>(__a) | __vector_bitcast<unsigned>(__b));
+}
+
+// }}}
+// __and{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>, typename... _Dummy>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp
+ __and(_Tp __a, typename _TVT::type __b, _Dummy...) noexcept
+{
+ static_assert(sizeof...(_Dummy) == 0);
+#if _GLIBCXX_SIMD_X86INTRIN
+ if constexpr (_TVT::template __is<float, 4> && __have_sse)
+ return _mm_and_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 2> && __have_sse2)
+ return _mm_and_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 8> && __have_avx)
+ return _mm256_and_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 4> && __have_avx)
+ return _mm256_and_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 16> && __have_avx512dq)
+ return _mm512_and_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 8> && __have_avx512dq)
+ return _mm512_and_pd(__a, __b);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ return reinterpret_cast<typename _TVT::type>(
+ __vector_bitcast<unsigned>(__a) & __vector_bitcast<unsigned>(__b));
+}
+
+template <typename _Tp, typename = decltype(_Tp() & _Tp())>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __and(_Tp __a, _Tp __b) noexcept
+{
+ return __a & __b;
+}
+
+// }}}
+// __andnot{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __andnot(_Tp __a, typename _TVT::type __b) noexcept
+{
+#if _GLIBCXX_SIMD_X86INTRIN
+ if constexpr (_TVT::template __is<float, 4> && __have_sse)
+ return _mm_andnot_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 2> && __have_sse2)
+ return _mm_andnot_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 8> && __have_avx)
+ return _mm256_andnot_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 4> && __have_avx)
+ return _mm256_andnot_pd(__a, __b);
+ else if constexpr (_TVT::template __is<float, 16> && __have_avx512dq)
+ return _mm512_andnot_ps(__a, __b);
+ else if constexpr (_TVT::template __is<double, 8> && __have_avx512dq)
+ return _mm512_andnot_pd(__a, __b);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ return reinterpret_cast<typename _TVT::type>(
+ ~__vector_bitcast<unsigned>(__a) & __vector_bitcast<unsigned>(__b));
+}
+
+// }}}
+// __not{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __not(_Tp __a) noexcept
+{
+ return reinterpret_cast<_Tp>(~__vector_bitcast<unsigned>(__a));
+}
+
+// }}}
+// __concat{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
+ typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_width * 2>>
+constexpr _R __concat(_Tp a_, _Tp b_) {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
+ using _W = std::conditional_t<std::is_floating_point_v<typename _TVT::value_type>,
+ double, long long>;
+ constexpr int input_width = sizeof(_Tp) / sizeof(_W);
+ const auto __a = __vector_bitcast<_W>(a_);
+ const auto __b = __vector_bitcast<_W>(b_);
+ using _U = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
+#else
+ constexpr int input_width = _TVT::_S_width;
+ const _Tp &__a = a_;
+ const _Tp &__b = b_;
+ using _U = _R;
+#endif
+ if constexpr(input_width == 2) {
+ return reinterpret_cast<_R>(_U{__a[0], __a[1], __b[0], __b[1]});
+ } else if constexpr (input_width == 4) {
+ return reinterpret_cast<_R>(_U{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
+ } else if constexpr (input_width == 8) {
+ return reinterpret_cast<_R>(_U{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7], __b[0],
+ __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
+ } else if constexpr (input_width == 16) {
+ return reinterpret_cast<_R>(
+ _U{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7], __a[8], __a[9], __a[10],
+ __a[11], __a[12], __a[13], __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4], __b[5],
+ __b[6], __b[7], __b[8], __b[9], __b[10], __b[11], __b[12], __b[13], __b[14], __b[15]});
+ } else if constexpr (input_width == 32) {
+ return reinterpret_cast<_R>(
+ _U{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7], __a[8], __a[9], __a[10],
+ __a[11], __a[12], __a[13], __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20], __a[21],
+ __a[22], __a[23], __a[24], __a[25], __a[26], __a[27], __a[28], __a[29], __a[30], __a[31], __b[0],
+ __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
+ __b[12], __b[13], __b[14], __b[15], __b[16], __b[17], __b[18], __b[19], __b[20], __b[21], __b[22],
+ __b[23], __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30], __b[31]});
+ }
+}
+
+// }}}
+// __zero_extend {{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC auto __zero_extend(_Tp __x)
+{
+ using value_type = typename _TVT::value_type;
+ constexpr size_t _N = _TVT::_S_width;
+ struct {
+ _Tp __x;
+ operator __vector_type_t<value_type, _N * 2>()
+ {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
+ if constexpr (__have_avx && _TVT::template __is<float, 4>) {
+ return __vector_bitcast<value_type>(_mm256_insertf128_ps(__m256(), __x, 0));
+ } else if constexpr (__have_avx && _TVT::template __is<double, 2>) {
+ return __vector_bitcast<value_type>(_mm256_insertf128_pd(__m256d(), __x, 0));
+ } else if constexpr (__have_avx2 && sizeof(__x) == 16) {
+ return __vector_bitcast<value_type>(_mm256_insertf128_si256(__m256i(), __x, 0));
+ } else if constexpr (__have_avx512f && _TVT::template __is<float, 8>) {
+ if constexpr (__have_avx512dq) {
+ return __vector_bitcast<value_type>(_mm512_insertf32x8(__m512(), __x, 0));
+ } else {
+ return reinterpret_cast<__m512>(
+ _mm512_insertf64x4(__m512d(), reinterpret_cast<__m256d>(__x), 0));
+ }
+ } else if constexpr (__have_avx512f && _TVT::template __is<double, 4>) {
+ return __vector_bitcast<value_type>(_mm512_insertf64x4(__m512d(), __x, 0));
+ } else if constexpr (__have_avx512f && sizeof(__x) == 32) {
+ return __vector_bitcast<value_type>(_mm512_inserti64x4(__m512i(), __x, 0));
+ }
+#endif
+ return __concat(__x, _Tp());
+ }
+ operator __vector_type_t<value_type, _N * 4>()
+ {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
+ if constexpr (__have_avx && _TVT::template __is<float, 4>) {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85480
+ asm("vmovaps %0, %0" : "+__x"(__x));
+ return __vector_bitcast<value_type>(_mm512_castps128_ps512(__x));
+#else
+ return __vector_bitcast<value_type>(_mm512_insertf32x4(__m512(), __x, 0));
+#endif
+ } else if constexpr (__have_avx && _TVT::template __is<double, 2>) {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85480
+ asm("vmovapd %0, %0" : "+__x"(__x));
+ return __vector_bitcast<value_type>(_mm512_castpd128_pd512(__x));
+#else
+ return __vector_bitcast<value_type>(_mm512_insertf64x2(__m512d(), __x, 0));
+#endif
+ } else if constexpr (__have_avx512f && sizeof(__x) == 16) {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85480
+ asm("vmovadq %0, %0" : "+__x"(__x));
+ return __vector_bitcast<value_type>(_mm512_castsi128_si512(__x));
+#else
+ return __vector_bitcast<value_type>(_mm512_inserti32x4(__m512i(), __x, 0));
+#endif
+ }
+#endif
+ return __concat(__concat(__x, _Tp()), __vector_type_t<value_type, _N * 2>());
+ }
+ operator __vector_type_t<value_type, _N * 8>()
+ {
+ return __concat(operator __vector_type_t<value_type, _N * 4>(),
+ __vector_type_t<value_type, _N * 4>());
+ }
+ operator __vector_type_t<value_type, _N * 16>()
+ {
+ return __concat(operator __vector_type_t<value_type, _N * 8>(),
+ __vector_type_t<value_type, _N * 8>());
+ }
+ } __r{__x};
+ return __r;
+}
+
+// }}}
+// __extract<_N, By>{{{
+template <int _Offset,
+ int _SplitBy,
+ typename _Tp,
+ typename _TVT = _VectorTraits<_Tp>,
+ typename _R = __vector_type_t<typename _TVT::value_type,
+ _TVT::_S_width / _SplitBy>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __extract(_Tp __in)
+{
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
+ using _W =
+ std::conditional_t<std::is_floating_point_v<typename _TVT::value_type>,
+ double, long long>;
+ constexpr int return_width = sizeof(_R) / sizeof(_W);
+ using _U = __vector_type_t<_W, return_width>;
+ const auto __x = __vector_bitcast<_W>(__in);
+#else
+ constexpr int return_width = _TVT::_S_width / _SplitBy;
+ using _U = _R;
+ const __vector_type_t<typename _TVT::value_type, _TVT::_S_width>& __x =
+ __in; // only needed for _Tp = _SimdWrapper<value_type, _N>
+#endif
+ constexpr int _O = _Offset * return_width;
+ if constexpr (return_width == 2)
+ {
+ return reinterpret_cast<_R>(_U{__x[_O + 0], __x[_O + 1]});
+ }
+ else if constexpr (return_width == 4)
+ {
+ return reinterpret_cast<_R>(
+ _U{__x[_O + 0], __x[_O + 1], __x[_O + 2], __x[_O + 3]});
+ }
+ else if constexpr (return_width == 8)
+ {
+ return reinterpret_cast<_R>(_U{__x[_O + 0], __x[_O + 1], __x[_O + 2],
+ __x[_O + 3], __x[_O + 4], __x[_O + 5],
+ __x[_O + 6], __x[_O + 7]});
+ }
+ else if constexpr (return_width == 16)
+ {
+ return reinterpret_cast<_R>(
+ _U{__x[_O + 0], __x[_O + 1], __x[_O + 2], __x[_O + 3], __x[_O + 4],
+ __x[_O + 5], __x[_O + 6], __x[_O + 7], __x[_O + 8], __x[_O + 9],
+ __x[_O + 10], __x[_O + 11], __x[_O + 12], __x[_O + 13], __x[_O + 14],
+ __x[_O + 15]});
+ }
+ else if constexpr (return_width == 32)
+ {
+ return reinterpret_cast<_R>(
+ _U{__x[_O + 0], __x[_O + 1], __x[_O + 2], __x[_O + 3], __x[_O + 4],
+ __x[_O + 5], __x[_O + 6], __x[_O + 7], __x[_O + 8], __x[_O + 9],
+ __x[_O + 10], __x[_O + 11], __x[_O + 12], __x[_O + 13], __x[_O + 14],
+ __x[_O + 15], __x[_O + 16], __x[_O + 17], __x[_O + 18], __x[_O + 19],
+ __x[_O + 20], __x[_O + 21], __x[_O + 22], __x[_O + 23], __x[_O + 24],
+ __x[_O + 25], __x[_O + 26], __x[_O + 27], __x[_O + 28], __x[_O + 29],
+ __x[_O + 30], __x[_O + 31]});
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+// }}}
+// __lo/__hi64[z]{{{
+template <
+ typename _Tp,
+ typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __lo64(_Tp __x)
+{
+ _R __r{};
+ __builtin_memcpy(&__r, &__x, 8);
+ return __r;
+}
+
+template <
+ typename _Tp,
+ typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __hi64(_Tp __x)
+{
+ static_assert(sizeof(_Tp) == 16);
+ _R __r{};
+ __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
+ return __r;
+}
+
+template <
+ typename _Tp,
+ typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _R __hi64z(_Tp __x)
+{
+ _R __r{};
+ if constexpr (sizeof(_Tp) == 16)
+ __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
+ return __r;
+}
+
+// }}}
+// __lo/__hi128{{{
+template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr auto __lo128(_Tp __x)
+{
+ return __extract<0, sizeof(_Tp) / 16>(__x);
+}
+template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr auto __hi128(_Tp __x)
+{
+ static_assert(sizeof(__x) == 32);
+ return __extract<1, 2>(__x);
+}
+
+// }}}
+// __lo/__hi256{{{
+template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr auto __lo256(_Tp __x)
+{
+ static_assert(sizeof(__x) == 64);
+ return __extract<0, 2>(__x);
+}
+template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr auto __hi256(_Tp __x)
+{
+ static_assert(sizeof(__x) == 64);
+ return __extract<1, 2>(__x);
+}
+
+// }}}
+// __intrin_bitcast{{{
+template <typename _To, typename _From> _GLIBCXX_SIMD_INTRINSIC constexpr _To __intrin_bitcast(_From __v)
+{
+ static_assert(__is_vector_type_v<_From> && __is_vector_type_v<_To>);
+ if constexpr (sizeof(_To) == sizeof(_From)) {
+ return reinterpret_cast<_To>(__v);
+ } else if constexpr (sizeof(_From) > sizeof(_To)) {
+ return reinterpret_cast<const _To &>(__v);
+#if _GLIBCXX_SIMD_X86INTRIN
+ } else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32) {
+ return reinterpret_cast<_To>(_mm256_castps128_ps256(
+ reinterpret_cast<__intrinsic_type_t<float, sizeof(_From) / sizeof(float)>>(__v)));
+ } else if constexpr (__have_avx512f && sizeof(_From) == 16 && sizeof(_To) == 64) {
+ return reinterpret_cast<_To>(_mm512_castps128_ps512(
+ reinterpret_cast<__intrinsic_type_t<float, sizeof(_From) / sizeof(float)>>(__v)));
+ } else if constexpr (__have_avx512f && sizeof(_From) == 32 && sizeof(_To) == 64) {
+ return reinterpret_cast<_To>(_mm512_castps256_ps512(
+ reinterpret_cast<__intrinsic_type_t<float, sizeof(_From) / sizeof(float)>>(__v)));
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ } else {
+ __assert_unreachable<_To>();
+ }
+}
+
+// }}}
+// __auto_bitcast{{{
+template <typename _Tp> struct auto_cast_t {
+ static_assert(__is_vector_type_v<_Tp>);
+ const _Tp __x;
+ template <typename _U> _GLIBCXX_SIMD_INTRINSIC constexpr operator _U() const
+ {
+ return __intrin_bitcast<_U>(__x);
+ }
+};
+template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr auto_cast_t<_Tp> __auto_bitcast(const _Tp &__x)
+{
+ return {__x};
+}
+template <typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto_cast_t<typename _SimdWrapper<_Tp, _N>::_BuiltinType> __auto_bitcast(
+ const _SimdWrapper<_Tp, _N> &__x)
+{
+ return {__x._M_data};
+}
+
+// }}}
+// __vector_to_bitset{{{
+_GLIBCXX_SIMD_INTRINSIC constexpr std::bitset<1> __vector_to_bitset(bool __x) { return unsigned(__x); }
+
+template <typename _Tp, typename = enable_if_t<__is_bitmask_v<_Tp> && __have_avx512f>>
+_GLIBCXX_SIMD_INTRINSIC constexpr std::bitset<8 * sizeof(_Tp)> __vector_to_bitset(_Tp __x)
+{
+ if constexpr (std::is_integral_v<_Tp>) {
+ return __x;
+ } else {
+ return __x._M_data;
+ }
+}
+
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC std::bitset<_TVT::_S_width> __vector_to_bitset(_Tp __x)
+{
+ constexpr int __w = sizeof(typename _TVT::value_type);
+
+#if _GLIBCXX_SIMD_HAVE_NEON // {{{
+ using _I = __int_with_sizeof_t<__w>;
+ if constexpr (__have_neon && sizeof(_Tp) == 16)
+ {
+ auto __asint = __vector_bitcast<_I>(__x);
+ [[maybe_unused]] constexpr auto __zero = decltype(__asint)();
+ if constexpr (__w == 1)
+ {
+ __asint &=
+ __make_vector<_I>(0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x1,
+ 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
+ return __vector_bitcast<_UShort>(vpaddq_s8(
+ vpaddq_s8(vpaddq_s8(__asint, __zero), __zero), __zero))[0];
+ }
+ else if constexpr (__w == 2)
+ {
+ __asint &=
+ __make_vector<_I>(0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
+#ifdef __aarch64__
+ return vpaddq_s16(vpaddq_s16(vpaddq_s16(__asint, __zero), __zero),
+ __zero)[0];
+#else
+ return vpadd_s16(vpadd_s16(
+ vpadd_s16(__lo64(__asint), __hi64(__asint)), __zero) __zero)[0];
+#endif
+ }
+ else if constexpr (__w == 4)
+ {
+ __asint &= __make_vector<_I>(0x1, 0x2, 0x4, 0x8);
+ return vpaddq_s32(vpaddq_s32(__asint, __zero), __zero)[0];
+ }
+ else if constexpr (__w == 8)
+ {
+ return (__asint[0] & 1) | (__asint[1] & 2);
+ }
+ }
+ else if constexpr (__have_neon && sizeof(_Tp) == 8)
+ {
+ auto __asint = __vector_bitcast<_I>(__x);
+ [[maybe_unused]] constexpr auto __zero = decltype(__asint)();
+ if constexpr (__w == 1)
+ {
+ __asint &=
+ __make_vector<_I>(0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
+ return vpadd_s8(vpadd_s8(vpadd_s8(__asint, __zero), __zero),
+ __zero)[0];
+ }
+ else if constexpr (__w == 2)
+ {
+ __asint &=
+ __make_vector<_I>(0x1, 0x2, 0x4, 0x8);
+ return vpadd_s16(vpadd_s16(__asint, __zero), __zero)[0];
+ }
+ else if constexpr (__w == 4)
+ {
+ __asint &= __make_vector<_I>(0x1, 0x2);
+ return vpadd_s32(__asint, __zero)[0];
+ }
+ else if constexpr (__w == 8)
+ {
+ return !!__asint[0];
+ }
+ }
+ else
+ __assert_unreachable<_Tp>();
+#endif // _GLIBCXX_SIMD_HAVE_NEON }}}
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ constexpr bool __is_sse = __have_sse && sizeof(_Tp) == 16;
+ constexpr bool __is_avx = __have_avx && sizeof(_Tp) == 32;
+ auto __intrin = __to_intrin(__x);
+ if constexpr (__is_sse && __w == 1)
+ {
+ return _mm_movemask_epi8(__intrin);
+ }
+ else if constexpr (__is_sse && __w == 2)
+ {
+ if constexpr (__have_avx512bw_vl)
+ {
+ return _mm_cmplt_epi16_mask(__intrin, __m128i());
+ }
+ else
+ {
+ return _mm_movemask_epi8(_mm_packs_epi16(__intrin, __m128i()));
+ }
+ }
+ else if constexpr (__is_sse && __w == 4)
+ {
+ if constexpr (__have_avx512vl && std::is_integral_v<_Tp>)
+ {
+ return _mm_cmplt_epi32_mask(__intrin, __m128i());
+ }
+ else
+ {
+ return _mm_movemask_ps(__vector_bitcast<float>(__x));
+ }
+ }
+ else if constexpr (__is_sse && __w == 8)
+ {
+ if constexpr (__have_avx512vl && std::is_integral_v<_Tp>)
+ {
+ return _mm_cmplt_epi64_mask(__intrin, __m128i());
+ }
+ else
+ {
+ return _mm_movemask_pd(__vector_bitcast<double>(__x));
+ }
+ }
+ else if constexpr (__is_avx && __w == 1)
+ {
+ return _mm256_movemask_epi8(__intrin);
+ }
+ else if constexpr (__is_avx && __w == 2)
+ {
+ if constexpr (__have_avx512bw_vl)
+ {
+ return _mm256_cmplt_epi16_mask(__intrin, __m256i());
+ }
+ else
+ {
+ return _mm_movemask_epi8(_mm_packs_epi16(__extract<0, 2>(__intrin),
+ __extract<1, 2>(__intrin)));
+ }
+ }
+ else if constexpr (__is_avx && __w == 4)
+ {
+ if constexpr (__have_avx512vl && std::is_integral_v<_Tp>)
+ {
+ return _mm256_cmplt_epi32_mask(__intrin, __m256i());
+ }
+ else
+ {
+ return _mm256_movemask_ps(__vector_bitcast<float>(__x));
+ }
+ }
+ else if constexpr (__is_avx && __w == 8)
+ {
+ if constexpr (__have_avx512vl && std::is_integral_v<_Tp>)
+ {
+ return _mm256_cmplt_epi64_mask(__intrin, __m256i());
+ }
+ else
+ {
+ return _mm256_movemask_pd(__vector_bitcast<double>(__x));
+ }
+ }
+ else
+ __assert_unreachable<_Tp>();
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ std::bitset<_TVT::_S_width> __r;
+ for (int __i = 0; __i < _TVT::_S_width; ++__i)
+ {
+ __r[__i] = !(__x[__i] == 0);
+ }
+ return __r;
+}
+
+// }}}
+// __blend{{{
+template <typename _K, typename _V0, typename _V1>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST auto
+ __blend(_K mask, _V0 at0, _V1 at1)
+{
+ using _V = _V0;
+ if constexpr (!std::is_same_v<_V0, _V1>)
+ {
+ static_assert(sizeof(_V0) == sizeof(_V1));
+ if constexpr (__is_vector_type_v<_V0> && !__is_vector_type_v<_V1>)
+ {
+ return __blend(mask, at0, reinterpret_cast<_V0>(at1._M_data));
+ }
+ else if constexpr (!__is_vector_type_v<_V0> && __is_vector_type_v<_V1>)
+ {
+ return __blend(mask, reinterpret_cast<_V1>(at0._M_data), at1);
+ }
+ else
+ {
+ __assert_unreachable<_K>();
+ }
+ }
+ else if constexpr (__is_bitmask_v<_V> && __is_bitmask_v<_K>)
+ {
+ static_assert(sizeof(_K) == sizeof(_V0) && sizeof(_V0) == sizeof(_V1));
+ return (mask & at1) | (~mask & at0);
+ }
+ else if constexpr (!__is_vector_type_v<_V>)
+ {
+ return __blend(mask, at0._M_data, at1._M_data);
+ }
+ else if constexpr (__is_bitmask_v<_K>) // blend via bitmask (AVX512)
+ {
+ using _Tp = typename _VectorTraits<_V>::value_type;
+ if constexpr (sizeof(_V) == 16 && __have_avx512bw_vl && sizeof(_Tp) <= 2)
+ {
+ if constexpr (sizeof(_Tp) == 1)
+ {
+ return __intrin_bitcast<_V>(
+ _mm_mask_mov_epi8(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (sizeof(_Tp) == 2)
+ {
+ return __intrin_bitcast<_V>(
+ _mm_mask_mov_epi16(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ }
+ else if constexpr (sizeof(_V) == 16 && __have_avx512vl && sizeof(_Tp) > 2)
+ {
+ if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(
+ _mm_mask_mov_epi32(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(
+ _mm_mask_mov_epi64(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm_mask_mov_ps(at0, mask, at1));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm_mask_mov_pd(at0, mask, at1));
+ }
+ }
+ else if constexpr (sizeof(_V) == 16 && __have_avx512f && sizeof(_Tp) > 2)
+ {
+ if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(__lo128(_mm512_mask_mov_epi32(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(__lo128(_mm512_mask_mov_epi64(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(__lo128(_mm512_mask_mov_ps(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(__lo128(_mm512_mask_mov_pd(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ }
+ else if constexpr (sizeof(_V) == 32 && __have_avx512bw_vl &&
+ sizeof(_Tp) <= 2)
+ {
+ if constexpr (sizeof(_Tp) == 1)
+ {
+ return __intrin_bitcast<_V>(
+ _mm256_mask_mov_epi8(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (sizeof(_Tp) == 2)
+ {
+ return __intrin_bitcast<_V>(_mm256_mask_mov_epi16(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ }
+ else if constexpr (sizeof(_V) == 32 && __have_avx512vl && sizeof(_Tp) > 2)
+ {
+ if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm256_mask_mov_epi32(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm256_mask_mov_epi64(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm256_mask_mov_ps(at0, mask, at1));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm256_mask_mov_pd(at0, mask, at1));
+ }
+ }
+ else if constexpr (sizeof(_V) == 32 && __have_avx512f && sizeof(_Tp) > 2)
+ {
+ if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(__lo256(_mm512_mask_mov_epi32(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(__lo256(_mm512_mask_mov_epi64(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(__lo256(_mm512_mask_mov_ps(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(__lo256(_mm512_mask_mov_pd(
+ __auto_bitcast(at0), mask, __auto_bitcast(at1))));
+ }
+ }
+ else if constexpr (sizeof(_V) == 64 && __have_avx512bw &&
+ sizeof(_Tp) <= 2)
+ {
+ if constexpr (sizeof(_Tp) == 1)
+ {
+ return __intrin_bitcast<_V>(
+ _mm512_mask_mov_epi8(__to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (sizeof(_Tp) == 2)
+ {
+ return __intrin_bitcast<_V>(_mm512_mask_mov_epi16(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ }
+ else if constexpr (sizeof(_V) == 64 && __have_avx512f && sizeof(_Tp) > 2)
+ {
+ if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm512_mask_mov_epi32(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_integral_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm512_mask_mov_epi64(
+ __to_intrin(at0), mask, __to_intrin(at1)));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm512_mask_mov_ps(at0, mask, at1));
+ }
+ else if constexpr (std::is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm512_mask_mov_pd(at0, mask, at1));
+ }
+ }
+ else
+ {
+ __assert_unreachable<_K>();
+ }
+ }
+ else if constexpr (((__have_avx512f && sizeof(_V) == 64) ||
+ __have_avx512vl) &&
+ (sizeof(typename _VectorTraits<_V>::value_type) >= 4 ||
+ __have_avx512bw))
+ { // convert mask to bitmask
+ return __blend(
+ __convert_mask<__bool_storage_member_type_t<_VectorTraits<_V>::_S_width>>(mask), at0,
+ at1);
+ }
+ else
+ {
+ const _V __k = __auto_bitcast(mask);
+ using _Tp = typename _VectorTraits<_V>::value_type;
+ if constexpr (sizeof(_V) == 16 && __have_sse4_1)
+ {
+ if constexpr (std::is_integral_v<_Tp>)
+ {
+ return __intrin_bitcast<_V>(_mm_blendv_epi8(
+ __to_intrin(at0), __to_intrin(at1), __to_intrin(__k)));
+ }
+ else if constexpr (sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm_blendv_ps(at0, at1, __k));
+ }
+ else if constexpr (sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm_blendv_pd(at0, at1, __k));
+ }
+ }
+ else if constexpr (sizeof(_V) == 32)
+ {
+ if constexpr (std::is_integral_v<_Tp>)
+ {
+ return __intrin_bitcast<_V>(_mm256_blendv_epi8(
+ __to_intrin(at0), __to_intrin(at1), __to_intrin(__k)));
+ }
+ else if constexpr (sizeof(_Tp) == 4)
+ {
+ return __intrin_bitcast<_V>(_mm256_blendv_ps(at0, at1, __k));
+ }
+ else if constexpr (sizeof(_Tp) == 8)
+ {
+ return __intrin_bitcast<_V>(_mm256_blendv_pd(at0, at1, __k));
+ }
+ }
+ else
+ {
+ return __or(__andnot(__k, at0), __and(__k, at1));
+ }
+ }
+}
+
+// }}}
+// __interleave(128)(__lo|__hi) {{{
+template <class _A,
+ class _B,
+ class _Tp = std::common_type_t<_A, _B>,
+ class _Trait = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp
+ __interleave_lo(const _A& _a, const _B& _b)
+{
+ const _Tp __a(_a);
+ const _Tp __b(_b);
+ if constexpr (_Trait::_S_width == 2)
+ return _Tp{__a[0], __b[0]};
+ else if constexpr (_Trait::_S_width == 4)
+ return _Tp{__a[0], __b[0], __a[1], __b[1]};
+ else if constexpr (_Trait::_S_width == 8)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3]};
+ else if constexpr (_Trait::_S_width == 16)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
+ __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]};
+ else if constexpr (_Trait::_S_width == 32)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
+ __b[3], __a[4], __b[4], __a[5], __b[5], __a[6], __b[6],
+ __a[7], __b[7], __a[8], __b[8], __a[9], __b[9], __a[10],
+ __b[10], __a[11], __b[11], __a[12], __b[12], __a[13], __b[13],
+ __a[14], __b[14], __a[15], __b[15]};
+ else if constexpr (_Trait::_S_width == 64)
+ return _Tp{
+ __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
+ __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7],
+ __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
+ __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15],
+ __a[16], __b[16], __a[17], __b[17], __a[18], __b[18], __a[19], __b[19],
+ __a[20], __b[20], __a[21], __b[21], __a[22], __b[22], __a[23], __b[23],
+ __a[24], __b[24], __a[25], __b[25], __a[26], __b[26], __a[27], __b[27],
+ __a[28], __b[28], __a[29], __b[29], __a[30], __b[30], __a[31], __b[31]};
+ else
+ __assert_unreachable<_Tp>();
+}
+
+template <class _A,
+ class _B,
+ class _Tp = std::common_type_t<_A, _B>,
+ class _Trait = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp
+ __interleave_hi(const _A& _a, const _B& _b)
+{
+ const _Tp __a(_a);
+ const _Tp __b(_b);
+ if constexpr (_Trait::_S_width == 2)
+ return _Tp{__a[1], __b[1]};
+ else if constexpr (_Trait::_S_width == 4)
+ return _Tp{__a[2], __b[2], __a[3], __b[3]};
+ else if constexpr (_Trait::_S_width == 8)
+ return _Tp{__a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]};
+ else if constexpr (_Trait::_S_width == 16)
+ return _Tp{__a[8], __b[8], __a[9], __b[9], __a[10], __b[10],
+ __a[11], __b[11], __a[12], __b[12], __a[13], __b[13],
+ __a[14], __b[14], __a[15], __b[15]};
+ else if constexpr (_Trait::_S_width == 32)
+ return _Tp{__a[16], __b[16], __a[17], __b[17], __a[18], __b[18], __a[19],
+ __b[19], __a[20], __b[20], __a[21], __b[21], __a[22], __b[22],
+ __a[23], __b[23], __a[24], __b[24], __a[25], __b[25], __a[26],
+ __b[26], __a[27], __b[27], __a[28], __b[28], __a[29], __b[29],
+ __a[30], __b[30], __a[31], __b[31]};
+ else if constexpr (_Trait::_S_width == 64)
+ return _Tp{
+ __a[32], __b[32], __a[33], __b[33], __a[34], __b[34], __a[35], __b[35],
+ __a[36], __b[36], __a[37], __b[37], __a[38], __b[38], __a[39], __b[39],
+ __a[40], __b[40], __a[41], __b[41], __a[42], __b[42], __a[43], __b[43],
+ __a[44], __b[44], __a[45], __b[45], __a[46], __b[46], __a[47], __b[47],
+ __a[48], __b[48], __a[49], __b[49], __a[50], __b[50], __a[51], __b[51],
+ __a[52], __b[52], __a[53], __b[53], __a[54], __b[54], __a[55], __b[55],
+ __a[56], __b[56], __a[57], __b[57], __a[58], __b[58], __a[59], __b[59],
+ __a[60], __b[60], __a[61], __b[61], __a[62], __b[62], __a[63], __b[63]};
+ else
+ __assert_unreachable<_Tp>();
+}
+
+template <class _A,
+ class _B,
+ class _Tp = std::common_type_t<_A, _B>,
+ class _Trait = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp
+ __interleave128_lo(const _A& _a, const _B& _b)
+{
+ const _Tp __a(_a);
+ const _Tp __b(_b);
+ if constexpr (sizeof(_Tp) == 16)
+ return __interleave_lo(__a, __b);
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 4)
+ return _Tp{__a[0], __b[0], __a[2], __b[2]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 8)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[4], __b[4], __a[5], __b[5]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 16)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2],
+ __a[3], __b[3], __a[8], __b[8], __a[9], __b[9],
+ __a[10], __b[10], __a[11], __b[11]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 32)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
+ __b[3], __a[4], __b[4], __a[5], __b[5], __a[6], __b[6],
+ __a[7], __b[7], __a[16], __b[16], __a[17], __b[17], __a[18],
+ __b[18], __a[19], __b[19], __a[20], __b[20], __a[21], __b[21],
+ __a[22], __b[22], __a[23], __b[23]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 64)
+ return _Tp{
+ __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
+ __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7],
+ __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
+ __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15],
+ __a[32], __b[32], __a[33], __b[33], __a[34], __b[34], __a[35], __b[35],
+ __a[36], __b[36], __a[37], __b[37], __a[38], __b[38], __a[39], __b[39],
+ __a[40], __b[40], __a[41], __b[41], __a[42], __b[42], __a[43], __b[43],
+ __a[44], __b[44], __a[45], __b[45], __a[46], __b[46], __a[47], __b[47]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 8)
+ return _Tp{__a[0], __b[0], __a[2], __b[2], __a[4], __b[4], __a[6], __b[6]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 16)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[4], __b[4],
+ __a[5], __b[5], __a[8], __b[8], __a[9], __b[9],
+ __a[12], __b[12], __a[13], __b[13]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 32)
+ return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
+ __b[3], __a[8], __b[8], __a[9], __b[9], __a[10], __b[10],
+ __a[11], __b[11], __a[16], __b[16], __a[17], __b[17], __a[18],
+ __b[18], __a[19], __b[19], __a[24], __b[24], __a[25], __b[25],
+ __a[26], __b[26], __a[27], __b[27]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 64)
+ return _Tp{
+ __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
+ __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7],
+ __a[16], __b[16], __a[17], __b[17], __a[18], __b[18], __a[19], __b[19],
+ __a[20], __b[20], __a[21], __b[21], __a[22], __b[22], __a[23], __b[23],
+ __a[32], __b[32], __a[33], __b[33], __a[34], __b[34], __a[35], __b[35],
+ __a[36], __b[36], __a[37], __b[37], __a[38], __b[38], __a[39], __b[39],
+ __a[48], __b[48], __a[49], __b[49], __a[50], __b[50], __a[51], __b[51],
+ __a[52], __b[52], __a[53], __b[53], __a[54], __b[54], __a[55], __b[55]};
+ else
+ __assert_unreachable<_Tp>();
+}
+
+template <class _A,
+ class _B,
+ class _Tp = std::common_type_t<_A, _B>,
+ class _Trait = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp
+ __interleave128_hi(const _A& _a, const _B& _b)
+{
+ const _Tp __a(_a);
+ const _Tp __b(_b);
+ if constexpr (sizeof(_Tp) == 16)
+ return __interleave_hi(__a, __b);
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 4)
+ return _Tp{__a[1], __b[1], __a[3], __b[3]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 8)
+ return _Tp{__a[2], __b[2], __a[3], __b[3], __a[6], __b[6], __a[7], __b[7]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 16)
+ return _Tp{__a[4], __b[4], __a[5], __b[5], __a[6], __b[6],
+ __a[7], __b[7], __a[12], __b[12], __a[13], __b[13],
+ __a[14], __b[14], __a[15], __b[15]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 32)
+ return _Tp{__a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11],
+ __b[11], __a[12], __b[12], __a[13], __b[13], __a[14], __b[14],
+ __a[15], __b[15], __a[24], __b[24], __a[25], __b[25], __a[26],
+ __b[26], __a[27], __b[27], __a[28], __b[28], __a[29], __b[29],
+ __a[30], __b[30], __a[31], __b[31]};
+ else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_width == 64)
+ return _Tp{
+ __a[16], __b[16], __a[17], __b[17], __a[18], __b[18], __a[19], __b[19],
+ __a[20], __b[20], __a[21], __b[21], __a[22], __b[22], __a[23], __b[23],
+ __a[24], __b[24], __a[25], __b[25], __a[26], __b[26], __a[27], __b[27],
+ __a[28], __b[28], __a[29], __b[29], __a[30], __b[30], __a[31], __b[31],
+ __a[48], __b[48], __a[49], __b[49], __a[50], __b[50], __a[51], __b[51],
+ __a[52], __b[52], __a[53], __b[53], __a[54], __b[54], __a[55], __b[55],
+ __a[56], __b[56], __a[57], __b[57], __a[58], __b[58], __a[59], __b[59],
+ __a[60], __b[60], __a[61], __b[61], __a[62], __b[62], __a[63], __b[63]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 8)
+ return _Tp{__a[1], __b[1], __a[3], __b[3], __a[5], __b[5], __a[7], __b[7]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 16)
+ return _Tp{__a[2], __b[2], __a[3], __b[3], __a[6], __b[6],
+ __a[7], __b[7], __a[10], __b[10], __a[11], __b[11],
+ __a[14], __b[14], __a[15], __b[15]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 32)
+ return _Tp{__a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7],
+ __b[7], __a[12], __b[12], __a[13], __b[13], __a[14], __b[14],
+ __a[15], __b[15], __a[20], __b[20], __a[21], __b[21], __a[22],
+ __b[22], __a[23], __b[23], __a[28], __b[28], __a[29], __b[29],
+ __a[30], __b[30], __a[31], __b[31]};
+ else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_width == 64)
+ return _Tp{
+ __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
+ __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15],
+ __a[24], __b[24], __a[25], __b[25], __a[26], __b[26], __a[27], __b[27],
+ __a[28], __b[28], __a[29], __b[29], __a[30], __b[30], __a[31], __b[31],
+ __a[40], __b[40], __a[41], __b[41], __a[42], __b[42], __a[43], __b[43],
+ __a[44], __b[44], __a[45], __b[45], __a[46], __b[46], __a[47], __b[47],
+ __a[56], __b[56], __a[57], __b[57], __a[58], __b[58], __a[59], __b[59],
+ __a[60], __b[60], __a[61], __b[61], __a[62], __b[62], __a[63], __b[63]};
+ else
+ __assert_unreachable<_Tp>();
+}
+// }}}
+// __vector_permute<Indices...>{{{
+// Index == -1 requests zeroing of the output element
+/*constexpr int __shuf_imm8(int __a, int __b,int __c,int __d)
+{
+ return (__a == -1 ? 0 : __a) * 0x01 + (__b == -1 ? 1 : __b) * 0x04 +
+ (__c == -1 ? 2 : __c) * 0x10 + (__d == -1 ? 3 : __d) * 0x40;
+}*/
+template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_Tp __vector_permute(_Tp __x)
+{
+ static_assert(sizeof...(_Indices) == _TVT::_S_width);
+ return __make_vector<typename _TVT::value_type>(
+ (_Indices == -1 ? 0 : __x[_Indices])...);
+}
+
+// }}}
+// __vector_shuffle<Indices...>{{{
+// Index == -1 requests zeroing of the output element
+template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_Tp __vector_shuffle(_Tp __x, _Tp __y)
+{
+ return _Tp{(_Indices == -1 ? 0
+ : _Indices < _TVT::_S_width
+ ? __x[_Indices]
+ : __y[_Indices - _TVT::_S_width])...};
+}
+
+// }}}
+// __is_zero{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC constexpr bool __is_zero(_Tp __a)
+{
+ const auto __b = __vector_bitcast<_LLong>(__a);
+ if constexpr (sizeof(__b) / sizeof(_LLong) == 2) {
+ return __b[0] == 0 && __b[1] == 0;
+ } else if constexpr (sizeof(__b) / sizeof(_LLong) == 4) {
+ return __b[0] == 0 && __b[1] == 0 && __b[2] == 0 && __b[3] == 0;
+ } else if constexpr (sizeof(__b) / sizeof(_LLong) == 8) {
+ return __b[0] == 0 && __b[1] == 0 && __b[2] == 0 && __b[3] == 0 && __b[4] == 0 &&
+ __b[5] == 0 && __b[6] == 0 && __b[7] == 0;
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+// }}}
+// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
+
+// __testz{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST int __testz(_Tp __a, _Tp __b)
+{
+ if constexpr (__have_avx) {
+ if constexpr (sizeof(_Tp) == 32 && _TVT::template __is<float>) {
+ return _mm256_testz_ps(__a, __b);
+ } else if constexpr (sizeof(_Tp) == 32 && _TVT::template __is<double>) {
+ return _mm256_testz_pd(__a, __b);
+ } else if constexpr (sizeof(_Tp) == 32) {
+ return _mm256_testz_si256(__vector_bitcast<_LLong>(__a),
+ __vector_bitcast<_LLong>(__b));
+ } else if constexpr(_TVT::template __is<float, 4>) {
+ return _mm_testz_ps(__a, __b);
+ } else if constexpr(_TVT::template __is<double, 2>) {
+ return _mm_testz_pd(__a, __b);
+ } else {
+ static_assert(sizeof(_Tp) == 16);
+ return _mm_testz_si128(__vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>(__b));
+ }
+ } else if constexpr (__have_sse4_1) {
+ return _mm_testz_si128(__vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>(__b));
+ } else if constexpr (__have_sse && _TVT::template __is<float, 4>) {
+ return _mm_movemask_ps(__and(__a, __b)) == 0;
+ } else if constexpr (__have_sse2 && _TVT::template __is<double, 2>) {
+ return _mm_movemask_pd(__and(__a, __b)) == 0;
+ } else if constexpr (__have_sse2) {
+ return _mm_movemask_epi8(__a & __b) == 0;
+ } else {
+ return __is_zero(__and(__a, __b));
+ }
+}
+
+// }}}
+// __testnzc{{{
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST int __testnzc(_Tp __a, _Tp __b)
+{
+ if constexpr (__have_avx) {
+ if constexpr (sizeof(_Tp) == 32 && _TVT::template __is<float>) {
+ return _mm256_testnzc_ps(__a, __b);
+ } else if constexpr (sizeof(_Tp) == 32 && _TVT::template __is<double>) {
+ return _mm256_testnzc_pd(__a, __b);
+ } else if constexpr (sizeof(_Tp) == 32) {
+ return _mm256_testnzc_si256(__vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>(__b));
+ } else if constexpr(_TVT::template __is<float, 4>) {
+ return _mm_testnzc_ps(__a, __b);
+ } else if constexpr(_TVT::template __is<double, 2>) {
+ return _mm_testnzc_pd(__a, __b);
+ } else {
+ static_assert(sizeof(_Tp) == 16);
+ return _mm_testnzc_si128(__vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>(__b));
+ }
+ } else if constexpr (__have_sse4_1) {
+ return _mm_testnzc_si128(__vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>(__b));
+ } else if constexpr (__have_sse && _TVT::template __is<float, 4>) {
+ return _mm_movemask_ps(__and(__a, __b)) == 0 && _mm_movemask_ps(__andnot(__a, __b)) == 0;
+ } else if constexpr (__have_sse2 && _TVT::template __is<double, 2>) {
+ return _mm_movemask_pd(__and(__a, __b)) == 0 && _mm_movemask_pd(__andnot(__a, __b)) == 0;
+ } else if constexpr (__have_sse2) {
+ return _mm_movemask_epi8(__and(__a, __b)) == 0 &&
+ _mm_movemask_epi8(__andnot(__a, __b)) == 0;
+ } else {
+ return !(__is_zero(__vector_bitcast<_LLong>(__and(__a, __b))) ||
+ __is_zero(__vector_bitcast<_LLong>(__andnot(__a, __b))));
+ }
+}
+
+// }}}
+// __movemask{{{
+#if _GLIBCXX_SIMD_X86INTRIN
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST int __movemask(_Tp __a)
+{
+ if constexpr (__have_sse && _TVT::template __is<float, 4>) {
+ return _mm_movemask_ps(__a);
+ } else if constexpr (__have_avx && _TVT::template __is<float, 8>) {
+ return _mm256_movemask_ps(__a);
+ } else if constexpr (__have_sse2 && _TVT::template __is<double, 2>) {
+ return _mm_movemask_pd(__a);
+ } else if constexpr (__have_avx && _TVT::template __is<double, 4>) {
+ return _mm256_movemask_pd(__a);
+ } else if constexpr (__have_sse2 && sizeof(_Tp) == 16) {
+ return _mm_movemask_epi8(__a);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 32) {
+ return _mm256_movemask_epi8(__a);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST int movemask_epi16(_Tp __a)
+{
+ static_assert(std::is_integral_v<typename _TVT::value_type>);
+ if constexpr(__have_avx512bw_vl && sizeof(_Tp) == 16) {
+ return _mm_cmp_epi16_mask(__a, __m128i(), _MM_CMPINT_NE);
+ } else if constexpr(__have_avx512bw_vl && sizeof(_Tp) == 32) {
+ return _mm256_cmp_epi16_mask(__a, __m256i(), _MM_CMPINT_NE);
+ } else if constexpr(sizeof(_Tp) == 32) {
+ return _mm_movemask_epi8(_mm_packs_epi16(__lo128(__a), __hi128(__a)));
+ } else {
+ static_assert(sizeof(_Tp) == 16);
+ return _mm_movemask_epi8(_mm_packs_epi16(__a, __m128i()));
+ }
+}
+#endif // _GLIBCXX_SIMD_X86INTRIN
+
+// }}}
+#if _GLIBCXX_SIMD_HAVE_SSE_ABI
+// __bool_storage_member_type{{{
+#if _GLIBCXX_SIMD_HAVE_AVX512F
+template <size_t _Size>
+struct __bool_storage_member_type
+{
+ static_assert((_Size & (_Size - 1)) == 0,
+ "This trait may only be used for non-power-of-2 sizes. "
+ "Power-of-2 sizes must be specialized.");
+ using type =
+ typename __bool_storage_member_type<__next_power_of_2(_Size)>::type;
+};
+template <> struct __bool_storage_member_type< 2> { using type = __mmask8 ; };
+template <> struct __bool_storage_member_type< 4> { using type = __mmask8 ; };
+template <> struct __bool_storage_member_type< 8> { using type = __mmask8 ; };
+template <> struct __bool_storage_member_type<16> { using type = __mmask16; };
+template <> struct __bool_storage_member_type<32> { using type = __mmask32; };
+template <> struct __bool_storage_member_type<64> { using type = __mmask64; };
+#endif // _GLIBCXX_SIMD_HAVE_AVX512F
+
+// }}}
+// __intrinsic_type (x86){{{
+// the following excludes bool via __is_vectorizable
+#if _GLIBCXX_SIMD_HAVE_AVX512F
+template <>
+struct __intrinsic_type<double, 64, void>
+{
+ using type [[__gnu__::__vector_size__(64)]] = double;
+};
+template <>
+struct __intrinsic_type<float, 64, void>
+{
+ using type [[__gnu__::__vector_size__(64)]] = float;
+};
+template <typename _Tp>
+struct __intrinsic_type<_Tp, 64, enable_if_t<is_integral_v<_Tp>>>
+{
+ using type [[__gnu__::__vector_size__(64)]] = long long int;
+};
+#endif // _GLIBCXX_SIMD_HAVE_AVX512F
+
+#if _GLIBCXX_SIMD_HAVE_AVX
+template <>
+struct __intrinsic_type<double, 32, void>
+{
+ using type [[__gnu__::__vector_size__(32)]] = double;
+};
+template <>
+struct __intrinsic_type<float, 32, void>
+{
+ using type [[__gnu__::__vector_size__(32)]] = float;
+};
+template <typename _Tp>
+struct __intrinsic_type<_Tp, 32, enable_if_t<is_integral_v<_Tp>>>
+{
+ using type [[__gnu__::__vector_size__(32)]] = long long int;
+};
+#endif // _GLIBCXX_SIMD_HAVE_AVX
+
+#if _GLIBCXX_SIMD_HAVE_SSE
+template <>
+struct __intrinsic_type<float, 16, void>
+{
+ using type [[__gnu__::__vector_size__(16)]] = float;
+};
+template <>
+struct __intrinsic_type<float, 8, void>
+{
+ using type [[__gnu__::__vector_size__(16)]] = float;
+};
+template <>
+struct __intrinsic_type<float, 4, void>
+{
+ using type [[__gnu__::__vector_size__(16)]] = float;
+};
+template <>
+struct __intrinsic_type<double, 16, void>
+{
+ using type [[__gnu__::__vector_size__(16)]] = double;
+};
+template <>
+struct __intrinsic_type<double, 8, void>
+{
+ using type [[__gnu__::__vector_size__(16)]] = double;
+};
+template <typename _Tp, size_t _Bytes>
+struct __intrinsic_type<
+ _Tp,
+ _Bytes,
+ enable_if_t<(_Bytes <= 16 && _Bytes >= sizeof(_Tp) &&
+ ((_Bytes - 1) & _Bytes) == 0 && is_integral_v<_Tp>)>>
+{
+ using type [[__gnu__::__vector_size__(16)]] = long long int;
+};
+#endif // _GLIBCXX_SIMD_HAVE_SSE
+
+// }}}
+// _(Sse|Avx|Avx512)(Simd|Mask)Member{{{
+template <typename _Tp> using _SseSimdMember = _SimdWrapper16<_Tp>;
+template <typename _Tp> using _SseMaskMember = _SimdWrapper16<_Tp>;
+
+template <typename _Tp> using _AvxSimdMember = _SimdWrapper32<_Tp>;
+template <typename _Tp> using _AvxMaskMember = _SimdWrapper32<_Tp>;
+
+template <typename _Tp> using _Avx512SimdMember = _SimdWrapper64<_Tp>;
+template <typename _Tp> using _Avx512MaskMember = _SimdWrapper<bool, 64 / sizeof(_Tp)>;
+template <size_t _N> using _Avx512MaskMemberN = _SimdWrapper<bool, _N>;
+
+//}}}
+#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
+// __intrinsic_type (ARM){{{
+#if _GLIBCXX_SIMD_HAVE_NEON
+#define _GLIBCXX_SIMD_NEON_INTRIN(_Tp) \
+ template <> \
+ struct __intrinsic_type<__remove_cvref_t<decltype(_Tp()[0])>, sizeof(_Tp), \
+ void> \
+ { \
+ using type = _Tp; \
+ }
+_GLIBCXX_SIMD_NEON_INTRIN(int8x8_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int8x16_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int16x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int16x8_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int32x2_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int32x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int64x1_t);
+_GLIBCXX_SIMD_NEON_INTRIN(int64x2_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint8x8_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint8x16_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint16x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint16x8_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint32x2_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint32x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint64x1_t);
+_GLIBCXX_SIMD_NEON_INTRIN(uint64x2_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float16x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float16x8_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float32x2_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float32x4_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float64x1_t);
+_GLIBCXX_SIMD_NEON_INTRIN(float64x2_t);
+#undef _GLIBCXX_SIMD_NEON_INTRIN
+
+template <typename _Tp, size_t _Bytes>
+struct __intrinsic_type<_Tp,
+ _Bytes,
+ enable_if_t<(_Bytes <= 16 && _Bytes >= sizeof(_Tp) &&
+ ((_Bytes - 1) & _Bytes) == 0)>>
+{
+ static constexpr int _VBytes = _Bytes <= 8 ? 8 : 16;
+ using _Tmp =
+ conditional_t<sizeof(_Tp) == 1, __remove_cvref_t<decltype(int8x16_t()[0])>,
+ conditional_t<sizeof(_Tp) == 2, short,
+ conditional_t<sizeof(_Tp) == 4, int,
+ conditional_t<sizeof(_Tp) == 8, __remove_cvref_t<decltype(int64x2_t()[0])>,
+ void>>>>;
+ using _U = conditional_t<is_floating_point_v<_Tp>, _Tp,
+ conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Tmp>, _Tmp>>;
+ using type = typename __intrinsic_type<_U, _VBytes>::type;
+};
+#endif // _GLIBCXX_SIMD_HAVE_NEON
+
+// }}}
+// _SimdWrapper<bool>{{{1
+template <size_t _Width>
+struct _SimdWrapper<
+ bool,
+ _Width,
+ std::void_t<typename __bool_storage_member_type<_Width>::type>>
+{
+ using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
+ using value_type = bool;
+ static constexpr size_t _S_width = _Width;
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
+ : _M_data(__k){};
+
+ _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_PURE
+ operator const _BuiltinType&() const
+ {
+ return _M_data;
+ }
+ _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_PURE operator _BuiltinType&()
+ {
+ return _M_data;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const { return _M_data; }
+
+ _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_PURE value_type
+ operator[](size_t __i) const
+ {
+ return _M_data & (_BuiltinType(1) << __i);
+ }
+ _GLIBCXX_SIMD_INTRINSIC void set(size_t __i, value_type __x)
+ {
+ if (__x)
+ _M_data |= (_BuiltinType(1) << __i);
+ else
+ _M_data &= ~(_BuiltinType(1) << __i);
+ }
+
+ _BuiltinType _M_data;
+};
+
+// _SimdWrapperBase{{{1
+template <
+ typename _Tp,
+ size_t _Width,
+ typename _RegisterType = __vector_type_t<_Tp, _Width>,
+ bool = std::disjunction_v<
+ std::is_same<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>,
+ std::is_same<_RegisterType, __intrinsic_type_t<_Tp, _Width>>>>
+struct _SimdWrapperBase;
+
+template <typename _Tp, size_t _Width, typename _RegisterType>
+struct _SimdWrapperBase<_Tp, _Width, _RegisterType, true>
+{
+ _RegisterType _M_data;
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(
+ __vector_type_t<_Tp, _Width> __x)
+ : _M_data(reinterpret_cast<_RegisterType>(__x))
+ {
+ }
+};
+
+template <typename _Tp, size_t _Width, typename _RegisterType>
+struct _SimdWrapperBase<_Tp, _Width, _RegisterType, false>
+{
+ using _IntrinType = __intrinsic_type_t<_Tp, _Width>;
+ _RegisterType _M_data;
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(
+ __vector_type_t<_Tp, _Width> __x)
+ : _M_data(reinterpret_cast<_RegisterType>(__x))
+ {
+ }
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_IntrinType __x)
+ : _M_data(reinterpret_cast<_RegisterType>(__x))
+ {
+ }
+};
+
+// }}}
+// _SimdWrapper{{{
+template <typename _Tp, size_t _Width>
+struct _SimdWrapper<
+ _Tp,
+ _Width,
+ std::void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
+: _SimdWrapperBase<_Tp, _Width>
+{
+ static_assert(__is_vectorizable_v<_Tp>);
+ static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
+ using _BuiltinType = __vector_type_t<_Tp, _Width>;
+ using value_type = _Tp;
+ static constexpr size_t _S_width = _Width;
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
+ template <
+ typename _U,
+ typename = decltype(_SimdWrapperBase<_Tp, _Width>(std::declval<_U>()))>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_U&& __x)
+ : _SimdWrapperBase<_Tp, _Width>(std::forward<_U>(__x))
+ {
+ }
+ // I want to use ctor inheritance, but it breaks always_inline. Having a
+ // function that does a single movaps is stupid.
+ // using _SimdWrapperBase<_Tp, _Width>::_SimdWrapperBase;
+ using _SimdWrapperBase<_Tp, _Width>::_M_data;
+
+ template <
+ typename... _As,
+ typename = enable_if_t<((std::is_same_v<simd_abi::scalar, _As> && ...) &&
+ sizeof...(_As) <= _Width)>>
+ _GLIBCXX_SIMD_INTRINSIC constexpr operator _SimdTuple<_Tp, _As...>() const
+ {
+ const auto& dd = _M_data; // workaround for GCC7 ICE
+ return __generate_from_n_evaluations<sizeof...(_As),
+ _SimdTuple<_Tp, _As...>>(
+ [&](auto __i) { return dd[int(__i)]; });
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
+ {
+ return _M_data;
+ }
+ _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&() { return _M_data; }
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
+ {
+ return _M_data[__i];
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC void set(size_t __i, _Tp __x) { _M_data[__i] = __x; }
+};
+
+// }}}
+// _ToWrapper {{{
+template <typename _Tp>
+class _ToWrapper
+{
+ _Tp _M_data;
+
+public:
+ constexpr _ToWrapper(_Tp __x)
+ : _M_data(__x)
+ {
+ }
+
+ template <size_t _N>
+ constexpr operator _SimdWrapper<bool, _N>() const
+ {
+ static_assert(std::is_integral_v<_Tp>);
+ return static_cast<__bool_storage_member_type_t<_N>>(_M_data);
+ }
+
+ template <typename _U, size_t _N>
+ constexpr operator _SimdWrapper<_U, _N>() const
+ {
+ static_assert(__is_vector_type_v<_Tp>);
+ static_assert(sizeof(__vector_type_t<_U, _N>) == sizeof(_Tp));
+ return {reinterpret_cast<__vector_type_t<_U, _N>>(_M_data)};
+ }
+};
+
+// }}}
+// __wrapper_bitcast{{{
+template <typename _Tp,
+ typename _U,
+ size_t _M,
+ size_t _N = sizeof(_U) * _M / sizeof(_Tp)>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N>
+ __wrapper_bitcast(_SimdWrapper<_U, _M> __x)
+{
+ static_assert(sizeof(__vector_type_t<_Tp, _N>) ==
+ sizeof(__vector_type_t<_U, _M>));
+ return reinterpret_cast<__vector_type_t<_Tp, _N>>(__x._M_data);
+}
+
+// }}}
+// __make_wrapper{{{
+template <typename _Tp, typename... _Args>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, sizeof...(_Args)>
+ __make_wrapper(_Args&&... args)
+{
+ return {typename _SimdWrapper<_Tp, sizeof...(_Args)>::_BuiltinType{
+ static_cast<_Tp>(args)...}};
+}
+
+// }}}
+// __generate_wrapper{{{
+template <typename _Tp, size_t _N, typename _G>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N>
+ __generate_wrapper(_G&& __gen)
+{
+ return __generate_vector<_Tp, _N>(std::forward<_G>(__gen));
+}
+
+//}}}
+// __fallback_abi_for_long_double {{{
+template <typename _Tp, typename _A0, typename _A1>
+struct __fallback_abi_for_long_double
+{
+ using type = _A0;
+};
+template <typename _A0, typename _A1>
+struct __fallback_abi_for_long_double<long double, _A0, _A1>
+{
+ using type = _A1;
+};
+template <typename _Tp, typename _A0, typename _A1>
+using __fallback_abi_for_long_double_t =
+ typename __fallback_abi_for_long_double<_Tp, _A0, _A1>::type;
+// }}}
+
+namespace simd_abi
+{
+// most of simd_abi is defined in simd_detail.h
+template <typename _Tp>
+inline constexpr int max_fixed_size = 32;
+// compatible {{{
+#if defined __x86_64__
+template <typename _Tp>
+using compatible = __fallback_abi_for_long_double_t<_Tp, __sse, scalar>;
+#elif defined _GLIBCXX_SIMD_IS_AARCH64
+template <typename _Tp>
+using compatible = __fallback_abi_for_long_double_t<_Tp, __neon, scalar>;
+#else
+template <typename>
+using compatible = scalar;
+#endif
+
+// }}}
+// native {{{
+#if _GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI
+template <typename _Tp>
+using native = __fallback_abi_for_long_double_t<_Tp, __avx512, scalar>;
+#elif _GLIBCXX_SIMD_HAVE_AVX512_ABI
+template <typename _Tp>
+using native =
+ std::conditional_t<(sizeof(_Tp) >= 4),
+ __fallback_abi_for_long_double_t<_Tp, __avx512, scalar>,
+ __avx>;
+#elif _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI
+template <typename _Tp>
+using native = __fallback_abi_for_long_double_t<_Tp, __avx, scalar>;
+#elif _GLIBCXX_SIMD_HAVE_AVX_ABI
+template <typename _Tp>
+using native =
+ std::conditional_t<std::is_floating_point<_Tp>::value,
+ __fallback_abi_for_long_double_t<_Tp, __avx, scalar>,
+ __sse>;
+#elif _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI
+template <typename _Tp>
+using native = __fallback_abi_for_long_double_t<_Tp, __sse, scalar>;
+#elif _GLIBCXX_SIMD_HAVE_SSE_ABI
+template <typename _Tp>
+using native =
+ std::conditional_t<std::is_same<float, _Tp>::value, __sse, scalar>;
+#elif defined _GLIBCXX_SIMD_HAVE_FULL_NEON_ABI
+template <typename _Tp>
+using native = __fallback_abi_for_long_double_t<_Tp, __neon, scalar>;
+#else
+template <typename>
+using native = scalar;
+#endif
+
+// }}}
+// __default_abi {{{
+#if defined _GLIBCXX_SIMD_DEFAULT_ABI
+template <typename _Tp>
+using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
+#else
+template <typename _Tp>
+using __default_abi = compatible<_Tp>;
+#endif
+
+// }}}
+} // namespace simd_abi
+
+// traits {{{1
+// is_abi_tag {{{2
+template <typename _Tp, typename = std::void_t<>>
+struct is_abi_tag : false_type
+{
+};
+template <typename _Tp>
+struct is_abi_tag<_Tp, std::void_t<typename _Tp::is_valid_abi_tag>>
+: public _Tp::is_valid_abi_tag
+{
+};
+template <typename _Tp>
+inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
+
+// is_simd(_mask) {{{2
+template <typename _Tp>
+struct is_simd : public false_type
+{
+};
+template <typename _Tp>
+inline constexpr bool is_simd_v = is_simd<_Tp>::value;
+
+template <typename _Tp>
+struct is_simd_mask : public false_type
+{
+};
+template <typename _Tp>
+inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
+
+// simd_size {{{2
+template <typename _Tp, typename _Abi, typename = void>
+struct __simd_size_impl
+{
+};
+template <typename _Tp, typename _Abi>
+struct __simd_size_impl<
+ _Tp,
+ _Abi,
+ enable_if_t<std::conjunction_v<__is_vectorizable<_Tp>,
+ std::experimental::is_abi_tag<_Abi>>>>
+: _SizeConstant<_Abi::template size<_Tp>>
+{
+};
+
+template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
+struct simd_size : __simd_size_impl<_Tp, _Abi>
+{
+};
+template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
+inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
+
+// simd_abi::deduce {{{2
+template <typename _Tp, std::size_t _N, typename = void>
+struct __deduce_impl;
+namespace simd_abi
+{
+/**
+ * \tparam _Tp The requested `value_type` for the elements.
+ * \tparam _N The requested number of elements.
+ * \tparam _Abis This parameter is ignored, since this implementation cannot
+ * make any use of it. Either __a good native ABI is matched and used as `type`
+ * alias, or the `fixed_size<_N>` ABI is used, which internally is built from
+ * the best matching native ABIs.
+ */
+template <typename _Tp, std::size_t _N, typename...>
+struct deduce : std::experimental::__deduce_impl<_Tp, _N>
+{
+};
+
+template <typename _Tp, size_t _N, typename... _Abis>
+using deduce_t = typename deduce<_Tp, _N, _Abis...>::type;
+} // namespace simd_abi
+
+// }}}2
+// rebind_simd {{{2
+template <typename _Tp, typename _V>
+struct rebind_simd;
+template <typename _Tp, typename _U, typename _Abi>
+struct rebind_simd<_Tp, simd<_U, _Abi>>
+{
+ using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_U, _Abi>, _Abi>>;
+};
+template <typename _Tp, typename _U, typename _Abi>
+struct rebind_simd<_Tp, simd_mask<_U, _Abi>>
+{
+ using type =
+ simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_U, _Abi>, _Abi>>;
+};
+template <typename _Tp, typename _V>
+using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
+
+// resize_simd {{{2
+template <int _N, typename _V>
+struct resize_simd;
+template <int _N, typename _Tp, typename _Abi>
+struct resize_simd<_N, simd<_Tp, _Abi>>
+{
+ using type = simd<_Tp, simd_abi::deduce_t<_Tp, _N, _Abi>>;
+};
+template <int _N, typename _Tp, typename _Abi>
+struct resize_simd<_N, simd_mask<_Tp, _Abi>>
+{
+ using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _N, _Abi>>;
+};
+template <int _N, typename _V>
+using resize_simd_t = typename resize_simd<_N, _V>::type;
+
+// }}}2
+// memory_alignment {{{2
+template <typename _Tp, typename _U = typename _Tp::value_type>
+struct memory_alignment
+: public _SizeConstant<__next_power_of_2(sizeof(_U) * _Tp::size())>
+{
+};
+template <typename _Tp, typename _U = typename _Tp::value_type>
+inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _U>::value;
+
+// class template simd [simd] {{{1
+template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> class simd;
+template <typename _Tp, typename _Abi> struct is_simd<simd<_Tp, _Abi>> : public true_type {};
+template <typename _Tp> using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
+template <typename _Tp, int _N> using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_N>>;
+template <typename _Tp, size_t _N> using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _N>>;
+
+// class template simd_mask [simd_mask] {{{1
+template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> class simd_mask;
+template <typename _Tp, typename _Abi> struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
+template <typename _Tp> using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
+template <typename _Tp, int _N> using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_N>>;
+template <typename _Tp, size_t _N>
+using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _N>>;
+
+// __get_impl specializations for simd(_mask) {{{1
+template <typename _Tp, typename _Abi>
+struct __get_impl<std::experimental::simd_mask<_Tp, _Abi>>
+{
+ using _Traits = _SimdTraits<_Tp, _Abi>;
+ using _Impl = typename _Traits::_MaskImpl;
+};
+template <typename _Tp, typename _Abi>
+struct __get_impl<std::experimental::simd<_Tp, _Abi>>
+{
+ using _Traits = _SimdTraits<_Tp, _Abi>;
+ using _Impl = typename _Traits::_SimdImpl;
+};
+
+// casts [simd.casts] {{{1
+// static_simd_cast {{{2
+template <typename _Tp, typename _U, typename _A, bool = is_simd_v<_Tp>, typename = void>
+struct __static_simd_cast_return_type;
+
+template <typename _Tp, typename _A0, typename _U, typename _A>
+struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _U, _A, false, void>
+ : __static_simd_cast_return_type<simd<_Tp, _A0>, _U, _A> {
+};
+
+template <typename _Tp, typename _U, typename _A>
+struct __static_simd_cast_return_type<_Tp, _U, _A, true,
+ enable_if_t<_Tp::size() == simd_size_v<_U, _A>>> {
+ using type = _Tp;
+};
+
+template <typename _Tp, typename _A>
+struct __static_simd_cast_return_type<_Tp, _Tp, _A, false,
+#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
+ enable_if_t<__is_vectorizable_v<_Tp>>
+#else
+ void
+#endif
+ > {
+ using type = simd<_Tp, _A>;
+};
+
+template <typename _Tp, typename = void> struct __safe_make_signed {
+ using type = _Tp;
+};
+template <typename _Tp> struct __safe_make_signed<_Tp, enable_if_t<std::is_integral_v<_Tp>>> {
+ // the extra make_unsigned_t is because of PR85951
+ using type = std::make_signed_t<std::make_unsigned_t<_Tp>>;
+};
+template <typename _Tp> using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
+
+template <typename _Tp, typename _U, typename _A>
+struct __static_simd_cast_return_type<_Tp, _U, _A, false,
+#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
+ enable_if_t<__is_vectorizable_v<_Tp>>
+#else
+ void
+#endif
+ > {
+ using type =
+ std::conditional_t<(std::is_integral_v<_U> && std::is_integral_v<_Tp> &&
+#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
+ std::is_signed_v<_U> != std::is_signed_v<_Tp> &&
+#endif
+ std::is_same_v<safe_make_signed_t<_U>, safe_make_signed_t<_Tp>>),
+ simd<_Tp, _A>, fixed_size_simd<_Tp, simd_size_v<_U, _A>>>;
+};
+
+template <typename _To, typename, typename, typename _Native, typename _From>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const _Native *, const _From &__x)
+{
+ static_assert(std::is_same_v<_Native, typename __get_traits_t<_To>::_MaskMember>);
+ if constexpr (std::is_same_v<_Native, bool>) {
+ return {std::experimental::__private_init, bool(__x[0])};
+ } else if constexpr (std::is_same_v<_From, bool>) {
+ _To __r{};
+ __r[0] = __x;
+ return __r;
+ } else {
+ return {__private_init,
+ __convert_mask<typename __get_traits_t<_To>::_MaskMember>(__x)};
+ }
+}
+template <typename _To, typename, typename, typename _Native, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const _Native *, const std::bitset<_N> &__x)
+{
+ static_assert(_N <= sizeof(_ULLong) * CHAR_BIT ||
+ _To::size() <= sizeof(_ULLong) * CHAR_BIT,
+ "bug in std::experimental::(static|resizing)_simd_cast");
+ return {std::experimental::__bitset_init, __x.to_ullong()};
+}
+template <typename _To, typename, typename>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const bool *, bool __x)
+{
+ return _To(__x);
+}
+template <typename _To, typename, typename>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const std::bitset<1> *, bool __x)
+{
+ return _To(__x);
+}
+template <typename _To, typename _Tp, typename _Abi, size_t _N, typename _From>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const std::bitset<_N> *, const _From &__x)
+{
+ return {std::experimental::__private_init, __vector_to_bitset(__x)};
+}
+template <typename _To, typename, typename, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _To __mask_cast_impl(const std::bitset<_N> *, const std::bitset<_N> &__x)
+{
+ return {std::experimental::__private_init, __x};
+}
+
+template <typename _Tp, typename _U, typename _A,
+ typename _R = typename __static_simd_cast_return_type<_Tp, _U, _A>::type>
+_GLIBCXX_SIMD_INTRINSIC _R static_simd_cast(const simd<_U, _A> &__x)
+{
+ if constexpr(std::is_same<_R, simd<_U, _A>>::value) {
+ return __x;
+ } else {
+ _SimdConverter<_U, _A, typename _R::value_type, typename _R::abi_type> __c;
+ return _R(__private_init, __c(__data(__x)));
+ }
+}
+
+template <typename _Tp, typename _U, typename _A,
+ typename _R = typename __static_simd_cast_return_type<_Tp, _U, _A>::type>
+_GLIBCXX_SIMD_INTRINSIC typename _R::mask_type static_simd_cast(const simd_mask<_U, _A> &__x)
+{
+ using _RM = typename _R::mask_type;
+ if constexpr(std::is_same<_RM, simd_mask<_U, _A>>::value) {
+ return __x;
+ } else {
+ using __traits = _SimdTraits<typename _R::value_type, typename _R::abi_type>;
+ const typename __traits::_MaskMember *tag = nullptr;
+ return __mask_cast_impl<_RM, _U, _A>(tag, __data(__x));
+ }
+}
+
+// simd_cast {{{2
+template <typename _Tp, typename _U, typename _A, typename _To = __value_type_or_identity_t<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC auto simd_cast(const simd<_ValuePreserving<_U, _To>, _A> &__x)
+ ->decltype(static_simd_cast<_Tp>(__x))
+{
+ return static_simd_cast<_Tp>(__x);
+}
+
+template <typename _Tp, typename _U, typename _A, typename _To = __value_type_or_identity_t<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC auto simd_cast(const simd_mask<_ValuePreserving<_U, _To>, _A> &__x)
+ ->decltype(static_simd_cast<_Tp>(__x))
+{
+ return static_simd_cast<_Tp>(__x);
+}
+
+namespace __proposed
+{
+template <typename _Tp, typename _U, typename _A>
+_GLIBCXX_SIMD_INTRINSIC _Tp resizing_simd_cast(const simd_mask<_U, _A> &__x)
+{
+ static_assert(is_simd_mask_v<_Tp>);
+ if constexpr (std::is_same_v<_Tp, simd_mask<_U, _A>>) {
+ return __x;
+ } else {
+ using __traits = _SimdTraits<typename _Tp::simd_type::value_type, typename _Tp::abi_type>;
+ const typename __traits::_MaskMember *tag = nullptr;
+ return __mask_cast_impl<_Tp, _U, _A>(tag, __data(__x));
+ }
+}
+} // namespace __proposed
+
+// to_fixed_size {{{2
+template <typename _Tp, int _N>
+_GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _N> to_fixed_size(const fixed_size_simd<_Tp, _N> &__x)
+{
+ return __x;
+}
+
+template <typename _Tp, int _N>
+_GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _N> to_fixed_size(const fixed_size_simd_mask<_Tp, _N> &__x)
+{
+ return __x;
+}
+
+template <typename _Tp, typename _A> _GLIBCXX_SIMD_INTRINSIC auto to_fixed_size(const simd<_Tp, _A> &__x)
+{
+ return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _A>>>(
+ [&__x](auto __i) { return __x[__i]; });
+}
+
+template <typename _Tp, typename _A> _GLIBCXX_SIMD_INTRINSIC auto to_fixed_size(const simd_mask<_Tp, _A> &__x)
+{
+ constexpr int _N = simd_mask<_Tp, _A>::size();
+ fixed_size_simd_mask<_Tp, _N> __r;
+ __execute_n_times<_N>([&](auto __i) { __r[__i] = __x[__i]; });
+ return __r;
+}
+
+// to_native {{{2
+template <typename _Tp, int _N>
+_GLIBCXX_SIMD_INTRINSIC enable_if_t<(_N == native_simd<_Tp>::size()), native_simd<_Tp>>
+to_native(const fixed_size_simd<_Tp, _N> &__x)
+{
+ alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_N];
+ __x.copy_to(__mem, vector_aligned);
+ return {__mem, vector_aligned};
+}
+
+template <typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC enable_if_t<(_N == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>> to_native(
+ const fixed_size_simd_mask<_Tp, _N> &__x)
+{
+ return native_simd_mask<_Tp>([&](auto __i) { return __x[__i]; });
+}
+
+// to_compatible {{{2
+template <typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC enable_if_t<(_N == simd<_Tp>::size()), simd<_Tp>> to_compatible(
+ const simd<_Tp, simd_abi::fixed_size<_N>> &__x)
+{
+ alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_N];
+ __x.copy_to(__mem, vector_aligned);
+ return {__mem, vector_aligned};
+}
+
+template <typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC enable_if_t<(_N == simd_mask<_Tp>::size()), simd_mask<_Tp>> to_compatible(
+ const simd_mask<_Tp, simd_abi::fixed_size<_N>> &__x)
+{
+ return simd_mask<_Tp>([&](auto __i) { return __x[__i]; });
+}
+
+// simd_reinterpret_cast {{{2
+template <typename _To, size_t _N> _GLIBCXX_SIMD_INTRINSIC _To __simd_reinterpret_cast_impl(std::bitset<_N> __x)
+{
+ return {__bitset_init, __x};
+}
+
+template <typename _To, typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _To __simd_reinterpret_cast_impl(_SimdWrapper<_Tp, _N> __x)
+{
+ return {__private_init, __x};
+}
+
+namespace __proposed
+{
+template <typename _To,
+ typename _Tp,
+ typename _A,
+ typename = enable_if_t<sizeof(_To) == sizeof(simd<_Tp, _A>) &&
+ (is_simd_v<_To> || is_simd_mask_v<_To>)>>
+_GLIBCXX_SIMD_INTRINSIC _To simd_reinterpret_cast(const simd<_Tp, _A>& __x)
+{
+ _To __r;
+ std::memcpy(&__data(__r), &__data(__x), sizeof(_To));
+ return __r;
+}
+
+template <typename _To, typename _Tp, typename _A,
+ typename = enable_if_t<(is_simd_v<_To> || is_simd_mask_v<_To>)>>
+_GLIBCXX_SIMD_INTRINSIC _To simd_reinterpret_cast(const simd_mask<_Tp, _A> &__x)
+{
+ return std::experimental::__simd_reinterpret_cast_impl<_To>(__data(__x));
+ //return reinterpret_cast<const _To &>(__x);
+}
+} // namespace __proposed
+
+// masked assignment [simd_mask.where] {{{1
+
+// where_expression {{{1
+template <class _M, class _Tp> class const_where_expression //{{{2
+{
+ using _V = _Tp;
+ static_assert(std::is_same_v<_V, __remove_cvref_t<_Tp>>);
+ struct Wrapper {
+ using value_type = _V;
+ };
+
+protected:
+ using value_type =
+ typename std::conditional_t<std::is_arithmetic<_V>::value, Wrapper, _V>::value_type;
+ _GLIBCXX_SIMD_INTRINSIC friend const _M &__get_mask(const const_where_expression &__x) { return __x.__k; }
+ _GLIBCXX_SIMD_INTRINSIC friend const _Tp &__get_lvalue(const const_where_expression &__x) { return __x._M_value; }
+ const _M &__k;
+ _Tp &_M_value;
+
+public:
+ const_where_expression(const const_where_expression &) = delete;
+ const_where_expression &operator=(const const_where_expression &) = delete;
+
+ _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M &kk, const _Tp &dd) : __k(kk), _M_value(const_cast<_Tp &>(dd)) {}
+
+ _GLIBCXX_SIMD_INTRINSIC _V operator-() const &&
+ {
+ return {__private_init,
+ __get_impl_t<_V>::template masked_unary<std::negate>(
+ __data(__k), __data(_M_value))};
+ }
+
+ template <class _U, class _Flags>
+ [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
+ copy_from(const _LoadStorePtr<_U, value_type> *__mem, _Flags __f) const &&
+ {
+ return {__private_init, __get_impl_t<_V>::masked_load(
+ __data(_M_value), __data(__k), __mem, __f)};
+ }
+
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_INTRINSIC void copy_to(_LoadStorePtr<_U, value_type> *__mem,
+ _Flags __f) const &&
+ {
+ __get_impl_t<_V>::masked_store(__data(_M_value), __mem, __f, __data(__k));
+ }
+};
+
+template <class _Tp> class const_where_expression<bool, _Tp> //{{{2
+{
+ using _M = bool;
+ using _V = _Tp;
+ static_assert(std::is_same_v<_V, __remove_cvref_t<_Tp>>);
+ struct Wrapper {
+ using value_type = _V;
+ };
+
+protected:
+ using value_type =
+ typename std::conditional_t<std::is_arithmetic<_V>::value, Wrapper, _V>::value_type;
+ _GLIBCXX_SIMD_INTRINSIC friend const _M &__get_mask(const const_where_expression &__x) { return __x.__k; }
+ _GLIBCXX_SIMD_INTRINSIC friend const _Tp &__get_lvalue(const const_where_expression &__x) { return __x._M_value; }
+ const bool __k;
+ _Tp &_M_value;
+
+public:
+ const_where_expression(const const_where_expression &) = delete;
+ const_where_expression &operator=(const const_where_expression &) = delete;
+
+ _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool kk, const _Tp &dd) : __k(kk), _M_value(const_cast<_Tp &>(dd)) {}
+
+ _GLIBCXX_SIMD_INTRINSIC _V operator-() const && { return __k ? -_M_value : _M_value; }
+
+ template <class _U, class _Flags>
+ [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
+ copy_from(const _LoadStorePtr<_U, value_type> *__mem, _Flags) const &&
+ {
+ return __k ? static_cast<_V>(__mem[0]) : _M_value;
+ }
+
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_INTRINSIC void copy_to(_LoadStorePtr<_U, value_type> *__mem,
+ _Flags) const &&
+ {
+ if (__k) {
+ __mem[0] = _M_value;
+ }
+ }
+};
+
+// where_expression {{{2
+template <class _M, class _Tp>
+class where_expression : public const_where_expression<_M, _Tp>
+{
+ static_assert(!std::is_const<_Tp>::value, "where_expression may only be instantiated with __a non-const _Tp parameter");
+ using typename const_where_expression<_M, _Tp>::value_type;
+ using const_where_expression<_M, _Tp>::__k;
+ using const_where_expression<_M, _Tp>::_M_value;
+ static_assert(std::is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
+ static_assert(_M::size() == _Tp::size(), "");
+
+ _GLIBCXX_SIMD_INTRINSIC friend _Tp &__get_lvalue(where_expression &__x) { return __x._M_value; }
+public:
+ where_expression(const where_expression &) = delete;
+ where_expression &operator=(const where_expression &) = delete;
+
+ _GLIBCXX_SIMD_INTRINSIC where_expression(const _M &kk, _Tp &dd)
+ : const_where_expression<_M, _Tp>(kk, dd)
+ {
+ }
+
+ template <class _U> _GLIBCXX_SIMD_INTRINSIC void operator=(_U &&__x) &&
+ {
+ std::experimental::__get_impl_t<_Tp>::masked_assign(
+ __data(__k), __data(_M_value),
+ __to_value_type_or_member_type<_Tp>(std::forward<_U>(__x)));
+ }
+
+#define _GLIBCXX_SIMD_OP_(op_, name_) \
+ template <class _U> _GLIBCXX_SIMD_INTRINSIC void operator op_##=(_U&& __x)&& \
+ { \
+ std::experimental::__get_impl_t<_Tp>::template __masked_cassign<name_>( \
+ __data(__k), __data(_M_value), \
+ __to_value_type_or_member_type<_Tp>(std::forward<_U>(__x))); \
+ } \
+ static_assert(true)
+ _GLIBCXX_SIMD_OP_(+, std::plus);
+ _GLIBCXX_SIMD_OP_(-, std::minus);
+ _GLIBCXX_SIMD_OP_(*, std::multiplies);
+ _GLIBCXX_SIMD_OP_(/, std::divides);
+ _GLIBCXX_SIMD_OP_(%, std::modulus);
+ _GLIBCXX_SIMD_OP_(&, std::bit_and);
+ _GLIBCXX_SIMD_OP_(|, std::bit_or);
+ _GLIBCXX_SIMD_OP_(^, std::bit_xor);
+ _GLIBCXX_SIMD_OP_(<<, __shift_left);
+ _GLIBCXX_SIMD_OP_(>>, __shift_right);
+#undef _GLIBCXX_SIMD_OP_
+
+ _GLIBCXX_SIMD_INTRINSIC void operator++() &&
+ {
+ __data(_M_value) = __get_impl_t<_Tp>::template masked_unary<__increment>(
+ __data(__k), __data(_M_value));
+ }
+ _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
+ {
+ __data(_M_value) = __get_impl_t<_Tp>::template masked_unary<__increment>(
+ __data(__k), __data(_M_value));
+ }
+ _GLIBCXX_SIMD_INTRINSIC void operator--() &&
+ {
+ __data(_M_value) = __get_impl_t<_Tp>::template masked_unary<__decrement>(
+ __data(__k), __data(_M_value));
+ }
+ _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
+ {
+ __data(_M_value) = __get_impl_t<_Tp>::template masked_unary<__decrement>(
+ __data(__k), __data(_M_value));
+ }
+
+ // intentionally hides const_where_expression::copy_from
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_INTRINSIC void copy_from(const _LoadStorePtr<_U, value_type> *__mem,
+ _Flags __f) &&
+ {
+ __data(_M_value) =
+ __get_impl_t<_Tp>::masked_load(__data(_M_value), __data(__k), __mem, __f);
+ }
+};
+
+// where_expression<bool> {{{2
+template <class _Tp>
+class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
+{
+ using _M = bool;
+ using typename const_where_expression<_M, _Tp>::value_type;
+ using const_where_expression<_M, _Tp>::__k;
+ using const_where_expression<_M, _Tp>::_M_value;
+
+public:
+ where_expression(const where_expression &) = delete;
+ where_expression &operator=(const where_expression &) = delete;
+
+ _GLIBCXX_SIMD_INTRINSIC where_expression(const _M &kk, _Tp &dd)
+ : const_where_expression<_M, _Tp>(kk, dd)
+ {
+ }
+
+#define _GLIBCXX_SIMD_OP_(op_) \
+ template <class _U> _GLIBCXX_SIMD_INTRINSIC void operator op_(_U&& __x)&& \
+ { \
+ if (__k) { \
+ _M_value op_ std::forward<_U>(__x); \
+ } \
+ } \
+ static_assert(true)
+ _GLIBCXX_SIMD_OP_(=);
+ _GLIBCXX_SIMD_OP_(+=);
+ _GLIBCXX_SIMD_OP_(-=);
+ _GLIBCXX_SIMD_OP_(*=);
+ _GLIBCXX_SIMD_OP_(/=);
+ _GLIBCXX_SIMD_OP_(%=);
+ _GLIBCXX_SIMD_OP_(&=);
+ _GLIBCXX_SIMD_OP_(|=);
+ _GLIBCXX_SIMD_OP_(^=);
+ _GLIBCXX_SIMD_OP_(<<=);
+ _GLIBCXX_SIMD_OP_(>>=);
+#undef _GLIBCXX_SIMD_OP_
+ _GLIBCXX_SIMD_INTRINSIC void operator++() && { if (__k) { ++_M_value; } }
+ _GLIBCXX_SIMD_INTRINSIC void operator++(int) && { if (__k) { ++_M_value; } }
+ _GLIBCXX_SIMD_INTRINSIC void operator--() && { if (__k) { --_M_value; } }
+ _GLIBCXX_SIMD_INTRINSIC void operator--(int) && { if (__k) { --_M_value; } }
+
+ // intentionally hides const_where_expression::copy_from
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_INTRINSIC void copy_from(const _LoadStorePtr<_U, value_type> *__mem,
+ _Flags) &&
+ {
+ if (__k) {
+ _M_value = __mem[0];
+ }
+ }
+};
+
+// where_expression<_M, tuple<...>> {{{2
+
+// where {{{1
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _A>, simd<_Tp, _A>> where(
+ const typename simd<_Tp, _A>::mask_type &__k, simd<_Tp, _A> &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC const_where_expression<simd_mask<_Tp, _A>, simd<_Tp, _A>> where(
+ const typename simd<_Tp, _A>::mask_type &__k, const simd<_Tp, _A> &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _A>, simd_mask<_Tp, _A>> where(
+ const std::remove_const_t<simd_mask<_Tp, _A>> &__k, simd_mask<_Tp, _A> &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC const_where_expression<simd_mask<_Tp, _A>, simd_mask<_Tp, _A>> where(
+ const std::remove_const_t<simd_mask<_Tp, _A>> &__k, const simd_mask<_Tp, _A> &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp>
+_GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp> where(_ExactBool __k, _Tp &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp>
+_GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp> where(_ExactBool __k, const _Tp &__value)
+{
+ return {__k, __value};
+}
+template <class _Tp, class _A> void where(bool __k, simd<_Tp, _A> &__value) = delete;
+template <class _Tp, class _A> void where(bool __k, const simd<_Tp, _A> &__value) = delete;
+
+// proposed mask iterations {{{1
+namespace __proposed
+{
+template <size_t _N> class where_range
+{
+ const std::bitset<_N> __bits;
+
+public:
+ where_range(std::bitset<_N> __b) : __bits(__b) {}
+
+ class iterator
+ {
+ size_t __mask;
+ size_t __bit;
+
+ _GLIBCXX_SIMD_INTRINSIC void __next_bit() { __bit = __builtin_ctzl(__mask); }
+ _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
+ {
+ // 01100100 - 1 = 01100011
+ __mask &= (__mask - 1);
+ // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
+ }
+
+ public:
+ iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
+ iterator(const iterator &) = default;
+ iterator(iterator &&) = default;
+
+ _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const { return __bit; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const { return __bit; }
+
+ _GLIBCXX_SIMD_ALWAYS_INLINE iterator &operator++()
+ {
+ __reset_lsb();
+ __next_bit();
+ return *this;
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
+ {
+ iterator __tmp = *this;
+ __reset_lsb();
+ __next_bit();
+ return __tmp;
+ }
+
+ _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator &__rhs) const
+ {
+ return __mask == __rhs.__mask;
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator &__rhs) const
+ {
+ return __mask != __rhs.__mask;
+ }
+ };
+
+ iterator begin() const { return __bits.to_ullong(); }
+ iterator end() const { return 0; }
+};
+
+template <class _Tp, class _A>
+where_range<simd_size_v<_Tp, _A>> where(const simd_mask<_Tp, _A> &__k)
+{
+ return __k.__to_bitset();
+}
+
+} // namespace __proposed
+
+// }}}1
+// reductions [simd.reductions] {{{1
+template <class _Tp, class _Abi, class _BinaryOperation = std::plus<>>
+_GLIBCXX_SIMD_INTRINSIC _Tp reduce(const simd<_Tp, _Abi>& __v,
+ _BinaryOperation __binary_op = _BinaryOperation())
+{
+ using _V = simd<_Tp, _Abi>;
+ return __get_impl_t<_V>::reduce(__v, __binary_op);
+}
+
+template <class _M, class _V, class _BinaryOperation = std::plus<>>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, typename _V::value_type __identity_element,
+ _BinaryOperation __binary_op)
+{
+ _V __tmp = __identity_element;
+ __get_impl_t<_V>::masked_assign(__data(__get_mask(__x)), __data(__tmp),
+ __data(__get_lvalue(__x)));
+ return reduce(__tmp, __binary_op);
+}
+
+template <class _M, class _V>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, std::plus<> __binary_op = {})
+{
+ return reduce(__x, 0, __binary_op);
+}
+
+template <class _M, class _V>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, std::multiplies<> __binary_op)
+{
+ return reduce(__x, 1, __binary_op);
+}
+
+template <class _M, class _V>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, std::bit_and<> __binary_op)
+{
+ return reduce(__x, ~typename _V::value_type(), __binary_op);
+}
+
+template <class _M, class _V>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, std::bit_or<> __binary_op)
+{
+ return reduce(__x, 0, __binary_op);
+}
+
+template <class _M, class _V>
+_GLIBCXX_SIMD_INTRINSIC typename _V::value_type reduce(
+ const const_where_expression<_M, _V>& __x, std::bit_xor<> __binary_op)
+{
+ return reduce(__x, 0, __binary_op);
+}
+
+// }}}1
+// algorithms [simd.alg] {{{
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A> min(const simd<_Tp, _A> &__a, const simd<_Tp, _A> &__b)
+{
+ return {__private_init,
+ _A::_SimdImpl::min(__data(__a), __data(__b))};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A> max(const simd<_Tp, _A> &__a, const simd<_Tp, _A> &__b)
+{
+ return {__private_init,
+ _A::_SimdImpl::max(__data(__a), __data(__b))};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr std::pair<simd<_Tp, _A>, simd<_Tp, _A>> minmax(const simd<_Tp, _A> &__a,
+ const simd<_Tp, _A> &__b)
+{
+ const auto pair_of_members =
+ _A::_SimdImpl::minmax(__data(__a), __data(__b));
+ return {simd<_Tp, _A>(__private_init, pair_of_members.first),
+ simd<_Tp, _A>(__private_init, pair_of_members.second)};
+}
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A> clamp(const simd<_Tp, _A> &__v, const simd<_Tp, _A> &__lo,
+ const simd<_Tp, _A> &__hi)
+{
+ using _Impl = typename _A::_SimdImpl;
+ return {__private_init,
+ _Impl::min(__data(__hi), _Impl::max(__data(__lo), __data(__v)))};
+}
+
+// }}}
+
+namespace __proposed
+{
+// shuffle {{{1
+template <int _Stride, int _Offset = 0> struct strided {
+ static constexpr int _S_stride = _Stride;
+ static constexpr int _S_offset = _Offset;
+ template <class _Tp, class _A>
+ using __shuffle_return_type = simd<
+ _Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _A> - _Offset + _Stride - 1) / _Stride, _A>>;
+ // alternative, always use fixed_size:
+ // fixed_size_simd<_Tp, (simd_size_v<_Tp, _A> - _Offset + _Stride - 1) / _Stride>;
+ template <class _Tp> static constexpr auto __src_index(_Tp __dst_index)
+ {
+ return _Offset + __dst_index * _Stride;
+ }
+};
+
+// SFINAE for the return type ensures _P is a type that provides the alias template member
+// __shuffle_return_type and the static member function __src_index
+template <class _P, class _Tp, class _A,
+ class _R = typename _P::template __shuffle_return_type<_Tp, _A>,
+ class = decltype(_P::__src_index(std::experimental::_SizeConstant<0>()))>
+_GLIBCXX_SIMD_INTRINSIC _R shuffle(const simd<_Tp, _A> &__x)
+{
+ return _R([&__x](auto __i) { return __x[_P::__src_index(__i)]; });
+}
+
+// }}}1
+} // namespace __proposed
+
+template <size_t... _Sizes, class _Tp, class _A,
+ class = enable_if_t<((_Sizes + ...) == simd<_Tp, _A>::size())>>
+inline std::tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...> split(const simd<_Tp, _A> &);
+
+// __extract_part {{{
+template <size_t _Index, size_t _Total, typename _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST
+ __vector_type_t<_Tp, std::max(__min_vector_size / sizeof(_Tp), _N / _Total)>
+ __extract_part(_SimdWrapper<_Tp, _N>);
+template <int Index, int Parts, typename _Tp, typename _A0, typename... _As>
+auto __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
+
+// }}}
+// __size_list {{{
+template <size_t V0, size_t... Values> struct __size_list {
+ static constexpr size_t size = sizeof...(Values) + 1;
+
+ template <size_t _I> static constexpr size_t at(_SizeConstant<_I> = {})
+ {
+ if constexpr (_I == 0) {
+ return V0;
+ } else {
+ return __size_list<Values...>::template at<_I - 1>();
+ }
+ }
+
+ template <size_t _I> static constexpr auto before(_SizeConstant<_I> = {})
+ {
+ if constexpr (_I == 0) {
+ return _SizeConstant<0>();
+ } else {
+ return _SizeConstant<V0 + __size_list<Values...>::template before<_I - 1>()>();
+ }
+ }
+
+ template <size_t _N> static constexpr auto pop_front(_SizeConstant<_N> = {})
+ {
+ if constexpr (_N == 0) {
+ return __size_list();
+ } else {
+ return __size_list<Values...>::template pop_front<_N-1>();
+ }
+ }
+};
+// }}}
+// __extract_center {{{
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _N / 2> __extract_center(_SimdWrapper<_Tp, _N> __x)
+{
+ if constexpr (__have_avx512f && sizeof(__x) == 64) {
+ const auto __intrin = __to_intrin(__x);
+ if constexpr (std::is_integral_v<_Tp>) {
+ return __vector_bitcast<_Tp>(_mm512_castsi512_si256(_mm512_shuffle_i32x4(
+ __intrin, __intrin, 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
+ } else if constexpr (sizeof(_Tp) == 4) {
+ return __vector_bitcast<_Tp>(_mm512_castps512_ps256(_mm512_shuffle_f32x4(
+ __intrin, __intrin, 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
+ } else if constexpr (sizeof(_Tp) == 8) {
+ return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(_mm512_shuffle_f64x2(
+ __intrin, __intrin, 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+template <class _Tp, class _A>
+inline _SimdWrapper<_Tp, simd_size_v<_Tp, _A>> __extract_center(
+ const _SimdTuple<_Tp, _A, _A>& __x)
+{
+ return __concat(__extract<1, 2>(__x.first._M_data), __extract<0, 2>(__x.second.first._M_data));
+}
+template <class _Tp, class _A>
+inline _SimdWrapper<_Tp, simd_size_v<_Tp, _A> / 2> __extract_center(
+ const _SimdTuple<_Tp, _A>& __x)
+{
+ return __extract_center(__x.first);
+}
+
+// }}}
+// __split_wrapper {{{
+template <size_t... _Sizes, class _Tp, class... _As>
+auto __split_wrapper(__size_list<_Sizes...>, const _SimdTuple<_Tp, _As...> &__x)
+{
+ return std::experimental::split<_Sizes...>(
+ fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::size()>(__private_init, __x));
+}
+
+// }}}
+
+// split<simd>(simd) {{{
+template <class _V, class _A,
+ size_t Parts = simd_size_v<typename _V::value_type, _A> / _V::size()>
+inline enable_if_t<(is_simd<_V>::value &&
+ simd_size_v<typename _V::value_type, _A> == Parts * _V::size()),
+ std::array<_V, Parts>>
+split(const simd<typename _V::value_type, _A> &__x)
+{
+ using _Tp = typename _V::value_type;
+ if constexpr (Parts == 1) {
+ return {simd_cast<_V>(__x)};
+ } else if constexpr (__is_fixed_size_abi_v<_A> &&
+ (std::is_same_v<typename _V::abi_type, simd_abi::scalar> ||
+ (__is_fixed_size_abi_v<typename _V::abi_type> &&
+ sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
+ ))) {
+ // fixed_size -> fixed_size (w/o padding) or scalar
+#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
+ const __may_alias<_Tp> *const __element_ptr =
+ reinterpret_cast<const __may_alias<_Tp> *>(&__data(__x));
+ return __generate_from_n_evaluations<Parts, std::array<_V, Parts>>(
+ [&](auto __i) { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
+#else
+ const auto &__xx = __data(__x);
+ return __generate_from_n_evaluations<Parts, std::array<_V, Parts>>(
+ [&](auto __i) {
+ constexpr size_t __offset = decltype(__i)::value * _V::size();
+ __unused(__offset); // not really
+ return _V([&](auto __j) {
+ constexpr _SizeConstant<__j + __offset> __k;
+ return __xx[__k];
+ });
+ });
+#endif
+ } else if constexpr (std::is_same_v<typename _V::abi_type, simd_abi::scalar>) {
+ // normally memcpy should work here as well
+ return __generate_from_n_evaluations<Parts, std::array<_V, Parts>>(
+ [&](auto __i) { return __x[__i]; });
+ } else {
+ return __generate_from_n_evaluations<Parts, std::array<_V, Parts>>([&](auto __i) {
+ if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>) {
+ return _V([&](auto __j) { return __x[__i * _V::size() + __j]; });
+ } else {
+ return _V(__private_init,
+ __extract_part<__i, Parts>(__data(__x)));
+ }
+ });
+ }
+}
+
+// }}}
+// split<simd_mask>(simd_mask) {{{
+template <typename _V,
+ typename _A,
+ size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _A> /
+ _V::size()>
+enable_if_t<(is_simd_mask_v<_V> &&
+ simd_size_v<typename _V::simd_type::value_type, _A> ==
+ _Parts * _V::size()),
+ std::array<_V, _Parts>>
+ split(const simd_mask<typename _V::simd_type::value_type, _A>& __x)
+{
+ if constexpr (std::is_same_v<_A, typename _V::abi_type>)
+ {
+ return {__x};
+ }
+ else if constexpr (_Parts == 1)
+ {
+ return {static_simd_cast<_V>(__x)};
+ }
+ else if constexpr (_Parts == 2 &&
+ __is_abi<typename _V::abi_type, simd_abi::__sse>() &&
+ __is_abi<_A, simd_abi::__avx>())
+ {
+ return {_V(__private_init, __lo128(__data(__x))),
+ _V(__private_init, __hi128(__data(__x)))};
+ }
+ else if constexpr (_V::size() <= CHAR_BIT * sizeof(_ULLong))
+ {
+ const std::bitset __bits = __x.__to_bitset();
+ return __generate_from_n_evaluations<_Parts, std::array<_V, _Parts>>(
+ [&](auto __i) {
+ constexpr size_t __offset = __i * _V::size();
+ return _V(__bitset_init, (__bits >> __offset).to_ullong());
+ });
+ }
+ else
+ {
+ return __generate_from_n_evaluations<_Parts, std::array<_V, _Parts>>(
+ [&](auto __i) {
+ constexpr size_t __offset = __i * _V::size();
+ return _V(__private_init,
+ [&](auto __j) { return __x[__j + __offset]; });
+ });
+ }
+}
+
+// }}}
+// split<_Sizes...>(simd) {{{
+template <size_t... _Sizes, class _Tp, class _A,
+ class = enable_if_t<((_Sizes + ...) == simd<_Tp, _A>::size())>>
+_GLIBCXX_SIMD_ALWAYS_INLINE std::tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...> split(
+ const simd<_Tp, _A> &__x)
+{
+ using _SL = __size_list<_Sizes...>;
+ using _Tuple = std::tuple<__deduced_simd<_Tp, _Sizes>...>;
+ constexpr size_t _N = simd_size_v<_Tp, _A>;
+ constexpr size_t _N0 = _SL::template at<0>();
+ using _V = __deduced_simd<_Tp, _N0>;
+
+ if constexpr (_N == _N0) {
+ static_assert(sizeof...(_Sizes) == 1);
+ return {simd_cast<_V>(__x)};
+ } else if constexpr (__is_fixed_size_abi_v<_A> &&
+ __fixed_size_storage_t<_Tp, _N>::_S_first_size == _N0) {
+ // if the first part of the _SimdTuple input matches the first output vector
+ // in the std::tuple, extract it and recurse
+ static_assert(!__is_fixed_size_abi_v<typename _V::abi_type>,
+ "How can <_Tp, _N> be __a single _SimdTuple entry but __a fixed_size_simd "
+ "when deduced?");
+ const __fixed_size_storage_t<_Tp, _N> &__xx = __data(__x);
+ return std::tuple_cat(
+ std::make_tuple(_V(__private_init, __xx.first)),
+ __split_wrapper(_SL::template pop_front<1>(), __xx.second));
+ } else if constexpr ((!std::is_same_v<simd_abi::scalar,
+ simd_abi::deduce_t<_Tp, _Sizes>> &&
+ ...) &&
+ (!__is_fixed_size_abi_v<simd_abi::deduce_t<_Tp, _Sizes>> &&
+ ...)) {
+ if constexpr (((_Sizes * 2 == _N)&&...)) {
+ return {{__private_init, __extract_part<0, 2>(__data(__x))},
+ {__private_init, __extract_part<1, 2>(__data(__x))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<_N / 3, _N / 3, _N / 3>>) {
+ return {{__private_init, __extract_part<0, 3>(__data(__x))},
+ {__private_init, __extract_part<1, 3>(__data(__x))},
+ {__private_init, __extract_part<2, 3>(__data(__x))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<2 * _N / 3, _N / 3>>) {
+ return {{__private_init,
+ __concat(__extract_part<0, 3>(__data(__x)),
+ __extract_part<1, 3>(__data(__x)))},
+ {__private_init, __extract_part<2, 3>(__data(__x))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<_N / 3, 2 * _N / 3>>) {
+ return {{__private_init, __extract_part<0, 3>(__data(__x))},
+ {__private_init,
+ __concat(__extract_part<1, 3>(__data(__x)),
+ __extract_part<2, 3>(__data(__x)))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<_N / 2, _N / 4, _N / 4>>) {
+ return {{__private_init, __extract_part<0, 2>(__data(__x))},
+ {__private_init, __extract_part<2, 4>(__data(__x))},
+ {__private_init, __extract_part<3, 4>(__data(__x))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<_N / 4, _N / 4, _N / 2>>) {
+ return {{__private_init, __extract_part<0, 4>(__data(__x))},
+ {__private_init, __extract_part<1, 4>(__data(__x))},
+ {__private_init, __extract_part<1, 2>(__data(__x))}};
+ } else if constexpr (std::is_same_v<__size_list<_Sizes...>,
+ __size_list<_N / 4, _N / 2, _N / 4>>) {
+ return {
+ {__private_init, __extract_part<0, 4>(__data(__x))},
+ {__private_init, __extract_center(__data(__x))},
+ {__private_init, __extract_part<3, 4>(__data(__x))}};
+ } else if constexpr (((_Sizes * 4 == _N) && ...)) {
+ return {{__private_init, __extract_part<0, 4>(__data(__x))},
+ {__private_init, __extract_part<1, 4>(__data(__x))},
+ {__private_init, __extract_part<2, 4>(__data(__x))},
+ {__private_init, __extract_part<3, 4>(__data(__x))}};
+ //} else if constexpr (__is_fixed_size_abi_v<_A>) {
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
+ const __may_alias<_Tp> *const __element_ptr =
+ reinterpret_cast<const __may_alias<_Tp> *>(&__x);
+ return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](auto __i) {
+ using _Vi = __deduced_simd<_Tp, _SL::at(__i)>;
+ constexpr size_t __offset = _SL::before(__i);
+ constexpr size_t __base_align = alignof(simd<_Tp, _A>);
+ constexpr size_t __a = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
+ constexpr size_t __b = ((__a - 1) & __a) ^ __a;
+ constexpr size_t __alignment = __b == 0 ? __a : __b;
+ return _Vi(__element_ptr + __offset, overaligned<__alignment>);
+ });
+#else
+ return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](auto __i) {
+ using _Vi = __deduced_simd<_Tp, _SL::at(__i)>;
+ const auto &__xx = __data(__x);
+ using _Offset = decltype(_SL::before(__i));
+ return _Vi([&](auto __j) {
+ constexpr _SizeConstant<_Offset::value + __j> __k;
+ return __xx[__k];
+ });
+ });
+#endif
+ }
+}
+
+// }}}
+
+// __subscript_in_pack {{{
+template <size_t _I, class _Tp, class _A, class... _As>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp __subscript_in_pack(const simd<_Tp, _A> &__x, const simd<_Tp, _As> &... __xs)
+{
+ if constexpr (_I < simd_size_v<_Tp, _A>) {
+ return __x[_I];
+ } else {
+ return __subscript_in_pack<_I - simd_size_v<_Tp, _A>>(__xs...);
+ }
+}
+// }}}
+
+// concat(simd...) {{{
+template <class _Tp, class... _As>
+simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>> concat(
+ const simd<_Tp, _As> &... __xs)
+{
+ return simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
+ [&](auto __i) { return __subscript_in_pack<__i>(__xs...); });
+}
+
+// }}}
+
+// _Smart_reference {{{
+template <class _U, class _Accessor = _U, class _ValueType = typename _U::value_type>
+class _Smart_reference
+{
+ friend _Accessor;
+ int index;
+ _U &obj;
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType __read() const noexcept
+ {
+ if constexpr (std::is_arithmetic_v<_U>) {
+ _GLIBCXX_DEBUG_ASSERT(index == 0);
+ return obj;
+ } else {
+ return obj[index];
+ }
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC constexpr void __write(_Tp &&__x) const
+ {
+ _Accessor::set(obj, index, std::forward<_Tp>(__x));
+ }
+
+public:
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Smart_reference(_U& __o, int __i) noexcept
+ : index(__i), obj(__o)
+ {
+ }
+
+ using value_type = _ValueType;
+
+ _GLIBCXX_SIMD_INTRINSIC _Smart_reference(const _Smart_reference &) = delete;
+
+ _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept { return __read(); }
+
+ template <class _Tp,
+ class = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Smart_reference operator=(_Tp &&__x) &&
+ {
+ __write(std::forward<_Tp>(__x));
+ return {obj, index};
+ }
+
+// TODO: improve with operator.()
+
+#define _GLIBCXX_SIMD_OP_(op_) \
+ template <class _Tp, \
+ class _TT = \
+ decltype(std::declval<value_type>() op_ std::declval<_Tp>()), \
+ class = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
+ class = _ValuePreservingOrInt<_TT, value_type>> \
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Smart_reference operator op_##=( \
+ _Tp&& __x)&& \
+ { \
+ const value_type& __lhs = __read(); \
+ __write(__lhs op_ __x); \
+ return {obj, index}; \
+ }
+ _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
+ _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
+ _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
+#undef _GLIBCXX_SIMD_OP_
+
+ template <class _Tp = void,
+ class = decltype(
+ ++std::declval<std::conditional_t<true, value_type, _Tp> &>())>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Smart_reference operator++() &&
+ {
+ value_type __x = __read();
+ __write(++__x);
+ return {obj, index};
+ }
+
+ template <class _Tp = void,
+ class = decltype(
+ std::declval<std::conditional_t<true, value_type, _Tp> &>()++)>
+ _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
+ {
+ const value_type __r = __read();
+ value_type __x = __r;
+ __write(++__x);
+ return __r;
+ }
+
+ template <class _Tp = void,
+ class = decltype(
+ --std::declval<std::conditional_t<true, value_type, _Tp> &>())>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Smart_reference operator--() &&
+ {
+ value_type __x = __read();
+ __write(--__x);
+ return {obj, index};
+ }
+
+ template <class _Tp = void,
+ class = decltype(
+ std::declval<std::conditional_t<true, value_type, _Tp> &>()--)>
+ _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
+ {
+ const value_type __r = __read();
+ value_type __x = __r;
+ __write(--__x);
+ return __r;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC friend void swap(_Smart_reference &&__a, _Smart_reference &&__b) noexcept(
+ conjunction<std::is_nothrow_constructible<value_type, _Smart_reference &&>,
+ std::is_nothrow_assignable<_Smart_reference &&, value_type &&>>::value)
+ {
+ value_type __tmp = static_cast<_Smart_reference &&>(__a);
+ static_cast<_Smart_reference &&>(__a) = static_cast<value_type>(__b);
+ static_cast<_Smart_reference &&>(__b) = std::move(__tmp);
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC friend void swap(value_type &__a, _Smart_reference &&__b) noexcept(
+ conjunction<std::is_nothrow_constructible<value_type, value_type &&>,
+ std::is_nothrow_assignable<value_type &, value_type &&>,
+ std::is_nothrow_assignable<_Smart_reference &&, value_type &&>>::value)
+ {
+ value_type __tmp(std::move(__a));
+ __a = static_cast<value_type>(__b);
+ static_cast<_Smart_reference &&>(__b) = std::move(__tmp);
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC friend void swap(_Smart_reference &&__a, value_type &__b) noexcept(
+ conjunction<std::is_nothrow_constructible<value_type, _Smart_reference &&>,
+ std::is_nothrow_assignable<value_type &, value_type &&>,
+ std::is_nothrow_assignable<_Smart_reference &&, value_type &&>>::value)
+ {
+ value_type __tmp(__a);
+ static_cast<_Smart_reference &&>(__a) = std::move(__b);
+ __b = std::move(__tmp);
+ }
+};
+
+// }}}
+// abi impl fwd decls {{{
+template <int _Bytes>
+struct __neon_simd_impl;
+template <int _Bytes>
+struct __neon_mask_impl;
+struct __sse_mask_impl;
+struct __sse_simd_impl;
+struct __avx_mask_impl;
+struct __avx_simd_impl;
+struct __avx512_mask_impl;
+struct __avx512_simd_impl;
+struct __scalar_simd_impl;
+struct __scalar_mask_impl;
+template <int _N> struct __fixed_size_simd_impl;
+template <int _N> struct __fixed_size_mask_impl;
+template <int _N, class _Abi> struct __combine_simd_impl;
+template <int _N, class _Abi> struct __combine_mask_impl;
+
+// }}}
+// __gnu_traits {{{1
+template <class _Tp, class _MT, class _Abi, size_t _N> struct __gnu_traits {
+ using _IsValid = true_type;
+ using _SimdImpl = typename _Abi::_SimdImpl;
+ using _MaskImpl = typename _Abi::_MaskImpl;
+
+ // simd and simd_mask member types {{{2
+ using _SimdMember = _SimdWrapper<_Tp, _N>;
+ using _MaskMember = _SimdWrapper<_MT, _N>;
+ static constexpr size_t _S_simd_align = alignof(_SimdMember);
+ static constexpr size_t _S_mask_align = alignof(_MaskMember);
+
+ // _Simd_base / base class for simd, providing extra conversions {{{2
+ struct simd_base2 {
+ explicit operator __intrinsic_type_t<_Tp, _N>() const
+ {
+ return static_cast<const simd<_Tp, _Abi> *>(this)->_M_data.__v();
+ }
+ explicit operator __vector_type_t<_Tp, _N>() const
+ {
+ return static_cast<const simd<_Tp, _Abi> *>(this)->_M_data.builtin();
+ }
+ };
+ struct simd_base1 {
+ explicit operator __intrinsic_type_t<_Tp, _N>() const
+ {
+ return __data(*static_cast<const simd<_Tp, _Abi> *>(this));
+ }
+ };
+ using _Simd_base = std::conditional_t<
+ std::is_same<__intrinsic_type_t<_Tp, _N>, __vector_type_t<_Tp, _N>>::value,
+ simd_base1, simd_base2>;
+
+ // _Mask_base {{{2
+ struct mask_base2 {
+ explicit operator __intrinsic_type_t<_Tp, _N>() const
+ {
+ return static_cast<const simd_mask<_Tp, _Abi> *>(this)->_M_data.__intrin();
+ }
+ explicit operator __vector_type_t<_Tp, _N>() const
+ {
+ return static_cast<const simd_mask<_Tp, _Abi> *>(this)->_M_data._M_data;
+ }
+ };
+ struct mask_base1 {
+ explicit operator __intrinsic_type_t<_Tp, _N>() const
+ {
+ return __data(*static_cast<const simd_mask<_Tp, _Abi> *>(this));
+ }
+ };
+ using _Mask_base = std::conditional_t<
+ std::is_same<__intrinsic_type_t<_Tp, _N>, __vector_type_t<_Tp, _N>>::value,
+ mask_base1, mask_base2>;
+
+ // _MaskCastType {{{2
+ // parameter type of one explicit simd_mask constructor
+ class _MaskCastType
+ {
+ using _U = __intrinsic_type_t<_Tp, _N>;
+ _U _M_data;
+
+ public:
+ _MaskCastType(_U __x) : _M_data(__x) {}
+ operator _MaskMember() const { return _M_data; }
+ };
+
+ // _SimdCastType {{{2
+ // parameter type of one explicit simd constructor
+ class simd_cast_type1
+ {
+ using _A = __intrinsic_type_t<_Tp, _N>;
+ _SimdMember _M_data;
+
+ public:
+ simd_cast_type1(_A __a) : _M_data(__vector_bitcast<_Tp>(__a)) {}
+ operator _SimdMember() const { return _M_data; }
+ };
+
+ class simd_cast_type2
+ {
+ using _A = __intrinsic_type_t<_Tp, _N>;
+ using _B = __vector_type_t<_Tp, _N>;
+ _SimdMember _M_data;
+
+ public:
+ simd_cast_type2(_A __a) : _M_data(__vector_bitcast<_Tp>(__a)) {}
+ simd_cast_type2(_B __b) : _M_data(__b) {}
+ operator _SimdMember() const { return _M_data; }
+ };
+
+ using _SimdCastType = std::conditional_t<
+ std::is_same<__intrinsic_type_t<_Tp, _N>, __vector_type_t<_Tp, _N>>::value,
+ simd_cast_type1, simd_cast_type2>;
+ //}}}2
+};
+
+// __neon_is_vectorizable {{{1
+#if _GLIBCXX_SIMD_HAVE_NEON_ABI
+template <class _Tp> struct __neon_is_vectorizable : __is_vectorizable<_Tp> {};
+template <> struct __neon_is_vectorizable<long double> : false_type {};
+#if !_GLIBCXX_SIMD_HAVE_FULL_NEON_ABI
+template <> struct __neon_is_vectorizable<double> : false_type {};
+#endif
+#else
+template <class _Tp> struct __neon_is_vectorizable : false_type {};
+#endif
+
+// __sse_is_vectorizable {{{1
+#if _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI
+template <class _Tp> struct __sse_is_vectorizable : __is_vectorizable<_Tp> {};
+template <> struct __sse_is_vectorizable<long double> : false_type {};
+#elif _GLIBCXX_SIMD_HAVE_SSE_ABI
+template <class _Tp> struct __sse_is_vectorizable : is_same<_Tp, float> {};
+#else
+template <class _Tp> struct __sse_is_vectorizable : false_type {};
+#endif
+
+// __avx_is_vectorizable {{{1
+#if _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI
+template <class _Tp> struct __avx_is_vectorizable : __is_vectorizable<_Tp> {};
+#elif _GLIBCXX_SIMD_HAVE_AVX_ABI
+template <class _Tp> struct __avx_is_vectorizable : std::is_floating_point<_Tp> {};
+#else
+template <class _Tp> struct __avx_is_vectorizable : false_type {};
+#endif
+template <> struct __avx_is_vectorizable<long double> : false_type {};
+
+// __avx512_is_vectorizable {{{1
+#if _GLIBCXX_SIMD_HAVE_AVX512_ABI
+template <class _Tp> struct __avx512_is_vectorizable : __is_vectorizable<_Tp> {};
+template <> struct __avx512_is_vectorizable<long double> : false_type {};
+#if !_GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI
+template <> struct __avx512_is_vectorizable< char> : false_type {};
+template <> struct __avx512_is_vectorizable< _UChar> : false_type {};
+template <> struct __avx512_is_vectorizable< _SChar> : false_type {};
+template <> struct __avx512_is_vectorizable< short> : false_type {};
+template <> struct __avx512_is_vectorizable<_UShort> : false_type {};
+template <> struct __avx512_is_vectorizable<char16_t> : false_type {};
+template <> struct __avx512_is_vectorizable<wchar_t> : __bool_constant<sizeof(wchar_t) >= 4> {};
+#endif
+#else
+template <class _Tp> struct __avx512_is_vectorizable : false_type {};
+#endif
+
+// }}}
+// __implicit_mask_abi_base {{{
+template <int _Bytes, class _Abi> struct __implicit_mask_abi_base {
+ template <class _Tp>
+ using implicit_mask_type =
+ __vector_type_t<__int_for_sizeof_t<_Tp>, simd_size_v<_Tp, _Abi>>;
+
+ template <class _Tp>
+ static constexpr auto implicit_mask =
+ reinterpret_cast<__vector_type_t<_Tp, simd_size_v<_Tp, _Abi>>>(
+ _Abi::is_partial ? __generate_vector<implicit_mask_type<_Tp>>([](auto __i) {
+ return __i < _Bytes / sizeof(_Tp) ? -1 : 0;
+ })
+ : ~implicit_mask_type<_Tp>());
+
+ template <class _Tp, class _TVT = _VectorTraits<_Tp>>
+ static constexpr auto masked(_Tp __x)
+ {
+ using _U = typename _TVT::value_type;
+ if constexpr (_Abi::is_partial) {
+ return __and(__x , implicit_mask<_U>);
+ } else {
+ return __x;
+ }
+ }
+};
+
+// }}}
+
+namespace simd_abi
+{
+// __combine {{{1
+template <int _N, class _Abi> struct __combine {
+ template <class _Tp> static constexpr size_t size = _N *_Abi::template size<_Tp>;
+ template <class _Tp> static constexpr size_t full_size = size<_Tp>;
+
+ static constexpr int factor = _N;
+ using member_abi = _Abi;
+
+ // validity traits {{{2
+ // allow 2x, 3x, and 4x "unroll"
+ struct is_valid_abi_tag
+ : __bool_constant<(_N > 1 && _N <= 4) && _Abi::is_valid_abi_tag> {
+ };
+ template <class _Tp> struct is_valid_size_for : _Abi::template is_valid_size_for<_Tp> {
+ };
+ template <class _Tp>
+ struct _IsValid : conjunction<is_valid_abi_tag, typename _Abi::template _IsValid<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __combine_simd_impl<_N, _Abi>;
+ using _MaskImpl = __combine_mask_impl<_N, _Abi>;
+
+ // __traits {{{2
+ template <class _Tp, bool = is_valid_v<_Tp>> struct __traits : _InvalidTraits {
+ };
+
+ template <class _Tp> struct __traits<_Tp, true> {
+ using _IsValid = true_type;
+ using _SimdImpl = __combine_simd_impl<_N, _Abi>;
+ using _MaskImpl = __combine_mask_impl<_N, _Abi>;
+
+ // simd and simd_mask member types {{{2
+ using _SimdMember =
+ std::array<typename _Abi::template __traits<_Tp>::_SimdMember, _N>;
+ using _MaskMember =
+ std::array<typename _Abi::template __traits<_Tp>::_MaskMember, _N>;
+ static constexpr size_t _S_simd_align =
+ _Abi::template __traits<_Tp>::_S_simd_align;
+ static constexpr size_t _S_mask_align =
+ _Abi::template __traits<_Tp>::_S_mask_align;
+
+ // _Simd_base / base class for simd, providing extra conversions {{{2
+ struct _Simd_base {
+ explicit operator const _SimdMember &() const
+ {
+ return static_cast<const simd<_Tp, __combine> *>(this)->_M_data;
+ }
+ };
+
+ // _Mask_base {{{2
+ // empty. The std::bitset interface suffices
+ struct _Mask_base {
+ explicit operator const _MaskMember &() const
+ {
+ return static_cast<const simd_mask<_Tp, __combine> *>(this)->_M_data;
+ }
+ };
+
+ // _SimdCastType {{{2
+ struct _SimdCastType {
+ _SimdCastType(const _SimdMember &dd) : _M_data(dd) {}
+ explicit operator const _SimdMember &() const { return _M_data; }
+
+ private:
+ const _SimdMember &_M_data;
+ };
+
+ // _MaskCastType {{{2
+ struct _MaskCastType {
+ _MaskCastType(const _MaskMember &dd) : _M_data(dd) {}
+ explicit operator const _MaskMember &() const { return _M_data; }
+
+ private:
+ const _MaskMember &_M_data;
+ };
+ //}}}2
+ };
+ //}}}2
+};
+// __neon_abi {{{1
+template <int _Bytes>
+struct __neon_abi : __implicit_mask_abi_base<_Bytes, __neon_abi<_Bytes>> {
+ template <class _Tp> static constexpr size_t size = _Bytes / sizeof(_Tp);
+ template <class _Tp>
+ static constexpr size_t full_size = (_Bytes > 8 ? 16 : 8) / sizeof(_Tp);
+ static constexpr bool is_partial =
+ _Bytes < 8 || (_Bytes > 8 && _Bytes < 16);
+
+ // validity traits {{{2
+ struct is_valid_abi_tag : __bool_constant<(_Bytes == 8 || _Bytes == 16)>
+ {
+ };
+ //struct is_valid_abi_tag : __bool_constant<(_Bytes > 0 && _Bytes <= 16)> {};
+ template <class _Tp>
+ struct is_valid_size_for
+ : __bool_constant<(_Bytes / sizeof(_Tp) > 1 && _Bytes % sizeof(_Tp) == 0)> {
+ };
+ template <class _Tp>
+ struct _IsValid : conjunction<is_valid_abi_tag, __neon_is_vectorizable<_Tp>,
+ is_valid_size_for<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __neon_simd_impl<_Bytes>;
+ using _MaskImpl = __neon_mask_impl<_Bytes>;
+
+ // __traits {{{2
+ template <class _Tp>
+ using __traits = std::conditional_t<is_valid_v<_Tp>,
+ __gnu_traits<_Tp, _Tp, __neon_abi, full_size<_Tp>>,
+ _InvalidTraits>;
+ //}}}2
+};
+
+// __sse_abi {{{1
+template <int _Bytes>
+struct __sse_abi : __implicit_mask_abi_base<_Bytes, __sse_abi<_Bytes>> {
+ template <class _Tp> static constexpr size_t size = _Bytes / sizeof(_Tp);
+ template <class _Tp> static constexpr size_t full_size = 16 / sizeof(_Tp);
+ static constexpr bool is_partial = _Bytes < 16;
+
+ // validity traits {{{2
+ // allow 2x, 3x, and 4x "unroll"
+ struct is_valid_abi_tag : __bool_constant<_Bytes == 16> {};
+ //struct is_valid_abi_tag : __bool_constant<(_Bytes > 0 && _Bytes <= 16)> {};
+ template <class _Tp>
+ struct is_valid_size_for
+ : __bool_constant<(_Bytes / sizeof(_Tp) > 1 && _Bytes % sizeof(_Tp) == 0)> {
+ };
+
+ template <class _Tp>
+ struct _IsValid
+ : conjunction<is_valid_abi_tag, __sse_is_vectorizable<_Tp>, is_valid_size_for<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __sse_simd_impl;
+ using _MaskImpl = __sse_mask_impl;
+
+ // __traits {{{2
+ template <class _Tp>
+ using __traits = std::conditional_t<is_valid_v<_Tp>,
+ __gnu_traits<_Tp, _Tp, __sse_abi, full_size<_Tp>>,
+ _InvalidTraits>;
+ //}}}2
+};
+
+// __avx_abi {{{1
+template <int _Bytes>
+struct __avx_abi : __implicit_mask_abi_base<_Bytes, __avx_abi<_Bytes>> {
+ template <class _Tp> static constexpr size_t size = _Bytes / sizeof(_Tp);
+ template <class _Tp> static constexpr size_t full_size = 32 / sizeof(_Tp);
+ static constexpr bool is_partial = _Bytes < 32;
+
+ // validity traits {{{2
+ // - allow 2x, 3x, and 4x "unroll"
+ // - disallow <= 16 _Bytes as that's covered by __sse_abi
+ struct is_valid_abi_tag : __bool_constant<_Bytes == 32> {};
+ /* TODO:
+ struct is_valid_abi_tag
+ : __bool_constant<((_Bytes > 16 && _Bytes <= 32) || _Bytes == 64 ||
+ _Bytes == 96 || _Bytes == 128)> {
+ };
+ */
+ template <class _Tp>
+ struct is_valid_size_for : __bool_constant<(_Bytes % sizeof(_Tp) == 0)> {
+ };
+ template <class _Tp>
+ struct _IsValid
+ : conjunction<is_valid_abi_tag, __avx_is_vectorizable<_Tp>, is_valid_size_for<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __avx_simd_impl;
+ using _MaskImpl = __avx_mask_impl;
+
+ // __traits {{{2
+ template <class _Tp>
+ using __traits = std::conditional_t<is_valid_v<_Tp>,
+ __gnu_traits<_Tp, _Tp, __avx_abi, full_size<_Tp>>,
+ _InvalidTraits>;
+ //}}}2
+};
+
+// __avx512_abi {{{1
+template <int _Bytes> struct __avx512_abi {
+ template <class _Tp> static constexpr size_t size = _Bytes / sizeof(_Tp);
+ template <class _Tp> static constexpr size_t full_size = 64 / sizeof(_Tp);
+ static constexpr bool is_partial = _Bytes < 64;
+
+ // validity traits {{{2
+ // - disallow <= 32 _Bytes as that's covered by __sse_abi and __avx_abi
+ // TODO: consider AVX512VL
+ struct is_valid_abi_tag : __bool_constant<_Bytes == 64> {};
+ /* TODO:
+ struct is_valid_abi_tag
+ : __bool_constant<(_Bytes > 32 && _Bytes <= 64)> {
+ };
+ */
+ template <class _Tp>
+ struct is_valid_size_for : __bool_constant<(_Bytes % sizeof(_Tp) == 0)> {
+ };
+ template <class _Tp>
+ struct _IsValid
+ : conjunction<is_valid_abi_tag, __avx512_is_vectorizable<_Tp>, is_valid_size_for<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // implicit mask {{{2
+ template <class _Tp>
+ using implicit_mask_type = __bool_storage_member_type_t<64 / sizeof(_Tp)>;
+
+ template <class _Tp>
+ static constexpr implicit_mask_type<_Tp> implicit_mask =
+ _Bytes == 64 ? ~implicit_mask_type<_Tp>()
+ : (implicit_mask_type<_Tp>(1) << (_Bytes / sizeof(_Tp))) - 1;
+
+ template <class _Tp, class = enable_if_t<__is_bitmask_v<_Tp>>>
+ static constexpr _Tp masked(_Tp __x)
+ {
+ if constexpr (is_partial) {
+ constexpr size_t _N = sizeof(_Tp) * 8;
+ return __x &
+ ((__bool_storage_member_type_t<_N>(1) << (_Bytes * _N / 64)) - 1);
+ } else {
+ return __x;
+ }
+ }
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __avx512_simd_impl;
+ using _MaskImpl = __avx512_mask_impl;
+
+ // __traits {{{2
+ template <class _Tp>
+ using __traits =
+ std::conditional_t<is_valid_v<_Tp>,
+ __gnu_traits<_Tp, bool, __avx512_abi, full_size<_Tp>>,
+ _InvalidTraits>;
+ //}}}2
+};
+
+// __scalar_abi {{{1
+struct __scalar_abi {
+ template <class _Tp> static constexpr size_t size = 1;
+ template <class _Tp> static constexpr size_t full_size = 1;
+ struct is_valid_abi_tag : true_type {};
+ template <class _Tp> struct is_valid_size_for : true_type {};
+ template <class _Tp> struct _IsValid : __is_vectorizable<_Tp> {};
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ using _SimdImpl = __scalar_simd_impl;
+ using _MaskImpl = __scalar_mask_impl;
+
+ template <class _Tp, bool = is_valid_v<_Tp>> struct __traits : _InvalidTraits {
+ };
+
+ template <class _Tp> struct __traits<_Tp, true> {
+ using _IsValid = true_type;
+ using _SimdImpl = __scalar_simd_impl;
+ using _MaskImpl = __scalar_mask_impl;
+ using _SimdMember = _Tp;
+ using _MaskMember = bool;
+ static constexpr size_t _S_simd_align = alignof(_SimdMember);
+ static constexpr size_t _S_mask_align = alignof(_MaskMember);
+
+ // nothing the user can spell converts to/from simd/simd_mask
+ struct _SimdCastType {
+ _SimdCastType() = delete;
+ };
+ struct _MaskCastType {
+ _MaskCastType() = delete;
+ };
+ struct _Simd_base {};
+ struct _Mask_base {};
+ };
+};
+
+// __fixed_abi {{{1
+template <int _N> struct __fixed_abi {
+ template <class _Tp> static constexpr size_t size = _N;
+ template <class _Tp> static constexpr size_t full_size = _N;
+ // validity traits {{{2
+ struct is_valid_abi_tag
+ : public __bool_constant<(_N > 0)> {
+ };
+ template <class _Tp>
+ struct is_valid_size_for
+ : __bool_constant<((_N <= simd_abi::max_fixed_size<_Tp>) ||
+ (simd_abi::__neon::is_valid_v<char> &&
+ _N == simd_size_v<char, simd_abi::__neon>) ||
+ (simd_abi::__sse::is_valid_v<char> &&
+ _N == simd_size_v<char, simd_abi::__sse>) ||
+ (simd_abi::__avx::is_valid_v<char> &&
+ _N == simd_size_v<char, simd_abi::__avx>) ||
+ (simd_abi::__avx512::is_valid_v<char> &&
+ _N == simd_size_v<char, simd_abi::__avx512>))> {
+ };
+ template <class _Tp>
+ struct _IsValid
+ : conjunction<is_valid_abi_tag, __is_vectorizable<_Tp>, is_valid_size_for<_Tp>> {
+ };
+ template <class _Tp> static constexpr bool is_valid_v = _IsValid<_Tp>::value;
+
+ // simd/_MaskImpl {{{2
+ using _SimdImpl = __fixed_size_simd_impl<_N>;
+ using _MaskImpl = __fixed_size_mask_impl<_N>;
+
+ // __traits {{{2
+ template <class _Tp, bool = is_valid_v<_Tp>> struct __traits : _InvalidTraits {
+ };
+
+ template <class _Tp> struct __traits<_Tp, true> {
+ using _IsValid = true_type;
+ using _SimdImpl = __fixed_size_simd_impl<_N>;
+ using _MaskImpl = __fixed_size_mask_impl<_N>;
+
+ // simd and simd_mask member types {{{2
+ using _SimdMember = __fixed_size_storage_t<_Tp, _N>;
+ using _MaskMember = std::bitset<_N>;
+ static constexpr size_t _S_simd_align =
+ __next_power_of_2(_N * sizeof(_Tp));
+ static constexpr size_t _S_mask_align = alignof(_MaskMember);
+
+ // _Simd_base / base class for simd, providing extra conversions {{{2
+ struct _Simd_base {
+ // The following ensures, function arguments are passed via the stack. This is
+ // important for ABI compatibility across TU boundaries
+ _Simd_base(const _Simd_base &) {}
+ _Simd_base() = default;
+
+ explicit operator const _SimdMember &() const
+ {
+ return static_cast<const simd<_Tp, __fixed_abi> *>(this)->_M_data;
+ }
+ explicit operator std::array<_Tp, _N>() const
+ {
+ std::array<_Tp, _N> __r;
+ // _SimdMember can be larger because of higher alignment
+ static_assert(sizeof(__r) <= sizeof(_SimdMember), "");
+ std::memcpy(__r.data(), &static_cast<const _SimdMember &>(*this),
+ sizeof(__r));
+ return __r;
+ }
+ };
+
+ // _Mask_base {{{2
+ // empty. The std::bitset interface suffices
+ struct _Mask_base {};
+
+ // _SimdCastType {{{2
+ struct _SimdCastType {
+ _SimdCastType(const std::array<_Tp, _N> &);
+ _SimdCastType(const _SimdMember &dd) : _M_data(dd) {}
+ explicit operator const _SimdMember &() const { return _M_data; }
+
+ private:
+ const _SimdMember &_M_data;
+ };
+
+ // _MaskCastType {{{2
+ class _MaskCastType
+ {
+ _MaskCastType() = delete;
+ };
+ //}}}2
+ };
+};
+
+//}}}
+} // namespace simd_abi
+
+// __scalar_abi_wrapper {{{1
+template <int _Bytes> struct __scalar_abi_wrapper : simd_abi::__scalar_abi {
+ template <class _Tp>
+ static constexpr bool is_valid_v = simd_abi::__scalar_abi::_IsValid<_Tp>::value &&
+ sizeof(_Tp) == _Bytes;
+};
+
+// __decay_abi metafunction {{{1
+template <class _Tp> struct __decay_abi {
+ using type = _Tp;
+};
+template <int _Bytes> struct __decay_abi<__scalar_abi_wrapper<_Bytes>> {
+ using type = simd_abi::scalar;
+};
+
+// __full_abi metafunction {{{1
+template <template <int> class, int _Bytes>
+struct __full_abi;
+
+template <int _Bytes>
+struct __full_abi<simd_abi::__neon_abi, _Bytes>
+{
+ using type = simd_abi::__neon_abi<(_Bytes >= 16 ? 16 : 8)>;
+};
+
+template <int _Bytes>
+struct __full_abi<simd_abi::__sse_abi, _Bytes>
+{
+ using type = simd_abi::__sse;
+};
+template <int _Bytes>
+struct __full_abi<simd_abi::__avx_abi, _Bytes>
+{
+ using type = simd_abi::__avx;
+};
+template <int _Bytes>
+struct __full_abi<simd_abi::__avx512_abi, _Bytes>
+{
+ using type = simd_abi::__avx512;
+};
+template <int _Bytes>
+struct __full_abi<__scalar_abi_wrapper, _Bytes>
+{
+ using type = simd_abi::scalar;
+};
+
+// __abi_list {{{1
+template <template <int> class...> struct __abi_list {
+ template <class, int> static constexpr bool __has_valid_abi = false;
+ template <class, int> using __first_valid_abi = void;
+ template <class, int> using __best_abi = void;
+};
+
+template <template <int> class _A0, template <int> class... _Rest>
+struct __abi_list<_A0, _Rest...> {
+ template <class _Tp, int _N>
+ static constexpr bool __has_valid_abi = _A0<sizeof(_Tp) * _N>::template is_valid_v<_Tp> ||
+ __abi_list<_Rest...>::template __has_valid_abi<_Tp, _N>;
+ template <class _Tp, int _N>
+ using __first_valid_abi =
+ std::conditional_t<_A0<sizeof(_Tp) * _N>::template is_valid_v<_Tp>,
+ typename __decay_abi<_A0<sizeof(_Tp) * _N>>::type,
+ typename __abi_list<_Rest...>::template __first_valid_abi<_Tp, _N>>;
+ template <class _Tp,
+ int _N,
+ int _Bytes = sizeof(_Tp) * _N,
+ typename _B = typename __full_abi<_A0, _Bytes>::type>
+ using __best_abi = std::conditional_t<
+ _A0<_Bytes>::template is_valid_v<_Tp>,
+ typename __decay_abi<_A0<_Bytes>>::type,
+ std::conditional_t<
+ (_B::template is_valid_v<_Tp> && _B::template size<_Tp> <= _N),
+ _B,
+ typename __abi_list<_Rest...>::template __best_abi<_Tp, _N>>>;
+};
+
+// }}}1
+
+// the following lists all native ABIs, which makes them accessible to simd_abi::deduce
+// and select_best_vector_type_t (for fixed_size). Order matters: Whatever comes first has
+// higher priority.
+using __all_native_abis =
+ __abi_list<simd_abi::__avx512_abi, simd_abi::__avx_abi, simd_abi::__sse_abi,
+ simd_abi::__neon_abi, __scalar_abi_wrapper>;
+
+// valid _SimdTraits specialization {{{1
+template <class _Tp, class _Abi>
+struct _SimdTraits<_Tp, _Abi, std::void_t<typename _Abi::template _IsValid<_Tp>>>
+ : _Abi::template __traits<_Tp> {
+};
+
+// __deduce_impl specializations {{{1
+// try all native ABIs (including scalar) first
+template <class _Tp, std::size_t _N>
+struct __deduce_impl<_Tp, _N,
+ enable_if_t<__all_native_abis::template __has_valid_abi<_Tp, _N>>> {
+ using type = __all_native_abis::__first_valid_abi<_Tp, _N>;
+};
+
+// fall back to fixed_size only if scalar and native ABIs don't match
+template <class _Tp, std::size_t _N, class = void> struct __deduce_fixed_size_fallback {};
+template <class _Tp, std::size_t _N>
+struct __deduce_fixed_size_fallback<
+ _Tp, _N, enable_if_t<simd_abi::fixed_size<_N>::template is_valid_v<_Tp>>> {
+ using type = simd_abi::fixed_size<_N>;
+};
+template <class _Tp, std::size_t _N, class>
+struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _N> {
+};
+
+//}}}1
+
+// simd_mask {{{
+template <class _Tp, class _Abi> class simd_mask : public _SimdTraits<_Tp, _Abi>::_Mask_base
+{
+ // types, tags, and friends {{{
+ using __traits = _SimdTraits<_Tp, _Abi>;
+ using __impl = typename __traits::_MaskImpl;
+ using __member_type = typename __traits::_MaskMember;
+ static constexpr _Tp *_S_type_tag = nullptr;
+ friend typename __traits::_Mask_base;
+ friend class simd<_Tp, _Abi>; // to construct masks on return
+ friend __impl;
+ friend typename __traits::_SimdImpl; // to construct masks on return and
+ // inspect data on masked operations
+ // }}}
+ // is_<abi> {{{
+ static constexpr bool __is_scalar() { return __is_abi<_Abi, simd_abi::scalar>(); }
+ static constexpr bool __is_sse() { return __is_abi<_Abi, simd_abi::__sse_abi>(); }
+ static constexpr bool __is_avx() { return __is_abi<_Abi, simd_abi::__avx_abi>(); }
+ static constexpr bool __is_avx512()
+ {
+ return __is_abi<_Abi, simd_abi::__avx512_abi>();
+ }
+ static constexpr bool __is_neon()
+ {
+ return __is_abi<_Abi, simd_abi::__neon_abi>();
+ }
+ static constexpr bool __is_fixed() { return __is_fixed_size_abi_v<_Abi>; }
+ static constexpr bool __is_combined() { return __is_combined_abi<_Abi>(); }
+
+ // }}}
+
+public:
+ // member types {{{
+ using value_type = bool;
+ using reference = _Smart_reference<__member_type, __impl, value_type>;
+ using simd_type = simd<_Tp, _Abi>;
+ using abi_type = _Abi;
+
+ // }}}
+ static constexpr size_t size() { return __size_or_zero_v<_Tp, _Abi>; }
+ // constructors & assignment {{{
+ simd_mask() = default;
+ simd_mask(const simd_mask &) = default;
+ simd_mask(simd_mask &&) = default;
+ simd_mask &operator=(const simd_mask &) = default;
+ simd_mask &operator=(simd_mask &&) = default;
+
+ // }}}
+
+ // access to internal representation (suggested extension) {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE explicit simd_mask(
+ typename __traits::_MaskCastType __init)
+ : _M_data{__init}
+ {
+ }
+ // conversions to internal type is done in _Mask_base
+
+ // }}}
+ // bitset interface (extension to be proposed) {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask __from_bitset(std::bitset<size()> bs)
+ {
+ return {__bitset_init, bs};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE std::bitset<size()> __to_bitset() const {
+ if constexpr (__is_scalar()) {
+ return unsigned(_M_data);
+ } else if constexpr (__is_fixed()) {
+ return _M_data;
+ } else {
+ return __vector_to_bitset(builtin());
+ }
+ }
+
+ // }}}
+ // explicit broadcast constructor {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE explicit constexpr simd_mask(value_type __x) : _M_data(__broadcast(__x)) {}
+
+ // }}}
+ // implicit type conversion constructor {{{
+ template <class _U, class = enable_if_t<
+ conjunction<is_same<abi_type, simd_abi::fixed_size<size()>>,
+ is_same<_U, _U>>::value>>
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask(
+ const simd_mask<_U, simd_abi::fixed_size<size()>> &__x)
+ : simd_mask{__bitset_init, __data(__x)}
+ {
+ }
+ // }}}
+ /* reference implementation for explicit simd_mask casts {{{
+ template <class _U, class = enable_if<
+ (size() == simd_mask<_U, _Abi>::size()) &&
+ conjunction<std::is_integral<_Tp>, std::is_integral<_U>,
+ __negation<std::is_same<_Abi, simd_abi::fixed_size<size()>>>,
+ __negation<std::is_same<_Tp, _U>>>::value>>
+ simd_mask(const simd_mask<_U, _Abi> &__x)
+ : _M_data{__x._M_data}
+ {
+ }
+ template <class _U, class _Abi2, class = enable_if<conjunction<
+ __negation<std::is_same<abi_type, _Abi2>>,
+ std::is_same<abi_type, simd_abi::fixed_size<size()>>>::value>>
+ simd_mask(const simd_mask<_U, _Abi2> &__x)
+ {
+ __x.copy_to(&_M_data[0], vector_aligned);
+ }
+ }}} */
+
+ // load __impl {{{
+private:
+ template <class _F>
+ _GLIBCXX_SIMD_INTRINSIC static __member_type load_wrapper(const value_type* __mem,
+ [[maybe_unused]] _F __f)
+ {
+ if constexpr (__is_scalar())
+ {
+ return __mem[0];
+ }
+ else if constexpr (__is_fixed())
+ {
+ const fixed_size_simd<unsigned char, size()> __bools(
+ reinterpret_cast<const __may_alias<unsigned char>*>(__mem), __f);
+ return __data(__bools != 0);
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (__is_sse())
+ {
+ if constexpr (size() == 2 && __have_sse2)
+ {
+ return _ToWrapper(_mm_set_epi32(-int(__mem[1]), -int(__mem[1]),
+ -int(__mem[0]), -int(__mem[0])));
+ } else if constexpr (size() == 4 && __have_sse2) {
+ __m128i __k = _mm_cvtsi32_si128(*reinterpret_cast<const int *>(__mem));
+ __k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), _mm_setzero_si128());
+ return _ToWrapper(_mm_unpacklo_epi16(__k, __k));
+ } else if constexpr (size() == 4 && __have_mmx) {
+ __m128 __k =
+ _mm_cvtpi8_ps(_mm_cvtsi32_si64(*reinterpret_cast<const int *>(__mem)));
+ _mm_empty();
+ return _ToWrapper(_mm_cmpgt_ps(__k, __m128()));
+ } else if constexpr (size() == 8 && __have_sse2) {
+ const auto __k = __make_vector<long long>(
+ *reinterpret_cast<const __may_alias<long long> *>(__mem), 0);
+ if constexpr (__have_sse2) {
+ return _ToWrapper(
+ __vector_bitcast<short>(_mm_unpacklo_epi8(__k, __k)) != 0);
+ }
+ } else if constexpr (size() == 16 && __have_sse2) {
+ return __vector_bitcast<_Tp>(
+ _mm_cmpgt_epi8(__vector_load<long long, 2>(__mem, __f), __m128i()));
+ } else {
+ __assert_unreachable<_F>();
+ }
+ }
+ else if constexpr (__is_avx())
+ {
+ if constexpr (size() == 4 && __have_avx)
+ {
+ int bool4;
+ if constexpr (__is_aligned_v<_F, 4>)
+ {
+ bool4 = *reinterpret_cast<const __may_alias<int>*>(__mem);
+ } else {
+ std::memcpy(&bool4, __mem, 4);
+ }
+ const auto __k = __to_intrin(
+ (__vector_broadcast<4>(bool4) &
+ __make_vector<int>(0x1, 0x100, 0x10000, 0x1000000)) != 0);
+ return _ToWrapper(
+ __concat(_mm_unpacklo_epi32(__k, __k), _mm_unpackhi_epi32(__k, __k)));
+ } else if constexpr (size() == 8 && __have_avx) {
+ auto __k = __vector_load<long long, 2, 8>(__mem, __f);
+ __k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), __m128i());
+ return _ToWrapper(
+ __concat(_mm_unpacklo_epi16(__k, __k), _mm_unpackhi_epi16(__k, __k)));
+ } else if constexpr (size() == 16 && __have_avx) {
+ const auto __k =
+ _mm_cmpgt_epi8(__vector_load<long long, 2>(__mem, __f), __m128i());
+ return __concat(_mm_unpacklo_epi8(__k, __k), _mm_unpackhi_epi8(__k, __k));
+ } else if constexpr (size() == 32 && __have_avx2) {
+ return __vector_bitcast<_Tp>(
+ _mm256_cmpgt_epi8(__vector_load<long long, 4>(__mem, __f), __m256i()));
+ } else {
+ __assert_unreachable<_F>();
+ }
+ }
+ else if constexpr (__is_avx512())
+ {
+ if constexpr (size() == 8)
+ {
+ const auto __a = __vector_load<long long, 2, 8>(__mem, __f);
+ if constexpr (__have_avx512bw_vl)
+ {
+ return _mm_test_epi8_mask(__a, __a);
+ } else {
+ const auto __b = _mm512_cvtepi8_epi64(__a);
+ return _mm512_test_epi64_mask(__b, __b);
+ }
+ } else if constexpr (size() == 16) {
+ const auto __a = __vector_load<long long, 2>(__mem, __f);
+ if constexpr (__have_avx512bw_vl) {
+ return _mm_test_epi8_mask(__a, __a);
+ } else {
+ const auto __b = _mm512_cvtepi8_epi32(__a);
+ return _mm512_test_epi32_mask(__b, __b);
+ }
+ } else if constexpr (size() == 32) {
+ if constexpr (__have_avx512bw_vl) {
+ const auto __a = __vector_load<long long, 4>(__mem, __f);
+ return _mm256_test_epi8_mask(__a, __a);
+ } else {
+ const auto __a =
+ _mm512_cvtepi8_epi32(__vector_load<long long, 2>(__mem, __f));
+ const auto __b =
+ _mm512_cvtepi8_epi32(__vector_load<long long, 2>(__mem + 16, __f));
+ return _mm512_test_epi32_mask(__a, __a) |
+ (_mm512_test_epi32_mask(__b, __b) << 16);
+ }
+ } else if constexpr (size() == 64) {
+ if constexpr (__have_avx512bw) {
+ const auto __a = __vector_load<long long, 8>(__mem, __f);
+ return _mm512_test_epi8_mask(__a, __a);
+ } else {
+ const auto __a =
+ _mm512_cvtepi8_epi32(__vector_load<long long, 2>(__mem, __f));
+ const auto __b = _mm512_cvtepi8_epi32(
+ __vector_load<long long, 2>(__mem + 16, __f));
+ const auto __c = _mm512_cvtepi8_epi32(
+ __vector_load<long long, 2>(__mem + 32, __f));
+ const auto __d = _mm512_cvtepi8_epi32(
+ __vector_load<long long, 2>(__mem + 48, __f));
+ return _mm512_test_epi32_mask(__a, __a) |
+ (_mm512_test_epi32_mask(__b, __b) << 16) |
+ (_mm512_test_epi32_mask(__c, __c) << 32) |
+ (_mm512_test_epi32_mask(__d, __d) << 48);
+ }
+ } else {
+ __assert_unreachable<_F>();
+ }
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ else if constexpr (sizeof(_Tp) == sizeof(value_type) &&
+ is_integral_v<_Tp>)
+ {
+ const auto __bools = __vector_load<_Tp, size()>(__mem, __f);
+ return __vector_bitcast<_Tp>(__bools > 0);
+ }
+ else
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ return __vector_bitcast<_Tp>(__generate_vector<_I, size()>(
+ [&](auto __i) { return __mem[__i] ? ~_I() : _I(); }));
+ }
+ }
+
+public :
+ // }}}
+ // load constructor {{{
+ template <class _Flags>
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask(const value_type* __mem, _Flags __f)
+ : _M_data(load_wrapper(__mem, __f))
+ {
+ }
+ template <class _Flags>
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask(const value_type *__mem, simd_mask __k, _Flags __f) : _M_data{}
+ {
+ _M_data = __impl::masked_load(_M_data, __k._M_data, __mem, __f);
+ }
+
+ // }}}
+ // loads [simd_mask.load] {{{
+ template <class _Flags> _GLIBCXX_SIMD_ALWAYS_INLINE void copy_from(const value_type *__mem, _Flags __f)
+ {
+ _M_data = load_wrapper(__mem, __f);
+ }
+
+ // }}}
+ // stores [simd_mask.store] {{{
+ template <class _Flags> _GLIBCXX_SIMD_ALWAYS_INLINE void copy_to(value_type *__mem, _Flags __f) const
+ {
+ __impl::store(_M_data, __mem, __f);
+ }
+
+ // }}}
+ // scalar access {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE reference operator[](size_t __i) { return {_M_data, int(__i)}; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE value_type operator[](size_t __i) const {
+ if constexpr (__is_scalar()) {
+ _GLIBCXX_DEBUG_ASSERT(__i == 0);
+ __unused(__i);
+ return _M_data;
+ } else {
+ return _M_data[__i];
+ }
+ }
+
+ // }}}
+ // negation {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask operator!() const
+ {
+ if constexpr (__is_scalar()) {
+ return {__private_init, !_M_data};
+ } else if constexpr (__is_fixed()) {
+ return {__private_init, ~builtin()};
+ } else if constexpr (__have_avx512dq && __is_avx512() && size() <= 8) {
+ return {__private_init, _knot_mask8(builtin())};
+ } else if constexpr (__is_avx512() && size() <= 16) {
+ // the following is a narrowing conversion on KNL for doubles (__mmask8)
+ return simd_mask(__private_init, _knot_mask16(builtin()));
+ } else if constexpr (__have_avx512bw && __is_avx512() && size() <= 32) {
+ return {__private_init, _knot_mask32(builtin())};
+ } else if constexpr (__have_avx512bw && __is_avx512() && size() <= 64) {
+ return {__private_init, _knot_mask64(builtin())};
+ } else {
+ return {__private_init,
+ _ToWrapper(~__vector_bitcast<_UInt>(builtin()))};
+ }
+ }
+
+ // }}}
+ // simd_mask binary operators [simd_mask.binary] {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator&&(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::logical_and(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator||(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::logical_or(__x._M_data, __y._M_data)};
+ }
+
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator&(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::bit_and(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator|(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::bit_or(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator^(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::bit_xor(__x._M_data, __y._M_data)};
+ }
+
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask &operator&=(simd_mask &__x, const simd_mask &__y)
+ {
+ __x._M_data = __impl::bit_and(__x._M_data, __y._M_data);
+ return __x;
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask &operator|=(simd_mask &__x, const simd_mask &__y)
+ {
+ __x._M_data = __impl::bit_or(__x._M_data, __y._M_data);
+ return __x;
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask &operator^=(simd_mask &__x, const simd_mask &__y)
+ {
+ __x._M_data = __impl::bit_xor(__x._M_data, __y._M_data);
+ return __x;
+ }
+
+ // }}}
+ // simd_mask compares [simd_mask.comparison] {{{
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator==(const simd_mask &__x, const simd_mask &__y)
+ {
+ return !operator!=(__x, __y);
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask operator!=(const simd_mask &__x, const simd_mask &__y)
+ {
+ return {__private_init, __impl::bit_xor(__x._M_data, __y._M_data)};
+ }
+
+ // }}}
+ // private_init ctor {{{
+ _GLIBCXX_SIMD_INTRINSIC simd_mask(_PrivateInit, typename __traits::_MaskMember __init)
+ : _M_data(__init)
+ {
+ }
+
+ // }}}
+ // private_init generator ctor {{{
+ template <class _F, class = decltype(bool(std::declval<_F>()(size_t())))>
+ _GLIBCXX_SIMD_INTRINSIC simd_mask(_PrivateInit, _F &&__gen)
+ {
+ for (size_t __i = 0; __i < size(); ++__i) {
+ __impl::set(_M_data, __i, __gen(__i));
+ }
+ }
+
+ // }}}
+ // bitset_init ctor {{{
+ _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, std::bitset<size()> __init)
+ : _M_data(__impl::__from_bitset(__init, _S_type_tag))
+ {
+ }
+
+ // }}}
+ // __cvt {{{
+ struct _CvtProxy
+ {
+ template <
+ typename _U,
+ typename _A2,
+ typename = enable_if_t<simd_size_v<_U, _A2> == simd_size_v<_Tp, _Abi>>>
+ operator simd_mask<_U, _A2>() &&
+ {
+ return static_simd_cast<simd_mask<_U, _A2>>(_M_data);
+ }
+
+ const simd_mask<_Tp, _Abi>& _M_data;
+ };
+ _GLIBCXX_SIMD_INTRINSIC _CvtProxy __cvt() const { return {*this}; }
+ // }}}
+
+private:
+ _GLIBCXX_SIMD_INTRINSIC static constexpr __member_type __broadcast(value_type __x) // {{{
+ {
+ if constexpr (__is_scalar()) {
+ return __x;
+ } else if constexpr (__is_fixed()) {
+ return __x ? ~__member_type() : __member_type();
+ } else if constexpr (__is_avx512()) {
+ using mmask_type = typename __bool_storage_member_type<size()>::type;
+ return __x ? _Abi::masked(static_cast<mmask_type>(~mmask_type())) : mmask_type();
+ } else {
+ using _U = __vector_type_t<__int_for_sizeof_t<_Tp>, size()>;
+ return _ToWrapper(__x ? _Abi::masked(~_U()) : _U());
+ }
+ }
+
+ // }}}
+ // TODO remove __intrin:
+ /*auto __intrin() const // {{{
+ {
+ if constexpr (!__is_scalar() && !__is_fixed()) {
+ return __to_intrin(_M_data._M_data);
+ }
+ }*/
+
+ // }}}
+ auto &builtin() { // {{{
+ if constexpr (__is_scalar() || __is_fixed()) {
+ return _M_data;
+ } else {
+ return _M_data._M_data;
+ }
+ }
+ const auto &builtin() const
+ {
+ if constexpr (__is_scalar() || __is_fixed()) {
+ return _M_data;
+ } else {
+ return _M_data._M_data;
+ }
+ }
+
+ // }}}
+ friend const auto &__data<_Tp, abi_type>(const simd_mask &);
+ friend auto &__data<_Tp, abi_type>(simd_mask &);
+ alignas(__traits::_S_mask_align) __member_type _M_data;
+};
+
+// }}}
+
+// __data(simd_mask) {{{
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr const auto &__data(const simd_mask<_Tp, _A> &__x)
+{
+ return __x._M_data;
+}
+template <class _Tp, class _A> _GLIBCXX_SIMD_INTRINSIC constexpr auto &__data(simd_mask<_Tp, _A> &__x)
+{
+ return __x._M_data;
+}
+// }}}
+// __all_of {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC bool __all_of(const _Data &__k)
+{
+ // _Data = decltype(__data(simd_mask))
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ {
+ return __k;
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ {
+ return __k.all();
+ }
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ for (int __i = 0; __i < _Abi::factor; ++__i) {
+ if (!__all_of<_Tp, typename _Abi::member_abi>(__k[__i])) {
+ return false;
+ }
+ }
+ return true;
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>() ||
+ __is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (__have_sse4_1)
+ {
+ constexpr auto __b = reinterpret_cast<__intrinsic_type_t<_Tp, _N>>(
+ _Abi::template implicit_mask<_Tp>);
+ if constexpr (std::is_same_v<_Tp, float> && _N > 4)
+ return 0 != _mm256_testc_ps(__to_intrin(__k), __b);
+ else if constexpr (std::is_same_v<_Tp, float> && __have_avx)
+ return 0 != _mm_testc_ps(__to_intrin(__k), __b);
+ else if constexpr (std::is_same_v<_Tp, float>)
+ return 0 != _mm_testc_si128(_mm_castps_si128(__to_intrin(__k)),
+ _mm_castps_si128(__b));
+ else if constexpr (std::is_same_v<_Tp, double> && _N > 2)
+ return 0 != _mm256_testc_pd(__to_intrin(__k), __b);
+ else if constexpr (std::is_same_v<_Tp, double> && __have_avx)
+ return 0 != _mm_testc_pd(__to_intrin(__k), __b);
+ else if constexpr (std::is_same_v<_Tp, double>)
+ return 0 != _mm_testc_si128(_mm_castpd_si128(__to_intrin(__k)),
+ _mm_castpd_si128(__b));
+ else if constexpr (sizeof(__b) == 32)
+ return _mm256_testc_si256(__to_intrin(__k), __b);
+ else
+ return _mm_testc_si128(__to_intrin(__k), __b);
+ }
+ else if constexpr (std::is_same_v<_Tp, float>)
+ {
+ return (_mm_movemask_ps(__to_intrin(__k)) & ((1 << _N) - 1)) ==
+ (1 << _N) - 1;
+ }
+ else if constexpr (std::is_same_v<_Tp, double>)
+ {
+ return (_mm_movemask_pd(__to_intrin(__k)) & ((1 << _N) - 1)) ==
+ (1 << _N) - 1;
+ }
+ else
+ {
+ return (_mm_movemask_epi8(__to_intrin(__k)) &
+ ((1 << (_N * sizeof(_Tp))) - 1)) ==
+ (1 << (_N * sizeof(_Tp))) - 1;
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ constexpr auto _Mask = _Abi::template implicit_mask<_Tp>;
+ if constexpr (std::is_same_v<_Data, _SimdWrapper<bool, 8>>)
+ {
+ if constexpr (__have_avx512dq)
+ return _kortestc_mask8_u8(
+ __k._M_data, _Mask == 0xff ? __k._M_data : __mmask8(~_Mask));
+ else
+ return __k._M_data == _Mask;
+ }
+ else if constexpr (std::is_same_v<_Data, _SimdWrapper<bool, 16>>)
+ {
+ return _kortestc_mask16_u8(
+ __k._M_data, _Mask == 0xffff ? __k._M_data : __mmask16(~_Mask));
+ }
+ else if constexpr (std::is_same_v<_Data, _SimdWrapper<bool, 32>>)
+ {
+ if constexpr (__have_avx512bw)
+ {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85538
+ return __k._M_data == _Mask;
+#else
+ return _kortestc_mask32_u8(__k._M_data, _Mask == 0xffffffffU
+ ? __k._M_data
+ : __mmask32(~_Mask));
+#endif
+ }
+ }
+ else if constexpr (std::is_same_v<_Data, _SimdWrapper<bool, 64>>)
+ {
+ if constexpr (__have_avx512bw)
+ {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85538
+ return __k._M_data == _Mask;
+#else
+ return _kortestc_mask64_u8(__k._M_data,
+ _Mask == 0xffffffffffffffffULL
+ ? __k._M_data
+ : __mmask64(~_Mask));
+#endif
+ }
+ }
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+#if _GLIBCXX_SIMD_HAVE_NEON
+ else if constexpr (__is_abi<_Abi, simd_abi::__neon_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ const auto __x = __vector_bitcast<long long>(__k);
+ if constexpr (sizeof(__k) == 16)
+ return __x[0] + __x[1] == -2;
+ else if constexpr (sizeof(__k) == 8)
+ return __x == -1;
+ else
+ __assert_unreachable<_Tp>();
+ }
+#endif // _GLIBCXX_SIMD_HAVE_NEON
+ else
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ return __call_with_subscripts(
+ __vector_bitcast<__int_for_sizeof_t<_Tp>>(__k),
+ make_index_sequence<_N>(),
+ [](const auto... __ent) { return (... && !(__ent == 0)); });
+ }
+}
+
+// }}}
+// __any_of {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC bool __any_of(const _Data &__k)
+{
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ {
+ return __k;
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ {
+ return __k.any();
+ }
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ for (int __i = 0; __i < _Abi::factor; ++__i)
+ {
+ if (__any_of<_Tp, typename _Abi::member_abi>(__k[__i]))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+#if _GLIBCXX_SIMD_X86INTRIN
+ else if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>() ||
+ __is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (__have_sse4_1)
+ {
+ return 0 == __testz(__k._M_data, _Abi::template implicit_mask<_Tp>);
+ } else if constexpr (std::is_same_v<_Tp, float>) {
+ return (_mm_movemask_ps(__to_intrin(__k)) & ((1 << _N) - 1)) != 0;
+ } else if constexpr (std::is_same_v<_Tp, double>) {
+ return (_mm_movemask_pd(__to_intrin(__k)) & ((1 << _N) - 1)) != 0;
+ } else {
+ return (_mm_movemask_epi8(__to_intrin(__k)) & ((1 << (_N * sizeof(_Tp))) - 1)) != 0;
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ return (__k & _Abi::template implicit_mask<_Tp>) != 0;
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ else
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ return __call_with_subscripts(
+ __vector_bitcast<__int_for_sizeof_t<_Tp>>(__k),
+ make_index_sequence<_N>(),
+ [](const auto... __ent) { return (... || !(__ent == 0)); });
+ }
+}
+
+// }}}
+// __none_of {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC bool __none_of(const _Data &__k)
+{
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ {
+ return !__k;
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ {
+ return __k.none();
+ }
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ for (int __i = 0; __i < _Abi::factor; ++__i)
+ {
+ if (__any_of<_Tp, typename _Abi::member_abi>(__k[__i]))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+#if _GLIBCXX_SIMD_X86INTRIN
+ else if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>() ||
+ __is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (__have_sse4_1)
+ {
+ return 0 != __testz(__k._M_data, _Abi::template implicit_mask<_Tp>);
+ } else if constexpr (std::is_same_v<_Tp, float>) {
+ return (_mm_movemask_ps(__to_intrin(__k)) & ((1 << _N) - 1)) == 0;
+ } else if constexpr (std::is_same_v<_Tp, double>) {
+ return (_mm_movemask_pd(__to_intrin(__k)) & ((1 << _N) - 1)) == 0;
+ } else {
+ return (_mm_movemask_epi8(__to_intrin(__k)) & ((1 << (_N * sizeof(_Tp))) - 1)) == 0;
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ return (__k & _Abi::template implicit_mask<_Tp>) == 0;
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ else
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ return __call_with_subscripts(
+ __vector_bitcast<__int_for_sizeof_t<_Tp>>(__k),
+ make_index_sequence<_N>(),
+ [](const auto... __ent) { return (... && (__ent == 0)); });
+ }
+}
+
+// }}}
+// __some_of {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC bool __some_of(const _Data &__k)
+{
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (_N == 1)
+ {
+ return false;
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ {
+ return __k.any() && !__k.all();
+ }
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ return __any_of<_Tp, _Abi>(__k) && !__all_of<_Tp, _Abi>(__k);
+ }
+#if _GLIBCXX_SIMD_X86INTRIN
+ else if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>() ||
+ __is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ if constexpr (__have_sse4_1)
+ {
+ return 0 != __testnzc(__k._M_data, _Abi::template implicit_mask<_Tp>);
+ }
+ else if constexpr (std::is_same_v<_Tp, float>)
+ {
+ constexpr int __allbits = (1 << _N) - 1;
+ const auto __tmp = _mm_movemask_ps(__to_intrin(__k)) & __allbits;
+ return __tmp > 0 && __tmp < __allbits;
+ }
+ else if constexpr (std::is_same_v<_Tp, double>)
+ {
+ constexpr int __allbits = (1 << _N) - 1;
+ const auto __tmp = _mm_movemask_pd(__to_intrin(__k)) & __allbits;
+ return __tmp > 0 && __tmp < __allbits;
+ }
+ else
+ {
+ constexpr int __allbits = (1 << (_N * sizeof(_Tp))) - 1;
+ const auto __tmp = _mm_movemask_epi8(__to_intrin(__k)) & __allbits;
+ return __tmp > 0 && __tmp < __allbits;
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ return __any_of<_Tp, _Abi>(__k) && !__all_of<_Tp, _Abi>(__k);
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ else
+ {
+ int __n_false = __call_with_subscripts(
+ __vector_bitcast<__int_for_sizeof_t<_Tp>>(__k),
+ make_index_sequence<_N>(),
+ [](const auto... __ent) { return (... + (__ent == 0)); });
+ return __n_false > 0 && __n_false < _N;
+ }
+}
+
+// }}}
+// __popcount {{{
+template <class _Tp, class _Abi, class _Data>
+_GLIBCXX_SIMD_INTRINSIC int __popcount(const _Data& __k)
+{
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ {
+ return __k;
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ {
+ return __k.count();
+ }
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ int __count = __popcount<_Tp, typename _Abi::member_abi>(__k[0]);
+ for (int __i = 1; __i < _Abi::factor; ++__i)
+ {
+ __count += __popcount<_Tp, typename _Abi::member_abi>(__k[__i]);
+ }
+ return __count;
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>() ||
+ __is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ const auto __kk = _Abi::masked(__k._M_data);
+ if constexpr (__have_popcnt)
+ {
+ int __bits = __movemask(__to_intrin(__vector_bitcast<_Tp>(__kk)));
+ const int __count = __builtin_popcount(__bits);
+ return std::is_integral_v<_Tp> ? __count / sizeof(_Tp) : __count;
+ }
+ else if constexpr (_N == 2)
+ {
+ const int mask = _mm_movemask_pd(__auto_bitcast(__kk));
+ return mask - (mask >> 1);
+ }
+ else if constexpr (_N == 4 && sizeof(__kk) == 16 && __have_sse2)
+ {
+ auto __x = __vector_bitcast<_LLong>(__kk);
+ __x =
+ _mm_add_epi32(__x, _mm_shuffle_epi32(__x, _MM_SHUFFLE(0, 1, 2, 3)));
+ __x = _mm_add_epi32(
+ __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(1, 0, 3, 2)));
+ return -_mm_cvtsi128_si32(__x);
+ }
+ else if constexpr (_N == 4 && sizeof(__kk) == 16)
+ {
+ return __builtin_popcount(_mm_movemask_ps(__auto_bitcast(__kk)));
+ }
+ else if constexpr (_N == 8 && sizeof(__kk) == 16)
+ {
+ auto __x = __vector_bitcast<_LLong>(__kk);
+ __x =
+ _mm_add_epi16(__x, _mm_shuffle_epi32(__x, _MM_SHUFFLE(0, 1, 2, 3)));
+ __x = _mm_add_epi16(
+ __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2, 3)));
+ __x = _mm_add_epi16(
+ __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0, 1)));
+ return -short(_mm_extract_epi16(__x, 0));
+ }
+ else if constexpr (_N == 16 && sizeof(__kk) == 16)
+ {
+ auto __x = __vector_bitcast<_LLong>(__kk);
+ __x =
+ _mm_add_epi8(__x, _mm_shuffle_epi32(__x, _MM_SHUFFLE(0, 1, 2, 3)));
+ __x = _mm_add_epi8(__x,
+ _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2, 3)));
+ __x = _mm_add_epi8(__x,
+ _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0, 1)));
+ auto __y = -__vector_bitcast<_UChar>(__x);
+ if constexpr (__have_sse4_1)
+ {
+ return __y[0] + __y[1];
+ }
+ else
+ {
+ unsigned __z =
+ _mm_extract_epi16(__vector_bitcast<_LLong>(__y), 0);
+ return (__z & 0xff) + (__z >> 8);
+ }
+ }
+ else if constexpr (_N == 4 && sizeof(__kk) == 32)
+ {
+ auto __x = -(__lo128(__kk) + __hi128(__kk));
+ return __x[0] + __x[1];
+ }
+ else if constexpr (sizeof(__kk) == 32)
+ {
+ return __popcount<_Tp, simd_abi::__sse>(
+ -(__lo128(__kk) + __hi128(__kk)));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ const auto __kk = _Abi::masked(__k._M_data);
+ if constexpr (_N <= 4)
+ {
+ return __builtin_popcount(__kk);
+ }
+ else if constexpr (_N <= 8)
+ {
+ return __builtin_popcount(__kk);
+ }
+ else if constexpr (_N <= 16)
+ {
+ return __builtin_popcount(__kk);
+ }
+ else if constexpr (_N <= 32)
+ {
+ return __builtin_popcount(__kk);
+ }
+ else if constexpr (_N <= 64)
+ {
+ return __builtin_popcountll(__kk);
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+#if _GLIBCXX_SIMD_HAVE_NEON // {{{
+ else if constexpr (sizeof(_Tp) == 1)
+ {
+ const auto __s8 = __vector_bitcast<_SChar>(__k);
+ int8x8_t __tmp = __lo64(__s8) + __hi64z(__s8);
+ return -vpadd_s8(vpadd_s8(vpadd_s8(__tmp, int8x8_t()), int8x8_t()),
+ int8x8_t())[0];
+ }
+ else if constexpr (sizeof(_Tp) == 2)
+ {
+ const auto __s16 = __vector_bitcast<short>(__k);
+ int16x4_t __tmp = __lo64(__s16) + __hi64z(__s16);
+ return -vpadd_s16(vpadd_s16(__tmp, int16x4_t()), int16x4_t())[0];
+ }
+ else if constexpr (sizeof(_Tp) == 4)
+ {
+ const auto __s32 = __vector_bitcast<int>(__k);
+ int32x2_t __tmp = __lo64(__s32) + __hi64z(__s32);
+ return -vpadd_s32(__tmp, int32x2_t())[0];
+ }
+ else if constexpr (sizeof(_Tp) == 8)
+ {
+ static_assert(sizeof(__k) == 16);
+ const auto __s64 = __vector_bitcast<long>(__k);
+ return -(__s64[0] + __s64[1]);
+ }
+#endif // _GLIBCXX_SIMD_HAVE_NEON }}}
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+// }}}
+// __find_first_set {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC int __find_first_set(const _Data &__k)
+{
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ return 0;
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ return __firstbit(__k.to_ullong());
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ using _A2 = typename _Abi::member_abi;
+ for (int __i = 0; __i < _Abi::factor - 1; ++__i)
+ {
+ if (__any_of<_Tp, _A2>(__k[__i]))
+ {
+ return __i * simd_size_v<_Tp, _A2> + __find_first_set(__k[__i]);
+ }
+ }
+ return (_Abi::factor - 1) * simd_size_v<_Tp, _A2> +
+ __find_first_set(__k[_Abi::factor - 1]);
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ if constexpr (simd_size_v<_Tp, _Abi> <= 32)
+ return _tzcnt_u32(__k._M_data);
+ else
+ return __firstbit(__k._M_data);
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ else
+ return __firstbit(__vector_to_bitset(__k._M_data).to_ullong());
+}
+
+// }}}
+// __find_last_set {{{
+template <class _Tp, class _Abi, class _Data> _GLIBCXX_SIMD_INTRINSIC int __find_last_set(const _Data &__k)
+{
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>())
+ return 0;
+ else if constexpr (__is_abi<_Abi, simd_abi::fixed_size>())
+ return __lastbit(__k.to_ullong());
+ else if constexpr (__is_combined_abi<_Abi>())
+ {
+ using _A2 = typename _Abi::member_abi;
+ for (int __i = 0; __i < _Abi::factor - 1; ++__i)
+ {
+ if (__any_of<_Tp, _A2>(__k[__i]))
+ {
+ return __i * simd_size_v<_Tp, _A2> + __find_last_set(__k[__i]);
+ }
+ }
+ return (_Abi::factor - 1) * simd_size_v<_Tp, _A2> +
+ __find_last_set(__k[_Abi::factor - 1]);
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ if constexpr (simd_size_v<_Tp, _Abi> <= 32)
+ return 31 - _lzcnt_u32(__k._M_data);
+ else
+ return __lastbit(__k._M_data);
+ }
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ else
+ return __lastbit(__vector_to_bitset(__k._M_data).to_ullong());
+}
+
+// }}}
+
+// reductions [simd_mask.reductions] {{{
+template <class _Tp, class _Abi> _GLIBCXX_SIMD_ALWAYS_INLINE bool all_of(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __all_of<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi> _GLIBCXX_SIMD_ALWAYS_INLINE bool any_of(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __any_of<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi> _GLIBCXX_SIMD_ALWAYS_INLINE bool none_of(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __none_of<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi> _GLIBCXX_SIMD_ALWAYS_INLINE bool some_of(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __some_of<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi> _GLIBCXX_SIMD_ALWAYS_INLINE int popcount(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __popcount<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE int find_first_set(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __find_first_set<_Tp, _Abi>(__data(__k));
+}
+template <class _Tp, class _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE int find_last_set(const simd_mask<_Tp, _Abi> &__k)
+{
+ return __find_last_set<_Tp, _Abi>(__data(__k));
+}
+
+constexpr bool all_of(_ExactBool __x) { return __x; }
+constexpr bool any_of(_ExactBool __x) { return __x; }
+constexpr bool none_of(_ExactBool __x) { return !__x; }
+constexpr bool some_of(_ExactBool) { return false; }
+constexpr int popcount(_ExactBool __x) { return __x; }
+constexpr int find_first_set(_ExactBool) { return 0; }
+constexpr int find_last_set(_ExactBool) { return 0; }
+
+// }}}
+
+template <class _Abi> struct __generic_simd_impl;
+// _Simd_int_operators{{{1
+template <class _V, bool> class _Simd_int_operators {};
+template <class _V> class _Simd_int_operators<_V, true>
+{
+ using __impl = __get_impl_t<_V>;
+
+ _GLIBCXX_SIMD_INTRINSIC const _V &__derived() const { return *static_cast<const _V *>(this); }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _V __make_derived(_Tp &&__d)
+ {
+ return {__private_init, std::forward<_Tp>(__d)};
+ }
+
+public:
+ constexpr friend _V &operator %=(_V &__lhs, const _V &__x) { return __lhs = __lhs % __x; }
+ constexpr friend _V &operator &=(_V &__lhs, const _V &__x) { return __lhs = __lhs & __x; }
+ constexpr friend _V &operator |=(_V &__lhs, const _V &__x) { return __lhs = __lhs | __x; }
+ constexpr friend _V &operator ^=(_V &__lhs, const _V &__x) { return __lhs = __lhs ^ __x; }
+ constexpr friend _V &operator<<=(_V &__lhs, const _V &__x) { return __lhs = __lhs << __x; }
+ constexpr friend _V &operator>>=(_V &__lhs, const _V &__x) { return __lhs = __lhs >> __x; }
+ constexpr friend _V &operator<<=(_V &__lhs, int __x) { return __lhs = __lhs << __x; }
+ constexpr friend _V &operator>>=(_V &__lhs, int __x) { return __lhs = __lhs >> __x; }
+
+ constexpr friend _V operator% (const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::modulus (__data(__x), __data(__y))); }
+ constexpr friend _V operator& (const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::bit_and (__data(__x), __data(__y))); }
+ constexpr friend _V operator| (const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::bit_or (__data(__x), __data(__y))); }
+ constexpr friend _V operator^ (const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::bit_xor (__data(__x), __data(__y))); }
+ constexpr friend _V operator<<(const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::bit_shift_left (__data(__x), __data(__y))); }
+ constexpr friend _V operator>>(const _V &__x, const _V &__y) { return _Simd_int_operators::__make_derived(__impl::bit_shift_right(__data(__x), __data(__y))); }
+ constexpr friend _V operator<<(const _V &__x, int __y) { return _Simd_int_operators::__make_derived(__impl::bit_shift_left (__data(__x), __y)); }
+ constexpr friend _V operator>>(const _V &__x, int __y) { return _Simd_int_operators::__make_derived(__impl::bit_shift_right(__data(__x), __y)); }
+
+ // unary operators (for integral _Tp)
+ constexpr _V operator~() const
+ {
+ return {__private_init, __impl::complement(__derived()._M_data)};
+ }
+};
+
+//}}}1
+
+// simd {{{
+template <class _Tp, class _Abi>
+class simd
+ : public _Simd_int_operators<
+ simd<_Tp, _Abi>, conjunction<std::is_integral<_Tp>,
+ typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
+ public _SimdTraits<_Tp, _Abi>::_Simd_base
+{
+ using __traits = _SimdTraits<_Tp, _Abi>;
+ using __impl = typename __traits::_SimdImpl;
+ using __member_type = typename __traits::_SimdMember;
+ using __cast_type = typename __traits::_SimdCastType;
+ static constexpr _Tp *_S_type_tag = nullptr;
+ friend typename __traits::_Simd_base;
+ friend __impl;
+ friend __generic_simd_impl<_Abi>;
+ friend _Simd_int_operators<simd, true>;
+
+public:
+ using value_type = _Tp;
+ using reference = _Smart_reference<__member_type, __impl, value_type>;
+ using mask_type = simd_mask<_Tp, _Abi>;
+ using abi_type = _Abi;
+
+ static constexpr size_t size() { return __size_or_zero_v<_Tp, _Abi>; }
+ constexpr simd() = default;
+ constexpr simd(const simd &) = default;
+ constexpr simd(simd &&) = default;
+ constexpr simd &operator=(const simd &) = default;
+ constexpr simd &operator=(simd &&) = default;
+
+ // implicit broadcast constructor
+ template <class _U, class = _ValuePreservingOrInt<_U, value_type>>
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd(_U &&__x)
+ : _M_data(__impl::__broadcast(static_cast<value_type>(std::forward<_U>(__x))))
+ {
+ }
+
+ // implicit type conversion constructor (convert from fixed_size to fixed_size)
+ template <class _U>
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd(
+ const simd<_U, simd_abi::fixed_size<size()>> &__x,
+ enable_if_t<
+ conjunction<std::is_same<simd_abi::fixed_size<size()>, abi_type>,
+ std::negation<__is_narrowing_conversion<_U, value_type>>,
+ __converts_to_higher_integer_rank<_U, value_type>>::value,
+ void *> = nullptr)
+ : simd{static_cast<std::array<_U, size()>>(__x).data(), vector_aligned}
+ {
+ }
+
+ // generator constructor
+ template <class _F>
+ _GLIBCXX_SIMD_ALWAYS_INLINE explicit constexpr simd(
+ _F &&__gen,
+ _ValuePreservingOrInt<
+ decltype(std::declval<_F>()(std::declval<_SizeConstant<0> &>())),
+ value_type> * = nullptr)
+ : _M_data(__impl::generator(std::forward<_F>(__gen), _S_type_tag))
+ {
+ }
+
+ // load constructor
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_ALWAYS_INLINE simd(const _U *__mem, _Flags __f)
+ : _M_data(__impl::load(__mem, __f, _S_type_tag))
+ {
+ }
+
+ // loads [simd.load]
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_ALWAYS_INLINE void copy_from(const _Vectorizable<_U> *__mem, _Flags __f)
+ {
+ _M_data = static_cast<decltype(_M_data)>(__impl::load(__mem, __f, _S_type_tag));
+ }
+
+ // stores [simd.store]
+ template <class _U, class _Flags>
+ _GLIBCXX_SIMD_ALWAYS_INLINE void copy_to(_Vectorizable<_U> *__mem, _Flags __f) const
+ {
+ __impl::store(_M_data, __mem, __f, _S_type_tag);
+ }
+
+ // scalar access
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr reference operator[](size_t __i) { return {_M_data, int(__i)}; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr value_type operator[](size_t __i) const
+ {
+ if constexpr (__is_scalar()) {
+ _GLIBCXX_DEBUG_ASSERT(__i == 0);
+ __unused(__i);
+ return _M_data;
+ } else {
+ return _M_data[__i];
+ }
+ }
+
+ // increment and decrement:
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd &operator++() { __impl::__increment(_M_data); return *this; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd operator++(int) { simd __r = *this; __impl::__increment(_M_data); return __r; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd &operator--() { __impl::__decrement(_M_data); return *this; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd operator--(int) { simd __r = *this; __impl::__decrement(_M_data); return __r; }
+
+ // unary operators (for any _Tp)
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr mask_type operator!() const
+ {
+ return {__private_init, __impl::negate(_M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd operator+() const { return *this; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr simd operator-() const
+ {
+ return {__private_init, __impl::unary_minus(_M_data)};
+ }
+
+ // access to internal representation (suggested extension)
+ _GLIBCXX_SIMD_ALWAYS_INLINE explicit simd(__cast_type __init) : _M_data(__init) {}
+
+ // compound assignment [simd.cassign]
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd &operator+=(simd &__lhs, const simd &__x) { return __lhs = __lhs + __x; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd &operator-=(simd &__lhs, const simd &__x) { return __lhs = __lhs - __x; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd &operator*=(simd &__lhs, const simd &__x) { return __lhs = __lhs * __x; }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd &operator/=(simd &__lhs, const simd &__x) { return __lhs = __lhs / __x; }
+
+ // binary operators [simd.binary]
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd operator+(const simd &__x, const simd &__y)
+ {
+ return {__private_init, __impl::plus(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd operator-(const simd &__x, const simd &__y)
+ {
+ return {__private_init, __impl::minus(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd operator*(const simd& __x, const simd& __y)
+ {
+ return {__private_init, __impl::multiplies(__x._M_data, __y._M_data)};
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE constexpr friend simd operator/(const simd &__x, const simd &__y)
+ {
+ return {__private_init, __impl::divides(__x._M_data, __y._M_data)};
+ }
+
+ // compares [simd.comparison]
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator==(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::equal_to(__x._M_data, __y._M_data));
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator!=(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::not_equal_to(__x._M_data, __y._M_data));
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator<(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::less(__x._M_data, __y._M_data));
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator<=(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::less_equal(__x._M_data, __y._M_data));
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator>(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::less(__y._M_data, __x._M_data));
+ }
+ _GLIBCXX_SIMD_ALWAYS_INLINE friend mask_type operator>=(const simd &__x, const simd &__y)
+ {
+ return simd::make_mask(__impl::less_equal(__y._M_data, __x._M_data));
+ }
+
+ // "private" because of the first arguments's namespace
+ _GLIBCXX_SIMD_INTRINSIC constexpr simd(_PrivateInit,
+ const __member_type& __init)
+ : _M_data(__init)
+ {
+ }
+
+ // "private" because of the first arguments's namespace
+ _GLIBCXX_SIMD_INTRINSIC simd(_BitsetInit, std::bitset<size()> __init) : _M_data() {
+ where(mask_type(__bitset_init, __init), *this) = ~*this;
+ }
+
+private:
+ static constexpr bool __is_scalar() { return std::is_same_v<abi_type, simd_abi::scalar>; }
+ static constexpr bool __is_fixed() { return __is_fixed_size_abi_v<abi_type>; }
+
+ _GLIBCXX_SIMD_INTRINSIC static mask_type make_mask(typename mask_type::__member_type __k)
+ {
+ return {__private_init, __k};
+ }
+ friend const auto &__data<value_type, abi_type>(const simd &);
+ friend auto &__data<value_type, abi_type>(simd &);
+ alignas(__traits::_S_simd_align) __member_type _M_data;
+};
+
+// }}}
+// __data {{{
+template <class _Tp, class _A> _GLIBCXX_SIMD_INTRINSIC constexpr const auto &__data(const simd<_Tp, _A> &__x)
+{
+ return __x._M_data;
+}
+template <class _Tp, class _A> _GLIBCXX_SIMD_INTRINSIC constexpr auto &__data(simd<_Tp, _A> &__x)
+{
+ return __x._M_data;
+}
+// }}}
+
+namespace __proposed
+{
+namespace float_bitwise_operators
+{
+// float_bitwise_operators {{{
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A>
+ operator^(const simd<_Tp, _A>& __a, const simd<_Tp, _A>& __b)
+{
+ return {__private_init,
+ __get_impl_t<simd<_Tp, _A>>::bit_xor(__data(__a), __data(__b))};
+}
+
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A> operator|(const simd<_Tp, _A> &__a, const simd<_Tp, _A> &__b)
+{
+ return {__private_init, __get_impl_t<simd<_Tp, _A>>::bit_or(__data(__a), __data(__b))};
+}
+
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A> operator&(const simd<_Tp, _A> &__a, const simd<_Tp, _A> &__b)
+{
+ return {__private_init, __get_impl_t<simd<_Tp, _A>>::bit_and(__data(__a), __data(__b))};
+}
+// }}}
+} // namespace float_bitwise_operators
+} // namespace __proposed
+
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/include/experimental/bits/simd_abis.h b/libstdc++-v3/include/experimental/bits/simd_abis.h
new file mode 100644
index 00000000000..8e9a2b4a2af
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_abis.h
@@ -0,0 +1,6096 @@
+// Simd Abi specific implementations -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
+#define _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
+
+#if __cplusplus >= 201703L
+
+#include "simd.h"
+#include <array>
+#include <cmath>
+#include <cstdlib>
+
+#include "simd_debug.h"
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+// __subscript_read/_write {{{1
+template <typename _Tp> _Tp __subscript_read(_Vectorizable<_Tp> __x, size_t) noexcept
+{
+ return __x;
+}
+template <typename _Tp>
+void __subscript_write(_Vectorizable<_Tp> &__x, size_t, __id<_Tp> __y) noexcept
+{
+ __x = __y;
+}
+
+template <typename _Tp>
+typename _Tp::value_type __subscript_read(const _Tp &__x, size_t __i) noexcept
+{
+ return __x[__i];
+}
+template <typename _Tp>
+void __subscript_write(_Tp &__x, size_t __i, typename _Tp::value_type __y) noexcept
+{
+ return __x.set(__i, __y);
+}
+
+// __simd_tuple_element {{{1
+template <size_t _I, typename _Tp> struct __simd_tuple_element;
+template <typename _Tp, typename _A0, typename... _As>
+struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>> {
+ using type = std::experimental::simd<_Tp, _A0>;
+};
+template <size_t _I, typename _Tp, typename _A0, typename... _As>
+struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>> {
+ using type = typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type;
+};
+template <size_t _I, typename _Tp>
+using __simd_tuple_element_t = typename __simd_tuple_element<_I, _Tp>::type;
+
+// __simd_tuple_concat {{{1
+template <typename _Tp, typename... _A1s>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A1s...>
+ __simd_tuple_concat(const _SimdTuple<_Tp>&,
+ const _SimdTuple<_Tp, _A1s...>& __right)
+{
+ return __right;
+}
+
+template <typename _Tp,
+ typename _A00,
+ typename... _A0s,
+ typename _A10,
+ typename... _A1s>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp,
+ _A00,
+ _A0s...,
+ _A10,
+ _A1s...>
+ __simd_tuple_concat(const _SimdTuple<_Tp, _A00, _A0s...>& __left,
+ const _SimdTuple<_Tp, _A10, _A1s...>& __right)
+{
+ return {__left.first, __simd_tuple_concat(__left.second, __right)};
+}
+
+template <typename _Tp, typename _A00, typename... _A0s>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A00, _A0s...>
+ __simd_tuple_concat(const _SimdTuple<_Tp, _A00, _A0s...>& __left,
+ const _SimdTuple<_Tp>&)
+{
+ return __left;
+}
+
+template <typename _Tp, typename _A10, typename... _A1s>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp,
+ simd_abi::scalar,
+ _A10,
+ _A1s...>
+ __simd_tuple_concat(const _Tp& __left,
+ const _SimdTuple<_Tp, _A10, _A1s...>& __right)
+{
+ return {__left, __right};
+}
+
+// __simd_tuple_pop_front {{{1
+template <typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr const _Tp &__simd_tuple_pop_front(_SizeConstant<0>,
+ const _Tp &__x)
+{
+ return __x;
+}
+template <typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr _Tp &__simd_tuple_pop_front(_SizeConstant<0>, _Tp &__x)
+{
+ return __x;
+}
+template <size_t _K, typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr const auto &__simd_tuple_pop_front(_SizeConstant<_K>,
+ const _Tp &__x)
+{
+ return __simd_tuple_pop_front(_SizeConstant<_K - 1>(), __x.second);
+}
+template <size_t _K, typename _Tp>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto &__simd_tuple_pop_front(_SizeConstant<_K>, _Tp &__x)
+{
+ return __simd_tuple_pop_front(_SizeConstant<_K - 1>(), __x.second);
+}
+
+// __get_simd_at<_N> {{{1
+struct __as_simd {};
+struct __as_simd_tuple {};
+template <typename _Tp, typename _A0, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A0> __simd_tuple_get_impl(
+ __as_simd, const _SimdTuple<_Tp, _A0, _Abis...> &__t, _SizeConstant<0>)
+{
+ return {__private_init, __t.first};
+}
+template <typename _Tp, typename _A0, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr const auto &__simd_tuple_get_impl(
+ __as_simd_tuple, const _SimdTuple<_Tp, _A0, _Abis...> &__t, _SizeConstant<0>)
+{
+ return __t.first;
+}
+template <typename _Tp, typename _A0, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto &__simd_tuple_get_impl(
+ __as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...> &__t, _SizeConstant<0>)
+{
+ return __t.first;
+}
+
+template <typename _R, size_t _N, typename _Tp, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto __simd_tuple_get_impl(
+ _R, const _SimdTuple<_Tp, _Abis...> &__t, _SizeConstant<_N>)
+{
+ return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_N - 1>());
+}
+template <size_t _N, typename _Tp, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto &__simd_tuple_get_impl(
+ __as_simd_tuple, _SimdTuple<_Tp, _Abis...> &__t, _SizeConstant<_N>)
+{
+ return __simd_tuple_get_impl(__as_simd_tuple(), __t.second, _SizeConstant<_N - 1>());
+}
+
+template <size_t _N, typename _Tp, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto __get_simd_at(const _SimdTuple<_Tp, _Abis...> &__t)
+{
+ return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_N>());
+}
+
+// }}}
+// __get_tuple_at<_N> {{{
+template <size_t _N, typename _Tp, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto __get_tuple_at(const _SimdTuple<_Tp, _Abis...> &__t)
+{
+ return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_N>());
+}
+
+template <size_t _N, typename _Tp, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC constexpr auto &__get_tuple_at(_SimdTuple<_Tp, _Abis...> &__t)
+{
+ return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_N>());
+}
+
+// __how_many_to_extract {{{1
+template <size_t _LeftN, typename _RightT> constexpr size_t __tuple_elements_for () {
+ if constexpr (_LeftN == 0) {
+ return 0;
+ } else {
+ return 1 + __tuple_elements_for<_LeftN - _RightT::_S_first_size,
+ typename _RightT::_Second_type>();
+ }
+}
+template <size_t _LeftN, typename _RightT, bool = (_RightT::_S_first_size < _LeftN)>
+struct __how_many_to_extract;
+template <size_t _LeftN, typename _RightT> struct __how_many_to_extract<_LeftN, _RightT, true> {
+ static constexpr std::make_index_sequence<__tuple_elements_for<_LeftN, _RightT>()> tag()
+ {
+ return {};
+ }
+};
+template <typename _Tp, size_t _Offset, size_t _Length, bool _Done, typename _IndexSeq>
+struct chunked {
+};
+template <size_t _LeftN, typename _RightT> struct __how_many_to_extract<_LeftN, _RightT, false> {
+ static_assert(_LeftN != _RightT::_S_first_size, "");
+ static constexpr chunked<typename _RightT::_First_type, 0, _LeftN, false,
+ std::make_index_sequence<_LeftN>>
+ tag()
+ {
+ return {};
+ }
+};
+
+// __tuple_element_meta {{{1
+template <typename _Tp, typename _Abi, size_t _Offset>
+struct __tuple_element_meta : public _Abi::_SimdImpl {
+ static_assert(is_same_v<typename _Abi::_SimdImpl::abi_type,
+ _Abi>); // this fails e.g. when _SimdImpl is an alias
+ // for __generic_simd_impl<_DifferentAbi>
+ using value_type = _Tp;
+ using abi_type = _Abi;
+ using __traits = _SimdTraits<_Tp, _Abi>;
+ using maskimpl = typename __traits::_MaskImpl;
+ using __member_type = typename __traits::_SimdMember;
+ using _MaskMember = typename __traits::_MaskMember;
+ using simd_type = std::experimental::simd<_Tp, _Abi>;
+ static constexpr size_t offset = _Offset;
+ static constexpr size_t size() { return simd_size<_Tp, _Abi>::value; }
+ static constexpr maskimpl simd_mask = {};
+
+ template <size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember make_mask(std::bitset<_N> __bits)
+ {
+ constexpr _Tp* __type_tag = nullptr;
+ return maskimpl::__from_bitset(
+ std::bitset<size()>((__bits >> _Offset).to_ullong()), __type_tag);
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC static _ULLong mask_to_shifted_ullong(_MaskMember __k)
+ {
+ return __vector_to_bitset(__k).to_ullong() << _Offset;
+ }
+};
+
+template <size_t _Offset, typename _Tp, typename _Abi, typename... _As>
+__tuple_element_meta<_Tp, _Abi, _Offset> make_meta(const _SimdTuple<_Tp, _Abi, _As...> &)
+{
+ return {};
+}
+
+// _SimdTuple specializations {{{1
+// empty {{{2
+template <typename _Tp> struct _SimdTuple<_Tp> {
+ using value_type = _Tp;
+ static constexpr size_t tuple_size = 0;
+ static constexpr size_t size() { return 0; }
+};
+
+// 1 member {{{2
+template <typename _Tp, typename _Abi0> struct _SimdTuple<_Tp, _Abi0> {
+ using value_type = _Tp;
+ using _First_type = typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
+ using _Second_type = _SimdTuple<_Tp>;
+ using _First_abi = _Abi0;
+ static constexpr size_t tuple_size = 1;
+ static constexpr size_t size() { return simd_size_v<_Tp, _Abi0>; }
+ static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
+ _First_type first;
+ static constexpr _Second_type second = {};
+
+ template <size_t _Offset = 0, typename _F>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ generate(_F&& __gen, _SizeConstant<_Offset> = {})
+ {
+ return {__gen(__tuple_element_meta<_Tp, _Abi0, _Offset>())};
+ }
+
+ template <size_t _Offset = 0, typename _F, typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC _SimdTuple apply_wrapped(_F &&__fun, const _More &... __more) const
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return {__fun(make_meta<_Offset>(*this), first, __more.first...)};
+ }
+
+ template <typename _F, typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC constexpr friend _SimdTuple
+ __simd_tuple_apply(_F&& __fun, const _SimdTuple& __x, _More&&... __more)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return _SimdTuple::apply_impl(
+ __bool_constant<conjunction<__is_equal<
+ size_t, _S_first_size, __remove_cvref_t<_More>::_S_first_size>...>::value>(),
+ std::forward<_F>(__fun), __x, std::forward<_More>(__more)...);
+ }
+
+ private:
+ template <typename _F, typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl(true_type, // _S_first_size is equal for all arguments
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&&... __more)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__more.first = ", __more.first..., "__more
+ //=
+ //", __more...);
+ return {__fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
+ __more.first...)};
+ }
+
+ template <typename _F, typename _More>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple apply_impl(
+ false_type, // at least one argument in _More has different _S_first_size,
+ // __x has only one member, so _More has 2 or more
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__y = ", __y);
+ return apply_impl(
+ std::make_index_sequence<__remove_cvref_t<_More>::tuple_size>(),
+ std::forward<_F>(__fun), __x, std::forward<_More>(__y));
+ }
+
+ template <typename _F, typename _More, size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl(std::index_sequence<_Indexes...>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__y = ", __y);
+ auto tmp = std::experimental::concat(__get_simd_at<_Indexes>(__y)...);
+ const auto first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, tmp);
+ if constexpr (std::is_lvalue_reference<_More>::value &&
+ !std::is_const<_More>::value) {
+ // if __y is non-const lvalue ref, assume write back is necessary
+ const auto __tup =
+ std::experimental::split<__simd_tuple_element_t<_Indexes, __remove_cvref_t<_More>>::size()...>(tmp);
+ auto &&ignore = {
+ (__get_tuple_at<_Indexes>(__y) = __data(std::get<_Indexes>(__tup)), 0)...};
+ __unused(ignore);
+ }
+ return {first};
+ }
+
+ public:
+ // apply_impl2 can only be called from a 2-element _SimdTuple
+ template <typename _Tuple, size_t _Offset, typename _F2>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ __extract(_SizeConstant<_Offset>,
+ _SizeConstant<__remove_cvref_t<_Tuple>::_S_first_size - _Offset>,
+ _Tuple&& __tup,
+ _F2&& __fun2)
+ {
+ static_assert(_Offset > 0, "");
+ auto __splitted =
+ split<_Offset, __remove_cvref_t<_Tuple>::_S_first_size - _Offset>(__get_simd_at<0>(__tup));
+ _SimdTuple __r = __fun2(__data(std::get<1>(__splitted)));
+ // if __tup is non-const lvalue ref, write __get_tuple_at<0>(__splitted) back
+ __tup.first = __data(concat(std::get<0>(__splitted), std::get<1>(__splitted)));
+ return __r;
+ }
+
+ template <typename _F,
+ typename _More,
+ typename _U,
+ size_t _Length,
+ size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl2(chunked<_U,
+ __remove_cvref_t<_More>::_S_first_size,
+ _Length,
+ true,
+ std::index_sequence<_Indexes...>>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return __simd_tuple_apply(std::forward<_F>(__fun), __x, __y.second);
+ }
+
+ template <class _F,
+ class _More,
+ class _U,
+ size_t _Offset,
+ size_t _Length,
+ size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple apply_impl2(
+ chunked<_U, _Offset, _Length, false, std::index_sequence<_Indexes...>>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ static_assert(_Offset < __remove_cvref_t<_More>::_S_first_size, "");
+ static_assert(_Offset > 0, "");
+ return __extract(_SizeConstant<_Offset>(), _SizeConstant<_Length>(), __y,
+ [&](auto &&__yy) -> _SimdTuple {
+ return {__fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, __yy)};
+ });
+ }
+
+ template <class _R = _Tp, class _F, class... _More>
+ _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_R, size()> apply_r(_F &&__fun,
+ const _More &... __more) const
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return {__fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first, __more.first...)};
+ }
+
+ template <class _F, class... _More>
+ _GLIBCXX_SIMD_INTRINSIC friend std::bitset<size()> test(_F &&__fun, const _SimdTuple &__x,
+ const _More &... __more)
+ {
+ return __vector_to_bitset(
+ __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, __more.first...));
+ }
+
+ _Tp operator[](size_t __i) const noexcept { return __subscript_read(first, __i); }
+ void set(size_t __i, _Tp val) noexcept { __subscript_write(first, __i, val); }
+};
+
+// 2 or more {{{2
+template <class _Tp, class _Abi0, class... _Abis> struct _SimdTuple<_Tp, _Abi0, _Abis...> {
+ using value_type = _Tp;
+ using _First_type = typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
+ using _First_abi = _Abi0;
+ using _Second_type = _SimdTuple<_Tp, _Abis...>;
+ static constexpr size_t tuple_size = sizeof...(_Abis) + 1;
+ static constexpr size_t size() { return simd_size_v<_Tp, _Abi0> + _Second_type::size(); }
+ static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
+ //static constexpr size_t alignment = __next_power_of_2(sizeof(_Tp) * size());
+ //alignas(alignment)
+ _First_type first;
+ _Second_type second;
+
+ template <size_t _Offset = 0, class _F>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdTuple generate(_F &&__gen, _SizeConstant<_Offset> = {})
+ {
+ return {__gen(__tuple_element_meta<_Tp, _Abi0, _Offset>()),
+ _Second_type::generate(
+ std::forward<_F>(__gen),
+ _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())};
+ }
+
+ template <size_t _Offset = 0, class _F, class... _More>
+ _GLIBCXX_SIMD_INTRINSIC _SimdTuple apply_wrapped(_F &&__fun, const _More &... __more) const
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return {__fun(make_meta<_Offset>(*this), first, __more.first...),
+ second.template apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>(
+ std::forward<_F>(__fun), __more.second...)};
+ }
+
+ template <class _F, class... _More>
+ _GLIBCXX_SIMD_INTRINSIC constexpr friend _SimdTuple
+ __simd_tuple_apply(_F&& __fun, const _SimdTuple& __x, _More&&... __more)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__more = ", __more...);
+ return _SimdTuple::apply_impl(
+ __bool_constant<conjunction<
+ __is_equal<size_t, _S_first_size,
+ __remove_cvref_t<_More>::_S_first_size>...>::value>(),
+ std::forward<_F>(__fun), __x, std::forward<_More>(__more)...);
+ }
+
+ private:
+ template <class _F, class... _More>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl(true_type, // _S_first_size is equal for all arguments
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&&... __more)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return {__fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
+ __more.first...),
+ __simd_tuple_apply(std::forward<_F>(__fun), __x.second,
+ __more.second...)};
+ }
+
+ template <class _F, class _More>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple apply_impl(
+ false_type, // at least one argument in _More has different _S_first_size
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__y = ", __y);
+ return apply_impl2(__how_many_to_extract<_S_first_size, __remove_cvref_t<_More>>::tag(),
+ std::forward<_F>(__fun), __x, __y);
+ }
+
+ template <class _F, class _More, size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl2(std::index_sequence<_Indexes...>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ //_GLIBCXX_SIMD_DEBUG_DEFERRED("__y = ", __y);
+ auto tmp = std::experimental::concat(__get_simd_at<_Indexes>(__y)...);
+ const auto first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, tmp);
+ if constexpr (std::is_lvalue_reference<_More>::value &&
+ !std::is_const<_More>::value) {
+ // if __y is non-const lvalue ref, assume write back is necessary
+ const auto __tup =
+ std::experimental::split<__simd_tuple_element_t<_Indexes, __remove_cvref_t<_More>>::size()...>(tmp);
+ [](std::initializer_list<int>) {
+ }({(__get_tuple_at<_Indexes>(__y) = __data(std::get<_Indexes>(__tup)), 0)...});
+ }
+ return {first, __simd_tuple_apply(
+ std::forward<_F>(__fun), __x.second,
+ __simd_tuple_pop_front(_SizeConstant<sizeof...(_Indexes)>(), __y))};
+ }
+
+ public:
+ template <typename _F,
+ typename _More,
+ typename _U,
+ size_t _Length,
+ size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
+ apply_impl2(chunked<_U,
+ __remove_cvref_t<_More>::_S_first_size,
+ _Length,
+ true,
+ std::index_sequence<_Indexes...>>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return __simd_tuple_apply(std::forward<_F>(__fun), __x, __y.second);
+ }
+
+ template <typename _Tuple, size_t _Length, typename _F2>
+ _GLIBCXX_SIMD_INTRINSIC static auto __extract(_SizeConstant<0>, _SizeConstant<_Length>, _Tuple &&__tup,
+ _F2 &&__fun2)
+ {
+ auto __splitted =
+ split<_Length, __remove_cvref_t<_Tuple>::_S_first_size - _Length>(__get_simd_at<0>(__tup));
+ auto __r = __fun2(__data(std::get<0>(__splitted)));
+ // if __tup is non-const lvalue ref, write __get_tuple_at<0>(__splitted) back
+ __tup.first = __data(concat(std::get<0>(__splitted), std::get<1>(__splitted)));
+ return __r;
+ }
+
+ template <typename _Tuple, size_t _Offset, typename _F2>
+ _GLIBCXX_SIMD_INTRINSIC static auto __extract(
+ _SizeConstant<_Offset>, _SizeConstant<__remove_cvref_t<_Tuple>::_S_first_size - _Offset>,
+ _Tuple &&__tup, _F2 &&__fun2)
+ {
+ auto __splitted =
+ split<_Offset, __remove_cvref_t<_Tuple>::_S_first_size - _Offset>(__get_simd_at<0>(__tup));
+ auto __r = __fun2(__data(std::get<1>(__splitted)));
+ // if __tup is non-const lvalue ref, write __get_tuple_at<0>(__splitted) back
+ __tup.first = __data(concat(std::get<0>(__splitted), std::get<1>(__splitted)));
+ return __r;
+ }
+
+ template <
+ typename _Tuple, size_t _Offset, size_t _Length, typename _F2,
+ typename = enable_if_t<(_Offset + _Length < __remove_cvref_t<_Tuple>::_S_first_size)>>
+ _GLIBCXX_SIMD_INTRINSIC static auto __extract(_SizeConstant<_Offset>, _SizeConstant<_Length>,
+ _Tuple &&__tup, _F2 &&__fun2)
+ {
+ static_assert(_Offset + _Length < __remove_cvref_t<_Tuple>::_S_first_size, "");
+ auto __splitted =
+ split<_Offset, _Length, __remove_cvref_t<_Tuple>::_S_first_size - _Offset - _Length>(
+ __get_simd_at<0>(__tup));
+ auto __r = __fun2(__data(std::get<1>(__splitted)));
+ // if __tup is non-const lvalue ref, write __get_tuple_at<0>(__splitted) back
+ __tup.first = __data(
+ concat(std::get<0>(__splitted), std::get<1>(__splitted), std::get<2>(__splitted)));
+ return __r;
+ }
+
+ template <typename _F,
+ typename _More,
+ typename _U,
+ size_t _Offset,
+ size_t _Length,
+ size_t... _Indexes>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple apply_impl2(
+ chunked<_U, _Offset, _Length, false, std::index_sequence<_Indexes...>>,
+ _F&& __fun,
+ const _SimdTuple& __x,
+ _More&& __y)
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ static_assert(_Offset < __remove_cvref_t<_More>::_S_first_size, "");
+ return {__extract(_SizeConstant<_Offset>(), _SizeConstant<_Length>(), __y,
+ [&](auto &&__yy) {
+ return __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, __yy);
+ }),
+ _Second_type::apply_impl2(
+ chunked<_U, _Offset + _Length, _Length,
+ _Offset + _Length == __remove_cvref_t<_More>::_S_first_size,
+ std::index_sequence<_Indexes...>>(),
+ std::forward<_F>(__fun), __x.second, __y)};
+ }
+
+ template <typename _R = _Tp, typename _F, typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC auto apply_r(_F &&__fun, const _More &... __more) const
+ {
+ _GLIBCXX_SIMD_DEBUG(_SIMD_TUPLE);
+ return __simd_tuple_concat<_R>(
+ __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first, __more.first...),
+ second.template apply_r<_R>(std::forward<_F>(__fun), __more.second...));
+ }
+
+ template <typename _F, typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC friend std::bitset<size()> test(_F &&__fun, const _SimdTuple &__x,
+ const _More &... __more)
+ {
+ return __vector_to_bitset(
+ __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, __more.first...))
+ .to_ullong() |
+ (test(__fun, __x.second, __more.second...).to_ullong() << simd_size_v<_Tp, _Abi0>);
+ }
+
+ template <typename _U, _U _I>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](std::integral_constant<_U, _I>) const noexcept
+ {
+ if constexpr (_I < simd_size_v<_Tp, _Abi0>) {
+ return __subscript_read(first, _I);
+ } else {
+ return second[std::integral_constant<_U, _I - simd_size_v<_Tp, _Abi0>>()];
+ }
+ }
+
+ _Tp operator[](size_t __i) const noexcept
+ {
+#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
+ return reinterpret_cast<const __may_alias<_Tp> *>(this)[__i];
+#else
+ if constexpr (__is_abi<_Abi0, simd_abi::scalar>())
+ {
+ const _Tp* ptr = &first;
+ return ptr[__i];
+ }
+ else
+ {
+ return __i < simd_size_v<_Tp, _Abi0>
+ ? __subscript_read(first, __i)
+ : second[__i - simd_size_v<_Tp, _Abi0>];
+ }
+#endif
+ }
+ void set(size_t __i, _Tp val) noexcept
+ {
+#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
+ reinterpret_cast<__may_alias<_Tp> *>(this)[__i] = val;
+#else
+ if (__i < simd_size_v<_Tp, _Abi0>) {
+ __subscript_write(first, __i, val);
+ } else {
+ second.set(__i - simd_size_v<_Tp, _Abi0>, val);
+ }
+#endif
+ }
+};
+
+// __make_simd_tuple {{{1
+template <typename _Tp, typename _A0>
+_GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0> __make_simd_tuple(
+ std::experimental::simd<_Tp, _A0> __x0)
+{
+ return {__data(__x0)};
+}
+template <typename _Tp, typename _A0, typename... _As>
+_GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _As...> __make_simd_tuple(
+ const std::experimental::simd<_Tp, _A0> &__x0,
+ const std::experimental::simd<_Tp, _As> &... __xs)
+{
+ return {__data(__x0), __make_simd_tuple(__xs...)};
+}
+
+template <typename _Tp, typename _A0>
+_GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0> __make_simd_tuple(
+ const typename _SimdTraits<_Tp, _A0>::_SimdMember &arg0)
+{
+ return {arg0};
+}
+
+template <typename _Tp, typename _A0, typename _A1, typename... _Abis>
+_GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _A1, _Abis...> __make_simd_tuple(
+ const typename _SimdTraits<_Tp, _A0>::_SimdMember &arg0,
+ const typename _SimdTraits<_Tp, _A1>::_SimdMember &arg1,
+ const typename _SimdTraits<_Tp, _Abis>::_SimdMember &... args)
+{
+ return {arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(arg1, args...)};
+}
+
+// __to_simd_tuple {{{1
+template <size_t, class _Tp> using __to_tuple_helper = _Tp;
+template <class _Tp, class _A0, size_t... _Indexes>
+_GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>
+__to_simd_tuple_impl(std::index_sequence<_Indexes...>,
+ const std::array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>,
+ sizeof...(_Indexes)> &args)
+{
+ return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>(args[_Indexes]...);
+}
+
+template <class _Tp, class _A0, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC auto __to_simd_tuple(
+ const std::array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _N> &args)
+{
+ return __to_simd_tuple_impl<_Tp, _A0>(std::make_index_sequence<_N>(), args);
+}
+
+// __optimize_simd_tuple {{{1
+template <class _Tp> _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp> __optimize_simd_tuple(const _SimdTuple<_Tp>)
+{
+ return {};
+}
+
+template <class _Tp, class _A>
+_GLIBCXX_SIMD_INTRINSIC const _SimdTuple<_Tp, _A> &__optimize_simd_tuple(const _SimdTuple<_Tp, _A> &__x)
+{
+ return __x;
+}
+
+template <class _Tp, class _A0, class _A1, class... _Abis,
+ class _R = __fixed_size_storage_t<_Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::size()>>
+_GLIBCXX_SIMD_INTRINSIC _R __optimize_simd_tuple(const _SimdTuple<_Tp, _A0, _A1, _Abis...> &__x)
+{
+ using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>;
+ if constexpr (std::is_same_v<_R, _Tup>)
+ {
+ return __x;
+ }
+ else if constexpr (_R::_S_first_size == simd_size_v<_Tp, _A0>)
+ {
+ return __simd_tuple_concat(_SimdTuple<_Tp, typename _R::_First_abi>{__x.first},
+ __optimize_simd_tuple(__x.second));
+ }
+ else if constexpr (_R::_S_first_size ==
+ simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>)
+ {
+ return __simd_tuple_concat(_SimdTuple<_Tp, typename _R::_First_abi>{__data(
+ std::experimental::concat(__get_simd_at<0>(__x), __get_simd_at<1>(__x)))},
+ __optimize_simd_tuple(__x.second.second));
+ }
+ else if constexpr (_R::_S_first_size ==
+ 4 * __simd_tuple_element_t<0, _Tup>::size())
+ {
+ return __simd_tuple_concat(
+ _SimdTuple<_Tp, typename _R::_First_abi>{
+ __data(concat(__get_simd_at<0>(__x), __get_simd_at<1>(__x),
+ __get_simd_at<2>(__x), __get_simd_at<3>(__x)))},
+ __optimize_simd_tuple(__x.second.second.second.second));
+ }
+ else if constexpr (_R::_S_first_size ==
+ 8 * __simd_tuple_element_t<0, _Tup>::size())
+ {
+ return __simd_tuple_concat(
+ _SimdTuple<_Tp, typename _R::_First_abi>{__data(concat(
+ __get_simd_at<0>(__x), __get_simd_at<1>(__x), __get_simd_at<2>(__x),
+ __get_simd_at<3>(__x), __get_simd_at<4>(__x), __get_simd_at<5>(__x),
+ __get_simd_at<6>(__x), __get_simd_at<7>(__x)))},
+ __optimize_simd_tuple(
+ __x.second.second.second.second.second.second.second.second));
+ }
+ else if constexpr (_R::_S_first_size ==
+ 16 * __simd_tuple_element_t<0, _Tup>::size())
+ {
+ return __simd_tuple_concat(
+ _SimdTuple<_Tp, typename _R::_First_abi>{__data(concat(
+ __get_simd_at<0>(__x), __get_simd_at<1>(__x), __get_simd_at<2>(__x),
+ __get_simd_at<3>(__x), __get_simd_at<4>(__x), __get_simd_at<5>(__x),
+ __get_simd_at<6>(__x), __get_simd_at<7>(__x), __get_simd_at<8>(__x),
+ __get_simd_at<9>(__x), __get_simd_at<10>(__x),
+ __get_simd_at<11>(__x), __get_simd_at<12>(__x),
+ __get_simd_at<13>(__x), __get_simd_at<14>(__x),
+ __get_simd_at<15>(__x)))},
+ __optimize_simd_tuple(
+ __x.second.second.second.second.second.second.second.second.second
+ .second.second.second.second.second.second.second));
+ }
+ else
+ {
+ return __x;
+ }
+}
+
+// __for_each(const _SimdTuple &, Fun) {{{1
+template <size_t _Offset = 0, class _Tp, class _A0, class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(const _SimdTuple<_Tp, _A0>& __t, _F&& __fun)
+{
+ std::forward<_F>(__fun)(make_meta<_Offset>(__t), __t.first);
+}
+template <size_t _Offset = 0,
+ class _Tp,
+ class _A0,
+ class _A1,
+ class... _As,
+ class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _F&& __fun)
+{
+ __fun(make_meta<_Offset>(__t), __t.first);
+ __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
+ std::forward<_F>(__fun));
+}
+
+// __for_each(_SimdTuple &, Fun) {{{1
+template <size_t _Offset = 0, class _Tp, class _A0, class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(_SimdTuple<_Tp, _A0>& __t, _F&& __fun)
+{
+ std::forward<_F>(__fun)(make_meta<_Offset>(__t), __t.first);
+}
+template <size_t _Offset = 0,
+ class _Tp,
+ class _A0,
+ class _A1,
+ class... _As,
+ class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _F&& __fun)
+{
+ __fun(make_meta<_Offset>(__t), __t.first);
+ __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
+ std::forward<_F>(__fun));
+}
+
+// __for_each(_SimdTuple &, const _SimdTuple &, Fun) {{{1
+template <size_t _Offset = 0, class _Tp, class _A0, class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(_SimdTuple<_Tp, _A0>& __a,
+ const _SimdTuple<_Tp, _A0>& __b,
+ _F&& __fun)
+{
+ std::forward<_F>(__fun)(make_meta<_Offset>(__a), __a.first, __b.first);
+}
+template <size_t _Offset = 0,
+ class _Tp,
+ class _A0,
+ class _A1,
+ class... _As,
+ class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a,
+ const _SimdTuple<_Tp, _A0, _A1, _As...>& __b,
+ _F&& __fun)
+{
+ __fun(make_meta<_Offset>(__a), __a.first, __b.first);
+ __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
+ std::forward<_F>(__fun));
+}
+
+// __for_each(const _SimdTuple &, const _SimdTuple &, Fun) {{{1
+template <size_t _Offset = 0, class _Tp, class _A0, class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(const _SimdTuple<_Tp, _A0>& __a,
+ const _SimdTuple<_Tp, _A0>& __b,
+ _F&& __fun)
+{
+ std::forward<_F>(__fun)(make_meta<_Offset>(__a), __a.first, __b.first);
+}
+template <size_t _Offset = 0,
+ class _Tp,
+ class _A0,
+ class _A1,
+ class... _As,
+ class _F>
+_GLIBCXX_SIMD_INTRINSIC constexpr void
+ __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __a,
+ const _SimdTuple<_Tp, _A0, _A1, _As...>& __b,
+ _F&& __fun)
+{
+ __fun(make_meta<_Offset>(__a), __a.first, __b.first);
+ __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
+ std::forward<_F>(__fun));
+}
+
+// }}}1
+// missing _mmXXX_mask_cvtepi16_storeu_epi8 intrinsics {{{
+#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 8
+_GLIBCXX_SIMD_INTRINSIC void _mm_mask_cvtepi16_storeu_epi8(void *p, __mmask8 __k, __m128i __x)
+{
+ asm("vpmovwb %0,(%2)%{%1%}" :: "__x"(__x), "k"(__k), "g"(p) : "k0");
+}
+_GLIBCXX_SIMD_INTRINSIC void _mm256_mask_cvtepi16_storeu_epi8(void *p, __mmask16 __k, __m256i __x)
+{
+ asm("vpmovwb %0,(%2)%{%1%}" :: "__x"(__x), "k"(__k), "g"(p) : "k0");
+}
+_GLIBCXX_SIMD_INTRINSIC void _mm512_mask_cvtepi16_storeu_epi8(void *p, __mmask32 __k, __m512i __x)
+{
+ asm("vpmovwb %0,(%2)%{%1%}" :: "__x"(__x), "k"(__k), "g"(p) : "k0");
+}
+#endif
+
+// }}}
+// __cmpord{{{
+template <class _Tp, class _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC auto __cmpord(_Tp __x, _Tp __y)
+{
+ static_assert(is_floating_point_v<typename _TVT::value_type>);
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ if constexpr (__have_sse && _TVT::template __is<float, 4>)
+ return __intrin_bitcast<_Tp>(_mm_cmpord_ps(__x, __y));
+ else if constexpr (__have_sse2 && _TVT::template __is<double, 2>)
+ return __intrin_bitcast<_Tp>(_mm_cmpord_pd(__x, __y));
+ else if constexpr (__have_avx && _TVT::template __is<float, 8>)
+ return __intrin_bitcast<_Tp>(_mm256_cmp_ps(__x, __y, _CMP_ORD_Q));
+ else if constexpr (__have_avx && _TVT::template __is<double, 4>)
+ return __intrin_bitcast<_Tp>(_mm256_cmp_pd(__x, __y, _CMP_ORD_Q));
+ else if constexpr (__have_avx512f && _TVT::template __is<float, 16>)
+ return _mm512_cmp_ps_mask(__x, __y, _CMP_ORD_Q);
+ else if constexpr (__have_avx512f && _TVT::template __is<double, 8>)
+ return _mm512_cmp_pd_mask(__x, __y, _CMP_ORD_Q);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ {
+ return reinterpret_cast<_Tp>((__x < __y) != (__x >= __y));
+ }
+}
+
+// }}}
+// __cmpunord{{{
+template <class _Tp, class _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC auto __cmpunord(_Tp __x, _Tp __y)
+{
+ static_assert(is_floating_point_v<typename _TVT::value_type>);
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ if constexpr (__have_sse && _TVT::template __is<float, 4>)
+ return __intrin_bitcast<_Tp>(_mm_cmpunord_ps(__x, __y));
+ else if constexpr (__have_sse2 && _TVT::template __is<double, 2>)
+ return __intrin_bitcast<_Tp>(_mm_cmpunord_pd(__x, __y));
+ else if constexpr (__have_avx && _TVT::template __is<float, 8>)
+ return __intrin_bitcast<_Tp>(_mm256_cmp_ps(__x, __y, _CMP_UNORD_Q));
+ else if constexpr (__have_avx && _TVT::template __is<double, 4>)
+ return __intrin_bitcast<_Tp>(_mm256_cmp_pd(__x, __y, _CMP_UNORD_Q));
+ else if constexpr (__have_avx512f && _TVT::template __is<float, 16>)
+ return _mm512_cmp_ps_mask(__x, __y, _CMP_UNORD_Q);
+ else if constexpr (__have_avx512f && _TVT::template __is<double, 8>)
+ return _mm512_cmp_pd_mask(__x, __y, _CMP_UNORD_Q);
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ {
+ return reinterpret_cast<_Tp>((__x < __y) == (__x >= __y));
+ }
+}
+
+// }}}
+// __maskstore (non-converting; with optimizations for SSE2-AVX512BWVL) {{{
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+template <class _Tp, class _F>
+_GLIBCXX_SIMD_INTRINSIC void __maskstore(_SimdWrapper64<_Tp> __v, _Tp* __mem, _F,
+ _SimdWrapper<bool, _SimdWrapper64<_Tp>::_S_width> __k)
+{
+ [[maybe_unused]] const auto __vi = __to_intrin(__v);
+ static_assert(sizeof(__v) == 64 && __have_avx512f);
+ if constexpr (__have_avx512bw && sizeof(_Tp) == 1) {
+ _mm512_mask_storeu_epi8(__mem, __k, __vi);
+ } else if constexpr (__have_avx512bw && sizeof(_Tp) == 2) {
+ _mm512_mask_storeu_epi16(__mem, __k, __vi);
+ } else if constexpr (__have_avx512f && sizeof(_Tp) == 4) {
+ if constexpr (__is_aligned_v<_F, 64> && std::is_integral_v<_Tp>) {
+ _mm512_mask_store_epi32(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 64> && std::is_floating_point_v<_Tp>) {
+ _mm512_mask_store_ps(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm512_mask_storeu_epi32(__mem, __k, __vi);
+ } else {
+ _mm512_mask_storeu_ps(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_avx512f && sizeof(_Tp) == 8) {
+ if constexpr (__is_aligned_v<_F, 64> && std::is_integral_v<_Tp>) {
+ _mm512_mask_store_epi64(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 64> && std::is_floating_point_v<_Tp>) {
+ _mm512_mask_store_pd(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm512_mask_storeu_epi64(__mem, __k, __vi);
+ } else {
+ _mm512_mask_storeu_pd(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_sse2) {
+ constexpr int _N = 16 / sizeof(_Tp);
+ using _M = __vector_type_t<_Tp, _N>;
+ _mm_maskmoveu_si128(__auto_bitcast(__extract<0, 4>(__v._M_data)),
+ __auto_bitcast(__convert_mask<_M>(__k._M_data)),
+ reinterpret_cast<char*>(__mem));
+ _mm_maskmoveu_si128(__auto_bitcast(__extract<1, 4>(__v._M_data)),
+ __auto_bitcast(__convert_mask<_M>(__k._M_data >> 1 * _N)),
+ reinterpret_cast<char*>(__mem) + 1 * 16);
+ _mm_maskmoveu_si128(__auto_bitcast(__extract<2, 4>(__v._M_data)),
+ __auto_bitcast(__convert_mask<_M>(__k._M_data >> 2 * _N)),
+ reinterpret_cast<char*>(__mem) + 2 * 16);
+ _mm_maskmoveu_si128(__auto_bitcast(__extract<3, 4>(__v._M_data)),
+ __auto_bitcast(__convert_mask<_M>(__k._M_data >> 3 * _N)),
+ reinterpret_cast<char*>(__mem) + 3 * 16);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+template <class _Tp, class _F>
+_GLIBCXX_SIMD_INTRINSIC void __maskstore(_SimdWrapper32<_Tp> __v, _Tp* __mem, _F,
+ _SimdWrapper32<_Tp> __k)
+{
+ [[maybe_unused]] const auto __vi = __vector_bitcast<_LLong>(__v);
+ [[maybe_unused]] const auto __ki = __vector_bitcast<_LLong>(__k);
+ if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1) {
+ _mm256_mask_storeu_epi8(__mem, _mm256_movepi8_mask(__ki), __vi);
+ } else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2) {
+ _mm256_mask_storeu_epi16(__mem, _mm256_movepi16_mask(__ki), __vi);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 4 && std::is_integral_v<_Tp>) {
+ _mm256_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
+ } else if constexpr (sizeof(_Tp) == 4) {
+ _mm256_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
+ __vector_bitcast<float>(__v));
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 8 && std::is_integral_v<_Tp>) {
+ _mm256_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki, __vi);
+ } else if constexpr (__have_avx && sizeof(_Tp) == 8) {
+ _mm256_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
+ __vector_bitcast<double>(__v));
+ } else if constexpr (__have_sse2) {
+ _mm_maskmoveu_si128(__lo128(__vi), __lo128(__ki), reinterpret_cast<char*>(__mem));
+ _mm_maskmoveu_si128(__hi128(__vi), __hi128(__ki),
+ reinterpret_cast<char*>(__mem) + 16);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+template <class _Tp, class _F>
+_GLIBCXX_SIMD_INTRINSIC void __maskstore(_SimdWrapper32<_Tp> __v, _Tp* __mem, _F,
+ _SimdWrapper<bool, _SimdWrapper32<_Tp>::_S_width> __k)
+{
+ [[maybe_unused]] const auto __vi = __to_intrin(__v);
+ if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1) {
+ _mm256_mask_storeu_epi8(__mem, __k, __vi);
+ } else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2) {
+ _mm256_mask_storeu_epi16(__mem, __k, __vi);
+ } else if constexpr (__have_avx512vl && sizeof(_Tp) == 4) {
+ if constexpr (__is_aligned_v<_F, 32> && std::is_integral_v<_Tp>) {
+ _mm256_mask_store_epi32(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 32> && std::is_floating_point_v<_Tp>) {
+ _mm256_mask_store_ps(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm256_mask_storeu_epi32(__mem, __k, __vi);
+ } else {
+ _mm256_mask_storeu_ps(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_avx512vl && sizeof(_Tp) == 8) {
+ if constexpr (__is_aligned_v<_F, 32> && std::is_integral_v<_Tp>) {
+ _mm256_mask_store_epi64(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 32> && std::is_floating_point_v<_Tp>) {
+ _mm256_mask_store_pd(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm256_mask_storeu_epi64(__mem, __k, __vi);
+ } else {
+ _mm256_mask_storeu_pd(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_avx512f && (sizeof(_Tp) >= 4 || __have_avx512bw)) {
+ // use a 512-bit maskstore, using zero-extension of the bitmask
+ __maskstore(
+ _SimdWrapper64<_Tp>(__intrin_bitcast<__vector_type64_t<_Tp>>(__v._M_data)),
+ __mem,
+ // careful, vector_aligned has a stricter meaning in the 512-bit maskstore:
+ std::conditional_t<std::is_same_v<_F, vector_aligned_tag>,
+ overaligned_tag<32>, _F>(),
+ _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
+ } else {
+ __maskstore(
+ __v, __mem, _F(),
+ _SimdWrapper32<_Tp>(__convert_mask<__vector_type_t<_Tp, 32 / sizeof(_Tp)>>(__k)));
+ }
+}
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+
+template <class _Tp, class _F>
+_GLIBCXX_SIMD_INTRINSIC void __maskstore(_SimdWrapper16<_Tp> __v, _Tp* __mem, _F,
+ _SimdWrapper16<_Tp> __k)
+{
+ [[maybe_unused]] const auto __vi = __vector_bitcast<_LLong>(__v);
+ [[maybe_unused]] const auto __ki = __vector_bitcast<_LLong>(__k);
+ if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1) {
+ _mm_mask_storeu_epi8(__mem, _mm_movepi8_mask(__ki), __vi);
+ } else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2) {
+ _mm_mask_storeu_epi16(__mem, _mm_movepi16_mask(__ki), __vi);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 4 && std::is_integral_v<_Tp>) {
+ _mm_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
+ } else if constexpr (__have_avx && sizeof(_Tp) == 4) {
+ _mm_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
+ __vector_bitcast<float>(__v));
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 8 && std::is_integral_v<_Tp>) {
+ _mm_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki, __vi);
+ } else if constexpr (__have_avx && sizeof(_Tp) == 8) {
+ _mm_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
+ __vector_bitcast<double>(__v));
+ } else if constexpr (__have_sse2) {
+ _mm_maskmoveu_si128(__vi, __ki, reinterpret_cast<char*>(__mem));
+ } else {
+ __bit_iteration(__vector_to_bitset(__k._M_data).to_ullong(),
+ [&](auto __i) { __mem[__i] = __v[__i]; });
+ }
+}
+
+template <class _Tp, class _F>
+_GLIBCXX_SIMD_INTRINSIC void __maskstore(_SimdWrapper16<_Tp> __v, _Tp* __mem, _F,
+ _SimdWrapper<bool, _SimdWrapper16<_Tp>::_S_width> __k)
+{
+ [[maybe_unused]] const auto __vi = __to_intrin(__v);
+ if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1) {
+ _mm_mask_storeu_epi8(__mem, __k, __vi);
+ } else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2) {
+ _mm_mask_storeu_epi16(__mem, __k, __vi);
+ } else if constexpr (__have_avx512vl && sizeof(_Tp) == 4) {
+ if constexpr (__is_aligned_v<_F, 16> && std::is_integral_v<_Tp>) {
+ _mm_mask_store_epi32(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 16> && std::is_floating_point_v<_Tp>) {
+ _mm_mask_store_ps(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm_mask_storeu_epi32(__mem, __k, __vi);
+ } else {
+ _mm_mask_storeu_ps(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_avx512vl && sizeof(_Tp) == 8) {
+ if constexpr (__is_aligned_v<_F, 16> && std::is_integral_v<_Tp>) {
+ _mm_mask_store_epi64(__mem, __k, __vi);
+ } else if constexpr (__is_aligned_v<_F, 16> && std::is_floating_point_v<_Tp>) {
+ _mm_mask_store_pd(__mem, __k, __vi);
+ } else if constexpr (std::is_integral_v<_Tp>) {
+ _mm_mask_storeu_epi64(__mem, __k, __vi);
+ } else {
+ _mm_mask_storeu_pd(__mem, __k, __vi);
+ }
+ } else if constexpr (__have_avx512f && (sizeof(_Tp) >= 4 || __have_avx512bw)) {
+ // use a 512-bit maskstore, using zero-extension of the bitmask
+ __maskstore(
+ _SimdWrapper64<_Tp>(__intrin_bitcast<__intrinsic_type64_t<_Tp>>(__v._M_data)), __mem,
+ // careful, vector_aligned has a stricter meaning in the 512-bit maskstore:
+ std::conditional_t<std::is_same_v<_F, vector_aligned_tag>,
+ overaligned_tag<16>, _F>(),
+ _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
+ } else {
+ __maskstore(
+ __v, __mem, _F(),
+ _SimdWrapper16<_Tp>(__convert_mask<__vector_type_t<_Tp, 16 / sizeof(_Tp)>>(__k)));
+ }
+}
+
+template <typename _Tp, typename _F, typename _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC enable_if_t<sizeof(_Tp) == 8, void>
+ __maskstore(_Tp __v, typename _TVT::value_type* __mem, _F, _Tp __k)
+{
+ __bit_iteration(__vector_to_bitset(__k).to_ulong(),
+ [&](auto __i) { __mem[__i] = __v[__i]; });
+}
+
+// }}}
+// __xzyw{{{
+// shuffles the complete vector, swapping the inner two quarters. Often useful for AVX for
+// fixing up a shuffle result.
+template <class _Tp, class _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _Tp __xzyw(_Tp __a)
+{
+ if constexpr (sizeof(_Tp) == 16) {
+ static_assert(sizeof(float) == 4 && sizeof(int) == 4);
+ const auto __x = __vector_bitcast<
+ conditional_t<is_floating_point_v<typename _TVT::value_type>, float, int>>(
+ __a);
+ return reinterpret_cast<_Tp>(decltype(__x){__x[0], __x[2], __x[1], __x[3]});
+ } else if constexpr (sizeof(_Tp) == 32) {
+ static_assert(sizeof(double) == 8 && sizeof(_LLong) == 8);
+ const auto __x =
+ __vector_bitcast<conditional_t<is_floating_point_v<typename _TVT::value_type>,
+ double, _LLong>>(__a);
+ return reinterpret_cast<_Tp>(decltype(__x){__x[0], __x[2], __x[1], __x[3]});
+ } else if constexpr (sizeof(_Tp) == 64) {
+ static_assert(sizeof(double) == 8 && sizeof(_LLong) == 8);
+ const auto __x =
+ __vector_bitcast<conditional_t<is_floating_point_v<typename _TVT::value_type>,
+ double, _LLong>>(__a);
+ return reinterpret_cast<_Tp>(decltype(__x){__x[0], __x[1], __x[4], __x[5], __x[2],
+ __x[3], __x[6], __x[7]});
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+// }}}
+// __shift_elements_right{{{
+// if (__shift % 2ⁿ == 0) => the low n Bytes are correct
+template <unsigned __shift, class _Tp, class _TVT = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _Tp __shift_elements_right(_Tp __v)
+{
+ static_assert(__shift <= sizeof(_Tp));
+ if constexpr (__shift == 0) {
+ return __v;
+ } else if constexpr(__shift == sizeof(_Tp)) {
+ return _Tp();
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ } else if constexpr (__have_sse && __shift == 8 && _TVT::template __is<float, 4>) {
+ return _mm_movehl_ps(__v, __v);
+ } else if constexpr (__have_sse2 && __shift == 8 && _TVT::template __is<double, 2>) {
+ return _mm_unpackhi_pd(__v, __v);
+ } else if constexpr (__have_sse2 && sizeof(_Tp) == 16) {
+ return __intrin_bitcast<_Tp>(
+ _mm_srli_si128(__intrin_bitcast<__m128i>(__v), __shift));
+/*
+ } else if constexpr (__shift == 16 && sizeof(_Tp) == 32) {
+ if constexpr (__have_avx && _TVT::template __is<double, 4>) {
+ return _mm256_permute2f128_pd(__v, __v, 0x81);
+ } else if constexpr (__have_avx && _TVT::template __is<float, 8>) {
+ return _mm256_permute2f128_ps(__v, __v, 0x81);
+ } else if constexpr (__have_avx) {
+ return _mm256_permute2f128_si256(__v, __v, 0x81);
+ } else {
+ return __auto_bitcast(__hi128(__v));
+ }
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 32) {
+ const auto __vi = __intrin_bitcast<__m256i>(__v);
+ return __intrin_bitcast<_Tp>(_mm256_srli_si256(
+ __shift < 16 ? __vi : _mm256_permute2x128_si256(__vi, __vi, 0x81),
+ __shift % 16));
+ } else if constexpr (sizeof(_Tp) == 32) {
+ __shift % 16
+ return __intrin_bitcast<_Tp>(
+ __extract<_shift/16, 2>(__v)
+ );
+ } else if constexpr (__have512f && sizeof(_Tp) == 64) {
+ if constexpr (__shift % 8 == 0) {
+ return __mm512_alignr_epi64(__m512i(), __intrin_bitcast<__m512i>(__v),
+ __shift / 8);
+ } else if constexpr (__shift % 4 == 0) {
+ return __mm512_alignr_epi32(__m512i(), __intrin_bitcast<__m512i>(__v),
+ __shift / 4);
+ } else {
+ const auto __shifted = __mm512_alignr_epi8(
+ __m512i(), __intrin_bitcast<__m512i>(__v), __shift % 16);
+ return __intrin_bitcast<_Tp>(
+ __shift < 16
+ ? __shifted
+ : _mm512_shuffle_i32x4(__shifted, __shifted, 0xe4 + (__shift / 16)));
+ }
+ } else if constexpr (__shift == 32 && sizeof(_Tp) == 64) {
+ return __auto_bitcast(__hi256(__v));
+ } else if constexpr (__shift % 16 == 0 && sizeof(_Tp) == 64) {
+ return __auto_bitcast(__extract<__shift / 16, 4>(__v));
+*/
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ }
+ else
+ {
+ constexpr int __chunksize =
+ __shift % 8 == 0 ? 8 : __shift % 4 == 0 ? 4 : __shift % 2 == 0 ? 2 : 1;
+ auto __w = __vector_bitcast<__int_with_sizeof_t<__chunksize>>(__v);
+ return __intrin_bitcast<_Tp>(decltype(__w){__w[__shift / __chunksize]});
+ }
+}
+
+// }}}
+// __extract_part(_SimdWrapper<_Tp, _N>) {{{
+template <size_t _Index, size_t _Total, class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONST
+ __vector_type_t<_Tp, std::max(__min_vector_size / sizeof(_Tp), _N / _Total)>
+ __extract_part(_SimdWrapper<_Tp, _N> __x)
+{
+ constexpr size_t _NewN = _N / _Total;
+ static_assert(_Total > _Index, "_Total must be greater than _Index");
+ static_assert(_NewN * _Total == _N, "_N must be divisible by _Total");
+ if constexpr (_Index == 0 && _Total == 1) {
+ return __x._M_data;
+ } else if constexpr (sizeof(_Tp) * _NewN >= __min_vector_size) {
+ return __extract<_Index, _Total>(__x._M_data);
+ } else {
+ constexpr int split = sizeof(__x) / __min_vector_size;
+ constexpr int shift = (sizeof(__x) / _Total * _Index) % __min_vector_size;
+ return __shift_elements_right<shift>(
+ __extract_part<_Index * split / _Total, split>(__x));
+ }
+}
+
+// }}}
+// __extract_part(_SimdWrapper<bool, _N>) {{{
+template <size_t _Index, size_t _Total, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr __bool_storage_member_type_t<_N / _Total>
+__extract_part(_SimdWrapper<bool, _N> __x)
+{
+ static_assert(__have_avx512f && _N == _N);
+ static_assert(_Total >= 2 && _Index < _Total && _Index >= 0);
+ return __x._M_data >> (_Index * _N / _Total);
+}
+
+// }}}
+// __extract_part(_SimdTuple) {{{
+template <int _Index, int _Parts, class _Tp, class _A0, class... _As>
+_GLIBCXX_SIMD_INTRINSIC auto // __vector_type_t or _SimdTuple
+__extract_part(const _SimdTuple<_Tp, _A0, _As...> &__x)
+{
+ // worst cases:
+ // (a) 4, 4, 4 => 3, 3, 3, 3 (_Parts = 4)
+ // (b) 2, 2, 2 => 3, 3 (_Parts = 2)
+ // (c) 4, 2 => 2, 2, 2 (_Parts = 3)
+ using _Tuple = _SimdTuple<_Tp, _A0, _As...>;
+ static_assert(_Index < _Parts && _Index >= 0 && _Parts >= 1);
+ constexpr size_t _N = _Tuple::size();
+ static_assert(_N >= _Parts && _N % _Parts == 0);
+ constexpr size_t values_per_part = _N / _Parts;
+ if constexpr (_Parts == 1) {
+ if constexpr (_Tuple::tuple_size == 1) {
+ return __x.first._M_data;
+ } else {
+ return __x;
+ }
+ } else if constexpr (simd_size_v<_Tp, _A0> % values_per_part != 0) {
+ // nasty case: The requested partition does not match the partition of the
+ // _SimdTuple. Fall back to construction via scalar copies.
+#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
+ const __may_alias<_Tp> *const element_ptr =
+ reinterpret_cast<const __may_alias<_Tp> *>(&__x) + _Index * values_per_part;
+ return __data(simd<_Tp, simd_abi::deduce_t<_Tp, values_per_part>>(
+ [&](auto __i) { return element_ptr[__i]; }))
+ ._M_data;
+#else
+ constexpr size_t offset = _Index * values_per_part;
+ __unused(offset); // not really
+ return __data(simd<_Tp, simd_abi::deduce_t<_Tp, values_per_part>>([&](auto __i) {
+ constexpr _SizeConstant<__i + offset> __k;
+ return __x[__k];
+ }))
+ ._M_data;
+#endif
+ } else if constexpr (values_per_part * _Index >= simd_size_v<_Tp, _A0>) { // recurse
+ constexpr int parts_in_first = simd_size_v<_Tp, _A0> / values_per_part;
+ return __extract_part<_Index - parts_in_first, _Parts - parts_in_first>(__x.second);
+ } else { // at this point we know that all of the return values are in __x.first
+ static_assert(values_per_part * (1 + _Index) <= simd_size_v<_Tp, _A0>);
+ if constexpr (simd_size_v<_Tp, _A0> == values_per_part) {
+ return __x.first._M_data;
+ } else {
+ return __extract_part<_Index, simd_size_v<_Tp, _A0> / values_per_part>(
+ __x.first);
+ }
+ }
+}
+// }}}
+// _ToWrapper specializations for bitset and __mmask<_N> {{{
+#if _GLIBCXX_SIMD_HAVE_AVX512_ABI
+template <size_t _N> class _ToWrapper<std::bitset<_N>>
+{
+ std::bitset<_N> _M_data;
+
+public:
+ // can convert to larger storage for _Abi::is_partial == true
+ template <class _U, size_t _M> constexpr operator _SimdWrapper<_U, _M>() const
+ {
+ static_assert(_M >= _N);
+ return __convert_mask<_SimdWrapper<_U, _M>>(_M_data);
+ }
+};
+
+#define _GLIBCXX_SIMD_TO_STORAGE(_Type) \
+ template <> class _ToWrapper<_Type> \
+ { \
+ _Type _M_data; \
+ \
+ public: \
+ template <class _U, size_t _N> constexpr operator _SimdWrapper<_U, _N>() const \
+ { \
+ static_assert(_N >= sizeof(_Type) * CHAR_BIT); \
+ return reinterpret_cast<__vector_type_t<_U, _N>>( \
+ __convert_mask<_SimdWrapper<_U, _N>>(_M_data)); \
+ } \
+ \
+ template <size_t _N> constexpr operator _SimdWrapper<bool, _N>() const \
+ { \
+ static_assert( \
+ std::is_same_v<_Type, typename __bool_storage_member_type<_N>::type>); \
+ return _M_data; \
+ } \
+ }
+_GLIBCXX_SIMD_TO_STORAGE(__mmask8);
+_GLIBCXX_SIMD_TO_STORAGE(__mmask16);
+_GLIBCXX_SIMD_TO_STORAGE(__mmask32);
+_GLIBCXX_SIMD_TO_STORAGE(__mmask64);
+#undef _GLIBCXX_SIMD_TO_STORAGE
+#endif // _GLIBCXX_SIMD_HAVE_AVX512_ABI
+
+// }}}
+
+#if _GLIBCXX_SIMD_HAVE_SSE && defined _GLIBCXX_SIMD_WORKAROUND_PR85048
+#include "simd_x86_conversions.h"
+#endif // SSE && _GLIBCXX_SIMD_WORKAROUND_PR85048
+
+// __convert function{{{
+template <class _To, class _From, class... _More>
+_GLIBCXX_SIMD_INTRINSIC auto __convert(_From __v0, _More... __vs)
+{
+ static_assert((true && ... && is_same_v<_From, _More>));
+ if constexpr (__is_vectorizable_v<_From>) {
+ if constexpr (__is_vector_type_v<_To>) {
+ return __make_vector(__v0, __vs...);
+ } else {
+ using _Tp = typename _To::value_type;
+ return __make_wrapper<_Tp>(__v0, __vs...);
+ }
+ } else if constexpr (!__is_vector_type_v<_From>) {
+ return __convert<_To>(__v0._M_data, __vs._M_data...);
+ } else if constexpr (!__is_vector_type_v<_To>) {
+ return _To(__convert<typename _To::_BuiltinType>(__v0, __vs...));
+ } else if constexpr (__is_vectorizable_v<_To>) {
+ return __convert<__vector_type_t<_To, (_VectorTraits<_From>::_S_width *
+ (1 + sizeof...(_More)))>>(__v0, __vs...)
+ ._M_data;
+ } else {
+ static_assert(sizeof...(_More) == 0 ||
+ _VectorTraits<_To>::_S_width >=
+ (1 + sizeof...(_More)) * _VectorTraits<_From>::_S_width,
+ "__convert(...) requires the input to fit into the output");
+ return __vector_convert<_To>(__v0, __vs...);
+ }
+}
+
+// }}}
+// __convert_all{{{
+template <typename _To, typename _From> _GLIBCXX_SIMD_INTRINSIC auto __convert_all(_From __v)
+{
+ static_assert(__is_vector_type_v<_To>);
+ if constexpr (__is_vector_type_v<_From>) {
+ using _Trait = _VectorTraits<_From>;
+ using _S = _SimdWrapper<typename _Trait::value_type, _Trait::_S_width>;
+ return __convert_all<_To>(_S(__v));
+ } else if constexpr (_From::_S_width > _VectorTraits<_To>::_S_width) {
+ constexpr size_t _N = _From::_S_width / _VectorTraits<_To>::_S_width;
+ return __generate_from_n_evaluations<_N, std::array<_To, _N>>([&](auto __i) {
+ auto __part = __extract_part<decltype(__i)::value, _N>(__v);
+ return __convert<_To>(__part);
+ });
+ } else {
+ return __convert<_To>(__v);
+ }
+}
+
+// }}}
+// __converts_via_decomposition{{{
+// This lists all cases where a __vector_convert needs to fall back to conversion of
+// individual scalars (i.e. decompose the input vector into scalars, convert, compose
+// output vector). In those cases, masked_load & masked_store prefer to use the
+// __bit_iteration implementation.
+template <class _From, class _To, size_t _ToSize> struct __converts_via_decomposition {
+private:
+ static constexpr bool __i_to_i = is_integral_v<_From> && is_integral_v<_To>;
+ static constexpr bool __f_to_i = is_floating_point_v<_From> && is_integral_v<_To>;
+ static constexpr bool __f_to_f = is_floating_point_v<_From> && is_floating_point_v<_To>;
+ static constexpr bool __i_to_f = is_integral_v<_From> && is_floating_point_v<_To>;
+
+ template <size_t _A, size_t _B>
+ static constexpr bool __sizes = sizeof(_From) == _A && sizeof(_To) == _B;
+
+public:
+ static constexpr bool value =
+ (__i_to_i && __sizes<8, 2> && !__have_ssse3 && _ToSize == 16) ||
+ (__i_to_i && __sizes<8, 1> && !__have_avx512f && _ToSize == 16) ||
+ (__f_to_i && __sizes<4, 8> && !__have_avx512dq) ||
+ (__f_to_i && __sizes<8, 8> && !__have_avx512dq) ||
+ (__f_to_i && __sizes<8, 4> && !__have_sse4_1 && _ToSize == 16) ||
+ (__i_to_f && __sizes<8, 4> && !__have_avx512dq && _ToSize == 16) ||
+ (__i_to_f && __sizes<8, 8> && !__have_avx512dq && _ToSize < 64);
+};
+
+template <class _From, class _To, size_t _ToSize>
+inline constexpr bool __converts_via_decomposition_v =
+ __converts_via_decomposition<_From, _To, _ToSize>::value;
+
+// }}}
+// __plus{{{
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N> __plus(_SimdWrapper<_Tp, _N> __a,
+ _SimdWrapper<_Tp, _N> __b)
+{
+ return __a._M_data + __b._M_data;
+}
+
+//}}}
+// __minus{{{
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N> __minus(_SimdWrapper<_Tp, _N> __a,
+ _SimdWrapper<_Tp, _N> __b)
+{
+ return __a._M_data - __b._M_data;
+}
+
+//}}}
+// __multiplies{{{
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N> __multiplies(_SimdWrapper<_Tp, _N> __a,
+ _SimdWrapper<_Tp, _N> __b)
+{
+ if constexpr (sizeof(_Tp) == 1) {
+ return __vector_bitcast<_Tp>(
+ ((__vector_bitcast<short>(__a) * __vector_bitcast<short>(__b)) &
+ __vector_bitcast<short>(~__vector_type_t<ushort, _N / 2>() >> 8)) |
+ (((__vector_bitcast<short>(__a) >> 8) * (__vector_bitcast<short>(__b) >> 8))
+ << 8));
+ }
+ return __a._M_data * __b._M_data;
+}
+
+//}}}
+// __abs{{{
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _N> __abs(_SimdWrapper<_Tp, _N> __v)
+{
+ // if (__builtin_is_constant_evaluated())
+ // {
+ // return __v._M_data < 0 ? -__v._M_data : __v._M_data;
+ // }
+ if constexpr (std::is_floating_point_v<_Tp>)
+ {
+ // `v < 0 ? -v : v` cannot compile to the efficient implementation of
+ // masking the signbit off because it must consider v == -0
+
+ // ~(-0.) & v would be easy, but breaks with fno-signed-zeros
+ // return __andnot(__vector_broadcast<_N, _Tp>(-0.), __v._M_data);
+
+ using _I = std::make_unsigned_t<__int_for_sizeof_t<_Tp>>;
+ return __and(__v._M_data, __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, _I>(~_I() >> 1)));
+ }
+ else
+ {
+ return __v._M_data < 0 ? -__v._M_data : __v._M_data;
+ }
+}
+
+//}}}
+// __is_bitset {{{
+template <class _Tp> struct __is_bitset : false_type {};
+template <size_t _N> struct __is_bitset<std::bitset<_N>> : true_type {};
+template <class _Tp> inline constexpr bool __is_bitset_v = __is_bitset<_Tp>::value;
+
+// }}}
+// __is_storage {{{
+template <class _Tp> struct __is_storage : false_type {};
+template <class _Tp, size_t _N> struct __is_storage<_SimdWrapper<_Tp, _N>> : true_type {};
+template <class _Tp> inline constexpr bool __is_storage_v = __is_storage<_Tp>::value;
+
+// }}}
+// __convert_mask{{{
+template <class _To, class _From>
+inline _To __convert_mask(_From __k)
+{
+ if constexpr (std::is_same_v<_To, _From>)
+ { // also covers bool -> bool
+ return __k;
+ } else if constexpr (std::is_unsigned_v<_From> && std::is_unsigned_v<_To>) {
+ // bits -> bits
+ return __k; // zero-extends or truncates
+ } else if constexpr (__is_bitset_v<_From>) {
+ // from std::bitset {{{
+ static_assert(__k.size() <= sizeof(_ULLong) * CHAR_BIT);
+ using _Tp = std::conditional_t<
+ (__k.size() <= sizeof(ushort) * CHAR_BIT),
+ std::conditional_t<(__k.size() <= CHAR_BIT), _UChar, ushort>,
+ std::conditional_t<(__k.size() <= sizeof(_UInt) * CHAR_BIT), _UInt, _ULLong>>;
+ return __convert_mask<_To>(static_cast<_Tp>(__k.to_ullong()));
+ // }}}
+ } else if constexpr (__is_bitset_v<_To>) {
+ // to std::bitset {{{
+ static_assert(_To().size() <= sizeof(_ULLong) * CHAR_BIT);
+ using _Tp = std::conditional_t<
+ (_To().size() <= sizeof(ushort) * CHAR_BIT),
+ std::conditional_t<(_To().size() <= CHAR_BIT), _UChar, ushort>,
+ std::conditional_t<(_To().size() <= sizeof(_UInt) * CHAR_BIT), _UInt, _ULLong>>;
+ return __convert_mask<_Tp>(__k);
+ // }}}
+ } else if constexpr (__is_storage_v<_From>) {
+ return __convert_mask<_To>(__k._M_data);
+ } else if constexpr (__is_storage_v<_To>) {
+ return __convert_mask<typename _To::_BuiltinType>(__k);
+ } else if constexpr (std::is_unsigned_v<_From> && __is_vector_type_v<_To>) {
+ // bits -> vector {{{
+ using _Trait = _VectorTraits<_To>;
+ constexpr size_t _N_in = sizeof(_From) * CHAR_BIT;
+ using _ToT = typename _Trait::value_type;
+ constexpr size_t _N_out = _Trait::_S_width;
+ constexpr size_t _N = std::min(_N_in, _N_out);
+ constexpr size_t bytes_per_output_element = sizeof(_ToT);
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ if constexpr (__have_avx512f) {
+ if constexpr (bytes_per_output_element == 1 && sizeof(_To) == 16) {
+ if constexpr (__have_avx512bw_vl) {
+ return __vector_bitcast<_ToT>(_mm_movm_epi8(__k));
+ } else if constexpr (__have_avx512bw) {
+ return __vector_bitcast<_ToT>(__lo128(_mm512_movm_epi8(__k)));
+ } else {
+ auto as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
+ auto as16bits = __xzyw(
+ _mm256_packs_epi32(__lo256(as32bits), __hi256(as32bits)));
+ return __vector_bitcast<_ToT>(
+ _mm_packs_epi16(__lo128(as16bits), __hi128(as16bits)));
+ }
+ } else if constexpr (bytes_per_output_element == 1 && sizeof(_To) == 32) {
+ if constexpr (__have_avx512bw_vl) {
+ return __vector_bitcast<_ToT>(_mm256_movm_epi8(__k));
+ } else if constexpr (__have_avx512bw) {
+ return __vector_bitcast<_ToT>(__lo256(_mm512_movm_epi8(__k)));
+ } else {
+ auto as16bits = // 0 16 1 17 ... 15 31
+ _mm512_srli_epi32(_mm512_maskz_mov_epi32(__k, ~__m512i()), 16) |
+ _mm512_slli_epi32(_mm512_maskz_mov_epi32(__k >> 16, ~__m512i()),
+ 16);
+ auto _0_16_1_17 = __xzyw(_mm256_packs_epi16(
+ __lo256(as16bits),
+ __hi256(as16bits)) // 0 16 1 17 2 18 3 19 8 24 9 25 ...
+ );
+ // deinterleave:
+ return __vector_bitcast<_ToT>(__xzyw(_mm256_shuffle_epi8(
+ _0_16_1_17, // 0 16 1 17 2 ...
+ _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13,
+ 15, 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11,
+ 13, 15)))); // 0-7 16-23 8-15 24-31 -> xzyw
+ // 0-3 8-11 16-19 24-27
+ // 4-7 12-15 20-23 28-31
+ }
+ } else if constexpr (bytes_per_output_element == 1 && sizeof(_To) == 64) {
+ return reinterpret_cast<__vector_type_t<_SChar, 64>>(_mm512_movm_epi8(__k));
+ } else if constexpr (bytes_per_output_element == 2 && sizeof(_To) == 16) {
+ if constexpr (__have_avx512bw_vl) {
+ return __vector_bitcast<_ToT>(_mm_movm_epi16(__k));
+ } else if constexpr (__have_avx512bw) {
+ return __vector_bitcast<_ToT>(__lo128(_mm512_movm_epi16(__k)));
+ } else {
+ __m256i as32bits;
+ if constexpr (__have_avx512vl) {
+ as32bits = _mm256_maskz_mov_epi32(__k, ~__m256i());
+ } else {
+ as32bits = __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i()));
+ }
+ return __vector_bitcast<_ToT>(
+ _mm_packs_epi32(__lo128(as32bits), __hi128(as32bits)));
+ }
+ } else if constexpr (bytes_per_output_element == 2 && sizeof(_To) == 32) {
+ if constexpr (__have_avx512bw_vl) {
+ return __vector_bitcast<_ToT>(_mm256_movm_epi16(__k));
+ } else if constexpr (__have_avx512bw) {
+ return __vector_bitcast<_ToT>(__lo256(_mm512_movm_epi16(__k)));
+ } else {
+ auto as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
+ return __vector_bitcast<_ToT>(__xzyw(
+ _mm256_packs_epi32(__lo256(as32bits), __hi256(as32bits))));
+ }
+ } else if constexpr (bytes_per_output_element == 2 && sizeof(_To) == 64) {
+ return __vector_bitcast<_ToT>(_mm512_movm_epi16(__k));
+ } else if constexpr (bytes_per_output_element == 4 && sizeof(_To) == 16) {
+ return __vector_bitcast<_ToT>(
+ __have_avx512dq_vl
+ ? _mm_movm_epi32(__k)
+ : __have_avx512dq
+ ? __lo128(_mm512_movm_epi32(__k))
+ : __have_avx512vl
+ ? _mm_maskz_mov_epi32(__k, ~__m128i())
+ : __lo128(_mm512_maskz_mov_epi32(__k, ~__m512i())));
+ } else if constexpr (bytes_per_output_element == 4 && sizeof(_To) == 32) {
+ return __vector_bitcast<_ToT>(
+ __have_avx512dq_vl
+ ? _mm256_movm_epi32(__k)
+ : __have_avx512dq
+ ? __lo256(_mm512_movm_epi32(__k))
+ : __have_avx512vl
+ ? _mm256_maskz_mov_epi32(__k, ~__m256i())
+ : __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i())));
+ } else if constexpr (bytes_per_output_element == 4 && sizeof(_To) == 64) {
+ return __vector_bitcast<_ToT>(__have_avx512dq
+ ? _mm512_movm_epi32(__k)
+ : _mm512_maskz_mov_epi32(__k, ~__m512i()));
+ } else if constexpr (bytes_per_output_element == 8 && sizeof(_To) == 16) {
+ return __vector_bitcast<_ToT>(
+ __have_avx512dq_vl
+ ? _mm_movm_epi64(__k)
+ : __have_avx512dq
+ ? __lo128(_mm512_movm_epi64(__k))
+ : __have_avx512vl
+ ? _mm_maskz_mov_epi64(__k, ~__m128i())
+ : __lo128(_mm512_maskz_mov_epi64(__k, ~__m512i())));
+ } else if constexpr (bytes_per_output_element == 8 && sizeof(_To) == 32) {
+ return __vector_bitcast<_ToT>(
+ __have_avx512dq_vl
+ ? _mm256_movm_epi64(__k)
+ : __have_avx512dq
+ ? __lo256(_mm512_movm_epi64(__k))
+ : __have_avx512vl
+ ? _mm256_maskz_mov_epi64(__k, ~__m256i())
+ : __lo256(_mm512_maskz_mov_epi64(__k, ~__m512i())));
+ } else if constexpr (bytes_per_output_element == 8 && sizeof(_To) == 64) {
+ return __vector_bitcast<_ToT>(__have_avx512dq
+ ? _mm512_movm_epi64(__k)
+ : _mm512_maskz_mov_epi64(__k, ~__m512i()));
+ } else {
+ __assert_unreachable<_To>();
+ }
+ } else if constexpr (__have_sse) {
+ using _U = std::make_unsigned_t<__int_for_sizeof_t<_ToT>>;
+ using _V = __vector_type_t<_U, _N>; // simd<_U, _Abi>;
+ static_assert(sizeof(_V) <= 32); // can't be AVX512
+ constexpr size_t bits_per_element = sizeof(_U) * CHAR_BIT;
+ if constexpr (!__have_avx2 && __have_avx && sizeof(_V) == 32) {
+ if constexpr (_N == 8) {
+ return _mm256_cmp_ps(
+ _mm256_and_ps(
+ _mm256_castsi256_ps(_mm256_set1_epi32(__k)),
+ _mm256_castsi256_ps(_mm256_setr_epi32(
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80))),
+ _mm256_setzero_ps(), _CMP_NEQ_UQ);
+ } else if constexpr (_N == 4) {
+ return _mm256_cmp_pd(
+ _mm256_and_pd(
+ _mm256_castsi256_pd(_mm256_set1_epi64x(__k)),
+ _mm256_castsi256_pd(
+ _mm256_setr_epi64x(0x01, 0x02, 0x04, 0x08))),
+ _mm256_setzero_pd(), _CMP_NEQ_UQ);
+ } else {
+ __assert_unreachable<_To>();
+ }
+ } else if constexpr (bits_per_element >= _N) {
+ constexpr auto bitmask = __generate_vector<__vector_type_t<_U, _N>>(
+ [](auto __i) -> _U { return 1ull << __i; });
+ return __vector_bitcast<_ToT>(
+ (__vector_broadcast<_N, _U>(__k) & bitmask) != 0);
+ } else if constexpr (sizeof(_V) == 16 && sizeof(_ToT) == 1 && __have_ssse3) {
+ const auto bitmask = __to_intrin(__make_vector<_UChar>(
+ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128));
+ return __vector_bitcast<_ToT>(
+ __vector_bitcast<_ToT>(
+ _mm_shuffle_epi8(
+ __to_intrin(__vector_type_t<_ULLong, 2>{__k}),
+ _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1,
+ 1)) &
+ bitmask) != 0);
+ } else if constexpr (sizeof(_V) == 32 && sizeof(_ToT) == 1 && __have_avx2) {
+ const auto bitmask =
+ _mm256_broadcastsi128_si256(__to_intrin(__make_vector<_UChar>(
+ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128)));
+ return __vector_bitcast<_ToT>(
+ __vector_bitcast<_ToT>(_mm256_shuffle_epi8(
+ _mm256_broadcastsi128_si256(__to_intrin(
+ __vector_type_t<_ULLong, 2>{__k})),
+ _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1,
+ 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
+ 2, 2, 3, 3, 3, 3, 3, 3, 3, 3)) &
+ bitmask) != 0);
+ /* TODO:
+ } else if constexpr (sizeof(_V) == 32 && sizeof(_ToT) == 2 && __have_avx2) {
+ constexpr auto bitmask = _mm256_broadcastsi128_si256(
+ _mm_setr_epi8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+ 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000)); return
+ __vector_bitcast<_ToT>( _mm256_shuffle_epi8(
+ _mm256_broadcastsi128_si256(__m128i{__k}),
+ _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
+ 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3)) & bitmask) != 0;
+ */
+ } else {
+ const _V tmp = __generate_vector<_V>([&](auto __i) {
+ return static_cast<_U>(
+ __k >> (bits_per_element * (__i / bits_per_element)));
+ }) &
+ __generate_vector<_V>([](auto __i) {
+ return static_cast<_U>(1ull << (__i % bits_per_element));
+ }); // mask bit index
+ return __vector_bitcast<_ToT>(tmp != _V());
+ }
+ } else
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ {
+ using _I = __int_for_sizeof_t<_ToT>;
+ return reinterpret_cast<_To>(
+ __generate_vector<__vector_type_t<_I, _N_out>>([&](auto __i) {
+ return ((__k >> __i) & 1) == 0 ? _I() : ~_I();
+ }));
+ }
+ // }}}
+ } else if constexpr (__is_vector_type_v<_From> && std::is_unsigned_v<_To>) {
+ // vector -> bits {{{
+ using _Trait = _VectorTraits<_From>;
+ using _Tp = typename _Trait::value_type;
+ constexpr size_t _FromN = _Trait::_S_width;
+ constexpr size_t cvt_id = _FromN * 10 + sizeof(_Tp);
+ constexpr bool __have_avx512_int = __have_avx512f && std::is_integral_v<_Tp>;
+ [[maybe_unused]] // PR85827
+ const auto __intrin = __to_intrin(__k);
+
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ if constexpr (cvt_id == 16'1 && __have_avx512bw_vl) { return _mm_movepi8_mask(__intrin); }
+ else if constexpr (cvt_id == 16'1 && __have_avx512bw ) { return _mm512_movepi8_mask(__zero_extend(__intrin)); }
+ else if constexpr (cvt_id == 16'1 ) { return _mm_movemask_epi8(__intrin); }
+ else if constexpr (cvt_id == 32'1 && __have_avx512bw_vl) { return _mm256_movepi8_mask(__intrin); }
+ else if constexpr (cvt_id == 32'1 && __have_avx512bw ) { return _mm512_movepi8_mask(__zero_extend(__intrin)); }
+ else if constexpr (cvt_id == 32'1 ) { return _mm256_movemask_epi8(__intrin); }
+ else if constexpr (cvt_id == 64'1 && __have_avx512bw ) { return _mm512_movepi8_mask(__intrin); }
+ else if constexpr (cvt_id == 8'2 && __have_avx512bw_vl) { return _mm_movepi16_mask(__intrin); }
+ else if constexpr (cvt_id == 8'2 && __have_avx512bw ) { return _mm512_movepi16_mask(__zero_extend(__intrin)); }
+ else if constexpr (cvt_id == 8'2 ) { return movemask_epi16(__intrin); }
+ else if constexpr (cvt_id == 16'2 && __have_avx512bw_vl) { return _mm256_movepi16_mask(__intrin); }
+ else if constexpr (cvt_id == 16'2 && __have_avx512bw ) { return _mm512_movepi16_mask(__zero_extend(__intrin)); }
+ else if constexpr (cvt_id == 16'2 ) { return movemask_epi16(__intrin); }
+ else if constexpr (cvt_id == 32'2 && __have_avx512bw ) { return _mm512_movepi16_mask(__intrin); }
+ else if constexpr (cvt_id == 4'4 && __have_avx512dq_vl) { return _mm_movepi32_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 4'4 && __have_avx512dq ) { return _mm512_movepi32_mask(__zero_extend(__vector_bitcast<_LLong>(__k))); }
+ else if constexpr (cvt_id == 4'4 && __have_avx512vl ) { return _mm_cmp_epi32_mask(__vector_bitcast<_LLong>(__k), __m128i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 4'4 && __have_avx512_int ) { return _mm512_cmp_epi32_mask(__zero_extend(__intrin), __m512i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 4'4 ) { return _mm_movemask_ps(__k); }
+ else if constexpr (cvt_id == 8'4 && __have_avx512dq_vl) { return _mm256_movepi32_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 8'4 && __have_avx512dq ) { return _mm512_movepi32_mask(__zero_extend(__vector_bitcast<_LLong>(__k))); }
+ else if constexpr (cvt_id == 8'4 && __have_avx512vl ) { return _mm256_cmp_epi32_mask(__vector_bitcast<_LLong>(__k), __m256i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 8'4 && __have_avx512_int ) { return _mm512_cmp_epi32_mask(__zero_extend(__intrin), __m512i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 8'4 ) { return _mm256_movemask_ps(__k); }
+ else if constexpr (cvt_id == 16'4 && __have_avx512dq ) { return _mm512_movepi32_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 16'4 ) { return _mm512_cmp_epi32_mask(__vector_bitcast<_LLong>(__k), __m512i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 2'8 && __have_avx512dq_vl) { return _mm_movepi64_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 2'8 && __have_avx512dq ) { return _mm512_movepi64_mask(__zero_extend(__vector_bitcast<_LLong>(__k))); }
+ else if constexpr (cvt_id == 2'8 && __have_avx512vl ) { return _mm_cmp_epi64_mask(__vector_bitcast<_LLong>(__k), __m128i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 2'8 && __have_avx512_int ) { return _mm512_cmp_epi64_mask(__zero_extend(__intrin), __m512i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 2'8 ) { return _mm_movemask_pd(__k); }
+ else if constexpr (cvt_id == 4'8 && __have_avx512dq_vl) { return _mm256_movepi64_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 4'8 && __have_avx512dq ) { return _mm512_movepi64_mask(__zero_extend(__vector_bitcast<_LLong>(__k))); }
+ else if constexpr (cvt_id == 4'8 && __have_avx512vl ) { return _mm256_cmp_epi64_mask(__vector_bitcast<_LLong>(__k), __m256i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 4'8 && __have_avx512_int ) { return _mm512_cmp_epi64_mask(__zero_extend(__intrin), __m512i(), _MM_CMPINT_LT); }
+ else if constexpr (cvt_id == 4'8 ) { return _mm256_movemask_pd(__k); }
+ else if constexpr (cvt_id == 8'8 && __have_avx512dq ) { return _mm512_movepi64_mask(__vector_bitcast<_LLong>(__k)); }
+ else if constexpr (cvt_id == 8'8 ) { return _mm512_cmp_epi64_mask(__vector_bitcast<_LLong>(__k), __m512i(), _MM_CMPINT_LT); }
+ else
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ __assert_unreachable<_To>();
+ // }}}
+ } else if constexpr (__is_vector_type_v<_From> && __is_vector_type_v<_To>) {
+ // vector -> vector {{{
+ using _ToTrait = _VectorTraits<_To>;
+ using _FromTrait = _VectorTraits<_From>;
+ using _ToT = typename _ToTrait::value_type;
+ using _Tp = typename _FromTrait::value_type;
+ constexpr size_t _FromN = _FromTrait::_S_width;
+ constexpr size_t _ToN = _ToTrait::_S_width;
+ constexpr int _FromBytes = sizeof(_Tp);
+ constexpr int _ToBytes = sizeof(_ToT);
+
+ if constexpr (_FromN == _ToN && sizeof(_From) == sizeof(_To))
+ { // reinterpret the bits
+ return reinterpret_cast<_To>(__k);
+ }
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+ else if constexpr (sizeof(_To) == 16 && sizeof(__k) == 16)
+ { // SSE -> SSE {{{
+ if constexpr (_FromBytes == 4 && _ToBytes == 8) {
+ if constexpr(std::is_integral_v<_Tp>) {
+ return __vector_bitcast<_ToT>(__interleave128_lo(__k, __k));
+ } else {
+ return __vector_bitcast<_ToT>(__interleave128_lo(__k, __k));
+ }
+ } else if constexpr (_FromBytes == 2 && _ToBytes == 8) {
+ const auto __y = __vector_bitcast<int>(__interleave128_lo(__k, __k));
+ return __vector_bitcast<_ToT>(__interleave128_lo(__y, __y));
+ } else if constexpr (_FromBytes == 1 && _ToBytes == 8) {
+ auto __y = __vector_bitcast<short>(__interleave128_lo(__k, __k));
+ auto __z = __vector_bitcast<int>(__interleave128_lo(__y, __y));
+ return __vector_bitcast<_ToT>(__interleave128_lo(__z, __z));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 4) {
+ if constexpr (__have_sse2)
+ return __vector_bitcast<_ToT>(
+ _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
+ else
+ return __vector_shuffle<1, 3, 6, 7>(
+ __vector_bitcast<_ToT>(__k), _To());
+ } else if constexpr (_FromBytes == 2 && _ToBytes == 4) {
+ return __vector_bitcast<_ToT>(__interleave128_lo(__k, __k));
+ } else if constexpr (_FromBytes == 1 && _ToBytes == 4) {
+ const auto __y = __vector_bitcast<short>(__interleave128_lo(__k, __k));
+ return __vector_bitcast<_ToT>(__interleave128_lo(__y, __y));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 2) {
+ if constexpr (__have_sse2 && !__have_ssse3)
+ return __vector_bitcast<_ToT>(_mm_packs_epi32(
+ _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()),
+ __m128i()));
+ else
+ return __vector_permute<3, 7, -1, -1, -1, -1, -1, -1>(
+ __vector_bitcast<_ToT>(__k));
+ } else if constexpr (_FromBytes == 4 && _ToBytes == 2) {
+ return __vector_bitcast<_ToT>(
+ _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
+ } else if constexpr (_FromBytes == 1 && _ToBytes == 2) {
+ return __vector_bitcast<_ToT>(__interleave128_lo(__k, __k));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 1) {
+ if constexpr(__have_ssse3) {
+ return __vector_bitcast<_ToT>(
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1)));
+ } else {
+ auto __y = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
+ __y = _mm_packs_epi32(__y, __m128i());
+ return __vector_bitcast<_ToT>(_mm_packs_epi16(__y, __m128i()));
+ }
+ return __vector_permute<7, 15, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1>(
+ __vector_bitcast<_ToT>(__k));
+ } else if constexpr (_FromBytes == 4 && _ToBytes == 1) {
+ if constexpr(__have_ssse3) {
+ return __vector_bitcast<_ToT>(
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1)));
+ } else {
+ const auto __y = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
+ return __vector_bitcast<_ToT>(_mm_packs_epi16(__y, __m128i()));
+ }
+ return __vector_permute<3, 7, 11, 15, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1>(
+ __vector_bitcast<_ToT>(__k));
+ } else if constexpr (_FromBytes == 2 && _ToBytes == 1) {
+ return __vector_bitcast<_ToT>(_mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()));
+ } else {
+ static_assert(!std::is_same_v<_Tp, _Tp>, "should be unreachable");
+ }
+ } // }}}
+ else if constexpr (sizeof(_To) == 32 && sizeof(__k) == 32)
+ { // AVX -> AVX {{{
+ if constexpr (_FromBytes == _ToBytes) { // keep low 1/2
+ __assert_unreachable<_Tp>();
+ } else if constexpr (_FromBytes == _ToBytes * 2) {
+ const auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(
+ _mm256_castsi128_si256(_mm_packs_epi16(__lo128(__y), __hi128(__y))));
+ } else if constexpr (_FromBytes == _ToBytes * 4) {
+ const auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(_mm256_castsi128_si256(
+ _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)), __m128i())));
+ } else if constexpr (_FromBytes == _ToBytes * 8) {
+ const auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(_mm256_castsi128_si256(
+ _mm_shuffle_epi8(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
+ _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1))));
+ } else if constexpr (_FromBytes * 2 == _ToBytes) {
+ auto __y = __xzyw(__to_intrin(__k));
+ if constexpr(std::is_floating_point_v<_Tp>) {
+ return __vector_bitcast<_ToT>(_mm256_unpacklo_ps(__y, __y));
+ } else {
+ return __vector_bitcast<_ToT>(_mm256_unpacklo_epi8(__y, __y));
+ }
+ } else if constexpr (_FromBytes * 4 == _ToBytes) {
+ auto __y = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
+ __lo128(__vector_bitcast<_LLong>(__k))); // drops 3/4 of input
+ return __vector_bitcast<_ToT>(
+ __concat(_mm_unpacklo_epi16(__y, __y), _mm_unpackhi_epi16(__y, __y)));
+ } else if constexpr (_FromBytes == 1 && _ToBytes == 8) {
+ auto __y = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
+ __lo128(__vector_bitcast<_LLong>(__k))); // drops 3/4 of input
+ __y = _mm_unpacklo_epi16(__y, __y); // drops another 1/2 => 7/8 total
+ return __vector_bitcast<_ToT>(
+ __concat(_mm_unpacklo_epi32(__y, __y), _mm_unpackhi_epi32(__y, __y)));
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } // }}}
+ else if constexpr (sizeof(_To) == 32 && sizeof(__k) == 16)
+ { // SSE -> AVX {{{
+ if constexpr (_FromBytes == _ToBytes) {
+ return __vector_bitcast<_ToT>(
+ __intrinsic_type_t<_Tp, 32 / sizeof(_Tp)>(__zero_extend(__to_intrin(__k))));
+ } else if constexpr (_FromBytes * 2 == _ToBytes) { // keep all
+ return __vector_bitcast<_ToT>(__concat(_mm_unpacklo_epi8(__vector_bitcast<_LLong>(__k), __vector_bitcast<_LLong>(__k)),
+ _mm_unpackhi_epi8(__vector_bitcast<_LLong>(__k), __vector_bitcast<_LLong>(__k))));
+ } else if constexpr (_FromBytes * 4 == _ToBytes) {
+ if constexpr (__have_avx2) {
+ return __vector_bitcast<_ToT>(_mm256_shuffle_epi8(
+ __concat(__vector_bitcast<_LLong>(__k), __vector_bitcast<_LLong>(__k)),
+ _mm256_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
+ 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7,
+ 7)));
+ } else {
+ return __vector_bitcast<_ToT>(
+ __concat(_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2,
+ 2, 2, 2, 3, 3, 3, 3)),
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(4, 4, 4, 4, 5, 5, 5, 5, 6,
+ 6, 6, 6, 7, 7, 7, 7))));
+ }
+ } else if constexpr (_FromBytes * 8 == _ToBytes) {
+ if constexpr (__have_avx2) {
+ return __vector_bitcast<_ToT>(_mm256_shuffle_epi8(
+ __concat(__vector_bitcast<_LLong>(__k), __vector_bitcast<_LLong>(__k)),
+ _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3,
+ 3)));
+ } else {
+ return __vector_bitcast<_ToT>(
+ __concat(_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1,
+ 1, 1, 1, 1, 1, 1, 1)),
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2, 3,
+ 3, 3, 3, 3, 3, 3, 3))));
+ }
+ } else if constexpr (_FromBytes == _ToBytes * 2) {
+ return __vector_bitcast<_ToT>(
+ __m256i(__zero_extend(_mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()))));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 2) {
+ return __vector_bitcast<_ToT>(__m256i(__zero_extend(
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1)))));
+ } else if constexpr (_FromBytes == 4 && _ToBytes == 1) {
+ return __vector_bitcast<_ToT>(__m256i(__zero_extend(
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1)))));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 1) {
+ return __vector_bitcast<_ToT>(__m256i(__zero_extend(
+ _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
+ _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1)))));
+ } else {
+ static_assert(!std::is_same_v<_Tp, _Tp>, "should be unreachable");
+ }
+ } // }}}
+ else if constexpr (sizeof(_To) == 16 && sizeof(__k) == 32)
+ { // AVX -> SSE {{{
+ if constexpr (_FromBytes == _ToBytes) { // keep low 1/2
+ return __vector_bitcast<_ToT>(__lo128(__k));
+ } else if constexpr (_FromBytes == _ToBytes * 2) { // keep all
+ auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(_mm_packs_epi16(__lo128(__y), __hi128(__y)));
+ } else if constexpr (_FromBytes == _ToBytes * 4) { // add 1/2 undef
+ auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(
+ _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)), __m128i()));
+ } else if constexpr (_FromBytes == 8 && _ToBytes == 1) { // add 3/4 undef
+ auto __y = __vector_bitcast<_LLong>(__k);
+ return __vector_bitcast<_ToT>(
+ _mm_shuffle_epi8(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
+ _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1)));
+ } else if constexpr (_FromBytes * 2 == _ToBytes) { // keep low 1/4
+ auto __y = __lo128(__vector_bitcast<_LLong>(__k));
+ return __vector_bitcast<_ToT>(_mm_unpacklo_epi8(__y, __y));
+ } else if constexpr (_FromBytes * 4 == _ToBytes) { // keep low 1/8
+ auto __y = __lo128(__vector_bitcast<_LLong>(__k));
+ __y = _mm_unpacklo_epi8(__y, __y);
+ return __vector_bitcast<_ToT>(_mm_unpacklo_epi8(__y, __y));
+ } else if constexpr (_FromBytes * 8 == _ToBytes) { // keep low 1/16
+ auto __y = __lo128(__vector_bitcast<_LLong>(__k));
+ __y = _mm_unpacklo_epi8(__y, __y);
+ __y = _mm_unpacklo_epi8(__y, __y);
+ return __vector_bitcast<_ToT>(_mm_unpacklo_epi8(__y, __y));
+ } else {
+ static_assert(!std::is_same_v<_Tp, _Tp>, "should be unreachable");
+ }
+ } // }}}
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+ else
+ {
+ using _I = __int_for_sizeof_t<_ToT>;
+ return reinterpret_cast<_To>(
+ __generate_vector<__vector_type_t<_I, _ToN>>([&](auto __i) {
+ return __i >= _FromN || __k[int(__i)] == 0 ? _I() : ~_I();
+ }));
+ }
+ /*
+ } else if constexpr (_FromBytes > _ToBytes) {
+ const _To __y = __vector_bitcast<_ToT>(__k);
+ return [&] <std::size_t... _Is> (std::index_sequence<_Is...>) {
+ constexpr int _Stride = _FromBytes / _ToBytes;
+ return _To{__y[(_Is + 1) * _Stride - 1]...};
+ }(std::make_index_sequence<std::min(_ToN, _FromN)>());
+ } else {
+ // {0, 0, 1, 1} (_Dups = 2, _Is<4>)
+ // {0, 0, 0, 0, 1, 1, 1, 1} (_Dups = 4, _Is<8>)
+ // {0, 0, 1, 1, 2, 2, 3, 3} (_Dups = 2, _Is<8>)
+ // ...
+ return [&] <std::size_t... _Is> (std::index_sequence<_Is...>) {
+ constexpr int __dup = _ToBytes / _FromBytes;
+ return __vector_bitcast<_ToT>(_From{__k[_Is / __dup]...});
+ }(std::make_index_sequence<_FromN>());
+ }
+ */
+ // }}}
+ } else {
+ __assert_unreachable<_To>();
+ }
+}
+
+// }}}
+
+template <class _Abi> struct __simd_math_fallback { //{{{
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __acos(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::acos(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __asin(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::asin(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __atan(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::atan(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __atan2(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::atan2(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __cos(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::cos(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __sin(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::sin(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __tan(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::tan(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __acosh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::acosh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __asinh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::asinh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __atanh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::atanh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __cosh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::cosh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __sinh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::sinh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __tanh(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::tanh(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __exp(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::exp(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __exp2(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::exp2(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __expm1(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::expm1(__x[__i]); });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __frexp(const simd<_Tp, _Abi> &__x,
+ fixed_size_simd<int, simd_size_v<_Tp, _Abi>> &exp)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) {
+ int tmp;
+ _Tp __r = std::frexp(__x[__i], &tmp);
+ exp[__i] = tmp;
+ return __r;
+ });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __ldexp(const simd<_Tp, _Abi> &__x,
+ const fixed_size_simd<int, simd_size_v<_Tp, _Abi>> &exp)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::ldexp(__x[__i], exp[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC
+ fixed_size_simd<int, simd_size_v<_Tp, _Abi>> __ilogb(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::ilogb(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __log(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::log(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __log10(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::log10(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __log1p(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::log1p(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __log2(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::log2(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __logb(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::logb(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __modf(const simd<_Tp, _Abi> &__x, simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) {
+ _Tp tmp;
+ _Tp __r = std::modf(__x[__i], &tmp);
+ __y[__i] = tmp;
+ return __r;
+ });
+ }
+
+ template <class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi>
+ __scalbn(const simd<_Tp, _Abi>& __x,
+ const fixed_size_simd<int, simd_size_v<_Tp, _Abi>>& __y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::scalbn(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi>
+ __scalbln(const simd<_Tp, _Abi>& __x,
+ const fixed_size_simd<long, simd_size_v<_Tp, _Abi>>& __y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::scalbln(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __cbrt(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::cbrt(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __abs(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::abs(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __fabs(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fabs(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __pow(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::pow(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __sqrt(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::sqrt(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __erf(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::erf(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __erfc(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::erfc(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __lgamma(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::lgamma(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __tgamma(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::tgamma(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __ceil(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::ceil(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __floor(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::floor(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __nearbyint(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::nearbyint(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __rint(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::rint(__x[__i]); });
+ }
+
+ template <class _Tp>
+ fixed_size_simd<long, simd_size_v<_Tp, _Abi>> __lrint(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::lrint(__x[__i]); });
+ }
+
+ template <class _Tp>
+ fixed_size_simd<long long, simd_size_v<_Tp, _Abi>> __llrint(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::llrint(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __round(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::round(__x[__i]); });
+ }
+
+ template <class _Tp>
+ fixed_size_simd<long, simd_size_v<_Tp, _Abi>> __lround(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::lround(__x[__i]); });
+ }
+
+ template <class _Tp>
+ fixed_size_simd<long long, simd_size_v<_Tp, _Abi>> __llround(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::llround(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __trunc(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::trunc(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __fmod(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fmod(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __remainder(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::remainder(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __remquo(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y,
+ fixed_size_simd<int, simd_size_v<_Tp, _Abi>> &__z)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) {
+ int tmp;
+ _Tp __r = std::remquo(__x[__i], __y[__i], &tmp);
+ __z[__i] = tmp;
+ return __r;
+ });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __copysign(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::copysign(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp>
+ simd<_Tp, _Abi> __nextafter(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::nextafter(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __fdim(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fdim(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __fmax(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fmax(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __fmin(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fmin(__x[__i], __y[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC
+ simd<_Tp, _Abi> __fma(const simd<_Tp, _Abi> &__x, const simd<_Tp, _Abi> &__y,
+ const simd<_Tp, _Abi> &__z)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fma(__x[__i], __y[__i], __z[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC
+ fixed_size_simd<int, simd_size_v<_Tp, _Abi>> __fpclassify(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::fpclassify(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isfinite(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isfinite(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isinf(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isinf(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isnan(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isnan(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isnormal(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isnormal(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __signbit(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::signbit(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isgreater(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isgreater(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isgreaterequal(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isgreaterequal(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isless(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isless(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __islessequal(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::islessequal(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __islessgreater(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::islessgreater(__x[__i]); });
+ }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC simd_mask<_Tp, _Abi> __isunordered(const simd<_Tp, _Abi> &__x)
+ {
+ return simd<_Tp, _Abi>([&](auto __i) { return std::isunordered(__x[__i]); });
+ }
+}; // }}}
+// __scalar_simd_impl {{{
+struct __scalar_simd_impl : __simd_math_fallback<simd_abi::scalar> {
+ // member types {{{2
+ using abi_type = std::experimental::simd_abi::scalar;
+ using _MaskMember = bool;
+ template <class _Tp> using _SimdMember = _Tp;
+ template <class _Tp> using simd = std::experimental::simd<_Tp, abi_type>;
+ template <class _Tp> using simd_mask = std::experimental::simd_mask<_Tp, abi_type>;
+ template <class _Tp> using __type_tag = _Tp *;
+
+ // broadcast {{{2
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static constexpr _Tp __broadcast(_Tp __x) noexcept
+ {
+ return __x;
+ }
+
+ // generator {{{2
+ template <class _F, class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp generator(_F &&__gen, __type_tag<_Tp>)
+ {
+ return __gen(_SizeConstant<0>());
+ }
+
+ // load {{{2
+ template <class _Tp, class _U, class _F>
+ static inline _Tp load(const _U *mem, _F, __type_tag<_Tp>) noexcept
+ {
+ return static_cast<_Tp>(mem[0]);
+ }
+
+ // masked load {{{2
+ template <class _Tp, class _U, class _F>
+ static inline _Tp masked_load(_Tp merge, bool __k, const _U *mem, _F) noexcept
+ {
+ if (__k) {
+ merge = static_cast<_Tp>(mem[0]);
+ }
+ return merge;
+ }
+
+ // store {{{2
+ template <class _Tp, class _U, class _F>
+ static inline void store(_Tp __v, _U *mem, _F, __type_tag<_Tp>) noexcept
+ {
+ mem[0] = static_cast<_Tp>(__v);
+ }
+
+ // masked store {{{2
+ template <class _Tp, class _U, class _F>
+ static inline void masked_store(const _Tp __v, _U *mem, _F, const bool __k) noexcept
+ {
+ if (__k) {
+ mem[0] = __v;
+ }
+ }
+
+ // negation {{{2
+ template <class _Tp> static inline bool negate(_Tp __x) noexcept { return !__x; }
+
+ // reductions {{{2
+ template <class _Tp, class _BinaryOperation>
+ static inline _Tp reduce(const simd<_Tp> &__x, _BinaryOperation &)
+ {
+ return __x._M_data;
+ }
+
+ // min, max, clamp {{{2
+ template <class _Tp> static inline _Tp min(const _Tp __a, const _Tp __b)
+ {
+ return std::min(__a, __b);
+ }
+
+ template <class _Tp> static inline _Tp max(const _Tp __a, const _Tp __b)
+ {
+ return std::max(__a, __b);
+ }
+
+ // complement {{{2
+ template <class _Tp> static inline _Tp complement(_Tp __x) noexcept
+ {
+ return static_cast<_Tp>(~__x);
+ }
+
+ // unary minus {{{2
+ template <class _Tp> static inline _Tp unary_minus(_Tp __x) noexcept
+ {
+ return static_cast<_Tp>(-__x);
+ }
+
+ // arithmetic operators {{{2
+ template <class _Tp> static inline _Tp plus(_Tp __x, _Tp __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) +
+ __promote_preserving_unsigned(__y));
+ }
+
+ template <class _Tp> static inline _Tp minus(_Tp __x, _Tp __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) -
+ __promote_preserving_unsigned(__y));
+ }
+
+ template <class _Tp> static inline constexpr _Tp multiplies(_Tp __x, _Tp __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) *
+ __promote_preserving_unsigned(__y));
+ }
+
+ template <class _Tp> static inline _Tp divides(_Tp __x, _Tp __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) /
+ __promote_preserving_unsigned(__y));
+ }
+
+ template <class _Tp> static inline _Tp modulus(_Tp __x, _Tp __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) %
+ __promote_preserving_unsigned(__y));
+ }
+
+ template <class _Tp>
+ static inline _Tp bit_and(_Tp __x, _Tp __y)
+ {
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ const _I __r = reinterpret_cast<const __may_alias<_I>&>(__x) &
+ reinterpret_cast<const __may_alias<_I>&>(__y);
+ return reinterpret_cast<const __may_alias<_Tp>&>(__r);
+ }
+ else
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) &
+ __promote_preserving_unsigned(__y));
+ }
+ }
+
+ template <class _Tp>
+ static inline _Tp bit_or(_Tp __x, _Tp __y)
+ {
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ const _I __r = reinterpret_cast<const __may_alias<_I>&>(__x) |
+ reinterpret_cast<const __may_alias<_I>&>(__y);
+ return reinterpret_cast<const __may_alias<_Tp>&>(__r);
+ }
+ else
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) |
+ __promote_preserving_unsigned(__y));
+ }
+ }
+
+ template <class _Tp>
+ static inline _Tp bit_xor(_Tp __x, _Tp __y)
+ {
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ const _I __r = reinterpret_cast<const __may_alias<_I>&>(__x) ^
+ reinterpret_cast<const __may_alias<_I>&>(__y);
+ return reinterpret_cast<const __may_alias<_Tp>&>(__r);
+ }
+ else
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) ^
+ __promote_preserving_unsigned(__y));
+ }
+ }
+
+ template <class _Tp> static inline _Tp bit_shift_left(_Tp __x, int __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) << __y);
+ }
+
+ template <class _Tp> static inline _Tp bit_shift_right(_Tp __x, int __y)
+ {
+ return static_cast<_Tp>(__promote_preserving_unsigned(__x) >> __y);
+ }
+
+ // math {{{2
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _Tp __abs(_Tp __x) { return _Tp(std::abs(__x)); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _Tp __sqrt(_Tp __x) { return std::sqrt(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _Tp __trunc(_Tp __x) { return std::trunc(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _Tp __floor(_Tp __x) { return std::floor(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _Tp __ceil(_Tp __x) { return std::ceil(__x); }
+
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static _SimdTuple<int, abi_type> __fpclassify(_Tp __x)
+ {
+ return {std::fpclassify(__x)};
+ }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __isfinite(_Tp __x) { return std::isfinite(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __isinf(_Tp __x) { return std::isinf(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __isnan(_Tp __x) { return std::isnan(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __isnormal(_Tp __x) { return std::isnormal(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __signbit(_Tp __x) { return std::signbit(__x); }
+ template <class _Tp> _GLIBCXX_SIMD_INTRINSIC static bool __isunordered(_Tp __x, _Tp __y) { return std::isunordered(__x, __y); }
+
+ // __increment & __decrement{{{2
+ template <class _Tp> static inline void __increment(_Tp &__x) { ++__x; }
+ template <class _Tp> static inline void __decrement(_Tp &__x) { --__x; }
+
+ // compares {{{2
+ template <class _Tp> static bool equal_to(_Tp __x, _Tp __y) { return __x == __y; }
+ template <class _Tp> static bool not_equal_to(_Tp __x, _Tp __y) { return __x != __y; }
+ template <class _Tp> static bool less(_Tp __x, _Tp __y) { return __x < __y; }
+ template <class _Tp> static bool greater(_Tp __x, _Tp __y) { return __x > __y; }
+ template <class _Tp> static bool less_equal(_Tp __x, _Tp __y) { return __x <= __y; }
+ template <class _Tp> static bool greater_equal(_Tp __x, _Tp __y) { return __x >= __y; }
+
+ // smart_reference access {{{2
+ template <class _Tp, class _U> static void set(_Tp &__v, int __i, _U &&__x) noexcept
+ {
+ _GLIBCXX_DEBUG_ASSERT(__i == 0);
+ __unused(__i);
+ __v = std::forward<_U>(__x);
+ }
+
+ // masked_assign {{{2
+ template <typename _Tp> _GLIBCXX_SIMD_INTRINSIC static void masked_assign(bool __k, _Tp &__lhs, _Tp __rhs)
+ {
+ if (__k) {
+ __lhs = __rhs;
+ }
+ }
+
+ // __masked_cassign {{{2
+ template <template <typename> class _Op, typename _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static void __masked_cassign(const bool __k, _Tp &__lhs, const _Tp __rhs)
+ {
+ if (__k) {
+ __lhs = _Op<_Tp>{}(__lhs, __rhs);
+ }
+ }
+
+ // masked_unary {{{2
+ template <template <typename> class _Op, typename _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp masked_unary(const bool __k, const _Tp __v)
+ {
+ return static_cast<_Tp>(__k ? _Op<_Tp>{}(__v) : __v);
+ }
+
+ // }}}2
+};
+
+// }}}
+// __scalar_mask_impl {{{
+struct __scalar_mask_impl {
+ // member types {{{2
+ template <class _Tp> using simd_mask = std::experimental::simd_mask<_Tp, simd_abi::scalar>;
+ template <class _Tp> using __type_tag = _Tp *;
+
+ // __from_bitset {{{2
+ template <class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static bool __from_bitset(std::bitset<1> bs, __type_tag<_Tp>) noexcept
+ {
+ return bs[0];
+ }
+
+ // masked load {{{2
+ template <class _F>
+ _GLIBCXX_SIMD_INTRINSIC static bool masked_load(bool merge, bool mask, const bool *mem,
+ _F) noexcept
+ {
+ if (mask) {
+ merge = mem[0];
+ }
+ return merge;
+ }
+
+ // store {{{2
+ template <class _F> _GLIBCXX_SIMD_INTRINSIC static void store(bool __v, bool *mem, _F) noexcept
+ {
+ mem[0] = __v;
+ }
+
+ // masked store {{{2
+ template <class _F>
+ _GLIBCXX_SIMD_INTRINSIC static void masked_store(const bool __v, bool *mem, _F,
+ const bool __k) noexcept
+ {
+ if (__k) {
+ mem[0] = __v;
+ }
+ }
+
+ // logical and bitwise operators {{{2
+ static constexpr bool logical_and(bool __x, bool __y) { return __x && __y; }
+ static constexpr bool logical_or(bool __x, bool __y) { return __x || __y; }
+ static constexpr bool bit_and(bool __x, bool __y) { return __x && __y; }
+ static constexpr bool bit_or(bool __x, bool __y) { return __x || __y; }
+ static constexpr bool bit_xor(bool __x, bool __y) { return __x != __y; }
+
+ // smart_reference access {{{2
+ static void set(bool &__k, int __i, bool __x) noexcept
+ {
+ _GLIBCXX_DEBUG_ASSERT(__i == 0);
+ __unused(__i);
+ __k = __x;
+ }
+
+ // masked_assign {{{2
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(bool __k, bool &__lhs, bool __rhs)
+ {
+ if (__k) {
+ __lhs = __rhs;
+ }
+ }
+
+ // }}}2
+};
+
+// }}}
+
+// ISA & type detection {{{1
+template <class _Tp, size_t _N> constexpr bool __is_sse_ps()
+{
+ return __have_sse && std::is_same_v<_Tp, float> && _N == 4;
+}
+template <class _Tp, size_t _N> constexpr bool __is_sse_pd()
+{
+ return __have_sse2 && std::is_same_v<_Tp, double> && _N == 2;
+}
+template <class _Tp, size_t _N> constexpr bool __is_avx_ps()
+{
+ return __have_avx && std::is_same_v<_Tp, float> && _N == 8;
+}
+template <class _Tp, size_t _N> constexpr bool __is_avx_pd()
+{
+ return __have_avx && std::is_same_v<_Tp, double> && _N == 4;
+}
+template <class _Tp, size_t _N> constexpr bool __is_avx512_ps()
+{
+ return __have_avx512f && std::is_same_v<_Tp, float> && _N == 16;
+}
+template <class _Tp, size_t _N> constexpr bool __is_avx512_pd()
+{
+ return __have_avx512f && std::is_same_v<_Tp, double> && _N == 8;
+}
+
+template <class _Tp, size_t _N> constexpr bool __is_neon_ps()
+{
+ return __have_neon && std::is_same_v<_Tp, float> && _N == 4;
+}
+template <class _Tp, size_t _N> constexpr bool __is_neon_pd()
+{
+ return __have_neon && std::is_same_v<_Tp, double> && _N == 2;
+}
+
+// __generic_simd_impl {{{1
+template <class _Abi> struct __generic_simd_impl : __simd_math_fallback<_Abi> {
+ // member types {{{2
+ using abi_type = _Abi;
+ template <class _Tp> using __type_tag = _Tp *;
+ template <class _Tp>
+ using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
+ template <class _Tp>
+ using _MaskMember = typename _Abi::template __traits<_Tp>::_MaskMember;
+ template <class _Tp> static constexpr size_t full_size = _SimdMember<_Tp>::_S_width;
+
+ // make_simd(_SimdWrapper/__intrinsic_type_t) {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static simd<_Tp, _Abi> make_simd(_SimdWrapper<_Tp, _N> __x)
+ {
+ return {__private_init, __x};
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static simd<_Tp, _Abi> make_simd(__intrinsic_type_t<_Tp, _N> __x)
+ {
+ return {__private_init, __vector_bitcast<_Tp>(__x)};
+ }
+
+ // broadcast {{{2
+ template <class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> __broadcast(_Tp __x) noexcept
+ {
+ return __vector_broadcast<full_size<_Tp>>(__x);
+ }
+
+ // generator {{{2
+ template <class _F, class _Tp>
+ inline static _SimdMember<_Tp> generator(_F &&__gen, __type_tag<_Tp>)
+ {
+ return __generate_wrapper<_Tp, full_size<_Tp>>(std::forward<_F>(__gen));
+ }
+
+ // load {{{2
+ template <class _Tp, class _U, class _F>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdMember<_Tp> load(const _U *mem, _F,
+ __type_tag<_Tp>) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ constexpr size_t _N = _SimdMember<_Tp>::_S_width;
+ constexpr size_t max_load_size =
+ (sizeof(_U) >= 4 && __have_avx512f) || __have_avx512bw
+ ? 64
+ : (std::is_floating_point_v<_U> && __have_avx) || __have_avx2 ? 32 : 16;
+ if constexpr (sizeof(_U) > 8) {
+ return __generate_wrapper<_Tp, _N>(
+ [&](auto __i) { return static_cast<_Tp>(mem[__i]); });
+ } else if constexpr (std::is_same_v<_U, _Tp>) {
+ return __vector_load<_U, _N>(mem, _F());
+ } else if constexpr (sizeof(_U) * _N < 16) {
+ return __convert<_SimdMember<_Tp>>(
+ __vector_load16<_U, sizeof(_U) * _N>(mem, _F()));
+ } else if constexpr (sizeof(_U) * _N <= max_load_size) {
+ return __convert<_SimdMember<_Tp>>(__vector_load<_U, _N>(mem, _F()));
+ } else if constexpr (sizeof(_U) * _N == 2 * max_load_size) {
+ return __convert<_SimdMember<_Tp>>(
+ __vector_load<_U, _N / 2>(mem, _F()),
+ __vector_load<_U, _N / 2>(mem + _N / 2, _F()));
+ } else if constexpr (sizeof(_U) * _N == 4 * max_load_size) {
+ return __convert<_SimdMember<_Tp>>(
+ __vector_load<_U, _N / 4>(mem, _F()),
+ __vector_load<_U, _N / 4>(mem + 1 * _N / 4, _F()),
+ __vector_load<_U, _N / 4>(mem + 2 * _N / 4, _F()),
+ __vector_load<_U, _N / 4>(mem + 3 * _N / 4, _F()));
+ } else if constexpr (sizeof(_U) * _N == 8 * max_load_size) {
+ return __convert<_SimdMember<_Tp>>(
+ __vector_load<_U, _N / 8>(mem, _F()),
+ __vector_load<_U, _N / 8>(mem + 1 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 2 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 3 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 4 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 5 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 6 * _N / 8, _F()),
+ __vector_load<_U, _N / 8>(mem + 7 * _N / 8, _F()));
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ }
+
+ // masked load {{{2
+ template <class _Tp, size_t _N, class _U, class _F>
+ static inline _SimdWrapper<_Tp, _N> masked_load(_SimdWrapper<_Tp, _N> __merge,
+ _MaskMember<_Tp> __k,
+ const _U *__mem,
+ _F) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ __bit_iteration(__vector_to_bitset(__k._M_data).to_ullong(), [&](auto __i) {
+ __merge.set(__i, static_cast<_Tp>(__mem[__i]));
+ });
+ return __merge;
+ }
+
+ // store {{{2
+ template <class _Tp, class _U, class _F>
+ _GLIBCXX_SIMD_INTRINSIC static void store(_SimdMember<_Tp> __v, _U *mem, _F,
+ __type_tag<_Tp>) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ // TODO: converting int -> "smaller int" can be optimized with AVX512
+ constexpr size_t _N = _SimdMember<_Tp>::_S_width;
+ constexpr size_t __max_store_size =
+ (sizeof(_U) >= 4 && __have_avx512f) || __have_avx512bw
+ ? 64
+ : (std::is_floating_point_v<_U> && __have_avx) || __have_avx2 ? 32 : 16;
+ if constexpr (sizeof(_U) > 8) {
+ __execute_n_times<_N>([&](auto __i) { mem[__i] = __v[__i]; });
+ } else if constexpr (std::is_same_v<_U, _Tp>) {
+ __vector_store(__v._M_data, mem, _F());
+ } else if constexpr (sizeof(_U) * _N < 16) {
+ __vector_store<sizeof(_U) * _N>(__convert<__vector_type16_t<_U>>(__v),
+ mem, _F());
+ } else if constexpr (sizeof(_U) * _N <= __max_store_size) {
+ __vector_store(__convert<__vector_type_t<_U, _N>>(__v), mem, _F());
+ } else {
+ constexpr size_t __vsize = __max_store_size / sizeof(_U);
+ constexpr size_t __stores = _N / __vsize;
+ using _V = __vector_type_t<_U, __vsize>;
+ const std::array<_V, __stores> __converted = __convert_all<_V>(__v);
+ __execute_n_times<__stores>([&](auto __i) {
+ __vector_store(__converted[__i], mem + __i * __vsize, _F());
+ });
+ }
+ }
+
+ // masked store {{{2
+ template <class _Tp, size_t _N, class _U, class _F>
+ static inline void masked_store(const _SimdWrapper<_Tp, _N> __v, _U *__mem, _F,
+ const _MaskMember<_Tp> __k) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ [[maybe_unused]] const auto __vi = __to_intrin(__v);
+ constexpr size_t __max_store_size =
+ (sizeof(_U) >= 4 && __have_avx512f) || __have_avx512bw
+ ? 64
+ : (std::is_floating_point_v<_U> && __have_avx) || __have_avx2 ? 32 : 16;
+ if constexpr (std::is_same_v<_Tp, _U> ||
+ (std::is_integral_v<_Tp> && std::is_integral_v<_U> &&
+ sizeof(_Tp) == sizeof(_U))) {
+ // bitwise or no conversion, reinterpret:
+ const auto kk = [&]() {
+ if constexpr (__is_bitmask_v<decltype(__k)>) {
+ return _MaskMember<_U>(__k._M_data);
+ } else {
+ return __wrapper_bitcast<_U>(__k);
+ }
+ }();
+ __maskstore(__wrapper_bitcast<_U>(__v), __mem, _F(), kk);
+ } else if constexpr (sizeof(_U) <= 8 && // no long double
+ !__converts_via_decomposition_v<
+ _Tp, _U, __max_store_size> // conversion via decomposition
+ // is better handled via the
+ // bit_iteration fallback below
+ ) {
+ using _VV = _SimdWrapper<_U, std::clamp(_N, 16 / sizeof(_U), __max_store_size / sizeof(_U))>;
+ using _V = typename _VV::_BuiltinType;
+ constexpr bool prefer_bitmask =
+ (__have_avx512f && sizeof(_U) >= 4) || __have_avx512bw;
+ using _M = _SimdWrapper<std::conditional_t<prefer_bitmask, bool, _U>, _VV::_S_width>;
+ constexpr size_t _VN = _VectorTraits<_V>::_S_width;
+
+ if constexpr (_VN >= _N) {
+ __maskstore(_VV(__convert<_V>(__v)), __mem,
+ // careful, if _V has more elements than the input __v (_N),
+ // vector_aligned is incorrect:
+ std::conditional_t<(_VectorTraits<_V>::_S_width > _N),
+ overaligned_tag<sizeof(_U) * _N>, _F>(),
+ __convert_mask<_M>(__k));
+ } else if constexpr (_VN * 2 == _N) {
+ const std::array<_V, 2> converted = __convert_all<_V>(__v);
+ __maskstore(_VV(converted[0]), __mem, _F(), __convert_mask<_M>(__extract_part<0, 2>(__k)));
+ __maskstore(_VV(converted[1]), __mem + _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<1, 2>(__k)));
+ } else if constexpr (_VN * 4 == _N) {
+ const std::array<_V, 4> converted = __convert_all<_V>(__v);
+ __maskstore(_VV(converted[0]), __mem, _F(), __convert_mask<_M>(__extract_part<0, 4>(__k)));
+ __maskstore(_VV(converted[1]), __mem + 1 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<1, 4>(__k)));
+ __maskstore(_VV(converted[2]), __mem + 2 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<2, 4>(__k)));
+ __maskstore(_VV(converted[3]), __mem + 3 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<3, 4>(__k)));
+ } else if constexpr (_VN * 8 == _N) {
+ const std::array<_V, 8> converted = __convert_all<_V>(__v);
+ __maskstore(_VV(converted[0]), __mem, _F(), __convert_mask<_M>(__extract_part<0, 8>(__k)));
+ __maskstore(_VV(converted[1]), __mem + 1 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<1, 8>(__k)));
+ __maskstore(_VV(converted[2]), __mem + 2 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<2, 8>(__k)));
+ __maskstore(_VV(converted[3]), __mem + 3 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<3, 8>(__k)));
+ __maskstore(_VV(converted[4]), __mem + 4 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<4, 8>(__k)));
+ __maskstore(_VV(converted[5]), __mem + 5 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<5, 8>(__k)));
+ __maskstore(_VV(converted[6]), __mem + 6 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<6, 8>(__k)));
+ __maskstore(_VV(converted[7]), __mem + 7 * _VV::_S_width, _F(), __convert_mask<_M>(__extract_part<7, 8>(__k)));
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+ __bit_iteration(__vector_to_bitset(__k._M_data).to_ullong(),
+ [&](auto __i) { __mem[__i] = static_cast<_U>(__v[__i]); });
+ }
+ }
+
+ // complement {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> complement(_SimdWrapper<_Tp, _N> __x) noexcept
+ {
+ return ~__x._M_data;
+ }
+
+ // unary minus {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> unary_minus(_SimdWrapper<_Tp, _N> __x) noexcept
+ {
+ // GCC doesn't use the psign instructions, but pxor & psub seem to be just as good
+ // a choice as pcmpeqd & psign. So meh.
+ return -__x._M_data;
+ }
+
+ // arithmetic operators {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> plus(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ return __plus(__x, __y);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> minus(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ return __minus(__x, __y);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> multiplies(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __multiplies(__x, __y);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> divides(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX4
+ return __divides(__x._M_data, __y._M_data);
+#else
+ return __x._M_data / __y._M_data;
+#endif
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> modulus(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ static_assert(std::is_integral<_Tp>::value, "modulus is only supported for integral types");
+ return __x._M_data % __y._M_data;
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> bit_and(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __and(__x._M_data, __y._M_data);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> bit_or(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __or(__x._M_data, __y._M_data);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> bit_xor(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __xor(__x._M_data, __y._M_data);
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> bit_shift_left(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __x._M_data << __y._M_data;
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> bit_shift_right(_SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return __x._M_data >> __y._M_data;
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> bit_shift_left(_SimdWrapper<_Tp, _N> __x, int __y)
+ {
+ return __x._M_data << __y;
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> bit_shift_right(_SimdWrapper<_Tp, _N> __x,
+ int __y)
+ {
+ return __x._M_data >> __y;
+ }
+
+ // compares {{{2
+ // equal_to {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> equal_to(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return _ToWrapper(__x._M_data == __y._M_data);
+ }
+
+ // not_equal_to {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> not_equal_to(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ return _ToWrapper(__x._M_data != __y._M_data);
+ }
+
+ // less {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> less(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ return _ToWrapper(__x._M_data < __y._M_data);
+ }
+
+ // less_equal {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> less_equal(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ return _ToWrapper(__x._M_data <= __y._M_data);
+ }
+
+ // negation {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> negate(_SimdWrapper<_Tp, _N> __x) noexcept
+ {
+ return _ToWrapper(!__x._M_data);
+ }
+
+ // min, max, clamp {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> min(_SimdWrapper<_Tp, _N> __a,
+ _SimdWrapper<_Tp, _N> __b)
+ {
+ return __a._M_data < __b._M_data ? __a._M_data : __b._M_data;
+ }
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N> max(_SimdWrapper<_Tp, _N> __a,
+ _SimdWrapper<_Tp, _N> __b)
+ {
+ return __a._M_data > __b._M_data ? __a._M_data : __b._M_data;
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr std::pair<_SimdWrapper<_Tp, _N>, _SimdWrapper<_Tp, _N>>
+ minmax(_SimdWrapper<_Tp, _N> __a, _SimdWrapper<_Tp, _N> __b)
+ {
+ return {__a._M_data < __b._M_data ? __a._M_data : __b._M_data, __a._M_data < __b._M_data ? __b._M_data : __a._M_data};
+ }
+
+ // reductions {{{2
+ template <class _Tp, class _BinaryOperation>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp
+ reduce(simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op)
+ {
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (sizeof(__x) > __min_vector_size && _N > 2)
+ {
+ using _A = simd_abi::deduce_t<_Tp, _N / 2>;
+ using _V = std::experimental::simd<_Tp, _A>;
+ return _SimdTraits<_Tp, _A>::_SimdImpl::reduce(
+ __binary_op(
+ _V(__private_init, __extract<0, 2>(__data(__x)._M_data)),
+ _V(__private_init, __extract<1, 2>(__data(__x)._M_data))),
+ std::forward<_BinaryOperation>(__binary_op));
+ }
+#if _GLIBCXX_SIMD_HAVE_NEON // {{{
+ else if constexpr (sizeof(__x) == 8 || sizeof(__x) == 16)
+ {
+ static_assert(_N <= 8); // either 64-bit vectors or 128-bit double
+ if constexpr (_N == 8)
+ {
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(
+ __vector_permute<1, 0, 3, 2, 5, 4, 7, 6>(__x._M_data)));
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(
+ __vector_permute<3, 2, 1, 0, 7, 6, 5, 4>(__x._M_data)));
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(
+ __vector_permute<7, 6, 5, 4, 3, 2, 1, 0>(__x._M_data)));
+ return __x[0];
+ }
+ else if constexpr (_N == 4)
+ {
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(
+ __vector_permute<1, 0, 3, 2>(__x._M_data)));
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(
+ __vector_permute<3, 2, 1, 0>(__x._M_data)));
+ return __x[0];
+ }
+ else
+ {
+ static_assert(_N == 2);
+ __x = __binary_op(
+ __x, make_simd<_Tp, _N>(__vector_permute<1, 0>(__x._M_data)));
+ return __x[0];
+ }
+ }
+#endif // _GLIBCXX_SIMD_HAVE_NEON }}}
+ else if constexpr (sizeof(__x) == 16)
+ {
+ if constexpr (_N == 16)
+ {
+ const auto __y = __x._M_data;
+ __x = __binary_op(
+ make_simd<_Tp, _N>(__vector_permute<0, 0, 1, 1, 2, 2, 3, 3, 4,
+ 4, 5, 5, 6, 6, 7, 7>(__y)),
+ make_simd<_Tp, _N>(
+ __vector_permute<8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
+ 14, 14, 15, 15>(__y)));
+ }
+ if constexpr (_N >= 8)
+ {
+ const auto __y = __vector_bitcast<short>(__x._M_data);
+ __x =
+ __binary_op(make_simd<_Tp, _N>(__vector_bitcast<_Tp>(
+ __vector_permute<0, 0, 1, 1, 2, 2, 3, 3>(__y))),
+ make_simd<_Tp, _N>(__vector_bitcast<_Tp>(
+ __vector_permute<4, 4, 5, 5, 6, 6, 7, 7>(__y))));
+ }
+ if constexpr (_N >= 4)
+ {
+ using _U =
+ std::conditional_t<std::is_floating_point_v<_Tp>, float, int>;
+ const auto __y = __vector_bitcast<_U>(__x._M_data);
+ __x = __binary_op(__x, make_simd<_Tp, _N>(__vector_bitcast<_Tp>(
+ __vector_permute<3, 2, 1, 0>(__y))));
+ }
+ using _U =
+ std::conditional_t<std::is_floating_point_v<_Tp>, double, _LLong>;
+ const auto __y = __vector_bitcast<_U>(__x._M_data);
+ __x = __binary_op(__x, make_simd<_Tp, _N>(__vector_bitcast<_Tp>(
+ __vector_permute<1, 1>(__y))));
+ return __x[0];
+ }
+ else
+ __assert_unreachable<_Tp>();
+ }
+
+ // math {{{2
+ // abs {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __abs(_SimdWrapper<_Tp, _N> __x) noexcept
+ {
+ return std::experimental::parallelism_v2::__abs(__x);
+ }
+
+ // trunc {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N>
+ __trunc(_SimdWrapper<_Tp, _N> __x)
+ {
+ __assert_unreachable<_Tp>();
+ }
+
+ // floor {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __floor(_SimdWrapper<_Tp, _N> __x)
+ {
+ const auto __y = __trunc(__x)._M_data;
+ const auto negative_input = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_N, _Tp>(0));
+ const auto mask = __andnot(__vector_bitcast<_Tp>(__y == __x._M_data), negative_input);
+ return __or(__andnot(mask, __y), __and(mask, __y - __vector_broadcast<_N, _Tp>(1)));
+ }
+
+ // ceil {{{3
+ template <class _Tp, size_t _N> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __ceil(_SimdWrapper<_Tp, _N> __x)
+ {
+ const auto __y = __trunc(__x)._M_data;
+ const auto negative_input = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_N, _Tp>(0));
+ const auto inv_mask = __or(__vector_bitcast<_Tp>(__y == __x._M_data), negative_input);
+ return __or(__and(inv_mask, __y),
+ __andnot(inv_mask, __y + __vector_broadcast<_N, _Tp>(1)));
+ }
+
+ // isnan {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __isnan(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ __unused(__x);
+ return {}; // false
+#else
+ return __cmpunord(__x._M_data, __x._M_data);
+#endif
+ }
+
+ // isfinite {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __isfinite(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ __unused(__x);
+ return __vector_bitcast<_N>(_Tp()) == __vector_bitcast<_N>(_Tp());
+#else
+ // if all exponent bits are set, __x is either inf or NaN
+ using _I = __int_for_sizeof_t<_Tp>;
+ const auto __inf = __vector_bitcast<_I>(
+ __vector_broadcast<_N>(std::numeric_limits<_Tp>::infinity()));
+ return __vector_bitcast<_Tp>(__inf >
+ (__vector_bitcast<_I>(__x) & __inf));
+#endif
+ }
+
+ // isunordered {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __isunordered(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ return __cmpunord(__x._M_data, __y._M_data);
+ }
+
+ // signbit {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __signbit(_SimdWrapper<_Tp, _N> __x)
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ const auto __xx = __vector_bitcast<_I>(__x._M_data);
+ return __vector_bitcast<_Tp>(__xx >> std::numeric_limits<_I>::digits);
+ }
+
+ // isinf {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __isinf(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ __unused(__x);
+ return {}; // false
+#else
+ return _Abi::_SimdImpl::template equal_to<_Tp, _N>(
+ __abs(__x),
+ __vector_broadcast<_N>(std::numeric_limits<_Tp>::infinity()));
+ // alternative:
+ // compare to inf using the corresponding integer type
+ /*
+ return
+ __vector_bitcast<_Tp>(__vector_bitcast<__int_for_sizeof_t<_Tp>>(__abs(__x)._M_data)
+ ==
+ __vector_bitcast<__int_for_sizeof_t<_Tp>>(__vector_broadcast<_N>(
+ std::numeric_limits<_Tp>::infinity())));
+ */
+#endif
+ }
+
+ // isnormal {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
+ __isnormal(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ return less_equal<_Tp, _N>(__vector_broadcast<_N>(std::numeric_limits<_Tp>::min()),
+ __abs(__x));
+#else
+ return __and(less_equal<_Tp, _N>(__vector_broadcast<_N>(std::numeric_limits<_Tp>::min()),
+ __abs(__x)),
+ less<_Tp, _N>(__abs(__x), __vector_broadcast<_N>(std::numeric_limits<_Tp>::infinity())));
+#endif
+ }
+
+ // fpclassify {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static __fixed_size_storage_t<int, _N> __fpclassify(_SimdWrapper<_Tp, _N> __x)
+ {
+ constexpr auto __fp_normal = __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, __int_for_sizeof_t<_Tp>>(FP_NORMAL));
+ constexpr auto __fp_nan = __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, __int_for_sizeof_t<_Tp>>(FP_NAN));
+ constexpr auto __fp_infinite = __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, __int_for_sizeof_t<_Tp>>(FP_INFINITE));
+ constexpr auto __fp_subnormal = __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, __int_for_sizeof_t<_Tp>>(FP_SUBNORMAL));
+ constexpr auto __fp_zero = __vector_bitcast<_Tp>(
+ __vector_broadcast<_N, __int_for_sizeof_t<_Tp>>(FP_ZERO));
+
+ const auto __tmp = __vector_bitcast<_LLong>(
+ __abs(__x)._M_data < std::numeric_limits<_Tp>::min()
+ ? (__x._M_data == 0 ? __fp_zero : __fp_subnormal)
+ : __blend(__isinf(__x)._M_data,
+ __blend(__isnan(__x)._M_data, __fp_normal, __fp_nan),
+ __fp_infinite));
+ if constexpr (sizeof(_Tp) == sizeof(int))
+ {
+ if constexpr (__fixed_size_storage_t<int, _N>::tuple_size == 1)
+ {
+ return {__vector_bitcast<int>(__tmp)};
+ }
+ else if constexpr (__fixed_size_storage_t<int, _N>::tuple_size == 2)
+ {
+ return {__extract<0, 2>(__vector_bitcast<int>(__tmp)),
+ __extract<1, 2>(__vector_bitcast<int>(__tmp))};
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (_N == 2 && sizeof(_Tp) == 8 &&
+ __fixed_size_storage_t<int, _N>::tuple_size == 2)
+ {
+ return {int(__tmp[0]), {int(__tmp[1])}};
+ }
+ else if constexpr (_N == 4 && sizeof(_Tp) == 8 &&
+ __fixed_size_storage_t<int, _N>::tuple_size == 1)
+ {
+#if _GLIBCXX_SIMD_X86INTRIN
+ return {_mm_packs_epi32(__lo128(__tmp), __hi128(__tmp))};
+#else // _GLIBCXX_SIMD_X86INTRIN
+ return {__make_wrapper<int>(__tmp[0], __tmp[1], __tmp[2], __tmp[3])};
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ }
+ else if constexpr (_N == 2 && sizeof(_Tp) == 8 &&
+ __fixed_size_storage_t<int, _N>::tuple_size == 1)
+ return {__make_wrapper<int>(__tmp[0], __tmp[1])};
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+
+ // __increment & __decrement{{{2
+ template <class _Tp, size_t _N> _GLIBCXX_SIMD_INTRINSIC static void __increment(_SimdWrapper<_Tp, _N> &__x)
+ {
+ __x = __x._M_data + 1;
+ }
+ template <class _Tp, size_t _N> _GLIBCXX_SIMD_INTRINSIC static void __decrement(_SimdWrapper<_Tp, _N> &__x)
+ {
+ __x = __x._M_data - 1;
+ }
+
+ // smart_reference access {{{2
+ template <class _Tp, size_t _N, class _U>
+ _GLIBCXX_SIMD_INTRINSIC static void set(_SimdWrapper<_Tp, _N> &__v, int __i, _U &&__x) noexcept
+ {
+ __v.set(__i, std::forward<_U>(__x));
+ }
+
+ // masked_assign{{{2
+ template <class _Tp, class _K, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(_SimdWrapper<_K, _N> __k,
+ _SimdWrapper<_Tp, _N> &__lhs,
+ __id<_SimdWrapper<_Tp, _N>> __rhs)
+ {
+ __lhs = __blend(__k._M_data, __lhs._M_data, __rhs._M_data);
+ }
+
+ template <class _Tp, class _K, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(_SimdWrapper<_K, _N> __k, _SimdWrapper<_Tp, _N> &__lhs,
+ __id<_Tp> __rhs)
+ {
+ if (__builtin_constant_p(__rhs) && __rhs == 0 && std::is_same<_K, _Tp>::value) {
+ if constexpr (!__is_bitmask(__k)) {
+ // the __andnot optimization only makes sense if __k._M_data is a vector register
+ __lhs._M_data = __andnot(__k._M_data, __lhs._M_data);
+ return;
+ } else {
+ // for AVX512/__mmask, a _mm512_maskz_mov is best
+ __lhs._M_data = __auto_bitcast(__blend(__k, __lhs, __intrinsic_type_t<_Tp, _N>()));
+ return;
+ }
+ }
+ __lhs._M_data = __blend(__k._M_data, __lhs._M_data, __vector_broadcast<_N>(__rhs));
+ }
+
+ // __masked_cassign {{{2
+ template <template <typename> class _Op, class _Tp, class _K, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void __masked_cassign(const _SimdWrapper<_K, _N> __k, _SimdWrapper<_Tp, _N> &__lhs,
+ const __id<_SimdWrapper<_Tp, _N>> __rhs)
+ {
+ __lhs._M_data = __blend(__k._M_data, __lhs._M_data, _Op<void>{}(__lhs._M_data, __rhs._M_data));
+ }
+
+ template <template <typename> class _Op, class _Tp, class _K, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void __masked_cassign(const _SimdWrapper<_K, _N> __k, _SimdWrapper<_Tp, _N> &__lhs,
+ const __id<_Tp> __rhs)
+ {
+ __lhs._M_data = __blend(__k._M_data, __lhs._M_data, _Op<void>{}(__lhs._M_data, __vector_broadcast<_N>(__rhs)));
+ }
+
+ // masked_unary {{{2
+ template <template <typename> class _Op, class _Tp, class _K, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> masked_unary(const _SimdWrapper<_K, _N> __k,
+ const _SimdWrapper<_Tp, _N> __v)
+ {
+ auto __vv = make_simd(__v);
+ _Op<decltype(__vv)> op;
+ return __blend(__k, __v, __data(op(__vv)));
+ }
+
+ //}}}2
+};
+
+// __generic_mask_impl {{{1
+template <class _Abi> struct __generic_mask_impl {
+ // member types {{{2
+ template <class _Tp> using __type_tag = _Tp *;
+ template <class _Tp> using simd_mask = std::experimental::simd_mask<_Tp, _Abi>;
+ template <class _Tp>
+ using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
+ template <class _Tp>
+ using _MaskMember = typename _Abi::template __traits<_Tp>::_MaskMember;
+
+ // masked load {{{2
+ template <class _Tp, size_t _N, class _F>
+ static inline _SimdWrapper<_Tp, _N> masked_load(_SimdWrapper<_Tp, _N> merge,
+ _SimdWrapper<_Tp, _N> mask,
+ const bool* mem,
+ _F) noexcept
+ {
+ // AVX(2) has 32/64 bit maskload, but nothing at 8 bit granularity
+ auto __tmp = __wrapper_bitcast<__int_for_sizeof_t<_Tp>>(merge);
+ __bit_iteration(__vector_to_bitset(mask._M_data).to_ullong(),
+ [&](auto __i) { __tmp.set(__i, -mem[__i]); });
+ merge = __wrapper_bitcast<_Tp>(__tmp);
+ return merge;
+ }
+
+ // store {{{2
+ template <class _Tp, size_t _N, class _F>
+ _GLIBCXX_SIMD_INTRINSIC static void store(_SimdWrapper<_Tp, _N> __v, bool *__mem, _F) noexcept
+ {
+ __execute_n_times<_N>([&](auto __i) { __mem[__i] = __v[__i]; });
+ }
+
+ // masked store {{{2
+ template <class _Tp, size_t _N, class _F>
+ static inline void masked_store(const _SimdWrapper<_Tp, _N> __v, bool *__mem, _F,
+ const _SimdWrapper<_Tp, _N> __k) noexcept
+ {
+ __bit_iteration(__vector_to_bitset(__k._M_data).to_ullong(),
+ [&](auto __i) { __mem[__i] = __v[__i]; });
+ }
+
+ // __from_bitset{{{2
+ template <size_t _N, class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __from_bitset(std::bitset<_N> __bits, __type_tag<_Tp>)
+ {
+ return __convert_mask<typename _MaskMember<_Tp>::_BuiltinType>(__bits);
+ }
+
+ // logical and bitwise operators {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ logical_and(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ return __and(__x._M_data, __y._M_data);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ logical_or(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ return __or(__x._M_data, __y._M_data);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_and(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ return __and(__x._M_data, __y._M_data);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_or(const _SimdWrapper<_Tp, _N>& __x, const _SimdWrapper<_Tp, _N>& __y)
+ {
+ return __or(__x._M_data, __y._M_data);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_xor(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ return __xor(__x._M_data, __y._M_data);
+ }
+
+ // smart_reference access {{{2
+ template <class _Tp, size_t _N> static void set(_SimdWrapper<_Tp, _N> &__k, int __i, bool __x) noexcept
+ {
+ if constexpr (std::is_same_v<_Tp, bool>) {
+ __k.set(__i, __x);
+ } else {
+ using _IntT = __vector_type_t<__int_for_sizeof_t<_Tp>, _N>;
+ auto tmp = reinterpret_cast<_IntT>(__k._M_data);
+ tmp[__i] = -__x;
+ __k._M_data = __auto_bitcast(tmp);
+ }
+ }
+ // masked_assign{{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(_SimdWrapper<_Tp, _N> __k, _SimdWrapper<_Tp, _N> &__lhs,
+ __id<_SimdWrapper<_Tp, _N>> __rhs)
+ {
+ __lhs = __blend(__k._M_data, __lhs._M_data, __rhs._M_data);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(_SimdWrapper<_Tp, _N> __k, _SimdWrapper<_Tp, _N> &__lhs, bool __rhs)
+ {
+ if (__builtin_constant_p(__rhs)) {
+ if (__rhs == false) {
+ __lhs = __andnot(__k._M_data, __lhs._M_data);
+ } else {
+ __lhs = __or(__k._M_data, __lhs._M_data);
+ }
+ return;
+ }
+ __lhs = __blend(__k, __lhs, __data(simd_mask<_Tp>(__rhs)));
+ }
+
+ //}}}2
+};
+
+//}}}1
+
+#if _GLIBCXX_SIMD_X86INTRIN // {{{
+// __x86_simd_impl {{{1
+template <class _Abi> struct __x86_simd_impl : __generic_simd_impl<_Abi> {
+ using _Base = __generic_simd_impl<_Abi>;
+ template <typename _Tp>
+ using _MaskMember = typename _Base::template _MaskMember<_Tp>;
+
+ // masked load {{{2
+ template <class _Tp, size_t _N, class _U, class _F>
+ static inline _SimdWrapper<_Tp, _N>
+ masked_load(_SimdWrapper<_Tp, _N> __merge,
+ _MaskMember<_Tp> __k,
+ const _U* __mem,
+ _F) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ if constexpr (std::is_same_v<_Tp, _U> || // no conversion
+ (sizeof(_Tp) == sizeof(_U) &&
+ std::is_integral_v<_Tp> ==
+ std::is_integral_v<_U>) // conversion via bit
+ // reinterpretation
+ )
+ {
+ [[maybe_unused]] const auto __intrin = __to_intrin(__merge);
+ constexpr bool __have_avx512bw_vl_or_zmm =
+ __have_avx512bw_vl || (__have_avx512bw && sizeof(__merge) == 64);
+ if constexpr (__have_avx512bw_vl_or_zmm && sizeof(_Tp) == 1)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge = __vector_bitcast<_Tp>(_mm_mask_loadu_epi8(
+ __intrin, _mm_movemask_epi8(__to_intrin(__k)), __mem));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge = __vector_bitcast<_Tp>(_mm256_mask_loadu_epi8(
+ __intrin, _mm256_movemask_epi8(__to_intrin(__k)), __mem));
+ }
+ else if constexpr (sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_epi8(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__have_avx512bw_vl_or_zmm && sizeof(_Tp) == 2)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge = __vector_bitcast<_Tp>(_mm_mask_loadu_epi16(
+ __intrin, movemask_epi16(__to_intrin(__k)), __mem));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge = __vector_bitcast<_Tp>(_mm256_mask_loadu_epi16(
+ __intrin, movemask_epi16(__to_intrin(__k)), __mem));
+ }
+ else if constexpr (sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_epi16(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__have_avx2 && sizeof(_Tp) == 4 &&
+ std::is_integral_v<_U>)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge =
+ (~__k._M_data & __merge._M_data) |
+ __vector_bitcast<_Tp>(_mm_maskload_epi32(
+ reinterpret_cast<const int*>(__mem), __to_intrin(__k)));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge =
+ (~__k._M_data & __merge._M_data) |
+ __vector_bitcast<_Tp>(_mm256_maskload_epi32(
+ reinterpret_cast<const int*>(__mem), __to_intrin(__k)));
+ }
+ else if constexpr (__have_avx512f && sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_epi32(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__have_avx && sizeof(_Tp) == 4)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge = __or(__andnot(__k._M_data, __merge._M_data),
+ __vector_bitcast<_Tp>(_mm_maskload_ps(
+ reinterpret_cast<const float*>(__mem),
+ __vector_bitcast<_LLong>(__k))));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge =
+ __or(__andnot(__k._M_data, __merge._M_data),
+ _mm256_maskload_ps(reinterpret_cast<const float*>(__mem),
+ __vector_bitcast<_LLong>(__k)));
+ }
+ else if constexpr (__have_avx512f && sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_ps(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__have_avx2 && sizeof(_Tp) == 8 &&
+ std::is_integral_v<_U>)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge =
+ (~__k._M_data & __merge._M_data) |
+ __vector_bitcast<_Tp>(_mm_maskload_epi64(
+ reinterpret_cast<const _LLong*>(__mem), __to_intrin(__k)));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge =
+ (~__k._M_data & __merge._M_data) |
+ __vector_bitcast<_Tp>(_mm256_maskload_epi64(
+ reinterpret_cast<const _LLong*>(__mem), __to_intrin(__k)));
+ }
+ else if constexpr (__have_avx512f && sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_epi64(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__have_avx && sizeof(_Tp) == 8)
+ {
+ if constexpr (sizeof(__merge) == 16)
+ {
+ __merge = __or(__andnot(__k._M_data, __merge._M_data),
+ __vector_bitcast<_Tp>(_mm_maskload_pd(
+ reinterpret_cast<const double*>(__mem),
+ __vector_bitcast<_LLong>(__k))));
+ }
+ else if constexpr (sizeof(__merge) == 32)
+ {
+ __merge = __or(
+ __andnot(__k._M_data, __merge._M_data),
+ _mm256_maskload_pd(reinterpret_cast<const double*>(__mem),
+ __vector_bitcast<_LLong>(__k)));
+ }
+ else if constexpr (__have_avx512f && sizeof(__merge) == 64)
+ {
+ __merge = __vector_bitcast<_Tp>(
+ _mm512_mask_loadu_pd(__intrin, __k, __mem));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else
+ {
+ __bit_iteration(__vector_to_bitset(__k._M_data).to_ullong(),
+ [&](auto __i) {
+ __merge.set(__i, static_cast<_Tp>(__mem[__i]));
+ });
+ }
+ }
+ else if constexpr (sizeof(_U) <= 8 && // no long double
+ !__converts_via_decomposition_v<
+ _U, _Tp,
+ sizeof(__merge)> // conversion via decomposition
+ // is better handled via the
+ // bit_iteration fallback below
+ )
+ {
+ // TODO: copy pattern from masked_store, which doesn't resort to
+ // fixed_size
+ using _A = simd_abi::deduce_t<
+ _U, std::max(_N, 16 / sizeof(_U)) // _N or more, so that at least a 16
+ // Byte vector is used instead of a
+ // fixed_size filled with scalars
+ >;
+ using _ATraits = _SimdTraits<_U, _A>;
+ using _AImpl = typename _ATraits::_SimdImpl;
+ typename _ATraits::_SimdMember uncvted{};
+ typename _ATraits::_MaskMember kk;
+ if constexpr (__is_fixed_size_abi_v<_A>)
+ {
+ kk = __vector_to_bitset(__k._M_data);
+ }
+ else
+ {
+ kk = __convert_mask<typename _ATraits::_MaskMember>(__k);
+ }
+ uncvted = _AImpl::masked_load(uncvted, kk, __mem, _F());
+ _SimdConverter<_U, _A, _Tp, _Abi> converter;
+ _Base::masked_assign(__k, __merge, converter(uncvted));
+ }
+ else
+ {
+ __bit_iteration(
+ __vector_to_bitset(__k._M_data).to_ullong(),
+ [&](auto __i) { __merge.set(__i, static_cast<_Tp>(__mem[__i])); });
+ }
+ return __merge;
+ }
+
+ // masked store {{{2
+ template <class _Tp, size_t _N, class _U, class _F>
+ static inline void masked_store(const _SimdWrapper<_Tp, _N> __v, _U *__mem, _F,
+ const _MaskMember<_Tp> __k) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ if constexpr (std::is_integral_v<_Tp> && std::is_integral_v<_U> &&
+ sizeof(_Tp) > sizeof(_U) && __have_avx512f &&
+ (sizeof(_Tp) >= 4 || __have_avx512bw) &&
+ (sizeof(__v) == 64 || __have_avx512vl)) { // truncating store
+ [[maybe_unused]] const auto __vi = __to_intrin(__v);
+ const auto kk = [&]() {
+ if constexpr (__is_bitmask_v<decltype(__k)>) {
+ return __k;
+ } else {
+ return __convert_mask<_SimdWrapper<bool, _N>>(__k);
+ }
+ }();
+ if constexpr (sizeof(_Tp) == 8 && sizeof(_U) == 4) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi64_storeu_epi32(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi64_storeu_epi32(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi64_storeu_epi32(__mem, kk, __vi);
+ }
+ } else if constexpr (sizeof(_Tp) == 8 && sizeof(_U) == 2) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi64_storeu_epi16(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi64_storeu_epi16(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi64_storeu_epi16(__mem, kk, __vi);
+ }
+ } else if constexpr (sizeof(_Tp) == 8 && sizeof(_U) == 1) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi64_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi64_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi64_storeu_epi8(__mem, kk, __vi);
+ }
+ } else if constexpr (sizeof(_Tp) == 4 && sizeof(_U) == 2) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi32_storeu_epi16(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi32_storeu_epi16(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi32_storeu_epi16(__mem, kk, __vi);
+ }
+ } else if constexpr (sizeof(_Tp) == 4 && sizeof(_U) == 1) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi32_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi32_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi32_storeu_epi8(__mem, kk, __vi);
+ }
+ } else if constexpr (sizeof(_Tp) == 2 && sizeof(_U) == 1) {
+ if constexpr (sizeof(__vi) == 64) {
+ _mm512_mask_cvtepi16_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 32) {
+ _mm256_mask_cvtepi16_storeu_epi8(__mem, kk, __vi);
+ } else if constexpr (sizeof(__vi) == 16) {
+ _mm_mask_cvtepi16_storeu_epi8(__mem, kk, __vi);
+ }
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+ _Base::masked_store(__v,__mem,_F(),__k);
+ }
+ }
+
+ // compares {{{2
+ // equal_to {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> equal_to(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ if constexpr (sizeof(__x) == 64) { // AVX512
+ [[maybe_unused]] const auto __xi = __to_intrin(__x);
+ [[maybe_unused]] const auto __yi = __to_intrin(__y);
+ if constexpr (std::is_floating_point_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmp_pd_mask(__xi, __yi, _CMP_EQ_OQ);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmp_ps_mask(__xi, __yi, _CMP_EQ_OQ);
+ } else { __assert_unreachable<_Tp>(); }
+ } else {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmpeq_epi64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmpeq_epi32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return _mm512_cmpeq_epi16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return _mm512_cmpeq_epi8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ }
+ } else
+ return _Base::equal_to(__x,__y);
+ }
+
+ // not_equal_to {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> not_equal_to(
+ _SimdWrapper<_Tp, _N> __x, _SimdWrapper<_Tp, _N> __y)
+ {
+ if constexpr (sizeof(__x) == 64) { // AVX512
+ [[maybe_unused]] const auto __xi = __to_intrin(__x);
+ [[maybe_unused]] const auto __yi = __to_intrin(__y);
+ if constexpr (std::is_floating_point_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmp_pd_mask(__xi, __yi, _CMP_NEQ_UQ);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmp_ps_mask(__xi, __yi, _CMP_NEQ_UQ);
+ } else { __assert_unreachable<_Tp>(); }
+ } else {
+ if constexpr (sizeof(_Tp) == 8) { return ~_mm512_cmpeq_epi64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return ~_mm512_cmpeq_epi32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return ~_mm512_cmpeq_epi16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return ~_mm512_cmpeq_epi8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ }
+ } else
+ return _Base::not_equal_to(__x, __y);
+ }
+
+ // less {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> less(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ if constexpr (sizeof(__x) == 64) { // AVX512
+ [[maybe_unused]] const auto __xi = __to_intrin(__x);
+ [[maybe_unused]] const auto __yi = __to_intrin(__y);
+ if constexpr (std::is_floating_point_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmp_pd_mask(__xi, __yi, _CMP_LT_OS);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmp_ps_mask(__xi, __yi, _CMP_LT_OS);
+ } else { __assert_unreachable<_Tp>(); }
+ } else if constexpr (std::is_signed_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmplt_epi64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmplt_epi32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return _mm512_cmplt_epi16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return _mm512_cmplt_epi8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ } else {
+ static_assert(std::is_unsigned_v<_Tp>);
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmplt_epu64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmplt_epu32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return _mm512_cmplt_epu16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return _mm512_cmplt_epu8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ }
+ } else
+ return _Base::less(__x, __y);
+ }
+
+ // less_equal {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> less_equal(_SimdWrapper<_Tp, _N> __x,
+ _SimdWrapper<_Tp, _N> __y)
+ {
+ if constexpr (sizeof(__x) == 64) { // AVX512
+ [[maybe_unused]] const auto __xi = __to_intrin(__x);
+ [[maybe_unused]] const auto __yi = __to_intrin(__y);
+ if constexpr (std::is_floating_point_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmp_pd_mask(__xi, __yi, _CMP_LE_OS);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmp_ps_mask(__xi, __yi, _CMP_LE_OS);
+ } else { __assert_unreachable<_Tp>(); }
+ } else if constexpr (std::is_signed_v<_Tp>) {
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmple_epi64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmple_epi32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return _mm512_cmple_epi16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return _mm512_cmple_epi8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ } else {
+ static_assert(std::is_unsigned_v<_Tp>);
+ if constexpr (sizeof(_Tp) == 8) { return _mm512_cmple_epu64_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 4) { return _mm512_cmple_epu32_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 2) { return _mm512_cmple_epu16_mask(__xi, __yi);
+ } else if constexpr (sizeof(_Tp) == 1) { return _mm512_cmple_epu8_mask(__xi, __yi);
+ } else { __assert_unreachable<_Tp>(); }
+ }
+ } else
+ return _Base::less_equal(__x, __y);
+ }
+
+ // negation {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp> negate(_SimdWrapper<_Tp, _N> __x) noexcept
+ {
+ if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>()) {
+ return equal_to(__x, _SimdWrapper<_Tp, _N>());
+ } else {
+ return _Base::negate(__x);
+ }
+ }
+
+ // math {{{2
+ // sqrt {{{3
+ template <class _Tp, size_t _N> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __sqrt(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__is_sse_ps <_Tp, _N>()) { return _mm_sqrt_ps(__x);
+ } else if constexpr (__is_sse_pd <_Tp, _N>()) { return _mm_sqrt_pd(__x);
+ } else if constexpr (__is_avx_ps <_Tp, _N>()) { return _mm256_sqrt_ps(__x);
+ } else if constexpr (__is_avx_pd <_Tp, _N>()) { return _mm256_sqrt_pd(__x);
+ } else if constexpr (__is_avx512_ps<_Tp, _N>()) { return _mm512_sqrt_ps(__x);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) { return _mm512_sqrt_pd(__x);
+ } else { __assert_unreachable<_Tp>(); }
+ }
+
+ // trunc {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __trunc(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__is_avx512_ps<_Tp, _N>()) {
+ return _mm512_roundscale_round_ps(__x, 0x03, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ return _mm512_roundscale_round_pd(__x, 0x03, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx_ps<_Tp, _N>()) {
+ return _mm256_round_ps(__x, 0x3);
+ } else if constexpr (__is_avx_pd<_Tp, _N>()) {
+ return _mm256_round_pd(__x, 0x3);
+ } else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _N>()) {
+ return _mm_round_ps(__x, 0x3);
+ } else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _N>()) {
+ return _mm_round_pd(__x, 0x3);
+ } else if constexpr (__is_sse_ps<_Tp, _N>()) {
+ auto truncated = _mm_cvtepi32_ps(_mm_cvttps_epi32(__x));
+ const auto no_fractional_values = __vector_bitcast<float>(
+ __vector_bitcast<int>(__vector_bitcast<_UInt>(__x._M_data) & 0x7f800000u) <
+ 0x4b000000); // the exponent is so large that no mantissa bits signify
+ // fractional values (0x3f8 + 23*8 = 0x4b0)
+ return __blend(no_fractional_values, __x, truncated);
+ } else if constexpr (__is_sse_pd<_Tp, _N>()) {
+ const auto abs_x = __abs(__x)._M_data;
+ const auto min_no_fractional_bits = __vector_bitcast<double>(
+ __vector_broadcast<2>(0x4330'0000'0000'0000ull)); // 0x3ff + 52 = 0x433
+ __vector_type16_t<double> truncated =
+ (abs_x + min_no_fractional_bits) - min_no_fractional_bits;
+ // due to rounding, the result can be too large. In this case `truncated >
+ // abs(__x)` holds, so subtract 1 to truncated if `abs(__x) < truncated`
+ truncated -=
+ __and(__vector_bitcast<double>(abs_x < truncated), __vector_broadcast<2>(1.));
+ // finally, fix the sign bit:
+ return __or(
+ __and(__vector_bitcast<double>(__vector_broadcast<2>(0x8000'0000'0000'0000ull)),
+ __x),
+ truncated);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ }
+
+ // floor {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __floor(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__is_avx512_ps<_Tp, _N>()) {
+ return _mm512_roundscale_round_ps(__x, 0x01, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ return _mm512_roundscale_round_pd(__x, 0x01, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx_ps<_Tp, _N>()) {
+ return _mm256_round_ps(__x, 0x1);
+ } else if constexpr (__is_avx_pd<_Tp, _N>()) {
+ return _mm256_round_pd(__x, 0x1);
+ } else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _N>()) {
+ return _mm_floor_ps(__x);
+ } else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _N>()) {
+ return _mm_floor_pd(__x);
+ } else {
+ return _Base::__floor(__x);
+ }
+ }
+
+ // ceil {{{3
+ template <class _Tp, size_t _N> _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _N> __ceil(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__is_avx512_ps<_Tp, _N>()) {
+ return _mm512_roundscale_round_ps(__x, 0x02, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ return _mm512_roundscale_round_pd(__x, 0x02, _MM_FROUND_CUR_DIRECTION);
+ } else if constexpr (__is_avx_ps<_Tp, _N>()) {
+ return _mm256_round_ps(__x, 0x2);
+ } else if constexpr (__is_avx_pd<_Tp, _N>()) {
+ return _mm256_round_pd(__x, 0x2);
+ } else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _N>()) {
+ return _mm_ceil_ps(__x);
+ } else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _N>()) {
+ return _mm_ceil_pd(__x);
+ } else {
+ return _Base::__ceil(__x);
+ }
+ }
+
+ // signbit {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __signbit(_SimdWrapper<_Tp, _N> __x)
+ {
+ using _I = __int_for_sizeof_t<_Tp>;
+ if constexpr (__have_avx512dq && __is_avx512_ps<_Tp, _N>()) {
+ return _mm512_movepi32_mask(__vector_bitcast<_LLong>(__x));
+ } else if constexpr (__have_avx512dq && __is_avx512_pd<_Tp, _N>()) {
+ return _mm512_movepi64_mask(__vector_bitcast<_LLong>(__x));
+ } else if constexpr (sizeof(__x) == 64) {
+ const auto signmask = __vector_broadcast<_N>(std::numeric_limits<_I>::min());
+ return equal_to(_SimdWrapper<_I, _N>(__vector_bitcast<_I>(__x._M_data) & signmask),
+ _SimdWrapper<_I, _N>(signmask));
+ } else {
+ const auto __xx = __vector_bitcast<_I>(__x._M_data);
+ constexpr _I signmask = std::numeric_limits<_I>::min();
+ if constexpr ((sizeof(_Tp) == 4 && (__have_avx2 || sizeof(__x) == 16)) ||
+ __have_avx512vl) {
+ (void)signmask;
+ return __vector_bitcast<_Tp>(__xx >> std::numeric_limits<_I>::digits);
+ } else if constexpr ((__have_avx2 || (__have_ssse3 && sizeof(__x) == 16))) {
+ return __vector_bitcast<_Tp>((__xx & signmask) == signmask);
+ } else { // SSE2/3 or AVX (w/o AVX2)
+ constexpr auto one = __vector_broadcast<_N, _Tp>(1);
+ return __vector_bitcast<_Tp>(
+ __vector_bitcast<_Tp>((__xx & signmask) | __vector_bitcast<_I>(one)) // -1 or 1
+ != one);
+ }
+ }
+ }
+
+ // isnonzerovalue_mask (isnormal | is subnormal == !isinf & !isnan & !is zero) {{{3
+ template <class _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static auto isnonzerovalue_mask(_Tp __x)
+ {
+ using _Traits = _VectorTraits<_Tp>;
+ if constexpr (__have_avx512dq_vl)
+ {
+ if constexpr (_Traits::template __is<float, 4>)
+ return _knot_mask8(_mm_fpclass_ps_mask(__x, 0x9f));
+ else if constexpr (_Traits::template __is<float, 8>)
+ return _knot_mask8(_mm256_fpclass_ps_mask(__x, 0x9f));
+ else if constexpr (_Traits::template __is<float, 16>)
+ return _knot_mask16(_mm512_fpclass_ps_mask(__x, 0x9f));
+ else if constexpr (_Traits::template __is<double, 2>)
+ return _knot_mask8(_mm_fpclass_pd_mask(__x, 0x9f));
+ else if constexpr (_Traits::template __is<double, 4>)
+ return _knot_mask8(_mm256_fpclass_pd_mask(__x, 0x9f));
+ else if constexpr (_Traits::template __is<double, 8>)
+ return _knot_mask8(_mm512_fpclass_pd_mask(__x, 0x9f));
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ {
+ using _U = typename _Traits::value_type;
+ constexpr size_t _N = _Traits::_S_width;
+ const auto __a =
+ __x * std::numeric_limits<_U>::infinity(); // NaN if __x == 0
+ const auto __b = __x * _U(); // NaN if __x == inf
+ if constexpr (__have_avx512vl && __is_sse_ps<_U, _N>())
+ {
+ return _mm_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else if constexpr (__have_avx512f && __is_sse_ps<_U, _N>())
+ {
+ return __mmask8(0xf & _mm512_cmp_ps_mask(__auto_bitcast(__a),
+ __auto_bitcast(__b),
+ _CMP_ORD_Q));
+ }
+ else if constexpr (__have_avx512vl && __is_sse_pd<_U, _N>())
+ {
+ return _mm_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else if constexpr (__have_avx512f && __is_sse_pd<_U, _N>())
+ {
+ return __mmask8(0x3 & _mm512_cmp_pd_mask(__auto_bitcast(__a),
+ __auto_bitcast(__b),
+ _CMP_ORD_Q));
+ }
+ else if constexpr (__have_avx512vl && __is_avx_ps<_U, _N>())
+ {
+ return _mm256_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else if constexpr (__have_avx512f && __is_avx_ps<_U, _N>())
+ {
+ return __mmask8(_mm512_cmp_ps_mask(
+ __auto_bitcast(__a), __auto_bitcast(__b), _CMP_ORD_Q));
+ }
+ else if constexpr (__have_avx512vl && __is_avx_pd<_U, _N>())
+ {
+ return _mm256_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else if constexpr (__have_avx512f && __is_avx_pd<_U, _N>())
+ {
+ return __mmask8(0xf & _mm512_cmp_pd_mask(__auto_bitcast(__a),
+ __auto_bitcast(__b),
+ _CMP_ORD_Q));
+ }
+ else if constexpr (__is_avx512_ps<_U, _N>())
+ {
+ return _mm512_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else if constexpr (__is_avx512_pd<_U, _N>())
+ {
+ return _mm512_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ }
+
+ // isfinite {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
+ __isfinite(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ __unused(__x);
+ return equal_to(_SimdWrapper<_Tp, _N>(), _SimdWrapper<_Tp, _N>());
+#else
+ return __cmpord(__x._M_data, __x._M_data * _Tp());
+#endif
+ }
+
+ // isinf {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp> __isinf(_SimdWrapper<_Tp, _N> __x)
+ {
+#if __FINITE_MATH_ONLY__
+ __unused(__x);
+ return {}; // false
+#else
+ if constexpr (__is_avx512_pd<_Tp, _N>() && __have_avx512dq)
+ return _mm512_fpclass_pd_mask(__x, 0x18);
+ else if constexpr (__is_avx512_ps<_Tp, _N>() && __have_avx512dq)
+ return _mm512_fpclass_ps_mask(__x, 0x18);
+ else if constexpr (__have_avx512dq_vl)
+ {
+ if constexpr (__is_sse_pd<_Tp, _N>())
+ return __vector_bitcast<double>(
+ _mm_movm_epi64(_mm_fpclass_pd_mask(__x, 0x18)));
+ else if constexpr (__is_avx_pd<_Tp, _N>())
+ return __vector_bitcast<double>(
+ _mm256_movm_epi64(_mm256_fpclass_pd_mask(__x, 0x18)));
+ else if constexpr (__is_sse_ps<_Tp, _N>())
+ return __vector_bitcast<float>(
+ _mm_movm_epi32(_mm_fpclass_ps_mask(__x, 0x18)));
+ else if constexpr (__is_avx_ps<_Tp, _N>())
+ return __vector_bitcast<float>(
+ _mm256_movm_epi32(_mm256_fpclass_ps_mask(__x, 0x18)));
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::__isinf(__x);
+#endif
+ }
+
+ // isnormal {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
+ __isnormal(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__have_avx512dq)
+ {
+ if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _N>())
+ return __vector_bitcast<float>(
+ _mm_movm_epi32(_knot_mask8(_mm_fpclass_ps_mask(__x, 0xbf))));
+ else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _N>())
+ return __vector_bitcast<float>(_mm256_movm_epi32(
+ _knot_mask8(_mm256_fpclass_ps_mask(__x, 0xbf))));
+ else if constexpr (__is_avx512_ps<_Tp, _N>())
+ return _knot_mask16(_mm512_fpclass_ps_mask(__x, 0xbf));
+ else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _N>())
+ return __vector_bitcast<double>(
+ _mm_movm_epi64(_knot_mask8(_mm_fpclass_pd_mask(__x, 0xbf))));
+ else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _N>())
+ return __vector_bitcast<double>(_mm256_movm_epi64(
+ _knot_mask8(_mm256_fpclass_pd_mask(__x, 0xbf))));
+ else if constexpr (__is_avx512_pd<_Tp, _N>())
+ return _knot_mask8(_mm512_fpclass_pd_mask(__x, 0xbf));
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::__isnormal(__x);
+ }
+
+ // isnan {{{3
+ using _Base::__isnan;
+
+ // fpclassify {{{3
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static __fixed_size_storage_t<int, _N> __fpclassify(_SimdWrapper<_Tp, _N> __x)
+ {
+ if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ // AVX512 is special because we want to use an __mmask to blend int vectors
+ // (coming from double vectors). GCC doesn't allow this combination on the
+ // ternary operator. Thus, resort to intrinsics:
+ if constexpr (__have_avx512vl) {
+ auto &&__b = [](int __y) { return __to_intrin(__vector_broadcast<_N>(__y)); };
+ return {_mm256_mask_mov_epi32(
+ _mm256_mask_mov_epi32(
+ _mm256_mask_mov_epi32(__b(FP_NORMAL), __isnan(__x), __b(FP_NAN)),
+ __isinf(__x), __b(FP_INFINITE)),
+ _mm512_cmp_pd_mask(
+ __abs(__x),
+ __vector_broadcast<_N>(std::numeric_limits<double>::min()),
+ _CMP_LT_OS),
+ _mm256_mask_mov_epi32(
+ __b(FP_SUBNORMAL),
+ _mm512_cmp_pd_mask(__x, _mm512_setzero_pd(), _CMP_EQ_OQ),
+ __b(FP_ZERO)))};
+ } else {
+ auto &&__b = [](int __y) {
+ return _mm512_castsi256_si512(__to_intrin(__vector_broadcast<_N>(__y)));
+ };
+ return {__lo256(_mm512_mask_mov_epi32(
+ _mm512_mask_mov_epi32(
+ _mm512_mask_mov_epi32(__b(FP_NORMAL), __isnan(__x), __b(FP_NAN)),
+ __isinf(__x), __b(FP_INFINITE)),
+ _mm512_cmp_pd_mask(
+ __abs(__x),
+ __vector_broadcast<_N>(std::numeric_limits<double>::min()),
+ _CMP_LT_OS),
+ _mm512_mask_mov_epi32(
+ __b(FP_SUBNORMAL),
+ _mm512_cmp_pd_mask(__x, _mm512_setzero_pd(), _CMP_EQ_OQ),
+ __b(FP_ZERO))))};
+ }
+ } else {
+ return _Base::__fpclassify(__x);
+ }
+ }
+
+ //}}}2
+};
+
+// __x86_mask_impl {{{1
+template <class _Abi>
+struct __x86_mask_impl : __generic_mask_impl<_Abi>
+{
+ using _Base = __generic_mask_impl<_Abi>;
+
+ // masked load {{{2
+ template <class _Tp, size_t _N, class _F>
+ static inline _SimdWrapper<_Tp, _N> masked_load(_SimdWrapper<_Tp, _N> merge,
+ _SimdWrapper<_Tp, _N> mask,
+ const bool* mem,
+ _F) noexcept
+ {
+ if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ if constexpr (__have_avx512bw_vl)
+ {
+ if constexpr (_N == 8)
+ {
+ const auto __a = _mm_mask_loadu_epi8(__m128i(), mask, mem);
+ return (merge & ~mask) | _mm_test_epi8_mask(__a, __a);
+ }
+ else if constexpr (_N == 16)
+ {
+ const auto __a = _mm_mask_loadu_epi8(__m128i(), mask, mem);
+ return (merge & ~mask) | _mm_test_epi8_mask(__a, __a);
+ }
+ else if constexpr (_N == 32)
+ {
+ const auto __a = _mm256_mask_loadu_epi8(__m256i(), mask, mem);
+ return (merge & ~mask) | _mm256_test_epi8_mask(__a, __a);
+ }
+ else if constexpr (_N == 64)
+ {
+ const auto __a = _mm512_mask_loadu_epi8(__m512i(), mask, mem);
+ return (merge & ~mask) | _mm512_test_epi8_mask(__a, __a);
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else
+ {
+ __bit_iteration(mask, [&](auto __i) { merge.set(__i, mem[__i]); });
+ return merge;
+ }
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 32 && sizeof(_Tp) == 1)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(
+ _mm256_mask_sub_epi8(__vector_bitcast<_LLong>(merge), __k, __m256i(),
+ _mm256_mask_loadu_epi8(__m256i(), __k, mem)));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 16 && sizeof(_Tp) == 1)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(
+ _mm_mask_sub_epi8(__vector_bitcast<_LLong>(merge), __k, __m128i(),
+ _mm_mask_loadu_epi8(__m128i(), __k, mem)));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 16 && sizeof(_Tp) == 2)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm256_mask_sub_epi16(
+ __vector_bitcast<_LLong>(merge), __k, __m256i(),
+ _mm256_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 8 && sizeof(_Tp) == 2)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm_mask_sub_epi16(
+ __vector_bitcast<_LLong>(merge), __k, __m128i(),
+ _mm_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 8 && sizeof(_Tp) == 4)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm256_mask_sub_epi32(
+ __vector_bitcast<_LLong>(merge), __k, __m256i(),
+ _mm256_cvtepi8_epi32(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 4 && sizeof(_Tp) == 4)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm_mask_sub_epi32(
+ __vector_bitcast<_LLong>(merge), __k, __m128i(),
+ _mm_cvtepi8_epi32(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 4 && sizeof(_Tp) == 8)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm256_mask_sub_epi64(
+ __vector_bitcast<_LLong>(merge), __k, __m256i(),
+ _mm256_cvtepi8_epi64(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else if constexpr (__have_avx512bw_vl && _N == 2 && sizeof(_Tp) == 8)
+ {
+ const auto __k = __convert_mask<_SimdWrapper<bool, _N>>(mask);
+ merge = _ToWrapper(_mm_mask_sub_epi64(
+ __vector_bitcast<_LLong>(merge), __k, __m128i(),
+ _mm_cvtepi8_epi64(_mm_mask_loadu_epi8(__m128i(), __k, mem))));
+ }
+ else
+ {
+ return _Base::masked_load(merge, mask, mem, _F{});
+ }
+ return merge;
+ }
+
+ // store {{{2
+ template <class _Tp, size_t _N, class _F>
+ _GLIBCXX_SIMD_INTRINSIC static void
+ store(_SimdWrapper<_Tp, _N> __v, bool* __mem, _F) noexcept
+ {
+ if constexpr (__is_abi<_Abi, simd_abi::__sse_abi>())
+ {
+ if constexpr (_N == 2 && __have_sse2)
+ {
+ const auto __k = __vector_bitcast<int>(__v);
+ __mem[0] = -__k[1];
+ __mem[1] = -__k[3];
+ }
+ else if constexpr (_N == 4 && __have_sse2)
+ {
+ const unsigned bool4 =
+ __vector_bitcast<_UInt>(_mm_packs_epi16(
+ _mm_packs_epi32(__vector_bitcast<_LLong>(__v), __m128i()),
+ __m128i()))[0] &
+ 0x01010101u;
+ std::memcpy(__mem, &bool4, 4);
+ }
+ else if constexpr (std::is_same_v<_Tp, float> && __have_mmx)
+ {
+ const __m128 __k = __to_intrin(__v);
+ const __m64 kk = _mm_cvtps_pi8(__and(__k, _mm_set1_ps(1.f)));
+ __vector_store<4>(kk, __mem, _F());
+ _mm_empty();
+ }
+ else if constexpr (_N == 8 && __have_sse2)
+ {
+ __vector_store<8>(
+ _mm_packs_epi16(__to_intrin(__vector_bitcast<ushort>(__v) >> 15),
+ __m128i()),
+ __mem, _F());
+ }
+ else if constexpr (_N == 16 && __have_sse2)
+ {
+ __vector_store(__v._M_data & 1, __mem, _F());
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx_abi>())
+ {
+ if constexpr (_N == 4 && __have_avx)
+ {
+ auto __k = __vector_bitcast<_LLong>(__v);
+ int bool4;
+ if constexpr (__have_avx2)
+ {
+ bool4 = _mm256_movemask_epi8(__k);
+ }
+ else
+ {
+ bool4 = (_mm_movemask_epi8(__lo128(__k)) |
+ (_mm_movemask_epi8(__hi128(__k)) << 16));
+ }
+ bool4 &= 0x01010101;
+ std::memcpy(__mem, &bool4, 4);
+ }
+ else if constexpr (_N == 8 && __have_avx)
+ {
+ const auto __k = __vector_bitcast<_LLong>(__v);
+ const auto k2 =
+ _mm_srli_epi16(_mm_packs_epi16(__lo128(__k), __hi128(__k)), 15);
+ const auto k3 = _mm_packs_epi16(k2, __m128i());
+ __vector_store<8>(k3, __mem, _F());
+ }
+ else if constexpr (_N == 16 && __have_avx2)
+ {
+ const auto __x = _mm256_srli_epi16(__to_intrin(__v), 15);
+ const auto bools = _mm_packs_epi16(__lo128(__x), __hi128(__x));
+ __vector_store<16>(bools, __mem, _F());
+ }
+ else if constexpr (_N == 16 && __have_avx)
+ {
+ const auto bools =
+ 1 & __vector_bitcast<_UChar>(_mm_packs_epi16(
+ __lo128(__to_intrin(__v)), __hi128(__to_intrin(__v))));
+ __vector_store<16>(bools, __mem, _F());
+ }
+ else if constexpr (_N == 32 && __have_avx)
+ {
+ __vector_store<32>(1 & __v._M_data, __mem, _F());
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ if constexpr (_N == 8)
+ {
+ __vector_store<8>(
+#if _GLIBCXX_SIMD_HAVE_AVX512VL && _GLIBCXX_SIMD_HAVE_AVX512BW
+ _mm_maskz_set1_epi8(__v._M_data, 1),
+#elif defined __x86_64__
+ __make_wrapper<_ULLong>(
+ _pdep_u64(__v._M_data, 0x0101010101010101ULL), 0ull),
+#else
+ __make_wrapper<_UInt>(_pdep_u32(__v._M_data, 0x01010101U),
+ _pdep_u32(__v._M_data >> 4, 0x01010101U)),
+#endif
+ __mem, _F());
+ }
+ else if constexpr (_N == 16 && __have_avx512bw_vl)
+ {
+ __vector_store(_mm_maskz_set1_epi8(__v._M_data, 1), __mem, _F());
+ }
+ else if constexpr (_N == 16 && __have_avx512f)
+ {
+ _mm512_mask_cvtepi32_storeu_epi8(
+ __mem, ~__mmask16(), _mm512_maskz_set1_epi32(__v._M_data, 1));
+ }
+ else if constexpr (_N == 32 && __have_avx512bw_vl)
+ {
+ __vector_store(_mm256_maskz_set1_epi8(__v._M_data, 1), __mem, _F());
+ }
+ else if constexpr (_N == 32 && __have_avx512bw)
+ {
+ __vector_store(__lo256(_mm512_maskz_set1_epi8(__v._M_data, 1)),
+ __mem, _F());
+ }
+ else if constexpr (_N == 64 && __have_avx512bw)
+ {
+ __vector_store(_mm512_maskz_set1_epi8(__v._M_data, 1), __mem, _F());
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+
+ // masked store {{{2
+ template <class _Tp, size_t _N, class _F>
+ static inline void masked_store(const _SimdWrapper<_Tp, _N> __v,
+ bool* __mem,
+ _F,
+ const _SimdWrapper<_Tp, _N> __k) noexcept
+ {
+ if constexpr (__is_abi<_Abi, simd_abi::__avx512_abi>())
+ {
+ if constexpr (_N == 8 && __have_avx512bw_vl)
+ {
+ _mm_mask_cvtepi16_storeu_epi8(__mem, __k,
+ _mm_maskz_set1_epi16(__v, 1));
+ }
+ else if constexpr (_N == 8 && __have_avx512vl)
+ {
+ _mm256_mask_cvtepi32_storeu_epi8(__mem, __k,
+ _mm256_maskz_set1_epi32(__v, 1));
+ }
+ else if constexpr (_N == 8)
+ {
+ // we rely on __k < 0x100:
+ _mm512_mask_cvtepi32_storeu_epi8(__mem, __k,
+ _mm512_maskz_set1_epi32(__v, 1));
+ }
+ else if constexpr (_N == 16 && __have_avx512bw_vl)
+ {
+ _mm_mask_storeu_epi8(__mem, __k, _mm_maskz_set1_epi8(__v, 1));
+ }
+ else if constexpr (_N == 16)
+ {
+ _mm512_mask_cvtepi32_storeu_epi8(__mem, __k,
+ _mm512_maskz_set1_epi32(__v, 1));
+ }
+ else if constexpr (_N == 32 && __have_avx512bw_vl)
+ {
+ _mm256_mask_storeu_epi8(__mem, __k, _mm256_maskz_set1_epi8(__v, 1));
+ }
+ else if constexpr (_N == 32 && __have_avx512bw)
+ {
+ _mm256_mask_storeu_epi8(__mem, __k,
+ __lo256(_mm512_maskz_set1_epi8(__v, 1)));
+ }
+ else if constexpr (_N == 64 && __have_avx512bw)
+ {
+ _mm512_mask_storeu_epi8(__mem, __k, _mm512_maskz_set1_epi8(__v, 1));
+ }
+ else
+ {
+ __assert_unreachable<_Tp>();
+ }
+ }
+ else
+ {
+ _Base::masked_store(__v, __mem, _F(), __k);
+ }
+ }
+
+ // logical and bitwise operators {{{2
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ logical_and(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ if constexpr (std::is_same_v<_Tp, bool>)
+ {
+ if constexpr (__have_avx512dq && _N <= 8)
+ return _kand_mask8(__x._M_data, __y._M_data);
+ else if constexpr (_N <= 16)
+ return _kand_mask16(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 32)
+ return _kand_mask32(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 64)
+ return _kand_mask64(__x._M_data, __y._M_data);
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::logical_and(__x, __y);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ logical_or(const _SimdWrapper<_Tp, _N>& __x,
+ const _SimdWrapper<_Tp, _N>& __y)
+ {
+ if constexpr (std::is_same_v<_Tp, bool>)
+ {
+ if constexpr (__have_avx512dq && _N <= 8)
+ return _kor_mask8(__x._M_data, __y._M_data);
+ else if constexpr (_N <= 16)
+ return _kor_mask16(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 32)
+ return _kor_mask32(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 64)
+ return _kor_mask64(__x._M_data, __y._M_data);
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::logical_or(__x, __y);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_and(const _SimdWrapper<_Tp, _N>& __x, const _SimdWrapper<_Tp, _N>& __y)
+ {
+ if constexpr (std::is_same_v<_Tp, bool>)
+ {
+ if constexpr (__have_avx512dq && _N <= 8)
+ return _kand_mask8(__x._M_data, __y._M_data);
+ else if constexpr (_N <= 16)
+ return _kand_mask16(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 32)
+ return _kand_mask32(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 64)
+ return _kand_mask64(__x._M_data, __y._M_data);
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::bit_and(__x, __y);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_or(const _SimdWrapper<_Tp, _N>& __x, const _SimdWrapper<_Tp, _N>& __y)
+ {
+ if constexpr (std::is_same_v<_Tp, bool>)
+ {
+ if constexpr (__have_avx512dq && _N <= 8)
+ return _kor_mask8(__x._M_data, __y._M_data);
+ else if constexpr (_N <= 16)
+ return _kor_mask16(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 32)
+ return _kor_mask32(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 64)
+ return _kor_mask64(__x._M_data, __y._M_data);
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::bit_or(__x, __y);
+ }
+
+ template <class _Tp, size_t _N>
+ _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _N>
+ bit_xor(const _SimdWrapper<_Tp, _N>& __x, const _SimdWrapper<_Tp, _N>& __y)
+ {
+ if constexpr (std::is_same_v<_Tp, bool>)
+ {
+ if constexpr (__have_avx512dq && _N <= 8)
+ return _kxor_mask8(__x._M_data, __y._M_data);
+ else if constexpr (_N <= 16)
+ return _kxor_mask16(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 32)
+ return _kxor_mask32(__x._M_data, __y._M_data);
+ else if constexpr (__have_avx512bw && _N <= 64)
+ return _kxor_mask64(__x._M_data, __y._M_data);
+ else
+ __assert_unreachable<_Tp>();
+ }
+ else
+ return _Base::bit_xor(__x, __y);
+ }
+
+ //}}}2
+};
+
+//}}}1
+
+struct __sse_mask_impl : __x86_mask_impl<simd_abi::__sse> {};
+struct __sse_simd_impl : __x86_simd_impl<simd_abi::__sse> {};
+
+struct __avx_mask_impl : __x86_mask_impl<simd_abi::__avx> {};
+struct __avx_simd_impl : __x86_simd_impl<simd_abi::__avx> {};
+
+struct __avx512_simd_impl : __x86_simd_impl<simd_abi::__avx512> {};
+struct __avx512_mask_impl : __x86_mask_impl<simd_abi::__avx512> {};
+#endif // _GLIBCXX_SIMD_X86INTRIN }}}
+
+#if _GLIBCXX_SIMD_HAVE_NEON // {{{
+// __neon_simd_impl {{{
+template <int _Bytes>
+struct __neon_simd_impl : __generic_simd_impl<simd_abi::__neon_abi<_Bytes>>
+{
+ using _Base = __generic_simd_impl<simd_abi::__neon_abi<_Bytes>>;
+ // math {{{
+ // sqrt {{{
+ template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp __sqrt(_Tp __x)
+ {
+ const auto __intrin = __to_intrin(__x);
+ if constexpr (_TVT::template __is<float, 2>)
+ return vsqrt_f32(__intrin);
+ else if constexpr (_TVT::template __is<float, 4>)
+ return vsqrtq_f32(__intrin);
+ else if constexpr (_TVT::template __is<double, 1>)
+ return vsqrt_f64(__intrin);
+ else if constexpr (_TVT::template __is<double, 2>)
+ return vsqrtq_f64(__intrin);
+ else
+ return _Base::__sqrt(__x);
+ } // }}}
+ // trunc {{{
+ template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp __trunc(_Tp __x)
+ {
+ const auto __intrin = __to_intrin(__x);
+ if constexpr (_TVT::template __is<float, 2>)
+ return vrnd_f32(__intrin);
+ else if constexpr (_TVT::template __is<float, 4>)
+ return vrndq_f32(__intrin);
+ else if constexpr (_TVT::template __is<double, 1>)
+ return vrnd_f64(__intrin);
+ else if constexpr (_TVT::template __is<double, 2>)
+ return vrndq_f64(__intrin);
+ else
+ return _Base::__trunc(__x);
+ } // }}}
+ // floor {{{
+ template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp __floor(_Tp __x)
+ {
+ const auto __intrin = __to_intrin(__x);
+ if constexpr (_TVT::template __is<float, 2>)
+ return vrndm_f32(__intrin);
+ else if constexpr (_TVT::template __is<float, 4>)
+ return vrndmq_f32(__intrin);
+ else if constexpr (_TVT::template __is<double, 1>)
+ return vrndm_f64(__intrin);
+ else if constexpr (_TVT::template __is<double, 2>)
+ return vrndmq_f64(__intrin);
+ else
+ return _Base::__floor(__x);
+ } // }}}
+ // ceil {{{
+ template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
+ _GLIBCXX_SIMD_INTRINSIC static _Tp __ceil(_Tp __x)
+ {
+ const auto __intrin = __to_intrin(__x);
+ if constexpr (_TVT::template __is<float, 2>)
+ return vrndp_f32(__intrin);
+ else if constexpr (_TVT::template __is<float, 4>)
+ return vrndpq_f32(__intrin);
+ else if constexpr (_TVT::template __is<double, 1>)
+ return vrndp_f64(__intrin);
+ else if constexpr (_TVT::template __is<double, 2>)
+ return vrndpq_f64(__intrin);
+ else
+ return _Base::__ceil(__x);
+ } //}}}
+ //}}}
+}; // }}}
+// __neon_mask_impl {{{
+template <int _Bytes>
+struct __neon_mask_impl : __generic_mask_impl<simd_abi::__neon_abi<_Bytes>>
+{
+}; // }}}
+#endif // _GLIBCXX_SIMD_HAVE_NEON }}}
+
+/**
+ * The fixed_size ABI gives the following guarantees:
+ * - simd objects are passed via the stack
+ * - memory layout of `simd<_Tp, _N>` is equivalent to `std::array<_Tp, _N>`
+ * - alignment of `simd<_Tp, _N>` is `_N * sizeof(_Tp)` if _N is __a power-of-2 value,
+ * otherwise `__next_power_of_2(_N * sizeof(_Tp))` (Note: if the alignment were to
+ * exceed the system/compiler maximum, it is bounded to that maximum)
+ * - simd_mask objects are passed like std::bitset<_N>
+ * - memory layout of `simd_mask<_Tp, _N>` is equivalent to `std::bitset<_N>`
+ * - alignment of `simd_mask<_Tp, _N>` is equal to the alignment of `std::bitset<_N>`
+ */
+// __autocvt_to_simd {{{
+template <class _Tp, bool = std::is_arithmetic_v<__remove_cvref_t<_Tp>>>
+struct __autocvt_to_simd {
+ _Tp _M_data;
+ using _TT = __remove_cvref_t<_Tp>;
+ operator _TT() { return _M_data; }
+ operator _TT &()
+ {
+ static_assert(std::is_lvalue_reference<_Tp>::value, "");
+ static_assert(!std::is_const<_Tp>::value, "");
+ return _M_data;
+ }
+ operator _TT *()
+ {
+ static_assert(std::is_lvalue_reference<_Tp>::value, "");
+ static_assert(!std::is_const<_Tp>::value, "");
+ return &_M_data;
+ }
+
+ constexpr inline __autocvt_to_simd(_Tp dd) : _M_data(dd) {}
+
+ template <class _Abi> operator simd<typename _TT::value_type, _Abi>()
+ {
+ return {__private_init, _M_data};
+ }
+
+ template <class _Abi> operator simd<typename _TT::value_type, _Abi> &()
+ {
+ return *reinterpret_cast<simd<typename _TT::value_type, _Abi> *>(&_M_data);
+ }
+
+ template <class _Abi> operator simd<typename _TT::value_type, _Abi> *()
+ {
+ return reinterpret_cast<simd<typename _TT::value_type, _Abi> *>(&_M_data);
+ }
+};
+template <class _Tp> __autocvt_to_simd(_Tp &&)->__autocvt_to_simd<_Tp>;
+
+template <class _Tp> struct __autocvt_to_simd<_Tp, true> {
+ using _TT = __remove_cvref_t<_Tp>;
+ _Tp _M_data;
+ fixed_size_simd<_TT, 1> fd;
+
+ constexpr inline __autocvt_to_simd(_Tp dd) : _M_data(dd), fd(_M_data) {}
+ ~__autocvt_to_simd()
+ {
+ _M_data = __data(fd).first;
+ }
+
+ operator fixed_size_simd<_TT, 1>()
+ {
+ return fd;
+ }
+ operator fixed_size_simd<_TT, 1> &()
+ {
+ static_assert(std::is_lvalue_reference<_Tp>::value, "");
+ static_assert(!std::is_const<_Tp>::value, "");
+ return fd;
+ }
+ operator fixed_size_simd<_TT, 1> *()
+ {
+ static_assert(std::is_lvalue_reference<_Tp>::value, "");
+ static_assert(!std::is_const<_Tp>::value, "");
+ return &fd;
+ }
+};
+
+// }}}
+// __fixed_size_storage_t<_Tp, _N>{{{1
+template <class _Tp, int _N, class _Tuple,
+ class _Next = simd<_Tp, __all_native_abis::__best_abi<_Tp, _N>>,
+ int _Remain = _N - int(_Next::size())>
+struct __fixed_size_storage_builder;
+
+template <class _Tp, int _N>
+struct __fixed_size_storage
+ : public __fixed_size_storage_builder<_Tp, _N, _SimdTuple<_Tp>> {
+};
+
+template <class _Tp, int _N, class... _As, class _Next>
+struct __fixed_size_storage_builder<_Tp, _N, _SimdTuple<_Tp, _As...>, _Next, 0> {
+ using type = _SimdTuple<_Tp, _As..., typename _Next::abi_type>;
+};
+
+template <class _Tp, int _N, class... _As, class _Next, int _Remain>
+struct __fixed_size_storage_builder<_Tp, _N, _SimdTuple<_Tp, _As...>, _Next, _Remain> {
+ using type = typename __fixed_size_storage_builder<
+ _Tp, _Remain, _SimdTuple<_Tp, _As..., typename _Next::abi_type>>::type;
+};
+
+// __n_abis_in_tuple {{{1
+template <class _Tp> struct __seq_op;
+template <size_t _I0, size_t... _Is> struct __seq_op<std::index_sequence<_I0, _Is...>> {
+ using _FirstPlusOne = std::index_sequence<_I0 + 1, _Is...>;
+ using _NotFirstPlusOne = std::index_sequence<_I0, (_Is + 1)...>;
+ template <size_t _First, size_t _Add>
+ using _Prepend = std::index_sequence<_First, _I0 + _Add, (_Is + _Add)...>;
+};
+
+template <class _Tp> struct __n_abis_in_tuple;
+template <class _Tp> struct __n_abis_in_tuple<_SimdTuple<_Tp>> {
+ using __counts = std::index_sequence<0>;
+ using __begins = std::index_sequence<0>;
+};
+template <class _Tp, class _A> struct __n_abis_in_tuple<_SimdTuple<_Tp, _A>> {
+ using __counts = std::index_sequence<1>;
+ using __begins = std::index_sequence<0>;
+};
+template <class _Tp, class _A0, class... _As>
+struct __n_abis_in_tuple<_SimdTuple<_Tp, _A0, _A0, _As...>> {
+ using __counts = typename __seq_op<typename __n_abis_in_tuple<
+ _SimdTuple<_Tp, _A0, _As...>>::__counts>::_FirstPlusOne;
+ using __begins = typename __seq_op<typename __n_abis_in_tuple<
+ _SimdTuple<_Tp, _A0, _As...>>::__begins>::_NotFirstPlusOne;
+};
+template <class _Tp, class _A0, class _A1, class... _As>
+struct __n_abis_in_tuple<_SimdTuple<_Tp, _A0, _A1, _As...>> {
+ using __counts = typename __seq_op<typename __n_abis_in_tuple<
+ _SimdTuple<_Tp, _A1, _As...>>::__counts>::template _Prepend<1, 0>;
+ using __begins = typename __seq_op<typename __n_abis_in_tuple<
+ _SimdTuple<_Tp, _A1, _As...>>::__begins>::template _Prepend<0, 1>;
+};
+
+// __tree_reduction {{{1
+template <size_t _Count, size_t _Begin> struct __tree_reduction {
+ static_assert(_Count > 0,
+ "__tree_reduction requires at least one simd object to work with");
+ template <class _Tp, class... _As, class _BinaryOperation>
+ auto operator()(const _SimdTuple<_Tp, _As...> &__tup,
+ const _BinaryOperation &__binary_op) const noexcept
+ {
+ constexpr size_t __left = __next_power_of_2(_Count) / 2;
+ constexpr size_t __right = _Count - __left;
+ return __binary_op(__tree_reduction<__left, _Begin>()(__tup, __binary_op),
+ __tree_reduction<__right, _Begin + __left>()(__tup, __binary_op));
+ }
+};
+template <size_t _Begin> struct __tree_reduction<1, _Begin> {
+ template <class _Tp, class... _As, class _BinaryOperation>
+ auto operator()(const _SimdTuple<_Tp, _As...> &__tup, const _BinaryOperation &) const
+ noexcept
+ {
+ return __get_simd_at<_Begin>(__tup);
+ }
+};
+template <size_t _Begin> struct __tree_reduction<2, _Begin> {
+ template <class _Tp, class... _As, class _BinaryOperation>
+ auto operator()(const _SimdTuple<_Tp, _As...> &__tup,
+ const _BinaryOperation &__binary_op) const noexcept
+ {
+ return __binary_op(__get_simd_at<_Begin>(__tup),
+ __get_simd_at<_Begin + 1>(__tup));
+ }
+};
+
+// __vec_to_scalar_reduction {{{1
+// This helper function implements the second step in a generic fixed_size reduction.
+// - Input: a tuple of native simd (or scalar) objects of decreasing size.
+// - Output: a scalar (the reduction).
+// - Approach:
+// 1. reduce the first two tuple elements
+// a) If the number of elements differs by a factor of 2, split the first object into
+// two objects of the second type and reduce all three to one object of second
+// type.
+// b) If the number of elements differs by a factor of 4, split the first object into
+// two equally sized objects, reduce, and split to two objects of the second type.
+// Finally, reduce all three remaining objects to one object of second type.
+// c) Otherwise use std::experimental::reduce to reduce both inputs to a scalar, and binary_op to
+// reduce to a single scalar.
+//
+// (This optimizes all native cases on x86, e.g. <AVX512, SSE, Scalar>.)
+//
+// 2. Concate the result of (1) with the remaining tuple elements to recurse into
+// __vec_to_scalar_reduction.
+//
+// 3. If __vec_to_scalar_reduction is called with a one-element tuple, call std::experimental::reduce to
+// reduce to a scalar and return.
+template <class _Tp, class _A0, class _A1, class _BinaryOperation>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, _A1> __vec_to_scalar_reduction_first_pair(
+ const simd<_Tp, _A0> __left, const simd<_Tp, _A1> __right, const _BinaryOperation &__binary_op,
+ _SizeConstant<2>) noexcept
+{
+ const std::array<simd<_Tp, _A1>, 2> __splitted = split<simd<_Tp, _A1>>(__left);
+ return __binary_op(__binary_op(__splitted[0], __right), __splitted[1]);
+}
+
+template <class _Tp, class _A0, class _A1, class _BinaryOperation>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, _A1> __vec_to_scalar_reduction_first_pair(
+ const simd<_Tp, _A0> __left, const simd<_Tp, _A1> __right, const _BinaryOperation &__binary_op,
+ _SizeConstant<4>) noexcept
+{
+ constexpr auto _N0 = simd_size_v<_Tp, _A0> / 2;
+ const auto __left2 = split<simd<_Tp, simd_abi::deduce_t<_Tp, _N0>>>(__left);
+ const std::array<simd<_Tp, _A1>, 2> __splitted =
+ split<simd<_Tp, _A1>>(__binary_op(__left2[0], __left2[1]));
+ return __binary_op(__binary_op(__splitted[0], __right), __splitted[1]);
+}
+
+template <class _Tp, class _A0, class _A1, class _BinaryOperation, size_t _Factor>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, simd_abi::scalar> __vec_to_scalar_reduction_first_pair(
+ const simd<_Tp, _A0> __left, const simd<_Tp, _A1> __right, const _BinaryOperation &__binary_op,
+ _SizeConstant<_Factor>) noexcept
+{
+ return __binary_op(std::experimental::reduce(__left, __binary_op), std::experimental::reduce(__right, __binary_op));
+}
+
+template <class _Tp, class _A0, class _BinaryOperation>
+_GLIBCXX_SIMD_INTRINSIC _Tp __vec_to_scalar_reduction(const _SimdTuple<_Tp, _A0> &__tup,
+ const _BinaryOperation &__binary_op) noexcept
+{
+ return std::experimental::reduce(simd<_Tp, _A0>(__private_init, __tup.first), __binary_op);
+}
+
+template <class _Tp, class _A0, class _A1, class... _As, class _BinaryOperation>
+_GLIBCXX_SIMD_INTRINSIC _Tp __vec_to_scalar_reduction(const _SimdTuple<_Tp, _A0, _A1, _As...> &__tup,
+ const _BinaryOperation &__binary_op) noexcept
+{
+ return __vec_to_scalar_reduction(
+ __simd_tuple_concat(
+ __make_simd_tuple(
+ __vec_to_scalar_reduction_first_pair<_Tp, _A0, _A1, _BinaryOperation>(
+ {__private_init, __tup.first}, {__private_init, __tup.second.first},
+ __binary_op,
+ _SizeConstant<simd_size_v<_Tp, _A0> / simd_size_v<_Tp, _A1>>())),
+ __tup.second.second),
+ __binary_op);
+}
+
+// __partial_bitset_to_member_type {{{1
+template <class _V, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC auto __partial_bitset_to_member_type(std::bitset<_N> shifted_bits)
+{
+ static_assert(_V::size() <= _N, "");
+ using _M = typename _V::mask_type;
+ using _Tp = typename _V::value_type;
+ constexpr _Tp *__type_tag = nullptr;
+ return __get_impl_t<_M>::__from_bitset(
+ std::bitset<_V::size()>(shifted_bits.to_ullong()), __type_tag);
+}
+
+// __fixed_size_simd_impl {{{1
+template <int _N> struct __fixed_size_simd_impl {
+ // member types {{{2
+ using _MaskMember = std::bitset<_N>;
+ template <class _Tp> using _SimdMember = __fixed_size_storage_t<_Tp, _N>;
+ template <class _Tp>
+ static constexpr std::size_t tuple_size = _SimdMember<_Tp>::tuple_size;
+ template <class _Tp>
+ static constexpr std::make_index_sequence<_SimdMember<_Tp>::tuple_size> index_seq = {};
+ template <class _Tp> using simd = std::experimental::simd<_Tp, simd_abi::fixed_size<_N>>;
+ template <class _Tp> using simd_mask = std::experimental::simd_mask<_Tp, simd_abi::fixed_size<_N>>;
+ template <class _Tp> using __type_tag = _Tp *;
+
+ // broadcast {{{2
+ template <class _Tp> static constexpr inline _SimdMember<_Tp> __broadcast(_Tp __x) noexcept
+ {
+ return _SimdMember<_Tp>::generate(
+ [&](auto meta) { return meta.__broadcast(__x); });
+ }
+
+ // generator {{{2
+ template <class _F, class _Tp>
+ inline static _SimdMember<_Tp> generator(_F &&__gen, __type_tag<_Tp>)
+ {
+ return _SimdMember<_Tp>::generate([&__gen](auto meta) {
+ return meta.generator(
+ [&](auto __i) {
+ static_assert(__i < _N);
+ return __gen(_SizeConstant<meta.offset + __i>());
+ },
+ __type_tag<_Tp>());
+ });
+ }
+
+ // load {{{2
+ template <class _Tp, class _U, class _F>
+ static inline _SimdMember<_Tp> load(const _U *mem, _F __f,
+ __type_tag<_Tp>) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ return _SimdMember<_Tp>::generate(
+ [&](auto meta) { return meta.load(&mem[meta.offset], __f, __type_tag<_Tp>()); });
+ }
+
+ // masked load {{{2
+ template <class _Tp, class... _As, class _U, class _F>
+ static inline _SimdTuple<_Tp, _As...>
+ masked_load(const _SimdTuple<_Tp, _As...>& __old,
+ const _MaskMember __bits,
+ const _U* __mem,
+ _F __f) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ auto __merge = __old;
+ __for_each(__merge, [&](auto __meta, auto& __native) {
+ __native = __meta.masked_load(__native, __meta.make_mask(__bits),
+ &__mem[__meta.offset], __f);
+ });
+ return __merge;
+ }
+
+ // store {{{2
+ template <class _Tp, class _U, class _F>
+ static inline void store(const _SimdMember<_Tp>& __v,
+ _U* __mem,
+ _F __f,
+ __type_tag<_Tp>) _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ __for_each(__v, [&](auto __meta, auto __native) {
+ __meta.store(__native, &__mem[__meta.offset], __f, __type_tag<_Tp>());
+ });
+ }
+
+ // masked store {{{2
+ template <class _Tp, class... _As, class _U, class _F>
+ static inline void masked_store(const _SimdTuple<_Tp, _As...>& __v,
+ _U* __mem,
+ _F __f,
+ const _MaskMember __bits)
+ _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+ {
+ __for_each(__v, [&](auto __meta, auto __native) {
+ __meta.masked_store(__native, &__mem[__meta.offset], __f,
+ __meta.make_mask(__bits));
+ });
+ }
+
+ // negation {{{2
+ template <class _Tp, class... _As>
+ static inline _MaskMember
+ negate(const _SimdTuple<_Tp, _As...>& __x) noexcept
+ {
+ _MaskMember __bits = 0;
+ __for_each(__x, [&__bits](auto __meta, auto __native) {
+ __bits |= __meta.mask_to_shifted_ullong(__meta.negate(__native));
+ });
+ return __bits;
+ }
+
+ // reductions {{{2
+private:
+ template <class _Tp, class... _As, class _BinaryOperation, size_t... _Counts,
+ size_t... _Begins>
+ static inline _Tp reduce(const _SimdTuple<_Tp, _As...> &__tup,
+ const _BinaryOperation &__binary_op,
+ std::index_sequence<_Counts...>, std::index_sequence<_Begins...>)
+ {
+ // 1. reduce all tuple elements with equal ABI to a single element in the output
+ // tuple
+ const auto reduced_vec =
+ __make_simd_tuple(__tree_reduction<_Counts, _Begins>()(__tup, __binary_op)...);
+ // 2. split and reduce until a scalar results
+ return __vec_to_scalar_reduction(reduced_vec, __binary_op);
+ }
+
+public:
+ template <class _Tp, class _BinaryOperation>
+ static inline _Tp reduce(const simd<_Tp> &__x, const _BinaryOperation &__binary_op)
+ {
+ using __ranges = __n_abis_in_tuple<_SimdMember<_Tp>>;
+ return __fixed_size_simd_impl::reduce(__x._M_data, __binary_op,
+ typename __ranges::__counts(),
+ typename __ranges::__begins());
+ }
+
+ // min, max, clamp {{{2
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ min(const _SimdTuple<_Tp, _As...>& __a,
+ const _SimdTuple<_Tp, _As...>& __b)
+ {
+ return __simd_tuple_apply(
+ [](auto __impl, auto __aa, auto __bb) {
+ return __impl.min(__aa, __bb);
+ },
+ __a, __b);
+ }
+
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ max(const _SimdTuple<_Tp, _As...>& __a,
+ const _SimdTuple<_Tp, _As...>& __b)
+ {
+ return __simd_tuple_apply(
+ [](auto __impl, auto __aa, auto __bb) {
+ return __impl.max(__aa, __bb);
+ },
+ __a, __b);
+ }
+
+ // complement {{{2
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ complement(const _SimdTuple<_Tp, _As...>& __x) noexcept
+ {
+ return __simd_tuple_apply(
+ [](auto __impl, auto __xx) { return __impl.complement(__xx); }, __x);
+ }
+
+ // unary minus {{{2
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ unary_minus(const _SimdTuple<_Tp, _As...>& __x) noexcept
+ {
+ return __simd_tuple_apply(
+ [](auto __impl, auto __xx) { return __impl.unary_minus(__xx); }, __x);
+ }
+
+ // arithmetic operators {{{2
+
+#define _GLIBCXX_SIMD_FIXED_OP(name_, op_) \
+ template <typename _Tp, typename... _As> \
+ static inline constexpr _SimdTuple<_Tp, _As...> name_( \
+ const _SimdTuple<_Tp, _As...>& __x, \
+ const _SimdTuple<_Tp, _As...>& __y) \
+ { \
+ return __simd_tuple_apply( \
+ [](auto __impl, auto __xx, auto __yy) { \
+ return __impl.name_(__xx, __yy); \
+ }, \
+ __x, __y); \
+ }
+
+ _GLIBCXX_SIMD_FIXED_OP(plus, +)
+ _GLIBCXX_SIMD_FIXED_OP(minus, -)
+ _GLIBCXX_SIMD_FIXED_OP(multiplies, *)
+ _GLIBCXX_SIMD_FIXED_OP(divides, /)
+ _GLIBCXX_SIMD_FIXED_OP(modulus, %)
+ _GLIBCXX_SIMD_FIXED_OP(bit_and, &)
+ _GLIBCXX_SIMD_FIXED_OP(bit_or, |)
+ _GLIBCXX_SIMD_FIXED_OP(bit_xor, ^)
+ _GLIBCXX_SIMD_FIXED_OP(bit_shift_left, <<)
+ _GLIBCXX_SIMD_FIXED_OP(bit_shift_right, >>)
+#undef _GLIBCXX_SIMD_FIXED_OP
+
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ bit_shift_left(const _SimdTuple<_Tp, _As...>& __x, int __y)
+ {
+ return __simd_tuple_apply(
+ [__y](auto __impl, auto __xx) {
+ return __impl.bit_shift_left(__xx, __y);
+ },
+ __x);
+ }
+
+ template <typename _Tp, typename... _As>
+ static inline constexpr _SimdTuple<_Tp, _As...>
+ bit_shift_right(const _SimdTuple<_Tp, _As...>& __x, int __y)
+ {
+ return __simd_tuple_apply(
+ [__y](auto __impl, auto __xx) {
+ return __impl.bit_shift_right(__xx, __y);
+ },
+ __x);
+ }
+
+ // math {{{2
+#define _GLIBCXX_SIMD_APPLY_ON_TUPLE_(name_) \
+ template <typename _Tp, typename... _As> \
+ static inline _SimdTuple<_Tp, _As...> __##name_( \
+ const _SimdTuple<_Tp, _As...>& __x) noexcept \
+ { \
+ return __simd_tuple_apply( \
+ [](auto __impl, auto __xx) { \
+ using _V = typename decltype(__impl)::simd_type; \
+ return __data(name_(_V(__private_init, __xx))); \
+ }, \
+ __x); \
+ }
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(sqrt)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(abs)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(trunc)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(floor)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(ceil)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(sin)
+ _GLIBCXX_SIMD_APPLY_ON_TUPLE_(cos)
+#undef _GLIBCXX_SIMD_APPLY_ON_TUPLE_
+
+ template <typename _Tp, typename... _As>
+ static inline _SimdTuple<_Tp, _As...> __frexp(const _SimdTuple<_Tp, _As...> &__x,
+ __fixed_size_storage_t<int, _N> &exp) noexcept
+ {
+ return __simd_tuple_apply(
+ [](auto __impl, const auto &__a, auto &__b) {
+ return __data(
+ __impl.__frexp(typename decltype(__impl)::simd_type(__private_init, __a),
+ __autocvt_to_simd(__b)));
+ },
+ __x, exp);
+ }
+
+ template <typename _Tp, typename... _As>
+ static inline __fixed_size_storage_t<int, _N>
+ __fpclassify(const _SimdTuple<_Tp, _As...>& __x) noexcept
+ {
+ return __optimize_simd_tuple(__x.template apply_r<int>(
+ [](auto __impl, auto __xx) { return __impl.__fpclassify(__xx); }));
+ }
+
+#define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_) \
+ template <typename _Tp, typename... _As> \
+ static inline _MaskMember __##name_( \
+ const _SimdTuple<_Tp, _As...>& __x) noexcept \
+ { \
+ return test([](auto __impl, auto __xx) { return __impl.__##name_(__xx); }, \
+ __x); \
+ }
+ _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf)
+ _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite)
+ _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan)
+ _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal)
+ _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit)
+#undef _GLIBCXX_SIMD_TEST_ON_TUPLE_
+
+ // __increment & __decrement{{{2
+ template <typename... _Ts>
+ static inline constexpr void __increment(_SimdTuple<_Ts...>& __x)
+ {
+ __for_each(__x,
+ [](auto meta, auto& native) { meta.__increment(native); });
+ }
+
+ template <typename... _Ts>
+ static inline constexpr void __decrement(_SimdTuple<_Ts...>& __x)
+ {
+ __for_each(__x,
+ [](auto meta, auto& native) { meta.__decrement(native); });
+ }
+
+ // compares {{{2
+#define _GLIBCXX_SIMD_CMP_OPERATIONS(cmp_) \
+ template <typename _Tp, typename... _As> \
+ static inline _MaskMember cmp_(const _SimdTuple<_Tp, _As...>& __x, \
+ const _SimdTuple<_Tp, _As...>& __y) \
+ { \
+ _MaskMember __bits = 0; \
+ __for_each(__x, __y, [&__bits](auto meta, auto native_x, auto native_y) { \
+ __bits |= meta.mask_to_shifted_ullong(meta.cmp_(native_x, native_y)); \
+ }); \
+ return __bits; \
+ }
+ _GLIBCXX_SIMD_CMP_OPERATIONS(equal_to)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(not_equal_to)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(less)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(greater)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(less_equal)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(greater_equal)
+ _GLIBCXX_SIMD_CMP_OPERATIONS(isunordered)
+#undef _GLIBCXX_SIMD_CMP_OPERATIONS
+
+ // smart_reference access {{{2
+ template <typename _Tp, typename... _As, typename _U>
+ _GLIBCXX_SIMD_INTRINSIC static void set(_SimdTuple<_Tp, _As...> &__v, int __i, _U &&__x) noexcept
+ {
+ __v.set(__i, std::forward<_U>(__x));
+ }
+
+ // masked_assign {{{2
+ template <typename _Tp, typename... _As>
+ _GLIBCXX_SIMD_INTRINSIC static void
+ masked_assign(const _MaskMember __bits,
+ _SimdTuple<_Tp, _As...>& __lhs,
+ const __id<_SimdTuple<_Tp, _As...>>& __rhs)
+ {
+ __for_each(__lhs, __rhs,
+ [&](auto __meta, auto& __native_lhs, auto __native_rhs) {
+ __meta.masked_assign(__meta.make_mask(__bits), __native_lhs,
+ __native_rhs);
+ });
+ }
+
+ // Optimization for the case where the RHS is a scalar. No need to broadcast the
+ // scalar to a simd first.
+ template <typename _Tp, typename... _As>
+ _GLIBCXX_SIMD_INTRINSIC static void
+ masked_assign(const _MaskMember __bits,
+ _SimdTuple<_Tp, _As...>& __lhs,
+ const __id<_Tp> __rhs)
+ {
+ __for_each(__lhs, [&](auto __meta, auto& __native_lhs) {
+ __meta.masked_assign(__meta.make_mask(__bits), __native_lhs, __rhs);
+ });
+ }
+
+ // __masked_cassign {{{2
+ template <template <typename> class _Op, typename _Tp, typename... _As>
+ static inline void __masked_cassign(const _MaskMember __bits,
+ _SimdTuple<_Tp, _As...>& __lhs,
+ const _SimdTuple<_Tp, _As...>& __rhs)
+ {
+ __for_each(__lhs, __rhs,
+ [&](auto __meta, auto& __native_lhs, auto __native_rhs) {
+ __meta.template __masked_cassign<_Op>(
+ __meta.make_mask(__bits), __native_lhs, __native_rhs);
+ });
+ }
+
+ // Optimization for the case where the RHS is a scalar. No need to broadcast
+ // the scalar to a simd first.
+ template <template <typename> class _Op, typename _Tp, typename... _As>
+ static inline void __masked_cassign(const _MaskMember __bits,
+ _SimdTuple<_Tp, _As...>& __lhs,
+ const _Tp& __rhs)
+ {
+ __for_each(__lhs, [&](auto __meta, auto& __native_lhs) {
+ __meta.template __masked_cassign<_Op>(__meta.make_mask(__bits),
+ __native_lhs, __rhs);
+ });
+ }
+
+ // masked_unary {{{2
+ template <template <typename> class _Op, typename _Tp, typename... _As>
+ static inline _SimdTuple<_Tp, _As...>
+ masked_unary(const _MaskMember __bits,
+ const _SimdTuple<_Tp, _As...> __v) // TODO: const-ref __v?
+ {
+ return __v.apply_wrapped([&__bits](auto __meta, auto __native) {
+ return __meta.template masked_unary<_Op>(__meta.make_mask(__bits),
+ __native);
+ });
+ }
+
+ // }}}2
+};
+
+// __fixed_size_mask_impl {{{1
+template <int _N> struct __fixed_size_mask_impl {
+ static_assert(sizeof(_ULLong) * CHAR_BIT >= _N,
+ "The fixed_size implementation relies on one "
+ "_ULLong being able to store all boolean "
+ "elements."); // required in load & store
+
+ // member types {{{2
+ static constexpr std::make_index_sequence<_N> index_seq = {};
+ using _MaskMember = std::bitset<_N>;
+ template <typename _Tp> using simd_mask = std::experimental::simd_mask<_Tp, simd_abi::fixed_size<_N>>;
+ template <typename _Tp> using __type_tag = _Tp *;
+
+ // __from_bitset {{{2
+ template <typename _Tp>
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember __from_bitset(const _MaskMember &bs,
+ __type_tag<_Tp>) noexcept
+ {
+ return bs;
+ }
+
+ // load {{{2
+ template <typename _F> static inline _MaskMember load(const bool *mem, _F __f) noexcept
+ {
+ // TODO: _UChar is not necessarily the best type to use here. For smaller _N ushort,
+ // _UInt, _ULLong, float, and double can be more efficient.
+ _ULLong __r = 0;
+ using _Vs = __fixed_size_storage_t<_UChar, _N>;
+ __for_each(_Vs{}, [&](auto meta, auto) {
+ __r |= meta.mask_to_shifted_ullong(
+ meta.simd_mask.load(&mem[meta.offset], __f, _SizeConstant<meta.size()>()));
+ });
+ return __r;
+ }
+
+ // masked load {{{2
+ template <typename _F>
+ static inline _MaskMember masked_load(_MaskMember merge,
+ _MaskMember mask, const bool *mem,
+ _F) noexcept
+ {
+ __bit_iteration(mask.to_ullong(), [&](auto __i) { merge[__i] = mem[__i]; });
+ return merge;
+ }
+
+ // store {{{2
+ template <typename _F>
+ static inline void store(_MaskMember bs, bool *mem, _F __f) noexcept
+ {
+#if _GLIBCXX_SIMD_HAVE_AVX512BW
+ const __m512i bool64 = _mm512_movm_epi8(bs.to_ullong()) & 0x0101010101010101ULL;
+ __vector_store<_N>(bool64, mem, __f);
+#elif _GLIBCXX_SIMD_HAVE_BMI2
+#ifdef __x86_64__
+ __unused(__f);
+ __execute_n_times<_N / 8>([&](auto __i) {
+ constexpr size_t offset = __i * 8;
+ const _ULLong bool8 =
+ _pdep_u64(bs.to_ullong() >> offset, 0x0101010101010101ULL);
+ std::memcpy(&mem[offset], &bool8, 8);
+ });
+ if (_N % 8 > 0) {
+ constexpr size_t offset = (_N / 8) * 8;
+ const _ULLong bool8 =
+ _pdep_u64(bs.to_ullong() >> offset, 0x0101010101010101ULL);
+ std::memcpy(&mem[offset], &bool8, _N % 8);
+ }
+#else // __x86_64__
+ __unused(__f);
+ __execute_n_times<_N / 4>([&](auto __i) {
+ constexpr size_t offset = __i * 4;
+ const _ULLong bool4 =
+ _pdep_u32(bs.to_ullong() >> offset, 0x01010101U);
+ std::memcpy(&mem[offset], &bool4, 4);
+ });
+ if (_N % 4 > 0) {
+ constexpr size_t offset = (_N / 4) * 4;
+ const _ULLong bool4 =
+ _pdep_u32(bs.to_ullong() >> offset, 0x01010101U);
+ std::memcpy(&mem[offset], &bool4, _N % 4);
+ }
+#endif // __x86_64__
+#elif _GLIBCXX_SIMD_HAVE_SSE2 // !AVX512BW && !BMI2
+ using _V = simd<_UChar, simd_abi::__sse>;
+ _ULLong __bits = bs.to_ullong();
+ __execute_n_times<(_N + 15) / 16>([&](auto __i) {
+ constexpr size_t offset = __i * 16;
+ constexpr size_t remaining = _N - offset;
+ if constexpr (remaining == 1) {
+ mem[offset] = static_cast<bool>(__bits >> offset);
+ } else if constexpr (remaining <= 4) {
+ const _UInt bool4 = ((__bits >> offset) * 0x00204081U) & 0x01010101U;
+ std::memcpy(&mem[offset], &bool4, remaining);
+ } else if constexpr (remaining <= 7) {
+ const _ULLong bool8 =
+ ((__bits >> offset) * 0x40810204081ULL) & 0x0101010101010101ULL;
+ std::memcpy(&mem[offset], &bool8, remaining);
+ } else if constexpr (__have_sse2) {
+ auto tmp = _mm_cvtsi32_si128(__bits >> offset);
+ tmp = _mm_unpacklo_epi8(tmp, tmp);
+ tmp = _mm_unpacklo_epi16(tmp, tmp);
+ tmp = _mm_unpacklo_epi32(tmp, tmp);
+ _V tmp2(tmp);
+ tmp2 &= _V([](auto __j) {
+ return static_cast<_UChar>(1 << (__j % CHAR_BIT));
+ }); // mask bit index
+ const __m128i bool16 = __intrin_bitcast<__m128i>(
+ __vector_bitcast<_UChar>(__data(tmp2 == 0)) +
+ 1); // 0xff -> 0x00 | 0x00 -> 0x01
+ if constexpr (remaining >= 16) {
+ __vector_store<16>(bool16, &mem[offset], __f);
+ } else if constexpr (remaining & 3) {
+ constexpr int to_shift = 16 - int(remaining);
+ _mm_maskmoveu_si128(bool16,
+ _mm_srli_si128(__allbits<__m128i>, to_shift),
+ reinterpret_cast<char *>(&mem[offset]));
+ } else // at this point: 8 < remaining < 16
+ if constexpr (remaining >= 8) {
+ __vector_store<8>(bool16, &mem[offset], __f);
+ if constexpr (remaining == 12) {
+ __vector_store<4>(_mm_unpackhi_epi64(bool16, bool16),
+ &mem[offset + 8], __f);
+ }
+ }
+ } else {
+ __assert_unreachable<_F>();
+ }
+ });
+#else
+ // TODO: _UChar is not necessarily the best type to use here. For smaller _N ushort,
+ // _UInt, _ULLong, float, and double can be more efficient.
+ using _Vs = __fixed_size_storage_t<_UChar, _N>;
+ __for_each(_Vs{}, [&](auto meta, auto) {
+ meta.simd_mask.store(meta.make_mask(bs), &mem[meta.offset], __f);
+ });
+//#else
+ //__execute_n_times<_N>([&](auto __i) { mem[__i] = bs[__i]; });
+#endif // _GLIBCXX_SIMD_HAVE_BMI2
+ }
+
+ // masked store {{{2
+ template <typename _F>
+ static inline void masked_store(const _MaskMember __v, bool *mem, _F,
+ const _MaskMember __k) noexcept
+ {
+ __bit_iteration(__k, [&](auto __i) { mem[__i] = __v[__i]; });
+ }
+
+ // logical and bitwise operators {{{2
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember logical_and(const _MaskMember &__x,
+ const _MaskMember &__y) noexcept
+ {
+ return __x & __y;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember logical_or(const _MaskMember &__x,
+ const _MaskMember &__y) noexcept
+ {
+ return __x | __y;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember bit_and(const _MaskMember &__x,
+ const _MaskMember &__y) noexcept
+ {
+ return __x & __y;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember bit_or(const _MaskMember &__x,
+ const _MaskMember &__y) noexcept
+ {
+ return __x | __y;
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC static _MaskMember bit_xor(const _MaskMember &__x,
+ const _MaskMember &__y) noexcept
+ {
+ return __x ^ __y;
+ }
+
+ // smart_reference access {{{2
+ _GLIBCXX_SIMD_INTRINSIC static void set(_MaskMember &__k, int __i, bool __x) noexcept
+ {
+ __k.set(__i, __x);
+ }
+
+ // masked_assign {{{2
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(const _MaskMember __k,
+ _MaskMember &__lhs,
+ const _MaskMember __rhs)
+ {
+ __lhs = (__lhs & ~__k) | (__rhs & __k);
+ }
+
+ // Optimization for the case where the RHS is a scalar.
+ _GLIBCXX_SIMD_INTRINSIC static void masked_assign(const _MaskMember __k,
+ _MaskMember &__lhs, const bool __rhs)
+ {
+ if (__rhs) {
+ __lhs |= __k;
+ } else {
+ __lhs &= ~__k;
+ }
+ }
+
+ // }}}2
+};
+// }}}1
+
+// _SimdConverter <From, A> -> <To, A> (same ABI) {{{
+template <typename _From, typename _To, typename _Abi>
+struct _SimdConverter<_From, _Abi, _To, _Abi>
+{
+ template <typename _Tp>
+ using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
+ using _Arg = _SimdMember<_From>;
+ using _Ret = _SimdMember<_To>;
+ using _V = __vector_type_t<_To, simd_size_v<_To, _Abi>>;
+
+ _GLIBCXX_SIMD_INTRINSIC decltype(auto) operator()(const _Arg& __a)
+ {
+ if constexpr (is_same_v<_To, _From>)
+ return __a;
+ else
+ return __convert<_V>(__a);
+ }
+ template <typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC _Ret operator()(_Arg __a, _More... __more)
+ {
+ static_assert(sizeof(_From) >= (sizeof...(_More)+1) * sizeof(_To));
+ return __convert<_V>(__a, __more...);
+ }
+
+ _GLIBCXX_SIMD_INTRINSIC auto __all(const _Arg& __a)
+ {
+ return __convert_all<_V>(__a);
+ }
+};
+// }}}
+// _SimdConverter scalar -> scalar {{{
+template <typename _From, typename _To>
+struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar>
+{
+ _GLIBCXX_SIMD_INTRINSIC _To operator()(_From __a)
+ {
+ return static_cast<_To>(__a);
+ }
+};
+
+// }}}
+// _SimdConverter "native" -> scalar {{{
+template <typename _From, typename _To, typename _Abi>
+struct _SimdConverter<_From, _Abi, _To, simd_abi::scalar>
+{
+ using _Arg = typename _Abi::template __traits<_From>::_SimdMember;
+ static constexpr size_t _S_n = _Arg::_S_width;
+
+ _GLIBCXX_SIMD_INTRINSIC std::array<_To, _S_n> __all(_Arg __a)
+ {
+ return __call_with_subscripts(
+ __a, make_index_sequence<_S_n>(),
+ [&](auto... __values) -> std::array<_To, _S_n> {
+ return {static_cast<_To>(__values)...};
+ });
+ }
+};
+
+// }}}
+// _SimdConverter scalar -> "native" {{{
+template <typename _From, typename _To, typename _Abi>
+struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi>
+{
+ using _Ret = typename _Abi::template __traits<_To>::_SimdMember;
+
+ template <typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC constexpr _Ret operator()(_From __a, _More... __more)
+ {
+ static_assert(sizeof...(_More) + 1 == _Ret::_S_width);
+ static_assert(std::conjunction_v<std::is_same<_From, _More>...>);
+ return __make_vector<_To>(__a, __more...);
+ }
+};
+
+// }}}
+// _SimdConverter "native 1" -> "native 2" {{{
+template <typename _From, typename _To, typename _AFrom, typename _ATo>
+struct _SimdConverter<_From, _AFrom, _To, _ATo>
+{
+ using _Arg = typename _AFrom::template __traits<_From>::_SimdMember;
+ using _Ret = typename _ATo::template __traits<_To>::_SimdMember;
+ using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>;
+
+ _GLIBCXX_SIMD_INTRINSIC auto __all(_Arg __a)
+ {
+ return __convert_all<_V>(__a);
+ }
+
+ template <typename... _More>
+ _GLIBCXX_SIMD_INTRINSIC _Ret operator()(_Arg __a, _More... __more)
+ {
+ static_assert(std::conjunction_v<std::is_same<_Arg, _More>...>);
+ return __convert<_V>(__a, __more...);
+ }
+};
+
+// }}}
+// _SimdConverter scalar -> fixed_size<1> {{{1
+template <typename _From, typename _To>
+struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>> {
+ _SimdTuple<_To, simd_abi::scalar> operator()(_From __x) { return {static_cast<_To>(__x)}; }
+};
+
+// _SimdConverter fixed_size<1> -> scalar {{{1
+template <typename _From, typename _To>
+struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar> {
+ _GLIBCXX_SIMD_INTRINSIC _To operator()(_SimdTuple<_From, simd_abi::scalar> __x)
+ {
+ return {static_cast<_To>(__x.first)};
+ }
+};
+
+// _SimdConverter fixed_size<_N> -> fixed_size<_N> {{{1
+template <typename _Tp, int _N>
+struct _SimdConverter<_Tp, simd_abi::fixed_size<_N>, _Tp, simd_abi::fixed_size<_N>> {
+ using arg = __fixed_size_storage_t<_Tp, _N>;
+ _GLIBCXX_SIMD_INTRINSIC const arg &operator()(const arg &__x) { return __x; }
+};
+
+template <size_t _ChunkSize, typename _Tp> struct determine_required_input_chunks;
+
+template <typename _Tp, typename... _Abis>
+struct determine_required_input_chunks<0, _SimdTuple<_Tp, _Abis...>>
+ : public std::integral_constant<size_t, 0> {
+};
+
+template <size_t _ChunkSize, typename _Tp, typename _Abi0, typename... _Abis>
+struct determine_required_input_chunks<_ChunkSize, _SimdTuple<_Tp, _Abi0, _Abis...>>
+ : public std::integral_constant<
+ size_t, determine_required_input_chunks<_ChunkSize - simd_size_v<_Tp, _Abi0>,
+ _SimdTuple<_Tp, _Abis...>>::value> {
+};
+
+template <typename _From, typename _To> struct _FixedSizeConverter {
+ struct _OneToMultipleChunks {
+ };
+ template <int _N> struct _MultipleToOneChunk {
+ };
+ struct _EqualChunks {
+ };
+ template <typename _FromAbi,
+ typename _ToAbi,
+ size_t _ToSize = simd_size_v<_To, _ToAbi>,
+ size_t _FromSize = simd_size_v<_From, _FromAbi>>
+ using _ChunkRelation = std::conditional_t<
+ (_ToSize < _FromSize),
+ _OneToMultipleChunks,
+ std::conditional_t<(_ToSize == _FromSize),
+ _EqualChunks,
+ _MultipleToOneChunk<int(_ToSize / _FromSize)>>>;
+
+ template <typename... _Abis>
+ using __return_type = __fixed_size_storage_t<_To, _SimdTuple<_From, _Abis...>::size()>;
+
+
+protected:
+ // _OneToMultipleChunks {{{2
+ template <typename _A0>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0> __impl(_OneToMultipleChunks, const _SimdTuple<_From, _A0> &__x)
+ {
+ using _R = __return_type<_A0>;
+ _SimdConverter<_From, _A0, _To, typename _R::_First_abi> __native_cvt;
+ auto &&multiple_return_chunks = __native_cvt.__all(__x.first);
+ return __to_simd_tuple<_To, typename _R::_First_abi>(multiple_return_chunks);
+ }
+
+ template <typename... _Abis>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_Abis...> __impl(_OneToMultipleChunks,
+ const _SimdTuple<_From, _Abis...> &__x)
+ {
+ using _R = __return_type<_Abis...>;
+ using arg = _SimdTuple<_From, _Abis...>;
+ constexpr size_t first_chunk = simd_size_v<_From, typename arg::_First_abi>;
+ _SimdConverter<_From, typename arg::_First_abi, _To, typename _R::_First_abi>
+ __native_cvt;
+ auto &&multiple_return_chunks = __native_cvt.__all(__x.first);
+ constexpr size_t n_output_chunks =
+ first_chunk / simd_size_v<_To, typename _R::_First_abi>;
+ return __simd_tuple_concat(
+ __to_simd_tuple<_To, typename _R::_First_abi>(multiple_return_chunks),
+ __impl(_ChunkRelation<typename arg::_Second_type::_First_abi,
+ typename __simd_tuple_element<n_output_chunks, _R>::type::abi_type>(),
+ __x.second));
+ }
+
+ // _MultipleToOneChunk {{{2
+ template <int _N, typename _A0, typename... _Abis>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0, _Abis...> __impl(_MultipleToOneChunk<_N>,
+ const _SimdTuple<_From, _A0, _Abis...> &__x)
+ {
+ return impl_mto(std::integral_constant<bool, sizeof...(_Abis) + 1 == _N>(),
+ std::make_index_sequence<_N>(), __x);
+ }
+
+ template <size_t... _Indexes, typename _A0, typename... _Abis>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0, _Abis...> impl_mto(true_type,
+ std::index_sequence<_Indexes...>,
+ const _SimdTuple<_From, _A0, _Abis...> &__x)
+ {
+ using _R = __return_type<_A0, _Abis...>;
+ _SimdConverter<_From, _A0, _To, typename _R::_First_abi> __native_cvt;
+ return {__native_cvt(__get_tuple_at<_Indexes>(__x)...)};
+ }
+
+ template <size_t... _Indexes, typename _A0, typename... _Abis>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0, _Abis...> impl_mto(false_type,
+ std::index_sequence<_Indexes...>,
+ const _SimdTuple<_From, _A0, _Abis...> &__x)
+ {
+ using _R = __return_type<_A0, _Abis...>;
+ _SimdConverter<_From, _A0, _To, typename _R::_First_abi> __native_cvt;
+ return {
+ __native_cvt(__get_tuple_at<_Indexes>(__x)...),
+ __impl(
+ _ChunkRelation<
+ typename __simd_tuple_element<sizeof...(_Indexes),
+ _SimdTuple<_From, _A0, _Abis...>>::type::abi_type,
+ typename _R::_Second_type::_First_abi>(),
+ __simd_tuple_pop_front(_SizeConstant<sizeof...(_Indexes)>(), __x))};
+ }
+
+ // _EqualChunks {{{2
+ template <typename _A0>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0> __impl(_EqualChunks, const _SimdTuple<_From, _A0> &__x)
+ {
+ _SimdConverter<_From, _A0, _To, typename __return_type<_A0>::_First_abi> __native_cvt;
+ return {__native_cvt(__x.first)};
+ }
+
+ template <typename _A0, typename _A1, typename... _Abis>
+ _GLIBCXX_SIMD_INTRINSIC __return_type<_A0, _A1, _Abis...> __impl(
+ _EqualChunks, const _SimdTuple<_From, _A0, _A1, _Abis...> &__x)
+ {
+ using _R = __return_type<_A0, _A1, _Abis...>;
+ using _Rem = typename _R::_Second_type;
+ _SimdConverter<_From, _A0, _To, typename _R::_First_abi> __native_cvt;
+ return {__native_cvt(__x.first),
+ __impl(_ChunkRelation<_A1, typename _Rem::_First_abi>(), __x.second)};
+ }
+
+ //}}}2
+};
+
+template <typename _From, typename _To, int _N>
+struct _SimdConverter<_From,
+ simd_abi::fixed_size<_N>,
+ _To,
+ simd_abi::fixed_size<_N>>
+: public _FixedSizeConverter<_From, _To>
+{
+ using _Base = _FixedSizeConverter<_From, _To>;
+ using _Ret = __fixed_size_storage_t<_To, _N>;
+ using _Arg = __fixed_size_storage_t<_From, _N>;
+
+ _GLIBCXX_SIMD_INTRINSIC _Ret operator()(const _Arg& __x)
+ {
+ if constexpr (__is_abi<typename _Ret::_First_abi, simd_abi::scalar>())
+ { // then all entries of _Ret are scalar
+ return __call_with_subscripts(
+ __x, make_index_sequence<_N>(), [](auto... __values) -> _Ret {
+ return __to_simd_tuple<_To, simd_abi::scalar>(
+ array<_To, _N>{static_cast<_To>(__values)...});
+ });
+ }
+ else
+ {
+ using _CR =
+ typename _Base::template _ChunkRelation<typename _Arg::_First_abi,
+ typename _Ret::_First_abi>;
+ return _Base::__impl(_CR(), __x);
+ }
+ }
+};
+
+// _SimdConverter "native" -> fixed_size<_N> {{{1
+// i.e. 1 register to ? registers
+template <typename _From, typename _A, typename _To, int _N>
+struct _SimdConverter<_From, _A, _To, simd_abi::fixed_size<_N>> {
+ using __traits = _SimdTraits<_From, _A>;
+ using arg = typename __traits::_SimdMember;
+ using __return_type = __fixed_size_storage_t<_To, _N>;
+ static_assert(_N == simd_size_v<_From, _A>,
+ "_SimdConverter to fixed_size only works for equal element counts");
+
+ _GLIBCXX_SIMD_INTRINSIC __return_type operator()(arg __x)
+ {
+ return __impl(std::make_index_sequence<__return_type::tuple_size>(), __x);
+ }
+
+private:
+ __return_type __impl(std::index_sequence<0>, arg __x)
+ {
+ _SimdConverter<_From, _A, _To, typename __return_type::_First_abi> __native_cvt;
+ return {__native_cvt(__x)};
+ }
+ template <size_t... _Indexes> __return_type __impl(std::index_sequence<_Indexes...>, arg __x)
+ {
+ _SimdConverter<_From, _A, _To, typename __return_type::_First_abi> __native_cvt;
+ const auto &tmp = __native_cvt.__all(__x);
+ return {tmp[_Indexes]...};
+ }
+};
+
+// _SimdConverter fixed_size<_N> -> "native" {{{1
+// i.e. ? register to 1 registers
+template <typename _From, int _N, typename _To, typename _A>
+struct _SimdConverter<_From, simd_abi::fixed_size<_N>, _To, _A> {
+ using __traits = _SimdTraits<_To, _A>;
+ using __return_type = typename __traits::_SimdMember;
+ using arg = __fixed_size_storage_t<_From, _N>;
+ static_assert(_N == simd_size_v<_To, _A>,
+ "_SimdConverter to fixed_size only works for equal element counts");
+
+ _GLIBCXX_SIMD_INTRINSIC __return_type operator()(arg __x)
+ {
+ return __impl(std::make_index_sequence<arg::tuple_size>(), __x);
+ }
+
+private:
+ template <size_t... _Indexes> __return_type __impl(std::index_sequence<_Indexes...>, arg __x)
+ {
+ _SimdConverter<_From, typename arg::_First_abi, _To, _A> __native_cvt;
+ return __native_cvt(__get_tuple_at<_Indexes>(__x)...);
+ }
+};
+
+// }}}1
+_GLIBCXX_SIMD_END_NAMESPACE
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
+// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
diff --git a/libstdc++-v3/include/experimental/bits/simd_debug.h b/libstdc++-v3/include/experimental/bits/simd_debug.h
new file mode 100644
index 00000000000..d361848ea8c
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_debug.h
@@ -0,0 +1,215 @@
+// Debug utilities for use in the simd implementation -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_DEBUG_H_
+#define _GLIBCXX_EXPERIMENTAL_SIMD_DEBUG_H_
+
+#if defined _GLIBCXX_SIMD_DEBUG && !defined _GLIBCXX_SIMD_ENABLE_DEBUG
+#define _GLIBCXX_SIMD_ENABLE_DEBUG 1
+#endif
+
+#ifdef _GLIBCXX_SIMD_ENABLE_DEBUG
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#endif // _GLIBCXX_SIMD_ENABLE_DEBUG
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+enum class __area : unsigned {
+ __disabled = 0,
+ __enabled = 1,
+ __ = __enabled,
+
+#ifdef _GLIBCXX_SIMD_DEBUG
+
+#define _Sine 0x0000000000000001ull
+#define _Cosine 0x0000000000000002ull
+#define _SIMD_TUPLE 0x0000000000000004ull
+#define _Simd_view 0x0000000000000008ull
+#define _Logarithm 0x0000000000000010ull
+#define _Frexp 0x0000000000000020ull
+
+ __Sine = ((_GLIBCXX_SIMD_DEBUG) & _Sine) ? __enabled : __disabled,
+ __Cosine = ((_GLIBCXX_SIMD_DEBUG) & _Cosine) ? __enabled : __disabled,
+ __SIMD_TUPLE = ((_GLIBCXX_SIMD_DEBUG) & _SIMD_TUPLE) ? __enabled : __disabled,
+ __Simd_view = ((_GLIBCXX_SIMD_DEBUG) & _Simd_view ) ? __enabled : __disabled,
+ __Logarithm = ((_GLIBCXX_SIMD_DEBUG) & _Logarithm ) ? __enabled : __disabled,
+ __Frexp = ((_GLIBCXX_SIMD_DEBUG) & _Frexp ) ? __enabled : __disabled,
+#undef _Sine
+#undef _Cosine
+#undef _SIMD_TUPLE
+#undef _Simd_view
+#undef _Logarithm
+#undef _Frexp
+
+#undef _GLIBCXX_SIMD_DEBUG
+
+#else // _GLIBCXX_SIMD_DEBUG
+ __Sine = __disabled,
+ __Cosine = __disabled,
+ __SIMD_TUPLE = __disabled,
+ __Simd_view = __disabled,
+ __Logarithm = __disabled,
+ __Frexp = __disabled,
+#endif // _GLIBCXX_SIMD_DEBUG
+};
+
+#define _GLIBCXX_SIMD_DEBUG(_Area) \
+ std::experimental::__debug_stream<std::experimental::__area::_##_Area>( \
+ __PRETTY_FUNCTION__, __FILE__, __LINE__, std::experimental::__debug_instr_ptr())
+
+#ifdef _GLIBCXX_SIMD_ENABLE_DEBUG
+#define _GLIBCXX_SIMD_PRETTY_PRINT(var_) std::setw(16), #var_ " = ", (var_)
+
+#define _GLIBCXX_SIMD_CONCAT_IMPL(a_, b_, c_) a_##b_##c_
+#define _GLIBCXX_SIMD_CONCAT(a_, b_, c_) _GLIBCXX_SIMD_CONCAT_IMPL(a_, b_, c_)
+
+#define _GLIBCXX_SIMD_DEBUG_DEFERRED(_Area, ...) \
+ const auto &_GLIBCXX_SIMD_CONCAT(_GLIBCXX_SIMD_deferred_, __LINE__, _) = \
+ __defer([&]() { _GLIBCXX_SIMD_DEBUG(_Area) \
+ (__VA_ARGS__); });
+#else // _GLIBCXX_SIMD_ENABLE_DEBUG
+#define _GLIBCXX_SIMD_PRETTY_PRINT(var_) (var_)
+
+#define _GLIBCXX_SIMD_DEBUG_DEFERRED(_Area, ...)
+#endif // _GLIBCXX_SIMD_ENABLE_DEBUG
+
+_GLIBCXX_SIMD_ALWAYS_INLINE void *__debug_instr_ptr()
+{
+ void* __ip = nullptr;
+#if defined _GLIBCXX_SIMD_ENABLE_DEBUG
+#ifdef __x86_64__
+ asm volatile("lea 0(%%rip),%0" : "=r"(__ip));
+#elif defined __i386__
+ asm volatile("1: movl $1b,%0" : "=r"(__ip));
+#elif defined __arm__
+ asm volatile("mov %0,pc" : "=r"(__ip));
+#elif defined __aarch64__
+ asm volatile("adr %0,." : "=r"(__ip));
+#endif
+#endif //__GNUC__
+ return __ip;
+}
+
+template <__area> class __debug_stream;
+
+#ifdef _GLIBCXX_SIMD_ENABLE_DEBUG
+template <> class __debug_stream<__area::__enabled>
+{
+ std::stringstream __buffer;
+ int __color = 31;
+
+public:
+ __debug_stream(const char *__func, const char *__file, int __line, void *__instr_ptr)
+ {
+ __buffer << "\033[1;40;" << __color << "mDEBUG: " << __file << ':' << __line
+ << " @ " << __instr_ptr << "\n " << __func;
+ }
+
+ ~__debug_stream()
+ {
+ __buffer << "\033[0m\n";
+ std::cout << __buffer.str() << std::flush;
+ }
+
+ template <class... _Ts>
+ __debug_stream& operator()(const _Ts&... __args)
+ {
+ __color = __color > 37 ? 30 : __color + 1;
+ __buffer << "\n\033[1;40;" << __color << "m ";
+ [](const std::initializer_list<int>&) {}({(__print(__args, int()), 0)...});
+ return *this;
+ }
+
+private:
+ template <class _Tp, class = decltype(__buffer << std::declval<const _Tp&>())>
+ void __print(const _Tp& __x, int)
+ {
+ __buffer << ' ' << __x;
+ }
+
+ template <class _Tp,
+ class = decltype(__buffer << std::declval<const _Tp&>()[0])>
+ void __print(const _Tp& __x, float)
+ {
+ using _U = __remove_cvref_t<decltype(__x[0])>;
+ __buffer << " {" << +__x[0];
+ for (size_t __i = 1; __i < sizeof(_Tp) / sizeof(_U); ++__i)
+ {
+ __buffer << ", " << +__x[__i];
+ }
+ __buffer << '}';
+ }
+
+ static char hexChar(char __x) { return __x + (__x > 9 ? 87 : 48); }
+ template <class _Tp>
+ void __print(const _Tp& __x, ...)
+ {
+ __buffer.put(' ');
+ using _Bytes = char[sizeof(_Tp)];
+ auto&& __bytes = reinterpret_cast<const _Bytes&>(__x);
+ int __i = -1;
+ for (const unsigned char __b : __bytes)
+ {
+ if (++__i && (__i & 0x3) == 0)
+ {
+ __buffer.put('\'');
+ }
+ __buffer.put(hexChar(__b >> 4));
+ __buffer.put(hexChar(__b & 0xf));
+ }
+ }
+};
+#endif // _GLIBCXX_SIMD_ENABLE_DEBUGGING
+
+template <> class __debug_stream<__area::__disabled>
+{
+public:
+ __debug_stream(const char *, const char *, int, void *) {}
+ template <class... _Ts> const __debug_stream &operator()(_Ts &&...) const { return *this; }
+};
+
+template <class _F> class __defer_raii
+{
+public:
+ // construct the object from the given callable
+ template <class _FF> __defer_raii(_FF &&__f) : __cleanup_function(std::forward<_FF>(__f))
+ {
+ }
+
+ // when the object goes out of scope call the cleanup function
+ ~__defer_raii() { __cleanup_function(); }
+
+private:
+ _F __cleanup_function;
+};
+
+template <typename _F> __defer_raii<_F> __defer(_F &&__f) { return {std::forward<_F>(__f)}; }
+
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_DEBUG_H_
diff --git a/libstdc++-v3/include/experimental/bits/simd_detail.h b/libstdc++-v3/include/experimental/bits/simd_detail.h
new file mode 100644
index 00000000000..e603833e337
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_detail.h
@@ -0,0 +1,355 @@
+// Internal macros for the simd implementation -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_
+#define _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_
+
+#if __cplusplus >= 201703L
+
+#include <cstddef>
+#include <cstdint>
+
+#define _GLIBCXX_SIMD_BEGIN_NAMESPACE \
+ namespace std _GLIBCXX_VISIBILITY(default) \
+ { \
+ _GLIBCXX_BEGIN_NAMESPACE_VERSION namespace experimental \
+ { \
+ inline namespace parallelism_v2 \
+ {
+#define _GLIBCXX_SIMD_END_NAMESPACE \
+ } \
+ } \
+ _GLIBCXX_END_NAMESPACE_VERSION \
+ }
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+namespace simd_abi // {{{
+{
+// implementation details:
+struct __scalar_abi;
+template <int _N> struct __fixed_abi;
+
+template <int Bytes = 16> struct __sse_abi;
+template <int Bytes = 32> struct __avx_abi;
+template <int Bytes = 64> struct __avx512_abi;
+template <int Bytes = 16> struct __neon_abi;
+
+template <int _N, class Abi> struct __combine;
+
+// implementation-defined:
+template <int NRegisters> using __sse_x = __combine<NRegisters, __sse_abi<>>;
+template <int NRegisters> using __avx_x = __combine<NRegisters, __avx_abi<>>;
+template <int NRegisters> using __avx512_x = __combine<NRegisters, __avx512_abi<>>;
+template <int NRegisters> using __neon_x = __combine<NRegisters, __neon_abi<>>;
+
+template <class _Tp, int _N> using __sse_n = __sse_abi<sizeof(_Tp) * _N>;
+template <class _Tp, int _N> using __avx_n = __avx_abi<sizeof(_Tp) * _N>;
+template <class _Tp, int _N> using __avx512_n = __avx512_abi<sizeof(_Tp) * _N>;
+template <class _Tp, int _N> using __neon_n = __neon_abi<sizeof(_Tp) * _N>;
+
+using __sse = __sse_abi<>;
+using __avx = __avx_abi<>;
+using __avx512 = __avx512_abi<>;
+using __neon = __neon_abi<>;
+
+using __neon128 = __neon_abi<16>;
+using __neon64 = __neon_abi<8>;
+
+// standard:
+template <class _Tp, size_t _N, class... > struct deduce;
+template <int _N> using fixed_size = __fixed_abi<_N>;
+using scalar = __scalar_abi;
+} // namespace simd_abi }}}
+// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
+template <class _Tp> struct is_simd;
+template <class _Tp> struct is_simd_mask;
+template <class _Tp, class Abi> class simd;
+template <class _Tp, class Abi> class simd_mask;
+template <class _Tp, class Abi> struct simd_size;
+// }}}
+
+// On Windows (WIN32) we might see macros called min and max. Just undefine them and hope
+// noone (re)defines them (defining NOMINMAX should help).
+// {{{
+#ifdef WIN32
+#define NOMINMAX 1
+#if defined min
+#undef min
+#endif
+#if defined max
+#undef max
+#endif
+#endif // WIN32
+// }}}
+
+// ISA extension detection. The following defines all the _GLIBCXX_SIMD_HAVE_XXX macros
+// ARM{{{
+#ifdef __aarch64__
+#define _GLIBCXX_SIMD_IS_AARCH64 1
+#endif // __aarch64__
+
+#ifdef __ARM_NEON
+#define _GLIBCXX_SIMD_HAVE_NEON 1
+#define _GLIBCXX_SIMD_HAVE_NEON_ABI 1
+#define _GLIBCXX_SIMD_HAVE_FULL_NEON_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_NEON 0
+#define _GLIBCXX_SIMD_HAVE_NEON_ABI 0
+#define _GLIBCXX_SIMD_HAVE_FULL_NEON_ABI 0
+#endif // _GLIBCXX_SIMD_HAVE_NEON
+//}}}
+// x86{{{
+#ifdef __MMX__
+#define _GLIBCXX_SIMD_HAVE_MMX 1
+#else
+#define _GLIBCXX_SIMD_HAVE_MMX 0
+#endif
+#if defined __SSE__ || defined __x86_64__
+#define _GLIBCXX_SIMD_HAVE_SSE 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE 0
+#endif
+#if defined __SSE2__ || defined __x86_64__
+#define _GLIBCXX_SIMD_HAVE_SSE2 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE2 0
+#endif
+#ifdef __SSE3__
+#define _GLIBCXX_SIMD_HAVE_SSE3 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE3 0
+#endif
+#ifdef __SSSE3__
+#define _GLIBCXX_SIMD_HAVE_SSSE3 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSSE3 0
+#endif
+#ifdef __SSE4_1__
+#define _GLIBCXX_SIMD_HAVE_SSE4_1 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE4_1 0
+#endif
+#ifdef __SSE4_2__
+#define _GLIBCXX_SIMD_HAVE_SSE4_2 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE4_2 0
+#endif
+#ifdef __XOP__
+#define _GLIBCXX_SIMD_HAVE_XOP 1
+#else
+#define _GLIBCXX_SIMD_HAVE_XOP 0
+#endif
+#ifdef __AVX__
+#define _GLIBCXX_SIMD_HAVE_AVX 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX 0
+#endif
+#ifdef __AVX2__
+#define _GLIBCXX_SIMD_HAVE_AVX2 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX2 0
+#endif
+#ifdef __BMI__
+#define _GLIBCXX_SIMD_HAVE_BMI1 1
+#else
+#define _GLIBCXX_SIMD_HAVE_BMI1 0
+#endif
+#ifdef __BMI2__
+#define _GLIBCXX_SIMD_HAVE_BMI2 1
+#else
+#define _GLIBCXX_SIMD_HAVE_BMI2 0
+#endif
+#ifdef __LZCNT__
+#define _GLIBCXX_SIMD_HAVE_LZCNT 1
+#else
+#define _GLIBCXX_SIMD_HAVE_LZCNT 0
+#endif
+#ifdef __SSE4A__
+#define _GLIBCXX_SIMD_HAVE_SSE4A 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE4A 0
+#endif
+#ifdef __FMA__
+#define _GLIBCXX_SIMD_HAVE_FMA 1
+#else
+#define _GLIBCXX_SIMD_HAVE_FMA 0
+#endif
+#ifdef __FMA4__
+#define _GLIBCXX_SIMD_HAVE_FMA4 1
+#else
+#define _GLIBCXX_SIMD_HAVE_FMA4 0
+#endif
+#ifdef __F16C__
+#define _GLIBCXX_SIMD_HAVE_F16C 1
+#else
+#define _GLIBCXX_SIMD_HAVE_F16C 0
+#endif
+#ifdef __POPCNT__
+#define _GLIBCXX_SIMD_HAVE_POPCNT 1
+#else
+#define _GLIBCXX_SIMD_HAVE_POPCNT 0
+#endif
+#ifdef __AVX512F__
+#define _GLIBCXX_SIMD_HAVE_AVX512F 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX512F 0
+#endif
+#ifdef __AVX512DQ__
+#define _GLIBCXX_SIMD_HAVE_AVX512DQ 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX512DQ 0
+#endif
+#ifdef __AVX512VL__
+#define _GLIBCXX_SIMD_HAVE_AVX512VL 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX512VL 0
+#endif
+#ifdef __AVX512BW__
+#define _GLIBCXX_SIMD_HAVE_AVX512BW 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX512BW 0
+#endif
+
+#if _GLIBCXX_SIMD_HAVE_SSE
+#define _GLIBCXX_SIMD_HAVE_SSE_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_SSE_ABI 0
+#endif
+#if _GLIBCXX_SIMD_HAVE_SSE2
+#define _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI 0
+#endif
+
+#if _GLIBCXX_SIMD_HAVE_AVX
+#define _GLIBCXX_SIMD_HAVE_AVX_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX_ABI 0
+#endif
+#if _GLIBCXX_SIMD_HAVE_AVX2
+#define _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI 0
+#endif
+
+#if _GLIBCXX_SIMD_HAVE_AVX512F
+#define _GLIBCXX_SIMD_HAVE_AVX512_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_AVX512_ABI 0
+#endif
+#if _GLIBCXX_SIMD_HAVE_AVX512BW
+#define _GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI 1
+#else
+#define _GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI 0
+#endif
+
+#if defined __x86_64__ && !_GLIBCXX_SIMD_HAVE_SSE2
+#error "Use of SSE2 is required on AMD64"
+#endif
+//}}}
+
+#define _GLIBCXX_SIMD_NORMAL_MATH [[gnu::__optimize__("finite-math-only,no-signed-zeros")]]
+#define _GLIBCXX_SIMD_NEVER_INLINE [[gnu::__noinline__]]
+#define _GLIBCXX_SIMD_INTRINSIC [[gnu::__always_inline__, gnu::__artificial__]] inline
+#define _GLIBCXX_SIMD_CONST __attribute__((__const__))
+#define _GLIBCXX_SIMD_PURE __attribute__((__pure__))
+#define _GLIBCXX_SIMD_ALWAYS_INLINE [[gnu::__always_inline__]] inline
+#define _GLIBCXX_SIMD_IS_UNLIKELY(__x) __builtin_expect(__x, 0)
+#define _GLIBCXX_SIMD_IS_LIKELY(__x) __builtin_expect(__x, 1)
+
+#ifdef COMPILE_FOR_UNIT_TESTS
+#define _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST
+#else
+#define _GLIBCXX_SIMD_NOEXCEPT_OR_IN_TEST noexcept
+#endif
+
+#define _GLIBCXX_SIMD_LIST_BINARY(macro) macro(|) macro(&) macro(^)
+#define _GLIBCXX_SIMD_LIST_SHIFTS(macro) macro(<<) macro(>>)
+#define _GLIBCXX_SIMD_LIST_ARITHMETICS(macro) macro(+) macro(-) macro(*) macro(/) macro(%)
+
+#define _GLIBCXX_SIMD_ALL_BINARY(macro) _GLIBCXX_SIMD_LIST_BINARY(macro) static_assert(true)
+#define _GLIBCXX_SIMD_ALL_SHIFTS(macro) _GLIBCXX_SIMD_LIST_SHIFTS(macro) static_assert(true)
+#define _GLIBCXX_SIMD_ALL_ARITHMETICS(macro) _GLIBCXX_SIMD_LIST_ARITHMETICS(macro) static_assert(true)
+
+#ifdef _GLIBCXX_SIMD_NO_ALWAYS_INLINE
+#undef _GLIBCXX_SIMD_ALWAYS_INLINE
+#define _GLIBCXX_SIMD_ALWAYS_INLINE inline
+#undef _GLIBCXX_SIMD_INTRINSIC
+#define _GLIBCXX_SIMD_INTRINSIC inline
+#endif
+
+#if _GLIBCXX_SIMD_HAVE_SSE || _GLIBCXX_SIMD_HAVE_MMX
+#define _GLIBCXX_SIMD_X86INTRIN 1
+#else
+#define _GLIBCXX_SIMD_X86INTRIN 0
+#endif
+
+// workaround macros {{{
+// vector conversions on x86 not optimized:
+#if _GLIBCXX_SIMD_X86INTRIN
+#define _GLIBCXX_SIMD_WORKAROUND_PR85048 1
+#endif
+
+// zero extension from xmm to zmm not optimized:
+//#define _GLIBCXX_SIMD_WORKAROUND_PR85480 1
+
+// incorrect use of k0 register for _kortestc_mask64_u8 and _kortestc_mask32_u8:
+#define _GLIBCXX_SIMD_WORKAROUND_PR85538 1
+
+// missed optimization for __abs(__vector_type_t<_LLong, 2>):
+#define _GLIBCXX_SIMD_WORKAROUND_PR85572 1
+
+// very bad codegen for extraction and concatenation of 128/256 "subregisters" with
+// sizeof(element type) < 8: https://godbolt.org/g/mqUsgM
+#if _GLIBCXX_SIMD_X86INTRIN
+#define _GLIBCXX_SIMD_WORKAROUND_XXX_1 1
+#endif
+
+// bad codegen for 8 Byte memcpy to __vector_type_t<char, 16>
+#define _GLIBCXX_SIMD_WORKAROUND_XXX_2 1
+
+// bad codegen for zero-extend using simple concat(__x, 0)
+#if _GLIBCXX_SIMD_X86INTRIN
+#define _GLIBCXX_SIMD_WORKAROUND_XXX_3 1
+#endif
+
+// bad codegen for integer division
+#define _GLIBCXX_SIMD_WORKAROUND_XXX_4 1
+
+// https://github.com/cplusplus/parallelism-ts/issues/65 (incorrect return type of
+// static_simd_cast)
+#define _GLIBCXX_SIMD_FIX_P2TS_ISSUE65 1
+
+// https://github.com/cplusplus/parallelism-ts/issues/66 (incorrect SFINAE constraint on
+// (static)_simd_cast)
+#define _GLIBCXX_SIMD_FIX_P2TS_ISSUE66 1
+// }}}
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/include/experimental/bits/simd_math.h b/libstdc++-v3/include/experimental/bits/simd_math.h
new file mode 100644
index 00000000000..fa781ea9192
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_math.h
@@ -0,0 +1,1402 @@
+// Math overloads for simd -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H_
+#define _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H_
+
+#if __cplusplus >= 201703L
+
+#include "simd_abis.h"
+#include <utility>
+#include <iomanip>
+
+static_assert(std::is_same_v<bool, decltype(std::isnan(double()))>);
+static_assert(std::is_same_v<bool, decltype(std::isinf(double()))>);
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+template <class _Tp, class _V> using __samesize = fixed_size_simd<_Tp, _V::size()>;
+// __math_return_type {{{
+template <class _DoubleR, class _Tp, class _Abi> struct __math_return_type;
+template <class _DoubleR, class _Tp, class _Abi>
+using __math_return_type_t = typename __math_return_type<_DoubleR, _Tp, _Abi>::type;
+
+template <class _Tp, class _Abi> struct __math_return_type<double, _Tp, _Abi> {
+ using type = std::experimental::simd<_Tp, _Abi>;
+};
+template <class _Tp, class _Abi> struct __math_return_type<bool, _Tp, _Abi> {
+ using type = std::experimental::simd_mask<_Tp, _Abi>;
+};
+template <class _DoubleR, class _Tp, class _Abi> struct __math_return_type {
+ using type = std::experimental::fixed_size_simd<_DoubleR, simd_size_v<_Tp, _Abi>>;
+};
+//}}}
+// TODO: rely on __simd_math_fallback to get rid of the SFINAE magic here:
+// _GLIBCXX_SIMD_MATH_CALL_ {{{
+#define _GLIBCXX_SIMD_MATH_CALL_(__name) \
+ template <class _Tp, class _Abi, class..., \
+ class _R = std::experimental::__math_return_type_t< \
+ decltype(std::__name(std::declval<double>())), _Tp, _Abi>> \
+ enable_if_t<std::is_floating_point_v<_Tp>, _R> __name( \
+ std::experimental::simd<_Tp, _Abi> __x) \
+ { \
+ using _V = std::experimental::simd<_Tp, _Abi>; \
+ return std::experimental::__impl_or_fallback( \
+ [](const auto& __xx) \
+ -> decltype( \
+ _R(std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__xx)>::__##__name( \
+ std::experimental::__data(__xx)))) { \
+ return {std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__xx)>::__##__name( \
+ std::experimental::__data(__xx))}; \
+ }, \
+ [](const _V& __xx) { \
+ if constexpr (std::experimental::is_simd_mask_v<_R>) { \
+ return _R(std::experimental::__private_init, \
+ [&](auto __i) { return std::__name(__xx[__i]); }); \
+ } else { \
+ return _R([&](auto __i) { return std::__name(__xx[__i]); }); \
+ } \
+ }, \
+ __x); \
+ }
+
+// }}}
+//__extra_argument_type{{{
+template <class _U, class _Tp, class _Abi> struct __extra_argument_type;
+
+template <class _Tp, class _Abi> struct __extra_argument_type<_Tp *, _Tp, _Abi> {
+ using type = std::experimental::simd<_Tp, _Abi> *;
+ static constexpr double *declval();
+ _GLIBCXX_SIMD_INTRINSIC static constexpr auto __data(type __x) { return &std::experimental::__data(*__x); }
+ static constexpr bool __needs_temporary_scalar = true;
+};
+template <class _U, class _Tp, class _Abi> struct __extra_argument_type<_U *, _Tp, _Abi> {
+ static_assert(std::is_integral_v<_U>);
+ using type = std::experimental::fixed_size_simd<_U, std::experimental::simd_size_v<_Tp, _Abi>> *;
+ static constexpr _U *declval();
+ _GLIBCXX_SIMD_INTRINSIC static constexpr auto __data(type __x) { return &std::experimental::__data(*__x); }
+ static constexpr bool __needs_temporary_scalar = true;
+};
+template <class _Tp, class _Abi> struct __extra_argument_type<_Tp, _Tp, _Abi> {
+ using type = std::experimental::simd<_Tp, _Abi>;
+ static constexpr double declval();
+ _GLIBCXX_SIMD_INTRINSIC static constexpr decltype(auto) __data(const type &__x)
+ {
+ return std::experimental::__data(__x);
+ }
+ static constexpr bool __needs_temporary_scalar = false;
+};
+template <class _U, class _Tp, class _Abi> struct __extra_argument_type {
+ static_assert(std::is_integral_v<_U>);
+ using type = std::experimental::fixed_size_simd<_U, std::experimental::simd_size_v<_Tp, _Abi>>;
+ static constexpr _U declval();
+ _GLIBCXX_SIMD_INTRINSIC static constexpr decltype(auto) __data(const type &__x)
+ {
+ return std::experimental::__data(__x);
+ }
+ static constexpr bool __needs_temporary_scalar = false;
+};
+//}}}
+// _GLIBCXX_SIMD_MATH_CALL2_ {{{
+#define _GLIBCXX_SIMD_MATH_CALL2_(__name, arg2_) \
+ template < \
+ class _Tp, class _Abi, class..., \
+ class _Arg2 = std::experimental::__extra_argument_type<arg2_, _Tp, _Abi>, \
+ class _R = std::experimental::__math_return_type_t< \
+ decltype(std::__name(std::declval<double>(), _Arg2::declval())), _Tp, _Abi>> \
+ enable_if_t<std::is_floating_point_v<_Tp>, _R> __name( \
+ const std::experimental::simd<_Tp, _Abi>& __xx, const typename _Arg2::type& __yy) \
+ { \
+ using _V = std::experimental::simd<_Tp, _Abi>; \
+ return std::experimental::__impl_or_fallback( \
+ [](const auto& __x, const auto& __y) \
+ -> decltype( \
+ _R(std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__x)>::__##__name( \
+ std::experimental::__data(__x), _Arg2::__data(__y)))) { \
+ return {std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__x)>::__##__name( \
+ std::experimental::__data(__x), _Arg2::__data(__y))}; \
+ }, \
+ [](const _V& __x, const auto& __y) { \
+ auto&& gen = [&](auto __i) { \
+ if constexpr (_Arg2::__needs_temporary_scalar) { \
+ const auto& yy = *__y; \
+ auto __tmp = yy[__i]; \
+ auto __ret = std::__name(__x[__i], &__tmp); \
+ (*__y)[__i] = __tmp; \
+ return __ret; \
+ } else { \
+ return std::__name(__x[__i], __y[__i]); \
+ } \
+ }; \
+ if constexpr (std::experimental::is_simd_mask_v<_R>) { \
+ return _R(std::experimental::__private_init, gen); \
+ } else { \
+ return _R(gen); \
+ } \
+ }, \
+ __xx, __yy); \
+ } \
+ template <class _U, class _Tp, class _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC std::experimental::__math_return_type_t< \
+ decltype(std::__name( \
+ std::declval<double>(), \
+ std::declval<enable_if_t< \
+ std::conjunction_v< \
+ std::is_same<arg2_, _Tp>, \
+ std::negation<std::is_same<__remove_cvref_t<_U>, \
+ std::experimental::simd<_Tp, _Abi>>>, \
+ std::is_convertible<_U, std::experimental::simd<_Tp, _Abi>>, \
+ std::is_floating_point<_Tp>>, \
+ double>>())), \
+ _Tp, _Abi> \
+ __name(_U&& __xx, const std::experimental::simd<_Tp, _Abi>& __yy) \
+ { \
+ return std::experimental::__name( \
+ std::experimental::simd<_Tp, _Abi>(std::forward<_U>(__xx)), __yy); \
+ }
+
+// }}}
+// _GLIBCXX_SIMD_MATH_CALL3_ {{{
+#define _GLIBCXX_SIMD_MATH_CALL3_(__name, arg2_, arg3_) \
+ template <class _Tp, class _Abi, class..., \
+ class _Arg2 = std::experimental::__extra_argument_type<arg2_, _Tp, _Abi>, \
+ class _Arg3 = std::experimental::__extra_argument_type<arg3_, _Tp, _Abi>, \
+ class _R = std::experimental::__math_return_type_t< \
+ decltype(std::__name(std::declval<double>(), _Arg2::declval(), \
+ _Arg3::declval())), \
+ _Tp, _Abi>> \
+ enable_if_t<std::is_floating_point_v<_Tp>, _R> __name( \
+ std::experimental::simd<_Tp, _Abi> __xx, typename _Arg2::type __yy, \
+ typename _Arg3::type __zz) \
+ { \
+ using _V = std::experimental::simd<_Tp, _Abi>; \
+ return std::experimental::__impl_or_fallback( \
+ [](const auto& __x, const auto& __y, const auto& __z) \
+ -> decltype( \
+ _R(std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__x)>::__##__name( \
+ std::experimental::__data(__x), _Arg2::__data(__y), \
+ _Arg3::__data(__z)))) { \
+ return {std::experimental::__private_init, \
+ std::experimental::__get_impl_t<decltype(__x)>::__##__name( \
+ std::experimental::__data(__x), _Arg2::__data(__y), \
+ _Arg3::__data(__z))}; \
+ }, \
+ [](const _V& __x, const auto& __y, const auto& __z) { \
+ return _R([&](auto __i) { \
+ if constexpr (_Arg3::__needs_temporary_scalar) { \
+ const auto& __ztmp = *__z; \
+ auto __tmp = __ztmp[__i]; \
+ auto __ret = std::__name(__x[__i], __y[__i], &__tmp); \
+ (*__z)[__i] = __tmp; \
+ return __ret; \
+ } else { \
+ return std::__name(__x[__i], __y[__i], __z[__i]); \
+ } \
+ }); \
+ }, \
+ __xx, __yy, __zz); \
+ } \
+ template <class _Tp, class _U, class _V, class..., class _TT = __remove_cvref_t<_Tp>,\
+ class _UU = __remove_cvref_t<_U>, class _VV = __remove_cvref_t<_V>, \
+ class _Simd = \
+ std::conditional_t<std::experimental::is_simd_v<_UU>, _UU, _VV>> \
+ _GLIBCXX_SIMD_INTRINSIC decltype( \
+ std::experimental::__name(_Simd(std::declval<_Tp>()), _Simd(std::declval<_U>()), \
+ _Simd(std::declval<_V>()))) \
+ __name(_Tp&& __xx, _U&& __yy, _V&& __zz) \
+ { \
+ return std::experimental::__name(_Simd(std::forward<_Tp>(__xx)), \
+ _Simd(std::forward<_U>(__yy)), \
+ _Simd(std::forward<_V>(__zz))); \
+ }
+
+// }}}
+// __cosSeries {{{
+template <typename _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE static simd<float, _Abi>
+ __cosSeries(const simd<float, _Abi>& __x)
+{
+ const simd<float, _Abi> __x2 = __x * __x;
+ simd<float, _Abi> __y;
+ __y = 0x1.ap-16f; // 1/8!
+ __y = __y * __x2 - 0x1.6c1p-10f; // -1/6!
+ __y = __y * __x2 + 0x1.555556p-5f; // 1/4!
+ return __y * (__x2 * __x2) - .5f * __x2 + 1.f;
+}
+template <typename _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE static simd<double, _Abi>
+ __cosSeries(const simd<double, _Abi>& __x)
+{
+ const simd<double, _Abi> __x2 = __x * __x;
+ simd<double, _Abi> __y;
+ __y = 0x1.AC00000000000p-45; // 1/16!
+ __y = __y * __x2 - 0x1.9394000000000p-37; // -1/14!
+ __y = __y * __x2 + 0x1.1EED8C0000000p-29; // 1/12!
+ __y = __y * __x2 - 0x1.27E4FB7400000p-22; // -1/10!
+ __y = __y * __x2 + 0x1.A01A01A018000p-16; // 1/8!
+ __y = __y * __x2 - 0x1.6C16C16C16C00p-10; // -1/6!
+ __y = __y * __x2 + 0x1.5555555555554p-5; // 1/4!
+ return (__y * __x2 - .5f) * __x2 + 1.f;
+}
+
+// }}}
+// __sinSeries {{{
+template <typename _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE static simd<float, _Abi>
+ __sinSeries(const simd<float, _Abi>& __x)
+{
+ const simd<float, _Abi> __x2 = __x * __x;
+ simd<float, _Abi> __y;
+ __y = -0x1.9CC000p-13f; // -1/7!
+ __y = __y * __x2 + 0x1.111100p-7f; // 1/5!
+ __y = __y * __x2 - 0x1.555556p-3f; // -1/3!
+ return __y * (__x2 * __x) + __x;
+}
+
+template <typename _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE static simd<double, _Abi>
+ __sinSeries(const simd<double, _Abi>& __x)
+{
+ // __x = [0, 0.7854 = pi/4]
+ // __x² = [0, 0.6169 = pi²/8]
+ const simd<double, _Abi> __x2 = __x * __x;
+ simd<double, _Abi> __y;
+ __y = -0x1.ACF0000000000p-41; // -1/15!
+ __y = __y * __x2 + 0x1.6124400000000p-33; // 1/13!
+ __y = __y * __x2 - 0x1.AE64567000000p-26; // -1/11!
+ __y = __y * __x2 + 0x1.71DE3A5540000p-19; // 1/9!
+ __y = __y * __x2 - 0x1.A01A01A01A000p-13; // -1/7!
+ __y = __y * __x2 + 0x1.1111111111110p-7; // 1/5!
+ __y = __y * __x2 - 0x1.5555555555555p-3; // -1/3!
+ return __y * (__x2 * __x) + __x;
+}
+
+// }}}
+// __bit_cast {{{
+template <typename _To, typename _From>
+_GLIBCXX_SIMD_INTRINSIC _To __bit_cast(const _From __x)
+{
+ static_assert(sizeof(_To) == sizeof(_From));
+ _To __r;
+ std::memcpy(&__r, &__x, sizeof(_To));
+ return __r;
+}
+
+// }}}
+// __zero_low_bits {{{
+template <int _Bits, typename _Tp, typename _Abi>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> __zero_low_bits(simd<_Tp, _Abi> __x)
+{
+ const simd<_Tp, _Abi> __bitmask =
+ __bit_cast<_Tp>(~std::make_unsigned_t<__int_for_sizeof_t<_Tp>>() << _Bits);
+ return {__private_init, __get_impl_t<simd<_Tp, _Abi>>::bit_and(
+ __data(__x), __data(__bitmask))};
+}
+
+// }}}
+// __fold_input {{{
+
+/**\internal
+ * Fold \p x into [-¼π, ¼π] and remember the quadrant it came from:
+ * quadrant 0: [-¼π, ¼π]
+ * quadrant 1: [ ¼π, ¾π]
+ * quadrant 2: [ ¾π, 1¼π]
+ * quadrant 3: [1¼π, 1¾π]
+ *
+ * The algorithm determines `y` as the multiple `x - y * ¼π = [-¼π, ¼π]`. Using a bitmask,
+ * `y` is reduced to `quadrant`. `y` can be calculated as
+ * ```
+ * y = trunc(x / ¼π);
+ * y += fmod(y, 2);
+ * ```
+ * This can be simplified by moving the (implicit) division by 2 into the truncation
+ * expression. The `+= fmod` effect can the be achieved by using rounding instead of
+ * truncation:
+ * `y = round(x / ½π) * 2`.
+ * If precision allows, `2/π * x` is better (faster).
+ */
+template <class _Tp, class _Abi>
+struct __folded
+{
+ simd<_Tp, _Abi> _M_x;
+ rebind_simd_t<int, simd<_Tp, _Abi>> _M_quadrant;
+};
+
+namespace __math_float
+{
+inline constexpr float __pi_over_4 = 0x1.921FB6p-1f; // π/4
+inline constexpr float __2_over_pi = 0x1.45F306p-1f; // 2/π
+inline constexpr float __pi_2_5bits0 =
+ 0x1.921fc0p0f; // π/2, 5 0-bits (least significant)
+inline constexpr float __pi_2_5bits0_rem =
+ -0x1.5777a6p-21f; // π/2 - __pi_2_5bits0
+}
+namespace __math_double
+{
+inline constexpr double __pi_over_4 = 0x1.921fb54442d18p-1; // π/4
+inline constexpr double __2_over_pi = 0x1.45F306DC9C883p-1; // 2/π
+inline constexpr double __pi_2 = 0x1.921fb54442d18p0; // π/2
+}
+
+template <class _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE __folded<float, _Abi>
+ __fold_input(const simd<float, _Abi>& __x)
+{
+ using _V = simd<float, _Abi>;
+ using _IV = rebind_simd_t<int, _V>;
+ using namespace __math_float;
+ __folded<float, _Abi> __r;
+ __r._M_x = abs(__x);
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(all_of(__r._M_x < __pi_over_4)))
+ {
+ __r._M_quadrant = 0;
+ }
+ else if (_GLIBCXX_SIMD_IS_LIKELY(all_of(__r._M_x < 33 * __pi_over_4)))
+ {
+ const _V __y = round(__r._M_x * __2_over_pi);
+ __r._M_quadrant = static_simd_cast<_IV>(__y) & 3; // __y mod 4
+ __r._M_x -= __y * __pi_2_5bits0;
+ __r._M_x -= __y * __pi_2_5bits0_rem;
+ }
+ else
+ {
+ using __math_double::__2_over_pi;
+ using __math_double::__pi_2;
+ using _VD = rebind_simd_t<double, _V>;
+ _VD __xd = static_simd_cast<_VD>(__r._M_x);
+ _VD __y = round(__xd * __2_over_pi);
+ __r._M_quadrant = static_simd_cast<_IV>(__y) & 3; // = __y mod 4
+ __r._M_x = static_simd_cast<_V>(__xd - __y * __pi_2);
+ }
+ return __r;
+}
+
+template <typename _Abi>
+_GLIBCXX_SIMD_ALWAYS_INLINE __folded<double, _Abi>
+ __fold_input(const simd<double, _Abi>& __x)
+{
+ using _V = simd<double, _Abi>;
+ using _IV = rebind_simd_t<int, _V>;
+ using namespace __math_double;
+
+ __folded<double, _Abi> __r;
+ __r._M_x = abs(__x);
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(all_of(__r._M_x < __pi_over_4))) {
+ __r._M_quadrant = 0;
+ return __r;
+ }
+ const _V __y = round(__r._M_x / (2 * __pi_over_4));
+ __r._M_quadrant = static_simd_cast<_IV>(__y) & 3;
+
+ if (_GLIBCXX_SIMD_IS_LIKELY(all_of(__r._M_x < 1025 * __pi_over_4)))
+ {
+ // x - y * pi/2, y uses no more than 11 mantissa bits
+ __r._M_x -= __y * 0x1.921FB54443000p0;
+ __r._M_x -= __y * -0x1.73DCB3B39A000p-43;
+ __r._M_x -= __y * 0x1.45C06E0E68948p-86;
+ }
+ else if (_GLIBCXX_SIMD_IS_LIKELY(all_of(__y <= 0x1.0p30)))
+ {
+ // x - y * pi/2, y uses no more than 29 mantissa bits
+ __r._M_x -= __y * 0x1.921FB40000000p0;
+ __r._M_x -= __y * 0x1.4442D00000000p-24;
+ __r._M_x -= __y * 0x1.8469898CC5170p-48;
+ }
+ else
+ {
+ // x - y * pi/2, y may require all mantissa bits
+ const _V __y_hi = __zero_low_bits<26>(__y);
+ const _V __y_lo = __y - __y_hi;
+ const auto __pi_2_1 = 0x1.921FB50000000p0;
+ const auto __pi_2_2 = 0x1.110B460000000p-26;
+ const auto __pi_2_3 = 0x1.1A62630000000p-54;
+ const auto __pi_2_4 = 0x1.8A2E03707344Ap-81;
+ __r._M_x = __r._M_x
+ - __y_hi * __pi_2_1
+ - max(__y_hi * __pi_2_2, __y_lo * __pi_2_1)
+ - min(__y_hi * __pi_2_2, __y_lo * __pi_2_1)
+ - max(__y_hi * __pi_2_3, __y_lo * __pi_2_2)
+ - min(__y_hi * __pi_2_3, __y_lo * __pi_2_2)
+ - max(__y * __pi_2_4, __y_lo * __pi_2_3)
+ - min(__y * __pi_2_4, __y_lo * __pi_2_3);
+ }
+ return __r;
+}
+
+// }}}
+// __extract_exponent_bits {{{
+template <class _Abi>
+rebind_simd_t<int, simd<float, _Abi>> __extract_exponent_bits(const simd<float, _Abi> &__v)
+{
+ using namespace std::experimental::__proposed;
+ using namespace std::experimental::__proposed::float_bitwise_operators;
+ constexpr simd<float, _Abi> __exponent_mask =
+ std::numeric_limits<float>::infinity(); // 0x7f800000
+ return simd_reinterpret_cast<rebind_simd_t<int, simd<float, _Abi>>>(__v & __exponent_mask);
+}
+
+template <class _Abi>
+rebind_simd_t<int, simd<double, _Abi>> __extract_exponent_bits(const simd<double, _Abi> &__v)
+{
+ using namespace std::experimental::__proposed;
+ using namespace std::experimental::__proposed::float_bitwise_operators;
+ const simd<double, _Abi> __exponent_mask =
+ std::numeric_limits<double>::infinity(); // 0x7ff0000000000000
+ constexpr auto _N = simd_size_v<double, _Abi> * 2;
+ constexpr auto _Max = simd_abi::max_fixed_size<int>;
+ if constexpr (_N > _Max) {
+ const auto tup = split<_Max / 2, (_N - _Max) / 2>(__v & __exponent_mask);
+ return concat(
+ shuffle<strided<2, 1>>(
+ simd_reinterpret_cast<simd<int, simd_abi::deduce_t<int, _Max>>>(
+ std::get<0>(tup))),
+ shuffle<strided<2, 1>>(
+ simd_reinterpret_cast<simd<int, simd_abi::deduce_t<int, _N - _Max>>>(
+ std::get<1>(tup))));
+ } else {
+ return shuffle<strided<2, 1>>(
+ simd_reinterpret_cast<simd<int, simd_abi::deduce_t<int, _N>>>(__v &
+ __exponent_mask));
+ }
+}
+
+// }}}
+// __impl_or_fallback {{{
+template <class ImplFun, class FallbackFun, class... _Args>
+_GLIBCXX_SIMD_INTRINSIC auto __impl_or_fallback_dispatch(int, ImplFun&& __impl_fun,
+ FallbackFun&&, _Args&&... __args)
+ -> decltype(__impl_fun(std::forward<_Args>(__args)...))
+{
+ return __impl_fun(std::forward<_Args>(__args)...);
+}
+
+template <class ImplFun, class FallbackFun, class... _Args>
+inline auto __impl_or_fallback_dispatch(float, ImplFun&&, FallbackFun&& __fallback_fun,
+ _Args&&... __args)
+ -> decltype(__fallback_fun(std::forward<_Args>(__args)...))
+{
+ return __fallback_fun(std::forward<_Args>(__args)...);
+}
+
+template <class... _Args> _GLIBCXX_SIMD_INTRINSIC auto __impl_or_fallback(_Args&&... __args)
+{
+ return __impl_or_fallback_dispatch(int(), std::forward<_Args>(__args)...);
+} //}}}
+
+// trigonometric functions {{{
+_GLIBCXX_SIMD_MATH_CALL_(acos)
+_GLIBCXX_SIMD_MATH_CALL_(asin)
+_GLIBCXX_SIMD_MATH_CALL_(atan)
+_GLIBCXX_SIMD_MATH_CALL2_(atan2, _Tp)
+
+/*
+ * algorithm for sine and cosine:
+ *
+ * The result can be calculated with sine or cosine depending on the π/4 section the input
+ * is in.
+ * sine ≈ __x + __x³
+ * cosine ≈ 1 - __x²
+ *
+ * sine:
+ * Map -__x to __x and invert the output
+ * Extend precision of __x - n * π/4 by calculating
+ * ((__x - n * p1) - n * p2) - n * p3 (p1 + p2 + p3 = π/4)
+ *
+ * Calculate Taylor series with tuned coefficients.
+ * Fix sign.
+ */
+//cos{{{
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>>
+ cos(const simd<_Tp, _Abi>& __x)
+{
+ using _V = simd<_Tp, _Abi>;
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>() ||
+ __is_fixed_size_abi_v<_Abi>)
+ {
+ return {__private_init, __get_impl_t<_V>::__cos(__data(__x))};
+ }
+ else
+ {
+ if constexpr (is_same_v<_Tp, float>)
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(any_of(abs(__x) >= 393382)))
+ return static_simd_cast<_V>(
+ cos(static_simd_cast<rebind_simd_t<double, _V>>(__x)));
+
+ const auto __f = __fold_input(__x);
+ // quadrant | effect
+ // 0 | cosSeries, +
+ // 1 | sinSeries, -
+ // 2 | cosSeries, -
+ // 3 | sinSeries, +
+ using namespace std::experimental::__proposed::float_bitwise_operators;
+ const _V __sign_flip =
+ _V(-0.f) & static_simd_cast<_V>((1 + __f._M_quadrant) << 30);
+
+ const auto __need_cos = (__f._M_quadrant & 1) == 0;
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(all_of(__need_cos)))
+ {
+ return __sign_flip ^ __cosSeries(__f._M_x);
+ }
+ else if (_GLIBCXX_SIMD_IS_UNLIKELY(none_of(__need_cos)))
+ {
+ return __sign_flip ^ __sinSeries(__f._M_x);
+ }
+ else // some_of(__need_cos)
+ {
+ _V __r = __sinSeries(__f._M_x);
+ where(__need_cos.__cvt(), __r) = __cosSeries(__f._M_x);
+ return __r ^ __sign_flip;
+ }
+ }
+}
+
+template <class _Tp>
+_GLIBCXX_SIMD_ALWAYS_INLINE
+ enable_if_t<std::is_floating_point<_Tp>::value, simd<_Tp, simd_abi::scalar>>
+ cos(simd<_Tp, simd_abi::scalar> __x)
+{
+ return std::cos(__data(__x));
+}
+//}}}
+//sin{{{
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>>
+ sin(const simd<_Tp, _Abi>& __x)
+{
+ using _V = simd<_Tp, _Abi>;
+ if constexpr (__is_abi<_Abi, simd_abi::scalar>() ||
+ __is_fixed_size_abi_v<_Abi>)
+ {
+ return {__private_init, __get_impl_t<_V>::__sin(__data(__x))};
+ }
+ else
+ {
+ if constexpr (is_same_v<_Tp, float>)
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(any_of(abs(__x) >= 527449)))
+ return static_simd_cast<_V>(
+ sin(static_simd_cast<rebind_simd_t<double, _V>>(__x)));
+
+ const auto __f = __fold_input(__x);
+ // quadrant | effect
+ // 0 | sinSeries
+ // 1 | cosSeries
+ // 2 | sinSeries, sign flip
+ // 3 | cosSeries, sign flip
+ using namespace std::experimental::__proposed::float_bitwise_operators;
+ const auto __sign_flip =
+ (__x ^ static_simd_cast<_V>(1 - __f._M_quadrant)) & _V(_Tp(-0.));
+
+ const auto __need_sin = (__f._M_quadrant & 1) == 0;
+ if (_GLIBCXX_SIMD_IS_UNLIKELY(all_of(__need_sin)))
+ {
+ return __sign_flip ^ __sinSeries(__f._M_x);
+ }
+ else if (_GLIBCXX_SIMD_IS_UNLIKELY(none_of(__need_sin)))
+ {
+ return __sign_flip ^ __cosSeries(__f._M_x);
+ }
+ else // some_of(__need_sin)
+ {
+ _V __r = __cosSeries(__f._M_x);
+ where(__need_sin.__cvt(), __r) = __sinSeries(__f._M_x);
+ return __sign_flip ^ __r;
+ }
+ }
+}
+
+template <class _Tp>
+_GLIBCXX_SIMD_ALWAYS_INLINE
+ enable_if_t<std::is_floating_point<_Tp>::value, simd<_Tp, simd_abi::scalar>>
+ sin(simd<_Tp, simd_abi::scalar> __x)
+{
+ return std::sin(__data(__x));
+}
+//}}}
+
+_GLIBCXX_SIMD_MATH_CALL_(tan)
+_GLIBCXX_SIMD_MATH_CALL_(acosh)
+_GLIBCXX_SIMD_MATH_CALL_(asinh)
+_GLIBCXX_SIMD_MATH_CALL_(atanh)
+_GLIBCXX_SIMD_MATH_CALL_(cosh)
+_GLIBCXX_SIMD_MATH_CALL_(sinh)
+_GLIBCXX_SIMD_MATH_CALL_(tanh)
+// }}}
+// exponential functions {{{
+_GLIBCXX_SIMD_MATH_CALL_(exp)
+_GLIBCXX_SIMD_MATH_CALL_(exp2)
+_GLIBCXX_SIMD_MATH_CALL_(expm1)
+// }}}
+// frexp {{{
+template <class _Tp, size_t _N> _SimdWrapper<_Tp, _N> __getexp(_SimdWrapper<_Tp, _N> __x)
+{
+ if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _N>()) {
+ return _mm_getexp_ps(__x);
+ } else if constexpr (__have_avx512f && __is_sse_ps<_Tp, _N>()) {
+ return __lo128(_mm512_getexp_ps(__auto_bitcast(__x)));
+ } else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _N>()) {
+ return _mm_getexp_pd(__x);
+ } else if constexpr (__have_avx512f && __is_sse_pd<_Tp, _N>()) {
+ return __lo128(_mm512_getexp_pd(__auto_bitcast(__x)));
+ } else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _N>()) {
+ return _mm256_getexp_ps(__x);
+ } else if constexpr (__have_avx512f && __is_avx_ps<_Tp, _N>()) {
+ return __lo256(_mm512_getexp_ps(__auto_bitcast(__x)));
+ } else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _N>()) {
+ return _mm256_getexp_pd(__x);
+ } else if constexpr (__have_avx512f && __is_avx_pd<_Tp, _N>()) {
+ return __lo256(_mm512_getexp_pd(__auto_bitcast(__x)));
+ } else if constexpr (__is_avx512_ps<_Tp, _N>()) {
+ return _mm512_getexp_ps(__x);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ return _mm512_getexp_pd(__x);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+
+#if _GLIBCXX_SIMD_X86INTRIN
+template <class _Tp, size_t _N> _SimdWrapper<_Tp, _N> __getmant_avx512(_SimdWrapper<_Tp, _N> __x)
+{
+ if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _N>()) {
+ return _mm_getmant_ps(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else if constexpr (__have_avx512f && __is_sse_ps<_Tp, _N>()) {
+ return __lo128(
+ _mm512_getmant_ps(__auto_bitcast(__x), _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src));
+ } else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _N>()) {
+ return _mm_getmant_pd(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else if constexpr (__have_avx512f && __is_sse_pd<_Tp, _N>()) {
+ return __lo128(
+ _mm512_getmant_pd(__auto_bitcast(__x), _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src));
+ } else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _N>()) {
+ return _mm256_getmant_ps(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else if constexpr (__have_avx512f && __is_avx_ps<_Tp, _N>()) {
+ return __lo256(
+ _mm512_getmant_ps(__auto_bitcast(__x), _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src));
+ } else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _N>()) {
+ return _mm256_getmant_pd(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else if constexpr (__have_avx512f && __is_avx_pd<_Tp, _N>()) {
+ return __lo256(
+ _mm512_getmant_pd(__auto_bitcast(__x), _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src));
+ } else if constexpr (__is_avx512_ps<_Tp, _N>()) {
+ return _mm512_getmant_ps(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else if constexpr (__is_avx512_pd<_Tp, _N>()) {
+ return _mm512_getmant_pd(__x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+}
+#endif // _GLIBCXX_SIMD_X86INTRIN
+
+/**
+ * splits \p __v into exponent and mantissa, the sign is kept with the mantissa
+ *
+ * The return value will be in the range [0.5, 1.0[
+ * The \p __e value will be an integer defining the power-of-two exponent
+ */
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> frexp(
+ const simd<_Tp, _Abi> &__x, __samesize<int, simd<_Tp, _Abi>> *__exp)
+{
+ if constexpr (simd_size_v<_Tp, _Abi> == 1) {
+ int __tmp;
+ const auto __r = std::frexp(__x[0], &__tmp);
+ (*__exp)[0] = __tmp;
+ return __r;
+ } else if constexpr (__is_fixed_size_abi_v<_Abi>) {
+ return {__private_init, __get_impl_t<simd<_Tp, _Abi>>::__frexp(__data(__x), __data(*__exp))};
+#if _GLIBCXX_SIMD_X86INTRIN
+ } else if constexpr (__have_avx512f) {
+ using _IV = __samesize<int, simd<_Tp, _Abi>>;
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ constexpr size_t NI = _N < 4 ? 4 : _N;
+ const auto __v = __data(__x);
+ const auto isnonzero = __get_impl_t<simd<_Tp, _Abi>>::isnonzerovalue_mask(__v._M_data);
+ const auto __e =
+ __to_intrin(__blend(isnonzero, __vector_type_t<int, NI>(),
+ 1 + __convert<_SimdWrapper<int, NI>>(__getexp(__v))._M_data));
+ _GLIBCXX_SIMD_DEBUG(_Frexp)
+ (std::hex, _GLIBCXX_SIMD_PRETTY_PRINT(int(isnonzero)), std::dec,
+ _GLIBCXX_SIMD_PRETTY_PRINT(__e), _GLIBCXX_SIMD_PRETTY_PRINT(__getexp(__v)),
+ _GLIBCXX_SIMD_PRETTY_PRINT(
+ __to_intrin(1 + __convert<_SimdWrapper<int, NI>>(__getexp(__v))._M_data)));
+ __vector_store<_N * sizeof(int)>(__e, __exp, overaligned<alignof(_IV)>);
+ return {__private_init, __blend(isnonzero, __v, __getmant_avx512(__v))};
+#endif // _GLIBCXX_SIMD_X86INTRIN
+ } else {
+ // fallback implementation
+ static_assert(sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
+ using _V = simd<_Tp, _Abi>;
+ using _IV = rebind_simd_t<int, _V>;
+ using _IM = typename _IV::mask_type;
+ using _Limits = std::numeric_limits<_Tp>;
+ using namespace std::experimental::__proposed;
+ using namespace std::experimental::__proposed::float_bitwise_operators;
+
+ constexpr int __exp_shift = sizeof(_Tp) == 4 ? 23 : 20;
+ constexpr int __exp_adjust = sizeof(_Tp) == 4 ? 0x7e : 0x3fe;
+ constexpr int __exp_offset = sizeof(_Tp) == 4 ? 0x70 : 0x200;
+ constexpr _Tp __subnorm_scale = sizeof(_Tp) == 4 ? 0x1p112 : 0x1p512;
+ constexpr _V __exponent_mask =
+ _Limits::infinity(); // 0x7f800000 or 0x7ff0000000000000
+ constexpr _V __p5_1_exponent =
+ _Tp(sizeof(_Tp) == 4 ? -0x1.fffffep-1 : -0x1.fffffffffffffp-1);
+
+ _V __mant = __p5_1_exponent & (__exponent_mask | __x);
+ const _IV __exponent_bits = __extract_exponent_bits(__x);
+ if (_GLIBCXX_SIMD_IS_LIKELY(all_of(isnormal(__x)))) {
+ *__exp = simd_cast<__samesize<int, _V>>((__exponent_bits >> __exp_shift) -
+ __exp_adjust);
+ return __mant;
+ }
+ const auto __iszero_inf_nan = isunordered(__x * _Limits::infinity(), __x * _V());
+ const _V __scaled_subnormal = __x * __subnorm_scale;
+ const _V __mant_subnormal = __p5_1_exponent & (__exponent_mask | __scaled_subnormal);
+ where(!isnormal(__x), __mant) = __mant_subnormal;
+ where(__iszero_inf_nan, __mant) = __x;
+ _IV __e = __extract_exponent_bits(__scaled_subnormal);
+ const _IM __value_isnormal = static_simd_cast<_IM>(isnormal(__x));
+ where(__value_isnormal, __e) = __exponent_bits;
+ const _IV __offset = (simd_reinterpret_cast<_IV>(__value_isnormal) & _IV(__exp_adjust)) |
+ (simd_reinterpret_cast<_IV>((__exponent_bits == 0) &
+ (static_simd_cast<_IM>(__x != 0))) &
+ _IV(__exp_adjust + __exp_offset));
+ *__exp = simd_cast<__samesize<int, _V>>((__e >> __exp_shift) - __offset);
+ return __mant;
+ }
+}
+// }}}
+_GLIBCXX_SIMD_MATH_CALL2_(ldexp, int)
+_GLIBCXX_SIMD_MATH_CALL_(ilogb)
+
+// logarithms {{{
+_GLIBCXX_SIMD_MATH_CALL_(log)
+_GLIBCXX_SIMD_MATH_CALL_(log10)
+_GLIBCXX_SIMD_MATH_CALL_(log1p)
+_GLIBCXX_SIMD_MATH_CALL_(log2)
+//}}}
+//logb{{{
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point<_Tp>::value, simd<_Tp, _Abi>> logb(
+ const simd<_Tp, _Abi> &__x)
+{
+ constexpr size_t _N = simd_size_v<_Tp, _Abi>;
+ if constexpr (_N == 1) {
+ return std::logb(__x[0]);
+ } else if constexpr (__is_fixed_size_abi_v<_Abi>) {
+ return {__private_init,
+ __simd_tuple_apply(
+ [](auto __impl, auto __xx) {
+ using _V = typename decltype(__impl)::simd_type;
+ return __data(std::experimental::logb(_V(__private_init, __xx)));
+ },
+ __data(__x))};
+ }
+ else if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _N>())
+ {
+ return {__private_init, _mm_getexp_ps(__data(__x))};
+ }
+ else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _N>())
+ {
+ return {__private_init, _mm_getexp_pd(__data(__x))};
+ }
+ else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _N>())
+ {
+ return {__private_init, _mm256_getexp_ps(__data(__x))};
+ }
+ else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _N>())
+ {
+ return {__private_init, _mm256_getexp_pd(__data(__x))};
+ }
+ else if constexpr (__have_avx512f && __is_avx_ps<_Tp, _N>())
+ {
+ return {__private_init,
+ __lo256(_mm512_getexp_ps(__auto_bitcast(__data(__x))))};
+ }
+ else if constexpr (__have_avx512f && __is_avx_pd<_Tp, _N>())
+ {
+ return {__private_init,
+ __lo256(_mm512_getexp_pd(__auto_bitcast(__data(__x))))};
+ }
+ else if constexpr (__is_avx512_ps<_Tp, _N>())
+ {
+ return {__private_init, _mm512_getexp_ps(__data(__x))};
+ }
+ else if constexpr (__is_avx512_pd<_Tp, _N>())
+ {
+ return {__private_init, _mm512_getexp_pd(__data(__x))};
+ }
+ else
+ {
+ using _V = simd<_Tp, _Abi>;
+ using namespace std::experimental::__proposed;
+ auto __is_normal = isnormal(__x);
+
+ // work on __abs(__x) to reflect the return value on Linux for negative
+ // inputs (domain-error => implementation-defined value is returned)
+ const _V abs_x = abs(__x);
+
+ // __exponent(__x) returns the exponent value (bias removed) as simd<_U>
+ // with integral _U
+ auto&& __exponent = [](const _V& __v) {
+ using namespace std::experimental::__proposed;
+ using _IV = rebind_simd_t<
+ std::conditional_t<sizeof(_Tp) == sizeof(_LLong), _LLong, int>,
+ _V>;
+ return (simd_reinterpret_cast<_IV>(__v) >>
+ (std::numeric_limits<_Tp>::digits - 1)) -
+ (std::numeric_limits<_Tp>::max_exponent - 1);
+ };
+ _V __r = static_simd_cast<_V>(__exponent(abs_x));
+ if (_GLIBCXX_SIMD_IS_LIKELY(all_of(__is_normal)))
+ {
+ // without corner cases (nan, inf, subnormal, zero) we have our
+ // answer:
+ return __r;
+ }
+ const auto __is_zero = __x == 0;
+ const auto __is_nan = isnan(__x);
+ const auto __is_inf = isinf(__x);
+ where(__is_zero, __r) = -std::numeric_limits<_Tp>::infinity();
+ where(__is_nan, __r) = __x;
+ where(__is_inf, __r) = std::numeric_limits<_Tp>::infinity();
+ __is_normal |= __is_zero || __is_nan || __is_inf;
+ if (all_of(__is_normal))
+ {
+ // at this point everything but subnormals is handled
+ return __r;
+ }
+ // subnormals repeat the exponent extraction after multiplication of the
+ // input with __a floating point value that has 112 (0x70) in its exponent
+ // (not too big for sp and large enough for dp)
+ const _V __scaled = abs_x * _Tp(0x1p112);
+ _V __scaled_exp = static_simd_cast<_V>(__exponent(__scaled) - 112);
+ _GLIBCXX_SIMD_DEBUG(_Logarithm)
+ (__x, __scaled)(__is_normal)(__r, __scaled_exp);
+ where(__is_normal, __scaled_exp) = __r;
+ return __scaled_exp;
+ }
+}
+//}}}
+_GLIBCXX_SIMD_MATH_CALL2_(modf, _Tp *)
+_GLIBCXX_SIMD_MATH_CALL2_(scalbn, int)
+_GLIBCXX_SIMD_MATH_CALL2_(scalbln, long)
+
+_GLIBCXX_SIMD_MATH_CALL_(cbrt)
+
+_GLIBCXX_SIMD_MATH_CALL_(abs)
+_GLIBCXX_SIMD_MATH_CALL_(fabs)
+
+// [parallel.simd.math] only asks for is_floating_point_v<_Tp> and forgot to allow
+// signed integral _Tp
+template <class _Tp, class _Abi>
+enable_if_t<!std::is_floating_point_v<_Tp> && std::is_signed_v<_Tp>, simd<_Tp, _Abi>> abs(
+ const simd<_Tp, _Abi> &__x)
+{
+ return {__private_init, _Abi::_SimdImpl::__abs(__data(__x))};
+}
+template <class _Tp, class _Abi>
+enable_if_t<!std::is_floating_point_v<_Tp> && std::is_signed_v<_Tp>, simd<_Tp, _Abi>> fabs(
+ const simd<_Tp, _Abi> &__x)
+{
+ return {__private_init, _Abi::_SimdImpl::__abs(__data(__x))};
+}
+
+// the following are overloads for functions in <cstdlib> and not covered by
+// [parallel.simd.math]. I don't see much value in making them work, though
+/*
+template <class _Abi> simd<long, _Abi> labs(const simd<long, _Abi> &__x)
+{
+ return {__private_init, _Abi::_SimdImpl::abs(__data(__x))};
+}
+template <class _Abi> simd<long long, _Abi> llabs(const simd<long long, _Abi> &__x)
+{
+ return {__private_init, _Abi::_SimdImpl::abs(__data(__x))};
+}
+*/
+
+#define _GLIBCXX_SIMD_CVTING2(_NAME) \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const simd<_Tp, _Abi>& __x, const __id<simd<_Tp, _Abi>>& __y) \
+ { \
+ return _NAME(__x, __y); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const __id<simd<_Tp, _Abi>>& __x, const simd<_Tp, _Abi>& __y) \
+ { \
+ return _NAME(__x, __y); \
+ }
+
+#define _GLIBCXX_SIMD_CVTING3(_NAME) \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const __id<simd<_Tp, _Abi>>& __x, const simd<_Tp, _Abi>& __y, \
+ const simd<_Tp, _Abi>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const simd<_Tp, _Abi>& __x, const __id<simd<_Tp, _Abi>>& __y, \
+ const simd<_Tp, _Abi>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const simd<_Tp, _Abi>& __x, const simd<_Tp, _Abi>& __y, \
+ const __id<simd<_Tp, _Abi>>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const simd<_Tp, _Abi>& __x, const __id<simd<_Tp, _Abi>>& __y, \
+ const __id<simd<_Tp, _Abi>>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const __id<simd<_Tp, _Abi>>& __x, const simd<_Tp, _Abi>& __y, \
+ const __id<simd<_Tp, _Abi>>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ } \
+ template <typename _Tp, typename _Abi> \
+ _GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> _NAME( \
+ const __id<simd<_Tp, _Abi>>& __x, const __id<simd<_Tp, _Abi>>& __y, \
+ const simd<_Tp, _Abi>& __z) \
+ { \
+ return _NAME(__x, __y, __z); \
+ }
+
+template <typename _R, typename _ToApply, typename... _Tps>
+_GLIBCXX_SIMD_INTRINSIC _R __fixed_size_apply(_ToApply&& __apply,
+ const _Tps&... __args)
+{
+ return {__private_init, __simd_tuple_apply(
+ [&](auto __impl, const auto&... __inner) {
+ using _V = typename decltype(__impl)::simd_type;
+ return __data(__apply(_V(__private_init, __inner)...));
+ },
+ __data(__args)...)};
+}
+
+template <typename _VV> __remove_cvref_t<_VV> __hypot(_VV __x, _VV __y)
+{
+ using _V = __remove_cvref_t<_VV>;
+ using _Tp = typename _V::value_type;
+ if constexpr (_V::size() == 1)
+ {
+ return std::hypot(_Tp(__x[0]), _Tp(__y[0]));
+ }
+ else if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
+ {
+ return __fixed_size_apply<_V>(
+ [](auto __a, auto __b) { return hypot(__a, __b); }, __x, __y);
+ }
+ else
+ {
+ // A simple solution for _Tp == float would be to cast to double and simply calculate
+ // sqrt(x²+y²) as it can't over-/underflow anymore with dp. It still needs the Annex F fixups
+ // though and isn't faster on Skylake-AVX512 (not even for SSE and AVX vectors, and really bad
+ // for AVX-512).
+ using namespace __proposed::float_bitwise_operators;
+ using _Limits = std::numeric_limits<_Tp>;
+ _V __absx = abs(__x); // no error
+ _V __absy = abs(__y); // no error
+ _V __hi = max(__absx, __absy); // no error
+ _V __lo = min(__absy, __absx); // no error
+
+ // round __hi down to the next power-of-2:
+ constexpr _V __inf(_Limits::infinity());
+
+ if (_GLIBCXX_SIMD_IS_LIKELY(all_of(isnormal(__x)) &&
+ all_of(isnormal(__y))))
+ {
+ const _V __hi_exp = __hi & __inf;
+ //((__hi + __hi) & __inf) ^ __inf almost works for computing __scale, except
+ //when (__hi + __hi) & __inf == __inf, in which case __scale becomes 0 (should
+ //be min/2 instead) and thus loses the information from __lo.
+ const _V __scale = (__hi_exp ^ __inf) * _Tp(.5);
+ constexpr _V __mant_mask = _Limits::min() - _Limits::denorm_min();
+ const _V __h1 = (__hi & __mant_mask) | _V(1);
+ const _V __l1 = __lo * __scale;
+ return __hi_exp * sqrt(__h1 * __h1 + __l1 * __l1);
+ }
+ else
+ {
+ // slower path to support subnormals
+ // if __hi is subnormal, avoid scaling by inf & final mul by 0 (which
+ // yields NaN) by using min()
+ _V __scale = _V(1 / _Limits::min());
+ // invert exponent w/o error and w/o using the slow divider unit:
+ // xor inverts the exponent but off by 1. Multiplication with .5
+ // adjusts for the discrepancy.
+ where(__hi >= _Limits::min(), __scale) =
+ ((__hi & __inf) ^ __inf) * _Tp(.5);
+ // adjust final exponent for subnormal inputs
+ _V __hi_exp = _Limits::min();
+ where(__hi >= _Limits::min(), __hi_exp) = __hi & __inf; // no error
+ _V __h1 = __hi * __scale; // no error
+ _V __l1 = __lo * __scale; // no error
+
+ // sqrt(x²+y²) = e*sqrt((x/e)²+(y/e)²):
+ // this ensures no overflow in the argument to sqrt
+ _V __r = __hi_exp * sqrt(__h1 * __h1 + __l1 * __l1);
+#ifdef __STDC_IEC_559__
+ // fixup for Annex F requirements
+ // the naive fixup goes like this:
+ //
+ // where(__l1 == 0, __r) = __hi;
+ // where(isunordered(__x, __y), __r) = _Limits::quiet_NaN();
+ // where(isinf(__absx) || isinf(__absy), __r) = __inf;
+ //
+ // The fixup can be prepared in parallel with the sqrt, requiring a single blend
+ // step after hi_exp * sqrt, reducing latency and throughput:
+ _V __fixup = __hi; // __lo == 0
+ where(isunordered(__x, __y), __fixup) = _Limits::quiet_NaN();
+ where(isinf(__absx) || isinf(__absy), __fixup) = __inf;
+ where(!(__lo == 0 || isunordered(__x, __y) ||
+ (isinf(__absx) || isinf(__absy))),
+ __fixup) = __r;
+ __r = __fixup;
+#endif
+ return __r;
+ }
+ }
+}
+
+template <typename _Tp, typename _Abi>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi>
+ hypot(const simd<_Tp, _Abi>& __x, const simd<_Tp, _Abi>& __y)
+{
+ return __hypot<conditional_t<__is_fixed_size_abi_v<_Abi>,
+ const simd<_Tp, _Abi>&, simd<_Tp, _Abi>>>(__x,
+ __y);
+}
+_GLIBCXX_SIMD_CVTING2(hypot)
+
+template <typename _VV> __remove_cvref_t<_VV> __hypot(_VV __x, _VV __y, _VV __z)
+{
+ using _V = __remove_cvref_t<_VV>;
+ using _Abi = typename _V::abi_type;
+ using _Tp = typename _V::value_type;
+ /* FIXME: enable after PR77776 is resolved
+ if constexpr (_V::size() == 1)
+ {
+ return std::hypot(_Tp(__x[0]), _Tp(__y[0]), _Tp(__z[0]));
+ }
+ else
+ */
+ if constexpr (__is_fixed_size_abi_v<_Abi> && _V::size() > 1)
+ {
+ return __fixed_size_apply<simd<_Tp, _Abi>>(
+ [](auto __a, auto __b, auto __c) { return hypot(__a, __b, __c); }, __x,
+ __y, __z);
+ }
+ else
+ {
+ using namespace __proposed::float_bitwise_operators;
+ using _Limits = std::numeric_limits<_Tp>;
+ const _V __absx = abs(__x); // no error
+ const _V __absy = abs(__y); // no error
+ const _V __absz = abs(__z); // no error
+ _V __hi = max(max(__absx, __absy), __absz); // no error
+ _V __l0 = min(__absz, max(__absx, __absy)); // no error
+ _V __l1 = min(__absy, __absx); // no error
+ if constexpr (numeric_limits<_Tp>::digits == 64 &&
+ numeric_limits<_Tp>::max_exponent == 0x4000 &&
+ numeric_limits<_Tp>::min_exponent == -0x3FFD &&
+ _V::size() == 1)
+ { // Seems like x87 fp80, where bit 63 is always 1 unless subnormal or
+ // NaN. In this case the bit-tricks don't work, they require IEC559
+ // binary32 or binary64 format.
+#ifdef __STDC_IEC_559__
+ // fixup for Annex F requirements
+ if (isinf(__absx[0]) || isinf(__absy[0]) || isinf(__absz[0]))
+ return _Limits::infinity();
+ else if (isunordered(__absx[0], __absy[0]+__absz[0]))
+ return _Limits::quiet_NaN();
+ else if (__l0[0] == 0 && __l1[0] == 0)
+ return __hi;
+#endif
+ _V __hi_exp = __hi;
+ const _ULLong __tmp = 0x8000'0000'0000'0000ull;
+ std::memcpy(&__hi_exp, &__tmp, 8);
+ const _V __scale = 1 / __hi_exp;
+ __hi *= __scale;
+ __l0 *= __scale;
+ __l1 *= __scale;
+ return __hi_exp * sqrt((__l0 * __l0 + __l1 * __l1) + __hi * __hi);
+ }
+ else
+ {
+ // round __hi down to the next power-of-2:
+ constexpr _V __inf(_Limits::infinity());
+
+ if (_GLIBCXX_SIMD_IS_LIKELY(all_of(isnormal(__x)) &&
+ all_of(isnormal(__y)) &&
+ all_of(isnormal(__z))))
+ {
+ const _V __hi_exp = __hi & __inf;
+ //((__hi + __hi) & __inf) ^ __inf almost works for computing
+ //__scale, except when (__hi + __hi) & __inf == __inf, in which
+ // case __scale
+ // becomes 0 (should be min/2 instead) and thus loses the
+ // information from __lo.
+ const _V __scale = (__hi_exp ^ __inf) * _Tp(.5);
+ constexpr _V __mant_mask = _Limits::min() - _Limits::denorm_min();
+ const _V __h1 = (__hi & __mant_mask) | _V(1);
+ __l0 *= __scale;
+ __l1 *= __scale;
+ const _V __lo =
+ __l0 * __l0 + __l1 * __l1; // add the two smaller values first
+ return __hi_exp * sqrt(__lo + __h1 * __h1);
+ }
+ else
+ {
+ // slower path to support subnormals
+ // if __hi is subnormal, avoid scaling by inf & final mul by 0
+ // (which yields NaN) by using min()
+ _V __scale = _V(1 / _Limits::min());
+ // invert exponent w/o error and w/o using the slow divider unit:
+ // xor inverts the exponent but off by 1. Multiplication with .5
+ // adjusts for the discrepancy.
+ where(__hi >= _Limits::min(), __scale) =
+ ((__hi & __inf) ^ __inf) * _Tp(.5);
+ // adjust final exponent for subnormal inputs
+ _V __hi_exp = _Limits::min();
+ where(__hi >= _Limits::min(), __hi_exp) =
+ __hi & __inf; // no error
+ _V __h1 = __hi * __scale; // no error
+ __l0 *= __scale; // no error
+ __l1 *= __scale; // no error
+ _V __lo =
+ __l0 * __l0 + __l1 * __l1; // add the two smaller values first
+ _V __r = __hi_exp * sqrt(__lo + __h1 * __h1);
+#ifdef __STDC_IEC_559__
+ // fixup for Annex F requirements
+ _V __fixup = __hi; // __lo == 0
+ // where(__lo == 0, __fixup) = __hi;
+ where(isunordered(__x, __y + __z), __fixup) =
+ _Limits::quiet_NaN();
+ where(isinf(__absx) || isinf(__absy) || isinf(__absz), __fixup) =
+ __inf;
+ where(!(__lo == 0 || isunordered(__x, __y + __z) ||
+ isinf(__absx) || isinf(__absy) || isinf(__absz)),
+ __fixup) = __r;
+ __r = __fixup;
+#endif
+ return __r;
+ }
+ }
+ }
+}
+
+template <typename _Tp, typename _Abi>
+_GLIBCXX_SIMD_INTRINSIC simd<_Tp, _Abi> hypot(const simd<_Tp, _Abi>& __x,
+ const simd<_Tp, _Abi>& __y,
+ const simd<_Tp, _Abi>& __z)
+{
+ return __hypot<conditional_t<__is_fixed_size_abi_v<_Abi>,
+ const simd<_Tp, _Abi>&, simd<_Tp, _Abi>>>(
+ __x, __y, __z);
+}
+_GLIBCXX_SIMD_CVTING3(hypot)
+
+_GLIBCXX_SIMD_MATH_CALL2_(pow, _Tp)
+
+_GLIBCXX_SIMD_MATH_CALL_(sqrt)
+_GLIBCXX_SIMD_MATH_CALL_(erf)
+_GLIBCXX_SIMD_MATH_CALL_(erfc)
+_GLIBCXX_SIMD_MATH_CALL_(lgamma)
+_GLIBCXX_SIMD_MATH_CALL_(tgamma)
+_GLIBCXX_SIMD_MATH_CALL_(ceil)
+_GLIBCXX_SIMD_MATH_CALL_(floor)
+_GLIBCXX_SIMD_MATH_CALL_(nearbyint)
+_GLIBCXX_SIMD_MATH_CALL_(rint)
+_GLIBCXX_SIMD_MATH_CALL_(lrint)
+_GLIBCXX_SIMD_MATH_CALL_(llrint)
+
+_GLIBCXX_SIMD_MATH_CALL_(round)
+_GLIBCXX_SIMD_MATH_CALL_(lround)
+_GLIBCXX_SIMD_MATH_CALL_(llround)
+
+_GLIBCXX_SIMD_MATH_CALL_(trunc)
+
+_GLIBCXX_SIMD_MATH_CALL2_(fmod, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(remainder, _Tp)
+_GLIBCXX_SIMD_MATH_CALL3_(remquo, _Tp, int *)
+_GLIBCXX_SIMD_MATH_CALL2_(copysign, _Tp)
+
+_GLIBCXX_SIMD_MATH_CALL2_(nextafter, _Tp)
+// not covered in [parallel.simd.math]:
+// _GLIBCXX_SIMD_MATH_CALL2_(nexttoward, long double)
+_GLIBCXX_SIMD_MATH_CALL2_(fdim, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(fmax, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(fmin, _Tp)
+
+_GLIBCXX_SIMD_MATH_CALL3_(fma, _Tp, _Tp)
+_GLIBCXX_SIMD_MATH_CALL_(fpclassify)
+_GLIBCXX_SIMD_MATH_CALL_(isfinite)
+_GLIBCXX_SIMD_MATH_CALL_(isinf)
+_GLIBCXX_SIMD_MATH_CALL_(isnan)
+_GLIBCXX_SIMD_MATH_CALL_(isnormal)
+_GLIBCXX_SIMD_MATH_CALL_(signbit)
+
+_GLIBCXX_SIMD_MATH_CALL2_(isgreater, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(isgreaterequal, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(isless, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(islessequal, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(islessgreater, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(isunordered, _Tp)
+
+/* not covered in [parallel.simd.math]
+template <class _Abi> __doublev<_Abi> nan(const char* tagp);
+template <class _Abi> __floatv<_Abi> nanf(const char* tagp);
+template <class _Abi> __ldoublev<_Abi> nanl(const char* tagp);
+
+template <class _V> struct simd_div_t {
+ _V quot, rem;
+};
+template <class _Abi>
+simd_div_t<_SCharv<_Abi>> div(_SCharv<_Abi> numer,
+ _SCharv<_Abi> denom);
+template <class _Abi>
+simd_div_t<__shortv<_Abi>> div(__shortv<_Abi> numer,
+ __shortv<_Abi> denom);
+template <class _Abi>
+simd_div_t<__intv<_Abi>> div(__intv<_Abi> numer, __intv<_Abi> denom);
+template <class _Abi>
+simd_div_t<__longv<_Abi>> div(__longv<_Abi> numer,
+ __longv<_Abi> denom);
+template <class _Abi>
+simd_div_t<__llongv<_Abi>> div(__llongv<_Abi> numer,
+ __llongv<_Abi> denom);
+*/
+
+// special math {{{
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> assoc_laguerre(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__m,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::assoc_laguerre(__n[__i], __m[__i], __x[__i]); });
+}
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> assoc_legendre(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__m,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::assoc_legendre(__n[__i], __m[__i], __x[__i]); });
+}
+
+_GLIBCXX_SIMD_MATH_CALL2_(beta, _Tp)
+_GLIBCXX_SIMD_MATH_CALL_(comp_ellint_1)
+_GLIBCXX_SIMD_MATH_CALL_(comp_ellint_2)
+_GLIBCXX_SIMD_MATH_CALL2_(comp_ellint_3, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(cyl_bessel_i, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(cyl_bessel_j, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(cyl_bessel_k, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(cyl_neumann, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(ellint_1, _Tp)
+_GLIBCXX_SIMD_MATH_CALL2_(ellint_2, _Tp)
+_GLIBCXX_SIMD_MATH_CALL3_(ellint_3, _Tp, _Tp)
+_GLIBCXX_SIMD_MATH_CALL_(expint)
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> hermite(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::hermite(__n[__i], __x[__i]); });
+}
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> laguerre(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::laguerre(__n[__i], __x[__i]); });
+}
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> legendre(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::legendre(__n[__i], __x[__i]); });
+}
+
+_GLIBCXX_SIMD_MATH_CALL_(riemann_zeta)
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> sph_bessel(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::sph_bessel(__n[__i], __x[__i]); });
+}
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> sph_legendre(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__l,
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__m,
+ const std::experimental::simd<_Tp, _Abi> &theta)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::assoc_legendre(__l[__i], __m[__i], theta[__i]); });
+}
+
+template <class _Tp, class _Abi>
+enable_if_t<std::is_floating_point_v<_Tp>, simd<_Tp, _Abi>> sph_neumann(
+ const std::experimental::fixed_size_simd<unsigned, std::experimental::simd_size_v<_Tp, _Abi>> &__n,
+ const std::experimental::simd<_Tp, _Abi> &__x)
+{
+ return std::experimental::simd<_Tp, _Abi>([&](auto __i) { return std::sph_neumann(__n[__i], __x[__i]); });
+}
+// }}}
+
+#undef _GLIBCXX_SIMD_MATH_CALL_
+#undef _GLIBCXX_SIMD_MATH_CALL2_
+#undef _GLIBCXX_SIMD_MATH_CALL3_
+
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H_
+// vim: foldmethod=marker sw=2 ts=8 noet sts=2
diff --git a/libstdc++-v3/include/experimental/bits/simd_workarounds.h b/libstdc++-v3/include/experimental/bits/simd_workarounds.h
new file mode 100644
index 00000000000..89306099f37
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_workarounds.h
@@ -0,0 +1,492 @@
+// Unused workarounds for bad codegen -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H
+#define _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H
+
+#pragma GCC system_header
+
+#if __cplusplus >= 201703L
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+// __divides {{{1
+template <class _Tp, class = _VectorTraits<_Tp>>
+_GLIBCXX_SIMD_INTRINSIC _Tp __divides(_Tp __a, _Tp __b)
+{
+ using _U = typename Traits::value_type;
+ constexpr bool is_byte = sizeof(_U) == 1;
+ constexpr bool is_word = sizeof(_U) == 2;
+ constexpr bool is_dword = sizeof(_U) == 4;
+ constexpr bool is_ymm = sizeof(_Tp) == 32;
+ constexpr bool is_xmm = sizeof(_Tp) == 16;
+
+ if constexpr (is_dword && ((is_xmm && __have_avx) || (is_ymm && __have_avx512f))) {
+ return convert<_U>(convert<double>(__a) / convert<double>(__b));
+ } else if constexpr (is_dword) { // really better with is_xmm?
+ auto __x = __convert_all<__vector_type_t<double, Traits::_S_width / 2>>(__a);
+ auto __y = __convert_all<__vector_type_t<double, Traits::_S_width / 2>>(__b);
+ return convert<_Tp>(__x[0] / __y[0], __x[1] / __y[1]);
+ } else if constexpr (is_word) {
+ if constexpr ((is_xmm && __have_avx) || (is_ymm && __have_avx512f)) {
+ return convert<_Tp>(convert<float>(__a) / convert<float>(__b));
+ } else {
+ auto __x = __convert_all<__vector_type_t<float, Traits::_S_width / 2>>(__a);
+ auto __y = __convert_all<__vector_type_t<float, Traits::_S_width / 2>>(__b);
+ return convert<_Tp>(__x[0] / __y[0], __x[1] / __y[1]);
+ }
+ } else if constexpr (is_byte && is_xmm && __have_avx512f) {
+ return convert<_Tp>(convert<float>(__a) / convert<float>(__b));
+ } else if constexpr (is_byte && ((is_xmm && __have_avx) || is_ymm && __have_avx512f)) {
+ auto __x = __convert_all<__vector_type_t<float, Traits::_S_width / 2>>(__a);
+ auto __y = __convert_all<__vector_type_t<float, Traits::_S_width / 2>>(__b);
+ return convert<_Tp>(__x[0] / __y[0], __x[1] / __y[1]);
+ } else if constexpr (is_byte) {
+ auto __x = __convert_all<__vector_type_t<float, Traits::_S_width / 4>>(__a);
+ auto __y = __convert_all<__vector_type_t<float, Traits::_S_width / 4>>(__b);
+ return convert<_Tp>(__x[0] / __y[0], __x[1] / __y[1], __x[2] / __y[2], __x[3] / __y[3]);
+ } else {
+ return __a / __b;
+ }
+}
+// __bit_shift_left{{{1
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _N> constexpr __bit_shift_left(_SimdWrapper<_Tp, _N> __a, int __b)
+{
+ static_assert(std::is_integral<_Tp>::value, "__bit_shift_left is only supported for integral types");
+ if constexpr (sizeof(_Tp) == 1) {
+ // (cf. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83894)
+ if (__builtin_constant_p(__b)) {
+ if (__b == 0) {
+ return __a;
+ } else if (__b == 1) {
+ return __a._M_data + __a._M_data;
+ } else if (__b > 1 && __b < 8) {
+ const _UChar mask = (0xff << __b) & 0xff;
+ using _V = decltype(__a);
+ using In = typename _V::_IntrinType;
+ return reinterpret_cast<In>(__wrapper_bitcast<ushort>(__a)._M_data << __b) &
+ _V::broadcast(mask).__intrin();
+ } else {
+ __builtin_unreachable();
+ }
+ }
+ if constexpr (_N == 16 && __have_sse2) {
+ if constexpr (__have_avx512bw_vl) {
+ return _mm256_cvtepi16_epi8(reinterpret_cast<__m256i>(
+ reinterpret_cast<__vector_type_t<ushort, 16>>(_mm256_cvtepi8_epi16(__a))
+ << __b));
+ } else {
+ using vshort = __vector_type_t<ushort, 8>;
+ const auto mask = ((~vshort() >> 8) << __b) ^ (~vshort() << 8);
+ return _ToWrapper((reinterpret_cast<vshort>(__a._M_data) << __b) & mask);
+ }
+ } else if constexpr (_N == 32 && __have_avx2) {
+ if constexpr(__have_avx512bw) {
+ return _mm512_cvtepi16_epi8(reinterpret_cast<__m512i>(
+ reinterpret_cast<__vector_type_t<ushort, 32>>(_mm512_cvtepi8_epi16(__a))
+ << __b));
+ } else {
+ using vshort = __vector_type_t<ushort, 16>;
+ const auto mask = ((~vshort() >> 8) << __b) ^ (~vshort() << 8);
+ return _ToWrapper((reinterpret_cast<vshort>(__a._M_data) << __b) & mask);
+ }
+ } else if constexpr (_N == 64 && __have_avx512bw) {
+ using vshort = __vector_type_t<ushort, 32>;
+ const auto mask = ((~vshort() >> 8) << __b) ^ (~vshort() << 8);
+ return _ToWrapper((reinterpret_cast<vshort>(__a._M_data) << __b) & mask);
+ } else {
+ static_assert(!std::is_same_v<_Tp, _Tp>);
+ }
+ } else {
+ return __a._M_data << __b;
+ }
+}
+
+template <class _Tp, size_t _N>
+_GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _N> __bit_shift_left(_SimdWrapper<_Tp, _N> __a, _SimdWrapper<_Tp, _N> __b)
+{
+ static_assert(std::is_integral<_Tp>::value,
+ "__bit_shift_left is only supported for integral types");
+ if constexpr (sizeof(_Tp) == 2 && sizeof(__a) == 16 && !__have_avx2) {
+ __vector_type_t<int, 4> shift = __wrapper_bitcast<int>(__b)._M_data + (0x03f8'03f8 >> 3);
+ return multiplies(
+ __a,
+ _SimdWrapper<_Tp, _N>(
+ _mm_cvttps_epi32(reinterpret_cast<__m128>(shift << 23)) |
+ (_mm_cvttps_epi32(reinterpret_cast<__m128>(shift >> 16 << 23)) << 16)));
+ } else if constexpr (sizeof(_Tp) == 4 && sizeof(__a) == 16 && !__have_avx2) {
+ return __wrapper_bitcast<_Tp>(
+ multiplies(__a, _SimdWrapper<_Tp, _N>(_mm_cvttps_epi32(
+ reinterpret_cast<__m128>((__b._M_data << 23) + 0x3f80'0000)))));
+ } else if constexpr (sizeof(_Tp) == 8 && sizeof(__a) == 16 && !__have_avx2) {
+ const auto __lo = _mm_sll_epi64(__a, __b);
+ const auto __hi = _mm_sll_epi64(__a, _mm_unpackhi_epi64(__b, __b));
+ if constexpr (__have_sse4_1) {
+ return _mm_blend_epi16(__lo, __hi, 0xf0);
+ } else {
+ // return __make_wrapper<_LLong>(reinterpret_cast<__vector_type_t<_LLong,
+ // 2>>(__lo)[0], reinterpret_cast<__vector_type_t<_LLong, 2>>(__hi)[1]);
+ return _ToWrapper(
+ _mm_move_sd(__intrin_bitcast<__m128d>(__hi), __intrin_bitcast<__m128d>(__lo)));
+ }
+ } else if constexpr (__have_avx512f && sizeof(_Tp) == 8 && _N == 8) {
+ return _mm512_sllv_epi64(__a, __b);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 8 && _N == 4) {
+ return _mm256_sllv_epi64(__a, __b);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 8 && _N == 2) {
+ return _mm_sllv_epi64(__a, __b);
+ } else if constexpr (__have_avx512f && sizeof(_Tp) == 4 && _N == 16) {
+ return _mm512_sllv_epi32(__a, __b);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 4 && _N == 8) {
+ return _mm256_sllv_epi32(__a, __b);
+ } else if constexpr (__have_avx2 && sizeof(_Tp) == 4 && _N == 4) {
+ return _mm_sllv_epi32(__a, __b);
+ } else if constexpr (sizeof(_Tp) == 2) {
+ if constexpr (_N == 32 && __have_avx512bw) {
+ return _mm512_sllv_epi16(__a, __b);
+ } else if constexpr (_N == 16 && __have_avx512bw_vl) {
+ return _mm256_sllv_epi16(__a, __b);
+ } else if constexpr (_N == 16 && __have_avx512bw) {
+ return __lo256(
+ _mm512_sllv_epi16(_mm512_castsi256_si512(__a), _mm512_castsi256_si512(__b)));
+ } else if constexpr (_N == 16) {
+ const auto aa = __vector_bitcast<unsigned>(__a._M_data);
+ const auto bb = __vector_bitcast<unsigned>(__b._M_data);
+ return _mm256_blend_epi16(__auto_bitcast(aa << (bb & 0x0000ffffu)),
+ __auto_bitcast((aa & 0xffff0000u) << (bb >> 16)), 0xaa);
+ } else if constexpr (_N == 8 && __have_avx512bw_vl) {
+ return _mm_sllv_epi16(__a, __b);
+ } else if constexpr (_N == 8 && __have_avx512bw) {
+ return _mm512_sllv_epi16(_mm512_castsi128_si512(__a),
+ _mm512_castsi128_si512(__b));
+ } else if constexpr (_N == 8) {
+ const auto aa = __vector_bitcast<unsigned>(__a._M_data);
+ const auto bb = __vector_bitcast<unsigned>(__b._M_data);
+ return _mm_blend_epi16(__auto_bitcast(aa << (bb & 0x0000ffffu)),
+ __auto_bitcast((aa & 0xffff0000u) << (bb >> 16)), 0xaa);
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else if constexpr (sizeof(_Tp) == 1) {
+ if constexpr (_N == 64 && __have_avx512bw) {
+ return concat(_mm512_cvtepi16_epi8(_mm512_sllv_epi16(
+ _mm512_cvtepu8_epi16(__lo256(__vector_bitcast<_LLong>(__a))),
+ _mm512_cvtepu8_epi16(__lo256(__vector_bitcast<_LLong>(__b))))),
+ _mm512_cvtepi16_epi8(_mm512_sllv_epi16(
+ _mm512_cvtepu8_epi16(__hi256(__vector_bitcast<_LLong>(__a))),
+ _mm512_cvtepu8_epi16(__hi256(__vector_bitcast<_LLong>(__b))))));
+ } else if constexpr (_N == 32 && __have_avx512bw) {
+ return _mm512_cvtepi16_epi8(
+ _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__a), _mm512_cvtepu8_epi16(__b)));
+ } else if constexpr (_N == 16 && __have_avx512bw_vl) {
+ return _mm256_cvtepi16_epi8(
+ _mm256_sllv_epi16(_mm256_cvtepu8_epi16(__a), _mm256_cvtepu8_epi16(__b)));
+ } else if constexpr (_N == 16 && __have_avx512bw) {
+ return __lo128(_mm512_cvtepi16_epi8(
+ _mm512_sllv_epi16(_mm512_cvtepu8_epi16(_mm512_castsi256_si512(__a)),
+ _mm512_cvtepu8_epi16(_mm512_castsi256_si512(__b)))));
+ } else {
+ auto mask_from_bit = [](__vector_type_t<_Tp, _N> __x, int bit) {
+ auto __y = __vector_bitcast<short>(__x) << bit;
+ if constexpr (__have_sse4_1) {
+ return __to_intrin(__y);
+ } else {
+ return __to_intrin(__vector_bitcast<_SChar>(__y) < 0);
+ }
+ };
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand. left
+ // => valid input range for each element of __b is [0, 7]
+ // => only the 3 low bits of __b are relevant
+ // do __a =<< 4 where __b[2] is set
+ auto a4 = __vector_bitcast<_UChar>(__vector_bitcast<short>(__a._M_data) << 4);
+ if constexpr (std::is_unsigned_v<_Tp>) {
+ // shift into or over the sign bit is UB => never spills into a neighbor
+ a4 &= 0xf0u;
+ }
+ __a = __blend(mask_from_bit(__b, 5), __a, __to_intrin(a4));
+ // do __a =<< 2 where __b[1] is set
+ // shift into or over the sign bit is UB => never spills into a neighbor
+ const auto a2 = std::is_signed_v<_Tp> ? __to_intrin(__vector_bitcast<short>(__a._M_data) << 2)
+ : __to_intrin(__a._M_data << 2);
+ __a = __blend(mask_from_bit(__b, 6), __a, a2);
+ // do __a =<< 1 where __b[0] is set
+ return __blend(mask_from_bit(__b, 7), __a, __to_intrin(__a._M_data + __a._M_data));
+ }
+ } else {
+ return __a._M_data << __b._M_data;
+ }
+}
+
+// }}}
+// __bit_shift_right{{{1
+template <class _Tp, class Traits = _VectorTraits<_Tp>> _Tp __bit_shift_right(_Tp __a, _Tp __b)
+{
+ using _U = typename Traits::value_type;
+ constexpr bool is_byte = sizeof(_U) == 1;
+ constexpr bool is_word = sizeof(_U) == 2;
+ constexpr bool is_dword = sizeof(_U) == 4;
+ constexpr bool is_signed = std::is_signed_v<_U>;
+ constexpr bool is_zmm = sizeof(_Tp) == 64;
+ constexpr bool is_ymm = sizeof(_Tp) == 32;
+ constexpr bool is_xmm = sizeof(_Tp) == 16;
+
+ const auto ai = __to_intrin(__a);
+ const auto bi = __to_intrin(__b);
+
+ if constexpr (is_byte && is_xmm && __have_avx512bw_vl) { //{{{2
+ return is_signed ? _mm256_cvtepi16_epi8(_mm256_srav_epi16(
+ _mm256_cvtepi8_epi16(__a), _mm256_cvtepi8_epi16(__b)))
+ : _mm256_cvtepi16_epi8(_mm256_srlv_epi16(
+ _mm256_cvtepu8_epi16(__a), _mm256_cvtepu8_epi16(__b)));
+ } else if constexpr (is_byte && is_xmm && __have_sse4_1) { //{{{2
+ if constexpr (is_signed) {
+ const auto aa = __vector_bitcast<short>(__a);
+ const auto bb = __vector_bitcast<short>(__b);
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 7]
+ // => only the 3 low bits of __b are relevant
+ // do __a =>> 4 where __b[2] is set
+ auto signbit = aa & 0x8080u;
+ __a = reinterpret_cast<_Tp>(_mm_blendv_epi8(
+ __vector_bitcast<_LLong>(__a),
+ __vector_bitcast<_LLong>((((signbit << 5) - signbit) | (aa & 0xf8f8u)) >> 4),
+ __vector_bitcast<_LLong>(bb << 5)));
+ // do __a =>> 2 where __b[1] is set
+ __a = reinterpret_cast<_Tp>(_mm_blendv_epi8(
+ __vector_bitcast<_LLong>(__a),
+ __vector_bitcast<_LLong>((((signbit << 3) - signbit) | (aa & 0xfcfcu)) >> 2),
+ __vector_bitcast<_LLong>(bb << 6)));
+ // do __a =>> 1 where __b[0] is set
+ return reinterpret_cast<_Tp>(
+ _mm_blendv_epi8(__vector_bitcast<_LLong>(__a),
+ __vector_bitcast<_LLong>(signbit | ((aa & 0xfefeu) >> 1)),
+ __vector_bitcast<_LLong>(bb << 7)));
+ } else {
+ const auto aa = __vector_bitcast<ushort>(__a);
+ const auto bb = __vector_bitcast<ushort>(__b);
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 7]
+ // => only the 3 low bits of __b are relevant
+ // do __a =>> 4 where __b[2] is set
+ __a = reinterpret_cast<_Tp>(_mm_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 4) & 0x0f0fu),
+ __vector_bitcast<_LLong>(bb << 5)));
+ // do __a =>> 2 where __b[1] is set
+ __a = reinterpret_cast<_Tp>(_mm_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 2) & 0x3f3fu),
+ __vector_bitcast<_LLong>(bb << 6)));
+ // do __a =>> 1 where __b[0] is set
+ return reinterpret_cast<_Tp>(_mm_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 1) & 0x7f7fu),
+ __vector_bitcast<_LLong>(bb << 7)));
+ }
+ } else if constexpr (is_byte && is_ymm && __have_avx512bw) { //{{{2
+ return _mm512_cvtepi16_epi8(
+ is_signed
+ ? _mm512_srav_epi16(_mm512_cvtepi8_epi16(ai), _mm512_cvtepi8_epi16(bi))
+ : _mm512_srlv_epi16(_mm512_cvtepu8_epi16(ai), _mm512_cvtepu8_epi16(bi)));
+ } else if constexpr (is_byte && is_ymm && __have_avx2) { //{{{2
+ if constexpr (is_signed) {
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 7]
+ // => only the 3 low bits of __b are relevant
+ // do __a =<< 4 where __b[2] is set
+ return __vector_convert<_Tp>(
+ __vector_bitcast<int>(_mm256_srav_epi32(_mm256_cvtepi8_epi32(__lo128(ai)),
+ _mm256_cvtepi8_epi32(__lo128(bi)))),
+ __vector_bitcast<int>(_mm256_srav_epi32(
+ _mm256_cvtepi8_epi32(_mm_unpackhi_epi64(__lo128(ai), __lo128(ai))),
+ _mm256_cvtepi8_epi32(_mm_unpackhi_epi64(__lo128(bi), __lo128(bi))))),
+ __vector_bitcast<int>(_mm256_srav_epi32(_mm256_cvtepi8_epi32(__hi128(ai)),
+ _mm256_cvtepi8_epi32(__hi128(bi)))),
+ __vector_bitcast<int>(_mm256_srav_epi32(
+ _mm256_cvtepi8_epi32(_mm_unpackhi_epi64(__hi128(ai), __hi128(ai))),
+ _mm256_cvtepi8_epi32(_mm_unpackhi_epi64(__hi128(bi), __hi128(bi))))));
+ } else {
+ const auto aa = __vector_bitcast<ushort>(__a);
+ const auto bb = __vector_bitcast<ushort>(__b);
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 7]
+ // => only the 3 low bits of __b are relevant
+ // do __a =>> 4 where __b[2] is set
+ __a = reinterpret_cast<_Tp>(_mm256_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 4) & 0x0f0fu),
+ __vector_bitcast<_LLong>(bb << 5)));
+ // do __a =>> 2 where __b[1] is set
+ __a = reinterpret_cast<_Tp>(_mm256_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 2) & 0x3f3fu),
+ __vector_bitcast<_LLong>(bb << 6)));
+ // do __a =>> 1 where __b[0] is set
+ return reinterpret_cast<_Tp>(_mm256_blendv_epi8(
+ __vector_bitcast<_LLong>(__a), __vector_bitcast<_LLong>((aa >> 1) & 0x7f7fu),
+ __vector_bitcast<_LLong>(bb << 7)));
+ }
+ } else if constexpr (is_byte && is_zmm && __have_avx512bw) { //{{{2
+ return concat(__bit_shift_right(__lo256(__a), __lo256(__b)),
+ __bit_shift_right(__hi256(__a), __hi256(__b)));
+ } else if constexpr (is_word && is_xmm && __have_avx512bw_vl) { //{{{2
+ return is_signed ? _mm_srav_epi16(ai, bi) : _mm_srlv_epi16(ai, bi);
+ } else if constexpr (is_word && is_xmm && __have_avx2) { //{{{2
+ return is_signed ? __vector_convert<short>(__vector_convert<int>(__a) >>
+ __vector_convert<int>(__b))
+ : __vector_convert<_UShort>(__vector_convert<_UInt>(__a) >>
+ __vector_convert<_UInt>(__b));
+ } else if constexpr (is_word && is_xmm && __have_sse4_1) { //{{{2
+ if constexpr (is_signed) {
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 15]
+ // => only the 4 low bits of __b are relevant
+ // shift by 4 and duplicate to high byte
+ __b = (__b << 4) | (__b << 12);
+ // do __a =>> 8 where __b[3] is set
+ __a = _mm_blendv_epi8(__a, _mm_srai_epi16(__a, 8), __b);
+ // do __a =>> 4 where __b[2] is set
+ __a = _mm_blendv_epi8(__a, _mm_srai_epi16(__a, 4), __b = _mm_add_epi16(__b, __b));
+ // do __a =>> 2 where __b[1] is set
+ __a = _mm_blendv_epi8(__a, _mm_srai_epi16(__a, 2), __b = _mm_add_epi16(__b, __b));
+ // do __a =>> 1 where __b[0] is set
+ return _mm_blendv_epi8(__a, _mm_srai_epi16(__a, 1), _mm_add_epi16(__b, __b));
+ } else {
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 15]
+ // => only the 4 low bits of __b are relevant
+ // shift by 4 and duplicate to high byte
+ __b = (__b << 4) | (__b << 12);
+ // do __a =>> 8 where __b[3] is set
+ __a = _mm_blendv_epi8(__a, __vector_bitcast<_LLong>(__a >> 8), __b);
+ // do __a =>> 4 where __b[2] is set
+ __a = _mm_blendv_epi8(__a, __vector_bitcast<_LLong>(__a >> 4), __b = _mm_add_epi16(__b, __b));
+ // do __a =>> 2 where __b[1] is set
+ __a = _mm_blendv_epi8(__a, __vector_bitcast<_LLong>(__a >> 2), __b = _mm_add_epi16(__b, __b));
+ // do __a =>> 1 where __b[0] is set
+ return _mm_blendv_epi8(__a, __vector_bitcast<_LLong>(__a >> 1), _mm_add_epi16(__b, __b));
+ }
+ } else if constexpr (is_word && is_xmm && __have_sse2) { //{{{2
+ auto &&blend = [](_Tp __a, _Tp __b, _Tp __c) { return (~__c & __a) | (__c & __b); };
+ if constexpr (is_signed) {
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 15]
+ // => only the 4 low bits of __b are relevant
+ // do __a =>> 8 where __b[3] is set
+ __a = blend(__a, _mm_srai_epi16(__a, 8),
+ _mm_cmpgt_epi16(__b, __auto_broadcast(0x00070007)));
+ // do __a =>> 4 where __b[2] is set
+ __a = blend(__a, _mm_srai_epi16(__a, 4),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00040004)),
+ _mm_setzero_si128()));
+ // do __a =>> 2 where __b[1] is set
+ __a = blend(__a, _mm_srai_epi16(__a, 2),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00020002)),
+ _mm_setzero_si128()));
+ // do __a =>> 1 where __b[0] is set
+ return blend(__a, _mm_srai_epi16(__a, 1),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00010001)),
+ _mm_setzero_si128()));
+ } else {
+ // exploit UB: The behavior is undefined if the right operand is [...] greater
+ // than or equal to the length in bits of the promoted left operand.
+ // => valid input range for each element of __b is [0, 15]
+ // => only the 4 low bits of __b are relevant
+ // do __a =>> 8 where __b[3] is set
+ __a = blend(__a, __vector_bitcast<_LLong>(__a >> 8),
+ _mm_cmpgt_epi16(__b, __auto_broadcast(0x00070007)));
+ // do __a =>> 4 where __b[2] is set
+ __a = blend(__a, __vector_bitcast<_LLong>(__a >> 4),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00040004)),
+ _mm_setzero_si128()));
+ // do __a =>> 2 where __b[1] is set
+ __a = blend(__a, __vector_bitcast<_LLong>(__a >> 2),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00020002)),
+ _mm_setzero_si128()));
+ // do __a =>> 1 where __b[0] is set
+ return blend(__a, __vector_bitcast<_LLong>(__a >> 1),
+ _mm_cmpgt_epi16(__and(__b, __auto_broadcast(0x00010001)),
+ _mm_setzero_si128()));
+ }
+ } else if constexpr (is_word && is_ymm && __have_avx512bw_vl) { //{{{2
+ return is_signed ? _mm256_srav_epi16(ai, bi) : _mm256_srlv_epi16(ai, bi);
+ } else if constexpr (is_word && is_ymm && __have_avx2) { //{{{2
+ if constexpr (is_signed) {
+ auto lo32 = _mm256_srli_epi32(
+ _mm256_srav_epi32(_mm256_unpacklo_epi16(_mm256_setzero_si256(), __a),
+ _mm256_unpacklo_epi16(__b, _mm256_setzero_si256())),
+ 16);
+ auto hi32 = _mm256_srli_epi32(
+ _mm256_srav_epi32(_mm256_unpackhi_epi16(_mm256_setzero_si256(), __a),
+ _mm256_unpackhi_epi16(__b, _mm256_setzero_si256())),
+ 16);
+ return _mm256_packs_epi32(lo32, hi32);
+ } else {
+ return _mm256_blend_epi16(
+ (__vector_bitcast<_UInt>(__a) & 0xffffu) >> (__vector_bitcast<_UInt>(__b) & 0xffffu),
+ __vector_bitcast<_UInt>(__a) >> (__vector_bitcast<_UInt>(__b) >> 16), 0xaa);
+ }
+ } else if constexpr (is_word && is_zmm && __have_avx512bw) { //{{{2
+ return is_signed ? _mm512_srav_epi16(ai, bi) : _mm512_srlv_epi16(ai, bi);
+ } else if constexpr (is_dword && is_xmm && !__have_avx2) { //{{{2
+ if constexpr (is_signed) {
+ const auto r0 = _mm_sra_epi32(__a, _mm_unpacklo_epi32(__b, _mm_setzero_si128()));
+ const auto r1 = _mm_sra_epi32(__a, _mm_srli_epi64(__b, 32));
+ const auto r2 = _mm_sra_epi32(__a, _mm_unpackhi_epi32(__b, _mm_setzero_si128()));
+ const auto r3 = _mm_sra_epi32(__a, _mm_srli_si128(__b, 12));
+ if constexpr (__have_sse4_1) {
+ return _mm_blend_epi16(_mm_blend_epi16(r1, r0, 0x3),
+ _mm_blend_epi16(r3, r2, 0x30), 0xf0);
+ } else {
+ return _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, _mm_srli_si128(r1, 4)),
+ _mm_unpackhi_epi32(r2, _mm_srli_si128(r3, 4)));
+ }
+ } else {
+ const auto r0 = _mm_srl_epi32(__a, _mm_unpacklo_epi32(__b, _mm_setzero_si128()));
+ const auto r1 = _mm_srl_epi32(__a, _mm_srli_epi64(__b, 32));
+ const auto r2 = _mm_srl_epi32(__a, _mm_unpackhi_epi32(__b, _mm_setzero_si128()));
+ const auto r3 = _mm_srl_epi32(__a, _mm_srli_si128(__b, 12));
+ if constexpr (__have_sse4_1) {
+ return _mm_blend_epi16(_mm_blend_epi16(r1, r0, 0x3),
+ _mm_blend_epi16(r3, r2, 0x30), 0xf0);
+ } else {
+ return _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, _mm_srli_si128(r1, 4)),
+ _mm_unpackhi_epi32(r2, _mm_srli_si128(r3, 4)));
+ }
+ }
+ } // }}}2
+ return __a << __b;
+}
+// }}}1
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_MATH_H
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/include/experimental/bits/simd_x86_conversions.h b/libstdc++-v3/include/experimental/bits/simd_x86_conversions.h
new file mode 100644
index 00000000000..e3bd39f4918
--- /dev/null
+++ b/libstdc++-v3/include/experimental/bits/simd_x86_conversions.h
@@ -0,0 +1,1358 @@
+// x86 specific conversion optimizations -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H
+#define _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H
+
+#if __cplusplus >= 201703L
+
+// work around PR85827
+// 1-arg __convert_x86 {{{1
+template <class _To, class _V, class _Traits> _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __vv)
+{
+ using _Tp = typename _Traits::value_type;
+ constexpr size_t _N = _Traits::_S_width;
+ [[maybe_unused]] const auto __intrin = __to_intrin(__vv);
+ _SimdWrapper<_Tp, _N> __v(__vv);
+ using _U = typename _VectorTraits<_To>::value_type;
+ constexpr size_t _M = _VectorTraits<_To>::_S_width;
+
+ // [xyz]_to_[xyz] {{{2
+ [[maybe_unused]] constexpr bool __x_to_x = sizeof(__v) == 16 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __x_to_y = sizeof(__v) == 16 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __x_to_z = sizeof(__v) == 16 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __y_to_x = sizeof(__v) == 32 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __y_to_y = sizeof(__v) == 32 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __y_to_z = sizeof(__v) == 32 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __z_to_x = sizeof(__v) == 64 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __z_to_y = sizeof(__v) == 64 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __z_to_z = sizeof(__v) == 64 && sizeof(_To) == 64;
+
+ // iX_to_iX {{{2
+ [[maybe_unused]] constexpr bool __i_to_i = is_integral_v<_U> && is_integral_v<_Tp>;
+ [[maybe_unused]] constexpr bool __i8_to_i16 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i8_to_i32 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i8_to_i64 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i16_to_i8 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i16_to_i32 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i16_to_i64 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i32_to_i8 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i32_to_i16 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i32_to_i64 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i64_to_i8 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i64_to_i16 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i64_to_i32 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 4;
+
+ // [fsu]X_to_[fsu]X {{{2
+ // ibw = integral && byte or word, i.e. char and short with any signedness
+ [[maybe_unused]] constexpr bool __s64_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s32_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s16_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s8_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u64_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u32_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u16_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u8_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s64_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s32_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u64_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u32_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __ibw_to_f32 = is_integral_v<_Tp> && sizeof(_Tp) <= 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __ibw_to_f64 = is_integral_v<_Tp> && sizeof(_Tp) <= 2 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_f64 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_f32 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+
+ if constexpr (__i_to_i && __y_to_x && !__have_avx2) { //{{{2
+ return __convert_x86<_To>(__lo128(__v), __hi128(__v));
+ } else if constexpr (__i_to_i && __x_to_y && !__have_avx2) { //{{{2
+ return __concat(__convert_x86<__vector_type_t<_U, _M / 2>>(__v),
+ __convert_x86<__vector_type_t<_U, _M / 2>>(
+ __extract_part<1, _N / _M * 2>(__v)));
+ } else if constexpr (__i_to_i) { //{{{2
+ static_assert(__x_to_x || __have_avx2,
+ "integral conversions with ymm registers require AVX2");
+ static_assert(__have_avx512bw || ((sizeof(_Tp) >= 4 || sizeof(__v) < 64) &&
+ (sizeof(_U) >= 4 || sizeof(_To) < 64)),
+ "8/16-bit integers in zmm registers require AVX512BW");
+ static_assert((sizeof(__v) < 64 && sizeof(_To) < 64) || __have_avx512f,
+ "integral conversions with ymm registers require AVX2");
+ }
+
+ if constexpr (is_floating_point_v<_Tp> == is_floating_point_v<_U> && //{{{2
+ sizeof(_Tp) == sizeof(_U)) {
+ // conversion uses simple bit reinterpretation (or no conversion at all)
+ if constexpr (_N >= _M) {
+ return __intrin_bitcast<_To>(__vv);
+ } else {
+ return __zero_extend(__vector_bitcast<_U>(__v));
+ }
+ } else if constexpr (_N < _M && sizeof(_To) > 16) { // zero extend (eg. xmm -> ymm){{{2
+ return __zero_extend(
+ __convert_x86<
+ __vector_type_t<_U, (16 / sizeof(_U) > _N) ? 16 / sizeof(_U) : _N>>(__v));
+ } else if constexpr (_N > _M && sizeof(__v) > 16) { // partial input (eg. ymm -> xmm){{{2
+ return __convert_x86<_To>(__extract_part<0, _N / _M>(__v));
+ } else if constexpr (__i64_to_i32) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi64_epi32(__intrin));
+ } else if constexpr (__x_to_x) {
+ return __auto_bitcast(_mm_shuffle_ps(__vector_bitcast<float>(__vv), __m128(), 8));
+ } else if constexpr (__y_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi64_epi32(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi64_epi32(__auto_bitcast(__vv))));
+ } else if constexpr (__y_to_x) {
+ return __intrin_bitcast<_To>(__lo128(_mm256_permute4x64_epi64(_mm256_shuffle_epi32(__intrin, 8), 0 + 4 * 2)));
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_epi32(__intrin));
+ }
+ } else if constexpr (__i64_to_i16) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi64_epi16(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v))));
+ } else if constexpr (__x_to_x && __have_ssse3) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ __intrin, _mm_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
+ // fallback without SSSE3
+ } else if constexpr (__y_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi64_epi16(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v))));
+ } else if constexpr (__y_to_x) {
+ const auto __a = _mm256_shuffle_epi8(
+ __intrin, _mm256_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, 0, 1, 8, 9, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80));
+ return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
+ } else if constexpr (__z_to_x) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_epi16(__intrin));
+ }
+ } else if constexpr (__i64_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi64_epi8(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi64_epi8(__zero_extend(__intrin))));
+ } else if constexpr (__y_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi64_epi8(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_epi8(__zero_extend(__intrin)));
+ } else if constexpr (__z_to_x) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_epi8(__intrin));
+ }
+ } else if constexpr (__i32_to_i64) { //{{{2
+ if constexpr (__have_sse4_1 && __x_to_x) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi32_epi64(__intrin) : _mm_cvtepu32_epi64(__intrin));
+ } else if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(
+ __intrin, is_signed_v<_Tp> ? _mm_srai_epi32(__intrin, 31) : __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi32_epi64(__intrin) : _mm256_cvtepu32_epi64(__intrin));
+ } else if constexpr (__y_to_z) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi32_epi64(__intrin) : _mm512_cvtepu32_epi64(__intrin));
+ }
+ } else if constexpr (__i32_to_i16) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi32_epi16(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v))));
+ } else if constexpr (__x_to_x && __have_ssse3) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ __intrin, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80)));
+ } else if constexpr (__x_to_x) {
+ auto __a = _mm_unpacklo_epi16(__intrin, __m128i()); // 0o.o 1o.o
+ auto __b = _mm_unpackhi_epi16(__intrin, __m128i()); // 2o.o 3o.o
+ auto __c = _mm_unpacklo_epi16(__a, __b); // 02oo ..oo
+ auto __d = _mm_unpackhi_epi16(__a, __b); // 13oo ..oo
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__c, __d)); // 0123 oooo
+ } else if constexpr (__y_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi32_epi16(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v))));
+ } else if constexpr (__y_to_x) {
+ auto __a = _mm256_shuffle_epi8(
+ __intrin,
+ _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, 0, 1, 4, 5, 8, 9, 12, 13,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80));
+ return __intrin_bitcast<_To>(
+ __lo128(_mm256_permute4x64_epi64(__a, 0xf8))); // __a[0] __a[2] | __a[3] __a[3]
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi32_epi16(__intrin));
+ }
+ } else if constexpr (__i32_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi32_epi8(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi32_epi8(__zero_extend(__intrin))));
+ } else if constexpr (__x_to_x && __have_ssse3) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ __intrin, _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
+ } else if constexpr (__x_to_x) {
+ const auto __a = _mm_unpacklo_epi8(__intrin, __intrin); // 0... .... 1... ....
+ const auto __b = _mm_unpackhi_epi8(__intrin, __intrin); // 2... .... 3... ....
+ const auto __c = _mm_unpacklo_epi8(__a, __b); // 02.. .... .... ....
+ const auto __d = _mm_unpackhi_epi8(__a, __b); // 13.. .... .... ....
+ const auto __e = _mm_unpacklo_epi8(__c, __d); // 0123 .... .... ....
+ return __intrin_bitcast<_To>(__e & _mm_cvtsi32_si128(-1));
+ } else if constexpr (__y_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi32_epi8(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi32_epi8(__zero_extend(__intrin)));
+ } else if constexpr (__z_to_x) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi32_epi8(__intrin));
+ }
+ } else if constexpr (__i16_to_i64) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi16_epi64(__intrin) : _mm_cvtepu16_epi64(__intrin));
+ } else if constexpr (__x_to_x && is_signed_v<_Tp>) {
+ auto __x = _mm_srai_epi16(__intrin, 15);
+ auto __y = _mm_unpacklo_epi16(__intrin, __x);
+ __x = _mm_unpacklo_epi16(__x, __x);
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__y, __x));
+ } else if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(_mm_unpacklo_epi16(__intrin, __m128i()), __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi16_epi64(__intrin) : _mm256_cvtepu16_epi64(__intrin));
+ } else if constexpr (__x_to_z) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi16_epi64(__intrin) : _mm512_cvtepu16_epi64(__intrin));
+ }
+ } else if constexpr (__i16_to_i32) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__intrin) : _mm_cvtepu16_epi32(__intrin));
+ } else if constexpr (__x_to_x && is_signed_v<_Tp>) {
+ return __intrin_bitcast<_To>(_mm_srai_epi32(_mm_unpacklo_epi16(__intrin, __intrin), 16));
+ } else if constexpr (__x_to_x && is_unsigned_v<_Tp>) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__intrin, __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi16_epi32(__intrin) : _mm256_cvtepu16_epi32(__intrin));
+ } else if constexpr (__y_to_z) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi16_epi32(__intrin) : _mm512_cvtepu16_epi32(__intrin));
+ }
+ } else if constexpr (__i16_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_avx512bw_vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi16_epi8(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512bw) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepi16_epi8(__zero_extend(__intrin))));
+ } else if constexpr (__x_to_x && __have_ssse3) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ __intrin, _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80)));
+ } else if constexpr (__x_to_x) {
+ auto __a = _mm_unpacklo_epi8(__intrin, __intrin); // 00.. 11.. 22.. 33..
+ auto __b = _mm_unpackhi_epi8(__intrin, __intrin); // 44.. 55.. 66.. 77..
+ auto __c = _mm_unpacklo_epi8(__a, __b); // 0404 .... 1515 ....
+ auto __d = _mm_unpackhi_epi8(__a, __b); // 2626 .... 3737 ....
+ auto __e = _mm_unpacklo_epi8(__c, __d); // 0246 0246 .... ....
+ auto __f = _mm_unpackhi_epi8(__c, __d); // 1357 1357 .... ....
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f));
+ } else if constexpr (__y_to_x && __have_avx512bw_vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi16_epi8(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512bw) {
+ return __intrin_bitcast<_To>(__lo256(_mm512_cvtepi16_epi8(__zero_extend(__intrin))));
+ } else if constexpr (__y_to_x) {
+ auto __a = _mm256_shuffle_epi8(
+ __intrin,
+ _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, 0, 2, 4, 6, 8, 10, 12, 14));
+ return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
+ } else if constexpr (__z_to_y && __have_avx512bw) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi16_epi8(__intrin));
+ } else if constexpr (__z_to_y) {
+ __assert_unreachable<_Tp>();
+ }
+ } else if constexpr (__i8_to_i64) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi8_epi64(__intrin) : _mm_cvtepu8_epi64(__intrin));
+ } else if constexpr (__x_to_x && is_signed_v<_Tp>) {
+ if constexpr (__have_ssse3) {
+ auto __dup = _mm_unpacklo_epi8(__intrin, __intrin);
+ auto __epi16 = _mm_srai_epi16(__dup, 8);
+ _mm_shuffle_epi8(
+ __epi16, _mm_setr_epi8(0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3));
+ } else {
+ auto __x = _mm_unpacklo_epi8(__intrin, __intrin);
+ __x = _mm_unpacklo_epi16(__x, __x);
+ return __intrin_bitcast<_To>(
+ _mm_unpacklo_epi32(_mm_srai_epi32(__x, 24), _mm_srai_epi32(__x, 31)));
+ }
+ } else if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(
+ _mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()), __m128i()),
+ __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi8_epi64(__intrin) : _mm256_cvtepu8_epi64(__intrin));
+ } else if constexpr (__x_to_z) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi8_epi64(__intrin) : _mm512_cvtepu8_epi64(__intrin));
+ }
+ } else if constexpr (__i8_to_i32) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__intrin) : _mm_cvtepu8_epi32(__intrin));
+ } else if constexpr (__x_to_x && is_signed_v<_Tp>) {
+ const auto __x = _mm_unpacklo_epi8(__intrin, __intrin);
+ return __intrin_bitcast<_To>(_mm_srai_epi32(_mm_unpacklo_epi16(__x, __x), 24));
+ } else if constexpr (__x_to_x && is_unsigned_v<_Tp>) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()), __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi8_epi32(__intrin) : _mm256_cvtepu8_epi32(__intrin));
+ } else if constexpr (__x_to_z) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi8_epi32(__intrin) : _mm512_cvtepu8_epi32(__intrin));
+ }
+ } else if constexpr (__i8_to_i16) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm_cvtepi8_epi16(__intrin) : _mm_cvtepu8_epi16(__intrin));
+ } else if constexpr (__x_to_x && is_signed_v<_Tp>) {
+ return __intrin_bitcast<_To>(_mm_srai_epi16(_mm_unpacklo_epi8(__intrin, __intrin), 8));
+ } else if constexpr (__x_to_x && is_unsigned_v<_Tp>) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__intrin, __m128i()));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm256_cvtepi8_epi16(__intrin) : _mm256_cvtepu8_epi16(__intrin));
+ } else if constexpr (__y_to_z && __have_avx512bw) {
+ return __intrin_bitcast<_To>(is_signed_v<_Tp> ? _mm512_cvtepi8_epi16(__intrin) : _mm512_cvtepu8_epi16(__intrin));
+ } else if constexpr (__y_to_z) {
+ __assert_unreachable<_Tp>();
+ }
+ } else if constexpr (__f32_to_s64) { //{{{2
+ if constexpr (__have_avx512dq_vl && __x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttps_epi64(__intrin));
+ } else if constexpr (__have_avx512dq_vl && __x_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvttps_epi64(__intrin));
+ } else if constexpr (__have_avx512dq && __y_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvttps_epi64(__intrin));
+ } // else use scalar fallback
+ } else if constexpr (__f32_to_u64) { //{{{2
+ if constexpr (__have_avx512dq_vl && __x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttps_epu64(__intrin));
+ } else if constexpr (__have_avx512dq_vl && __x_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvttps_epu64(__intrin));
+ } else if constexpr (__have_avx512dq && __y_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvttps_epu64(__intrin));
+ } // else use scalar fallback
+ } else if constexpr (__f32_to_s32) { //{{{2
+ if constexpr (__x_to_x || __y_to_y || __z_to_z) {
+ // go to fallback, it does the right thing
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else if constexpr (__f32_to_u32) { //{{{2
+ // the __builtin_constant_p hack enables constant propagation
+ if constexpr (__have_avx512vl && __x_to_x) {
+ const __vector_type_t<float, 4> __x = __v._M_data;
+ return __builtin_constant_p(__x) ? __make_vector<_U>(__x[0], __x[1], __x[2], __x[3])
+ : __vector_bitcast<_U>(_mm_cvttps_epu32(__intrin));
+ } else if constexpr (__have_avx512f && __x_to_x) {
+ const __vector_type_t<float, 4> __x = __v._M_data;
+ return __builtin_constant_p(__x)
+ ? __make_vector<_U>(__x[0], __x[1], __x[2], __x[3])
+ : __vector_bitcast<_U>(__lo128(_mm512_cvttps_epu32(__auto_bitcast(__v))));
+ } else if constexpr (__have_avx512vl && __y_to_y) {
+ const __vector_type_t<float, 8> __x = __v._M_data;
+ return __builtin_constant_p(__x) ? __make_vector<_U>(__x[0], __x[1], __x[2], __x[3],
+ __x[4], __x[5], __x[6], __x[7])
+ : __vector_bitcast<_U>(_mm256_cvttps_epu32(__intrin));
+ } else if constexpr (__have_avx512f && __y_to_y) {
+ const __vector_type_t<float, 8> __x = __v._M_data;
+ return __builtin_constant_p(__x)
+ ? __make_vector<_U>(__x[0], __x[1], __x[2], __x[3], __x[4], __x[5], __x[6],
+ __x[7])
+ : __vector_bitcast<_U>(__lo256(_mm512_cvttps_epu32(__auto_bitcast(__v))));
+ } else if constexpr (__x_to_x || __y_to_y || __z_to_z) {
+ // go to fallback, it does the right thing. We can't use the _mm_floor_ps -
+ // 0x8000'0000 trick for f32->u32 because it would discard small input values
+ // (only 24 mantissa bits)
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else if constexpr (__f32_to_ibw) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N>>(__v));
+ } else if constexpr (__f64_to_s64) { //{{{2
+ if constexpr (__have_avx512dq_vl && __x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttpd_epi64(__intrin));
+ } else if constexpr (__have_avx512dq_vl && __y_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvttpd_epi64(__intrin));
+ } else if constexpr (__have_avx512dq && __z_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvttpd_epi64(__intrin));
+ } // else use scalar fallback
+ } else if constexpr (__f64_to_u64) { //{{{2
+ if constexpr (__have_avx512dq_vl && __x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttpd_epu64(__intrin));
+ } else if constexpr (__have_avx512dq_vl && __y_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvttpd_epu64(__intrin));
+ } else if constexpr (__have_avx512dq && __z_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvttpd_epu64(__intrin));
+ } // else use scalar fallback
+ } else if constexpr (__f64_to_s32) { //{{{2
+ if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttpd_epi32(__intrin));
+ } else if constexpr (__y_to_x) {
+ return __intrin_bitcast<_To>(_mm256_cvttpd_epi32(__intrin));
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvttpd_epi32(__intrin));
+ }
+ } else if constexpr (__f64_to_u32) { //{{{2
+ if constexpr (__have_avx512vl && __x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvttpd_epu32(__intrin));
+ } else if constexpr (__have_sse4_1 && __x_to_x) {
+ return __vector_bitcast<_U>(_mm_cvttpd_epi32(_mm_floor_pd(__intrin) - 0x8000'0000u)) ^
+ 0x8000'0000u;
+ } else if constexpr (__x_to_x) {
+ // use scalar fallback: it's only 2 values to convert, can't get much better
+ // than scalar decomposition
+ } else if constexpr (__have_avx512vl && __y_to_x) {
+ return __intrin_bitcast<_To>(_mm256_cvttpd_epu32(__intrin));
+ } else if constexpr (__y_to_x) {
+ return __intrin_bitcast<_To>(__vector_bitcast<_U>(_mm256_cvttpd_epi32(
+ _mm256_floor_pd(__intrin) - 0x8000'0000u)) ^
+ 0x8000'0000u);
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvttpd_epu32(__intrin));
+ }
+ } else if constexpr (__f64_to_ibw) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, (_N < 4 ? 4 : _N)>>(__v));
+ } else if constexpr (__s64_to_f32) { //{{{2
+ if constexpr (__x_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi64_ps(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi64_ps(__intrin));
+ } else if constexpr (__z_to_y && __have_avx512dq) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_ps(__intrin));
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvtpd_ps(__convert_x86<__vector_type_t<double, 8>>(__v)));
+ }
+ } else if constexpr (__u64_to_f32) { //{{{2
+ if constexpr (__x_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepu64_ps(__intrin));
+ } else if constexpr (__y_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepu64_ps(__intrin));
+ } else if constexpr (__z_to_y && __have_avx512dq) {
+ return __intrin_bitcast<_To>(_mm512_cvtepu64_ps(__intrin));
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(
+ __lo256(_mm512_cvtepu32_ps(
+ __auto_bitcast(_mm512_cvtepi64_epi32(_mm512_srai_epi64(__intrin, 32))))) *
+ 0x100000000LL +
+ __lo256(_mm512_cvtepu32_ps(__auto_bitcast(_mm512_cvtepi64_epi32(__intrin)))));
+ }
+ } else if constexpr (__s32_to_f32) { //{{{2
+ // use fallback (builtin conversion)
+ } else if constexpr (__u32_to_f32) { //{{{2
+ if constexpr(__x_to_x && __have_avx512vl) {
+ // use fallback
+ } else if constexpr(__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepu32_ps(__auto_bitcast(__v))));
+ } else if constexpr(__x_to_x && (__have_fma || __have_fma4)) {
+ // work around PR85819
+ return 0x10000 * _mm_cvtepi32_ps(__to_intrin(__vv >> 16)) +
+ _mm_cvtepi32_ps(__to_intrin(__vv & 0xffff));
+ } else if constexpr(__y_to_y && __have_avx512vl) {
+ // use fallback
+ } else if constexpr(__y_to_y && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo256(_mm512_cvtepu32_ps(__auto_bitcast(__v))));
+ } else if constexpr(__y_to_y) {
+ // work around PR85819
+ return 0x10000 * _mm256_cvtepi32_ps(__to_intrin(__vv >> 16)) +
+ _mm256_cvtepi32_ps(__to_intrin(__vv & 0xffff));
+ } // else use fallback (builtin conversion)
+ } else if constexpr (__ibw_to_f32) { //{{{2
+ if constexpr (_M == 4 || __have_avx2) {
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _M>>(__v));
+ } else {
+ static_assert(__x_to_y);
+ __m128i __a, __b;
+ if constexpr (__have_sse4_1) {
+ __a = sizeof(_Tp) == 2
+ ? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__intrin) : _mm_cvtepu16_epi32(__intrin))
+ : (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__intrin) : _mm_cvtepu8_epi32(__intrin));
+ const auto __w = _mm_shuffle_epi32(__intrin, sizeof(_Tp) == 2 ? 0xee : 0xe9);
+ __b = sizeof(_Tp) == 2
+ ? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__w) : _mm_cvtepu16_epi32(__w))
+ : (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__w) : _mm_cvtepu8_epi32(__w));
+ } else {
+ __m128i __tmp;
+ if constexpr (sizeof(_Tp) == 1) {
+ __tmp = is_signed_v<_Tp> ? _mm_srai_epi16(_mm_unpacklo_epi8(__intrin, __intrin), 8):
+ _mm_unpacklo_epi8(__intrin, __m128i());
+ } else {
+ static_assert(sizeof(_Tp) == 2);
+ __tmp = __intrin;
+ }
+ __a = is_signed_v<_Tp> ? _mm_srai_epi32(_mm_unpacklo_epi16(__tmp, __tmp), 16)
+ : _mm_unpacklo_epi16(__tmp, __m128i());
+ __b = is_signed_v<_Tp> ? _mm_srai_epi32(_mm_unpackhi_epi16(__tmp, __tmp), 16)
+ : _mm_unpackhi_epi16(__tmp, __m128i());
+ }
+ return __convert_x86<_To>(__vector_bitcast<int>(__a), __vector_bitcast<int>(__b));
+ }
+ } else if constexpr (__s64_to_f64) { //{{{2
+ if constexpr (__x_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepi64_pd(__intrin));
+ } else if constexpr (__y_to_y && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi64_pd(__intrin));
+ } else if constexpr (__z_to_z && __have_avx512dq) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi64_pd(__intrin));
+ } else if constexpr (__z_to_z) {
+ return __intrin_bitcast<_To>(
+ _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(__to_intrin(__vv._M_data >> 32))) *
+ 0x100000000LL +
+ _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin)));
+ }
+ } else if constexpr (__u64_to_f64) { //{{{2
+ if constexpr (__x_to_x && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepu64_pd(__intrin));
+ } else if constexpr (__y_to_y && __have_avx512dq_vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepu64_pd(__intrin));
+ } else if constexpr (__z_to_z && __have_avx512dq) {
+ return __intrin_bitcast<_To>(_mm512_cvtepu64_pd(__intrin));
+ } else if constexpr (__z_to_z) {
+ return __intrin_bitcast<_To>(
+ _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__to_intrin(__vv >> 32))) *
+ 0x100000000LL +
+ _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin)));
+ }
+ } else if constexpr (__s32_to_f64) { //{{{2
+ if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvtepi32_pd(__intrin));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi32_pd(__intrin));
+ } else if constexpr (__y_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvtepi32_pd(__intrin));
+ }
+ } else if constexpr (__u32_to_f64) { //{{{2
+ if constexpr (__x_to_x && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm_cvtepu32_pd(__intrin));
+ } else if constexpr (__x_to_x && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo128(_mm512_cvtepu32_pd(__auto_bitcast(__v))));
+ } else if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvtepi32_pd(__to_intrin(__vv ^ 0x8000'0000u)) + 0x8000'0000u);
+ } else if constexpr (__x_to_y && __have_avx512vl) {
+ return __intrin_bitcast<_To>(_mm256_cvtepu32_pd(__intrin));
+ } else if constexpr (__x_to_y && __have_avx512f) {
+ return __intrin_bitcast<_To>(__lo256(_mm512_cvtepu32_pd(__auto_bitcast(__v))));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvtepi32_pd(__to_intrin(__vv ^ 0x8000'0000u)) + 0x8000'0000u);
+ } else if constexpr (__y_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvtepu32_pd(__intrin));
+ }
+ } else if constexpr (__ibw_to_f64) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, std::max(size_t(4), _M)>>(__v));
+ } else if constexpr (__f32_to_f64) { //{{{2
+ if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvtps_pd(__intrin));
+ } else if constexpr (__x_to_y) {
+ return __intrin_bitcast<_To>(_mm256_cvtps_pd(__intrin));
+ } else if constexpr (__y_to_z) {
+ return __intrin_bitcast<_To>(_mm512_cvtps_pd(__intrin));
+ }
+ } else if constexpr (__f64_to_f32) { //{{{2
+ if constexpr (__x_to_x) {
+ return __intrin_bitcast<_To>(_mm_cvtpd_ps(__intrin));
+ } else if constexpr (__y_to_x) {
+ return __intrin_bitcast<_To>(_mm256_cvtpd_ps(__intrin));
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(_mm512_cvtpd_ps(__intrin));
+ }
+ } else { //{{{2
+ __assert_unreachable<_Tp>();
+ }
+
+ // fallback:{{{2
+ return __vector_convert<_To>(__vv, make_index_sequence<std::min(_M, _N)>());
+ //}}}
+} // }}}
+// 2-arg __convert_x86 {{{1
+template <class _To, class _V, class _Traits> _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __vv0, _V __vv1)
+{
+ using _Tp = typename _Traits::value_type;
+ constexpr size_t _N = _Traits::_S_width;
+ _SimdWrapper<_Tp, _N> __v0(__vv0);
+ _SimdWrapper<_Tp, _N> __v1(__vv1);
+ [[maybe_unused]] const auto __i0 = __to_intrin(__vv0);
+ [[maybe_unused]] const auto __i1 = __to_intrin(__vv1);
+ using _U = typename _VectorTraits<_To>::value_type;
+ constexpr size_t _M = _VectorTraits<_To>::_S_width;
+
+ static_assert(
+ 2 * _N <= _M,
+ "__v1 would be discarded; use the one-argument __convert_x86 overload instead");
+
+ // [xyz]_to_[xyz] {{{2
+ [[maybe_unused]] constexpr bool __x_to_x = sizeof(__v0) == 16 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __x_to_y = sizeof(__v0) == 16 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __x_to_z = sizeof(__v0) == 16 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __y_to_x = sizeof(__v0) == 32 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __y_to_y = sizeof(__v0) == 32 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __y_to_z = sizeof(__v0) == 32 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __z_to_x = sizeof(__v0) == 64 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __z_to_y = sizeof(__v0) == 64 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __z_to_z = sizeof(__v0) == 64 && sizeof(_To) == 64;
+
+ // iX_to_iX {{{2
+ [[maybe_unused]] constexpr bool __i_to_i = std::is_integral_v<_U> && std::is_integral_v<_Tp>;
+ [[maybe_unused]] constexpr bool __i8_to_i16 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i8_to_i32 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i8_to_i64 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i16_to_i8 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i16_to_i32 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i16_to_i64 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i32_to_i8 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i32_to_i16 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i32_to_i64 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i64_to_i8 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i64_to_i16 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i64_to_i32 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 4;
+
+ // [fsu]X_to_[fsu]X {{{2
+ // ibw = integral && byte or word, i.e. char and short with any signedness
+ [[maybe_unused]] constexpr bool __i64_to_f32 = is_integral_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s32_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s16_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s8_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u32_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u16_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u8_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s64_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s32_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s16_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s8_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u64_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u32_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u16_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u8_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_f64 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_f32 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+
+ if constexpr (__i_to_i && __y_to_x && !__have_avx2) { //{{{2
+ // <double, 4>, <double, 4> => <short, 8>
+ return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1), __hi128(__v1));
+ } else if constexpr (__i_to_i) { // assert ISA {{{2
+ static_assert(__x_to_x || __have_avx2,
+ "integral conversions with ymm registers require AVX2");
+ static_assert(__have_avx512bw || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) &&
+ (sizeof(_U) >= 4 || sizeof(_To) < 64)),
+ "8/16-bit integers in zmm registers require AVX512BW");
+ static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
+ "integral conversions with ymm registers require AVX2");
+ }
+ // concat => use 1-arg __convert_x86 {{{2
+ if constexpr ((sizeof(__v0) == 16 && __have_avx2) ||
+ (sizeof(__v0) == 16 && __have_avx && std::is_floating_point_v<_Tp>) ||
+ (sizeof(__v0) == 32 && __have_avx512f && (sizeof(_Tp) >= 4 || __have_avx512bw))) {
+ // The ISA can handle wider input registers, so concat and use one-arg
+ // implementation. This reduces code duplication considerably.
+ return __convert_x86<_To>(__concat(__v0, __v1));
+ } else { //{{{2
+ // conversion using bit reinterpretation (or no conversion at all) should all go
+ // through the concat branch above:
+ static_assert(!(std::is_floating_point_v<_Tp> == std::is_floating_point_v<_U> &&
+ sizeof(_Tp) == sizeof(_U)));
+ if constexpr (2 * _N < _M && sizeof(_To) > 16) { // handle all zero extension{{{2
+ constexpr size_t Min = 16 / sizeof(_U);
+ return __zero_extend(
+ __convert_x86<__vector_type_t<_U, (Min > 2 * _N) ? Min : 2 * _N>>(__v0, __v1));
+ } else if constexpr (__i64_to_i32) { //{{{2
+ if constexpr (__x_to_x) {
+ return __auto_bitcast(_mm_shuffle_ps(__auto_bitcast(__v0), __auto_bitcast(__v1), 0x88));
+ } else if constexpr (__y_to_y) {
+ // AVX512F is not available (would concat otherwise)
+ return __auto_bitcast(__xzyw(
+ _mm256_shuffle_ps(__auto_bitcast(__v0), __auto_bitcast(__v1), 0x88)));
+ // alternative:
+ // const auto v0_abxxcdxx = _mm256_shuffle_epi32(__v0, 8);
+ // const auto v1_efxxghxx = _mm256_shuffle_epi32(__v1, 8);
+ // const auto v_abefcdgh = _mm256_unpacklo_epi64(v0_abxxcdxx,
+ // v1_efxxghxx); return _mm256_permute4x64_epi64(v_abefcdgh,
+ // 0x01 * 0 + 0x04 * 2 + 0x10 * 1 + 0x40 * 3); // abcdefgh
+ } else if constexpr (__z_to_z) {
+ return __intrin_bitcast<_To>(
+ __concat(_mm512_cvtepi64_epi32(__i0), _mm512_cvtepi64_epi32(__i1)));
+ }
+ } else if constexpr (__i64_to_i16) { //{{{2
+ if constexpr (__x_to_x) {
+ // AVX2 is not available (would concat otherwise)
+ if constexpr (__have_sse4_1) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44),
+ _mm_setr_epi8(0, 1, 8, 9, 4, 5, 12, 13, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80)));
+ } else {
+ return __vector_type_t<_U, _M>{_U(__v0[0]), _U(__v0[1]), _U(__v1[0]), _U(__v1[1])};
+ }
+ } else if constexpr (__y_to_x) {
+ auto __a = _mm256_unpacklo_epi16(__i0, __i1); // 04.. .... 26.. ....
+ auto __b = _mm256_unpackhi_epi16(__i0, __i1); // 15.. .... 37.. ....
+ auto __c = _mm256_unpacklo_epi16(__a, __b); // 0145 .... 2367 ....
+ return __intrin_bitcast<_To>(
+ _mm_unpacklo_epi32(__lo128(__c), __hi128(__c))); // 0123 4567
+ } else if constexpr (__z_to_y) {
+ return __intrin_bitcast<_To>(
+ __concat(_mm512_cvtepi64_epi16(__i0), _mm512_cvtepi64_epi16(__i1)));
+ }
+ } else if constexpr (__i64_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44),
+ _mm_setr_epi8(0, 8, 4, 12, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
+ } else if constexpr (__x_to_x && __have_ssse3) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(
+ _mm_shuffle_epi8(
+ __i0, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80)),
+ _mm_shuffle_epi8(
+ __i1, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80))));
+ } else if constexpr (__x_to_x) {
+ return __vector_type_t<_U, _M>{_U(__v0[0]), _U(__v0[1]), _U(__v1[0]), _U(__v1[1])};
+ } else if constexpr (__y_to_x) {
+ const auto __a = _mm256_shuffle_epi8(
+ _mm256_blend_epi32(__i0, _mm256_slli_epi64(__i1, 32), 0xAA),
+ _mm256_setr_epi8(0, 8, -0x80, -0x80, 4, 12, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, 0, 8, -0x80, -0x80, 4, 12, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80));
+ return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
+ } // __z_to_x uses concat fallback
+ } else if constexpr (__i32_to_i16) { //{{{2
+ if constexpr (__x_to_x) {
+ // AVX2 is not available (would concat otherwise)
+ if constexpr (__have_sse4_1) {
+ return __intrin_bitcast<_To>(
+ _mm_shuffle_epi8(_mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0xaa),
+ _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6,
+ 7, 10, 11, 14, 15)));
+ } else if constexpr (__have_ssse3) {
+ return __intrin_bitcast<_To>(
+ _mm_hadd_epi16(__to_intrin(__vv0 << 16), __to_intrin(__vv1 << 16)));
+ /*
+ return _mm_unpacklo_epi64(
+ _mm_shuffle_epi8(__i0, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 8, 9,
+ 12, 13, 12, 13, 14, 15)),
+ _mm_shuffle_epi8(__i1, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 8, 9,
+ 12, 13, 12, 13, 14, 15)));
+ */
+ } else {
+ auto __a = _mm_unpacklo_epi16(__i0, __i1); // 04.. 15..
+ auto __b = _mm_unpackhi_epi16(__i0, __i1); // 26.. 37..
+ auto __c = _mm_unpacklo_epi16(__a, __b); // 0246 ....
+ auto __d = _mm_unpackhi_epi16(__a, __b); // 1357 ....
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__c, __d)); // 0123 4567
+ }
+ } else if constexpr (__y_to_y) {
+ const auto __shuf = _mm256_setr_epi8(
+ 0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, 0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80);
+ auto __a = _mm256_shuffle_epi8(__i0, __shuf);
+ auto __b = _mm256_shuffle_epi8(__i1, __shuf);
+ return __intrin_bitcast<_To>(__xzyw(_mm256_unpacklo_epi64(__a, __b)));
+ } // __z_to_z uses concat fallback
+ } else if constexpr (__i32_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_ssse3) {
+ const auto shufmask =
+ _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, -0x80, -0x80, -0x80, -0x80, -0x80);
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(
+ _mm_shuffle_epi8(__i0, shufmask), _mm_shuffle_epi8(__i1, shufmask)));
+ } else if constexpr (__x_to_x) {
+ auto __a = _mm_unpacklo_epi8(__i0, __i1); // 04.. .... 15.. ....
+ auto __b = _mm_unpackhi_epi8(__i0, __i1); // 26.. .... 37.. ....
+ auto __c = _mm_unpacklo_epi8(__a, __b); // 0246 .... .... ....
+ auto __d = _mm_unpackhi_epi8(__a, __b); // 1357 .... .... ....
+ auto __e = _mm_unpacklo_epi8(__c, __d); // 0123 4567 .... ....
+ return __intrin_bitcast<_To>(__e & __m128i{-1, 0});
+ } else if constexpr (__y_to_x) {
+ const auto __a = _mm256_shuffle_epi8(
+ _mm256_blend_epi16(__i0, _mm256_slli_epi32(__i1, 16), 0xAA),
+ _mm256_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, 2, 6, 10,
+ 14, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
+ -0x80, 0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, 2, 6,
+ 10, 14));
+ return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
+ } // __z_to_y uses concat fallback
+ } else if constexpr (__i16_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_ssse3) {
+ const auto __shuf = reinterpret_cast<__m128i>(
+ __vector_type_t<_UChar, 16>{0, 2, 4, 6, 8, 10, 12, 14, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80});
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi64(_mm_shuffle_epi8(__i0, __shuf),
+ _mm_shuffle_epi8(__i1, __shuf)));
+ } else if constexpr (__x_to_x) {
+ auto __a = _mm_unpacklo_epi8(__i0, __i1); // 08.. 19.. 2A.. 3B..
+ auto __b = _mm_unpackhi_epi8(__i0, __i1); // 4C.. 5D.. 6E.. 7F..
+ auto __c = _mm_unpacklo_epi8(__a, __b); // 048C .... 159D ....
+ auto __d = _mm_unpackhi_epi8(__a, __b); // 26AE .... 37BF ....
+ auto __e = _mm_unpacklo_epi8(__c, __d); // 0246 8ACE .... ....
+ auto __f = _mm_unpackhi_epi8(__c, __d); // 1357 9BDF .... ....
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f));
+ } else if constexpr (__y_to_y) {
+ return __intrin_bitcast<_To>(__xzyw(_mm256_shuffle_epi8(
+ (__to_intrin(__v0) & _mm256_set1_epi32(0x00ff00ff)) |
+ _mm256_slli_epi16(__i1, 8),
+ _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
+ 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13,
+ 15))));
+ } // __z_to_z uses concat fallback
+ } else if constexpr (__i64_to_f32) { //{{{2
+ if constexpr (__x_to_x) {
+ return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1]);
+ } else if constexpr (__y_to_y) {
+ static_assert(__y_to_y && __have_avx2);
+ const auto __a = _mm256_unpacklo_epi32(__i0, __i1); // aeAE cgCG
+ const auto __b = _mm256_unpackhi_epi32(__i0, __i1); // bfBF dhDH
+ const auto __lo32 = _mm256_unpacklo_epi32(__a, __b); // abef cdgh
+ const auto __hi32 =
+ __vector_bitcast<conditional_t<is_signed_v<_Tp>, int, _UInt>>(
+ _mm256_unpackhi_epi32(__a, __b)); // ABEF CDGH
+ const auto __hi = 0x100000000LL * __convert_x86<__vector_type_t<float, 8>>(__hi32);
+ const auto __mid =
+ 0x10000 * _mm256_cvtepi32_ps(_mm256_srli_epi32(__lo32, 16));
+ const auto __lo = _mm256_cvtepi32_ps(_mm256_set1_epi32(0x0000ffffu) & __lo32);
+ return __xzyw((__hi + __mid) + __lo);
+ } else if constexpr (__z_to_z && __have_avx512dq) {
+ return std::is_signed_v<_Tp> ? __concat(_mm512_cvtepi64_ps(__i0),
+ _mm512_cvtepi64_ps(__i1))
+ : __concat(_mm512_cvtepu64_ps(__i0),
+ _mm512_cvtepu64_ps(__i1));
+ } else if constexpr (__z_to_z && std::is_signed_v<_Tp>) {
+ const __m512 __hi32 = _mm512_cvtepi32_ps(
+ __concat(_mm512_cvtepi64_epi32(__to_intrin(__vv0 >> 32)),
+ _mm512_cvtepi64_epi32(__to_intrin(__vv1 >> 32))));
+ const __m512i __lo32 =
+ __concat(_mm512_cvtepi64_epi32(__i0), _mm512_cvtepi64_epi32(__i1));
+ // split low 32-bits, because if __hi32 is a small negative number, the
+ // 24-bit mantissa may lose important information if any of the high 8
+ // bits of __lo32 is set, leading to catastrophic cancelation in the FMA
+ const __m512 __hi16 =
+ _mm512_cvtepu32_ps(_mm512_set1_epi32(0xffff0000u) & __lo32);
+ const __m512 __lo16 =
+ _mm512_cvtepi32_ps(_mm512_set1_epi32(0x0000ffffu) & __lo32);
+ return (__hi32 * 0x100000000LL + __hi16) + __lo16;
+ } else if constexpr (__z_to_z && std::is_unsigned_v<_Tp>) {
+ return __intrin_bitcast<_To>(
+ _mm512_cvtepu32_ps(
+ __concat(_mm512_cvtepi64_epi32(_mm512_srai_epi64(__i0, 32)),
+ _mm512_cvtepi64_epi32(_mm512_srai_epi64(__i1, 32)))) *
+ 0x100000000LL +
+ _mm512_cvtepu32_ps(
+ __concat(_mm512_cvtepi64_epi32(__i0), _mm512_cvtepi64_epi32(__i1))));
+ }
+ } else if constexpr (__f64_to_s32) { //{{{2
+ // use concat fallback
+ } else if constexpr (__f64_to_u32) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __vector_bitcast<_U>(_mm_unpacklo_epi64(
+ _mm_cvttpd_epi32(_mm_floor_pd(__i0) - 0x8000'0000u),
+ _mm_cvttpd_epi32(_mm_floor_pd(__i1) - 0x8000'0000u))) ^
+ 0x8000'0000u;
+ // without SSE4.1 just use the scalar fallback, it's only four values
+ } else if constexpr (__y_to_y) {
+ return __vector_bitcast<_U>(__concat(
+ _mm256_cvttpd_epi32(_mm256_floor_pd(__i0) - 0x8000'0000u),
+ _mm256_cvttpd_epi32(_mm256_floor_pd(__i1) - 0x8000'0000u))) ^
+ 0x8000'0000u;
+ } // __z_to_z uses fallback
+ } else if constexpr (__f64_to_ibw) { //{{{2
+ // one-arg __f64_to_ibw goes via _SimdWrapper<int, ?>. The fallback would go via two
+ // independet conversions to _SimdWrapper<_To> and subsequent interleaving. This is
+ // better, because f64->__i32 allows to combine __v0 and __v1 into one register:
+ //if constexpr (__z_to_x || __y_to_x) {
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N * 2>>(__v0, __v1));
+ //}
+ } else if constexpr (__f32_to_ibw) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N>>(__v0),
+ __convert_x86<__vector_type_t<int, _N>>(__v1));
+ //}}}
+ }
+
+ // fallback: {{{2
+ if constexpr (sizeof(_To) >= 32) {
+ // if _To is ymm or zmm, then _SimdWrapper<_U, _M / 2> is xmm or ymm
+ return __concat(__convert_x86<__vector_type_t<_U, _M / 2>>(__v0),
+ __convert_x86<__vector_type_t<_U, _M / 2>>(__v1));
+ } else if constexpr (sizeof(_To) == 16) {
+ const auto __lo = __to_intrin(__convert_x86<_To>(__v0));
+ const auto __hi = __to_intrin(__convert_x86<_To>(__v1));
+ if constexpr (sizeof(_U) * _N == 8) {
+ if constexpr (is_floating_point_v<_U>) {
+ return __auto_bitcast(_mm_unpacklo_pd(__vector_bitcast<double>(__lo),
+ __vector_bitcast<double>(__hi)));
+ } else {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
+ }
+ } else if constexpr (sizeof(_U) * _N == 4) {
+ if constexpr (is_floating_point_v<_U>) {
+ return __auto_bitcast(_mm_unpacklo_ps(__vector_bitcast<float>(__lo),
+ __vector_bitcast<float>(__hi)));
+ } else {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi));
+ }
+ } else if constexpr (sizeof(_U) * _N == 2) {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__lo, __hi));
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+ return __vector_convert<_To>(__vv0, __vv1, make_index_sequence<_N>());
+ } //}}}
+ }
+}//}}}1
+// 4-arg __convert_x86 {{{1
+template <class _To, class _V, class _Traits> _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __vv0, _V __vv1,_V __vv2,_V __vv3)
+{
+ using _Tp = typename _Traits::value_type;
+ constexpr size_t _N = _Traits::_S_width;
+ _SimdWrapper<_Tp, _N> __v0(__vv0);
+ _SimdWrapper<_Tp, _N> __v1(__vv1);
+ _SimdWrapper<_Tp, _N> __v2(__vv2);
+ _SimdWrapper<_Tp, _N> __v3(__vv3);
+ [[maybe_unused]] const auto __i0 = __to_intrin(__vv0);
+ [[maybe_unused]] const auto __i1 = __to_intrin(__vv1);
+ [[maybe_unused]] const auto __i2 = __to_intrin(__vv2);
+ [[maybe_unused]] const auto __i3 = __to_intrin(__vv3);
+ using _U = typename _VectorTraits<_To>::value_type;
+ constexpr size_t _M = _VectorTraits<_To>::_S_width;
+
+ static_assert(
+ 4 * _N <= _M,
+ "__v2/__v3 would be discarded; use the two/one-argument __convert_x86 overload instead");
+
+ // [xyz]_to_[xyz] {{{2
+ [[maybe_unused]] constexpr bool __x_to_x = sizeof(__v0) == 16 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __x_to_y = sizeof(__v0) == 16 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __x_to_z = sizeof(__v0) == 16 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __y_to_x = sizeof(__v0) == 32 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __y_to_y = sizeof(__v0) == 32 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __y_to_z = sizeof(__v0) == 32 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __z_to_x = sizeof(__v0) == 64 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __z_to_y = sizeof(__v0) == 64 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __z_to_z = sizeof(__v0) == 64 && sizeof(_To) == 64;
+
+ // iX_to_iX {{{2
+ [[maybe_unused]] constexpr bool __i_to_i = std::is_integral_v<_U> && std::is_integral_v<_Tp>;
+ [[maybe_unused]] constexpr bool __i8_to_i16 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i8_to_i32 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i8_to_i64 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i16_to_i8 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i16_to_i32 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __i16_to_i64 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i32_to_i8 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i32_to_i16 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i32_to_i64 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __i64_to_i8 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __i64_to_i16 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 2;
+ [[maybe_unused]] constexpr bool __i64_to_i32 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 4;
+
+ // [fsu]X_to_[fsu]X {{{2
+ // ibw = integral && byte or word, i.e. char and short with any signedness
+ [[maybe_unused]] constexpr bool __i64_to_f32 = is_integral_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s32_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s16_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s8_to_f32 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u32_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u16_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __u8_to_f32 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 4;
+ [[maybe_unused]] constexpr bool __s64_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s32_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s16_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __s8_to_f64 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u64_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u32_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u16_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __u8_to_f64 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f32_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_s64 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_s32 = is_integral_v<_U> && is_signed_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u64 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 8 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_u32 = is_integral_v<_U> && is_unsigned_v<_U> && sizeof(_U) == 4 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
+ [[maybe_unused]] constexpr bool __f64_to_ibw = is_integral_v<_U> && sizeof(_U) <= 2 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+ [[maybe_unused]] constexpr bool __f32_to_f64 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 && is_floating_point_v<_U> && sizeof(_U) == 8;
+ [[maybe_unused]] constexpr bool __f64_to_f32 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 && is_floating_point_v<_U> && sizeof(_U) == 4;
+
+ if constexpr (__i_to_i && __y_to_x && !__have_avx2) { //{{{2
+ // <double, 4>, <double, 4>, <double, 4>, <double, 4> => <char, 16>
+ return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1), __hi128(__v1),
+ __lo128(__v2), __hi128(__v2), __lo128(__v3), __hi128(__v3));
+ } else if constexpr (__i_to_i) { // assert ISA {{{2
+ static_assert(__x_to_x || __have_avx2,
+ "integral conversions with ymm registers require AVX2");
+ static_assert(__have_avx512bw || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) &&
+ (sizeof(_U) >= 4 || sizeof(_To) < 64)),
+ "8/16-bit integers in zmm registers require AVX512BW");
+ static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
+ "integral conversions with ymm registers require AVX2");
+ }
+ // concat => use 2-arg __convert_x86 {{{2
+ if constexpr ((sizeof(__v0) == 16 && __have_avx2) ||
+ (sizeof(__v0) == 16 && __have_avx && std::is_floating_point_v<_Tp>) ||
+ (sizeof(__v0) == 32 && __have_avx512f)) {
+ // The ISA can handle wider input registers, so concat and use two-arg
+ // implementation. This reduces code duplication considerably.
+ return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3));
+ } else { //{{{2
+ // conversion using bit reinterpretation (or no conversion at all) should all go
+ // through the concat branch above:
+ static_assert(!(std::is_floating_point_v<_Tp> == std::is_floating_point_v<_U> &&
+ sizeof(_Tp) == sizeof(_U)));
+ if constexpr (4 * _N < _M && sizeof(_To) > 16) { // handle all zero extension{{{2
+ constexpr size_t Min = 16 / sizeof(_U);
+ return __zero_extend(
+ __convert_x86<__vector_type_t<_U, (Min > 4 * _N) ? Min : 4 * _N>>(__v0, __v1,
+ __v2, __v3));
+ } else if constexpr (__i64_to_i16) { //{{{2
+ if constexpr (__x_to_x && __have_sse4_1) {
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ _mm_blend_epi16(_mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0x22),
+ _mm_blend_epi16(_mm_slli_si128(__i2, 4),
+ _mm_slli_si128(__i3, 6), 0x88),
+ 0xcc),
+ _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15)));
+ } else if constexpr (__y_to_y) {
+ return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
+ __xzyw(_mm256_blend_epi16(
+ __auto_bitcast(_mm256_shuffle_ps(__vector_bitcast<float>(__v0),
+ __vector_bitcast<float>(__v2),
+ 0x88)), // 0.1. 8.9. 2.3. A.B.
+ __to_intrin(
+ __vector_bitcast<int>(_mm256_shuffle_ps(
+ __vector_bitcast<float>(__v1), __vector_bitcast<float>(__v3), 0x88))
+ << 16), // .4.5 .C.D .6.7 .E.F
+ 0xaa) // 0415 8C9D 2637 AEBF
+ ), // 0415 2637 8C9D AEBF
+ _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
+ 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14,
+ 15)));
+ /*
+ auto __a = _mm256_unpacklo_epi16(__v0, __v1); // 04.. .... 26.. ....
+ auto __b = _mm256_unpackhi_epi16(__v0, __v1); // 15.. .... 37.. ....
+ auto __c = _mm256_unpacklo_epi16(__v2, __v3); // 8C.. .... AE.. ....
+ auto __d = _mm256_unpackhi_epi16(__v2, __v3); // 9D.. .... BF.. ....
+ auto __e = _mm256_unpacklo_epi16(__a, __b); // 0145 .... 2367 ....
+ auto __f = _mm256_unpacklo_epi16(__c, __d); // 89CD .... ABEF ....
+ auto __g = _mm256_unpacklo_epi64(__e, __f); // 0145 89CD 2367 ABEF
+ return __concat(
+ _mm_unpacklo_epi32(__lo128(__g), __hi128(__g)),
+ _mm_unpackhi_epi32(__lo128(__g), __hi128(__g))); // 0123 4567 89AB CDEF
+ */
+ } // else use fallback
+ } else if constexpr (__i64_to_i8) { //{{{2
+ if constexpr (__x_to_x) {
+ // TODO: use fallback for now
+ } else if constexpr (__y_to_x) {
+ auto __a =
+ _mm256_srli_epi32(_mm256_slli_epi32(__i0, 24), 24) |
+ _mm256_srli_epi32(_mm256_slli_epi32(__i1, 24), 16) |
+ _mm256_srli_epi32(_mm256_slli_epi32(__i2, 24), 8) |
+ _mm256_slli_epi32(__i3, 24); // 048C .... 159D .... 26AE .... 37BF ....
+ /*return _mm_shuffle_epi8(
+ _mm_blend_epi32(__lo128(__a) << 32, __hi128(__a), 0x5),
+ _mm_setr_epi8(4, 12, 0, 8, 5, 13, 1, 9, 6, 14, 2, 10, 7, 15, 3, 11));*/
+ auto __b = _mm256_unpackhi_epi64(__a, __a); // 159D .... 159D .... 37BF .... 37BF ....
+ auto __c = _mm256_unpacklo_epi8(__a, __b); // 0145 89CD .... .... 2367 ABEF .... ....
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__lo128(__c), __hi128(__c))); // 0123 4567 89AB CDEF
+ }
+ } else if constexpr (__i32_to_i8) { //{{{2
+ if constexpr (__x_to_x) {
+ if constexpr (__have_ssse3) {
+ const auto __x0 = __vector_bitcast<_UInt>(__v0._M_data) & 0xff;
+ const auto __x1 = (__vector_bitcast<_UInt>(__v1._M_data) & 0xff) << 8;
+ const auto __x2 = (__vector_bitcast<_UInt>(__v2._M_data) & 0xff) << 16;
+ const auto __x3 = __vector_bitcast<_UInt>(__v3._M_data) << 24;
+ return __intrin_bitcast<_To>(
+ _mm_shuffle_epi8(__to_intrin(__x0 | __x1 | __x2 | __x3),
+ _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10,
+ 14, 3, 7, 11, 15)));
+ } else {
+ auto __a = _mm_unpacklo_epi8(__i0, __i2); // 08.. .... 19.. ....
+ auto __b = _mm_unpackhi_epi8(__i0, __i2); // 2A.. .... 3B.. ....
+ auto __c = _mm_unpacklo_epi8(__i1, __i3); // 4C.. .... 5D.. ....
+ auto __d = _mm_unpackhi_epi8(__i1, __i3); // 6E.. .... 7F.. ....
+ auto __e = _mm_unpacklo_epi8(__a, __c); // 048C .... .... ....
+ auto __f = _mm_unpackhi_epi8(__a, __c); // 159D .... .... ....
+ auto __g = _mm_unpacklo_epi8(__b, __d); // 26AE .... .... ....
+ auto __h = _mm_unpackhi_epi8(__b, __d); // 37BF .... .... ....
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi8(
+ _mm_unpacklo_epi8(__e, __g), // 0246 8ACE .... ....
+ _mm_unpacklo_epi8(__f, __h) // 1357 9BDF .... ....
+ )); // 0123 4567 89AB CDEF
+ }
+ } else if constexpr (__y_to_y) {
+ const auto __a = _mm256_shuffle_epi8(
+ __to_intrin((__vector_bitcast<ushort>(_mm256_blend_epi16(
+ __i0, _mm256_slli_epi32(__i1, 16), 0xAA)) &
+ 0xff) |
+ (__vector_bitcast<ushort>(_mm256_blend_epi16(
+ __i2, _mm256_slli_epi32(__i3, 16), 0xAA))
+ << 8)),
+ _mm256_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15,
+ 0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15));
+ return __intrin_bitcast<_To>(_mm256_permutevar8x32_epi32(
+ __a, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)));
+ }
+ } else if constexpr (__i64_to_f32) { //{{{2
+ // this branch is only relevant with AVX and w/o AVX2 (i.e. no ymm integers)
+ if constexpr (__x_to_y) {
+ return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1], __v2[0], __v2[1], __v3[0], __v3[1]);
+
+ const auto __a = _mm_unpacklo_epi32(__i0, __i1); // acAC
+ const auto __b = _mm_unpackhi_epi32(__i0, __i1); // bdBD
+ const auto __c = _mm_unpacklo_epi32(__i2, __i3); // egEG
+ const auto __d = _mm_unpackhi_epi32(__i2, __i3); // fhFH
+ const auto __lo32a = _mm_unpacklo_epi32(__a, __b); // abcd
+ const auto __lo32b = _mm_unpacklo_epi32(__c, __d); // efgh
+ const auto __hi32 =
+ __vector_bitcast<conditional_t<is_signed_v<_Tp>, int, _UInt>>(
+ __concat(_mm_unpackhi_epi32(__a, __b),
+ _mm_unpackhi_epi32(__c, __d))); // ABCD EFGH
+ const auto __hi =
+ 0x100000000LL * __convert_x86<__vector_type_t<float, 8>>(__hi32);
+ const auto __mid =
+ 0x10000 * _mm256_cvtepi32_ps(__concat(_mm_srli_epi32(__lo32a, 16),
+ _mm_srli_epi32(__lo32b, 16)));
+ const auto __lo =
+ _mm256_cvtepi32_ps(__concat(_mm_set1_epi32(0x0000ffffu) & __lo32a,
+ _mm_set1_epi32(0x0000ffffu) & __lo32b));
+ return (__hi + __mid) + __lo;
+ }
+ } else if constexpr (__f64_to_ibw) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N * 2>>(__v0, __v1),
+ __convert_x86<__vector_type_t<int, _N * 2>>(__v2, __v3));
+ } else if constexpr (__f32_to_ibw) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N>>(__v0),
+ __convert_x86<__vector_type_t<int, _N>>(__v1),
+ __convert_x86<__vector_type_t<int, _N>>(__v2),
+ __convert_x86<__vector_type_t<int, _N>>(__v3));
+ } //}}}
+
+ // fallback: {{{2
+ if constexpr (sizeof(_To) >= 32) {
+ // if _To is ymm or zmm, then _SimdWrapper<_U, _M / 2> is xmm or ymm
+ return __concat(__convert_x86<__vector_type_t<_U, _M / 2>>(__v0, __v1),
+ __convert_x86<__vector_type_t<_U, _M / 2>>(__v2, __v3));
+ } else if constexpr (sizeof(_To) == 16) {
+ const auto __lo = __to_intrin(__convert_x86<_To>(__v0, __v1));
+ const auto __hi = __to_intrin(__convert_x86<_To>(__v2, __v3));
+ if constexpr (sizeof(_U) * _N * 2 == 8) {
+ if constexpr (is_floating_point_v<_U>) {
+ return __auto_bitcast(_mm_unpacklo_pd(__lo, __hi));
+ } else {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
+ }
+ } else if constexpr (sizeof(_U) * _N * 2 == 4) {
+ if constexpr (is_floating_point_v<_U>) {
+ return __auto_bitcast(_mm_unpacklo_ps(__lo, __hi));
+ } else {
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi));
+ }
+ } else {
+ __assert_unreachable<_Tp>();
+ }
+ } else {
+ return __vector_convert<_To>(__v0._M_data, __v1._M_data, __v2._M_data, __v3._M_data,
+ make_index_sequence<_N>());
+ } //}}}2
+ }
+}//}}}
+// 8-arg __convert_x86 {{{1
+template <class _To, class _V, class _Traits> _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __vv0, _V __vv1,_V __vv2,_V __vv3,_V __vv4,_V __vv5, _V __vv6, _V __vv7)
+{
+ using _Tp = typename _Traits::value_type;
+ constexpr size_t _N = _Traits::_S_width;
+ _SimdWrapper<_Tp, _N> __v0(__vv0);
+ _SimdWrapper<_Tp, _N> __v1(__vv1);
+ _SimdWrapper<_Tp, _N> __v2(__vv2);
+ _SimdWrapper<_Tp, _N> __v3(__vv3);
+ _SimdWrapper<_Tp, _N> __v4(__vv4);
+ _SimdWrapper<_Tp, _N> __v5(__vv5);
+ _SimdWrapper<_Tp, _N> __v6(__vv6);
+ _SimdWrapper<_Tp, _N> __v7(__vv7);
+ [[maybe_unused]] const auto __i0 = __to_intrin(__vv0);
+ [[maybe_unused]] const auto __i1 = __to_intrin(__vv1);
+ [[maybe_unused]] const auto __i2 = __to_intrin(__vv2);
+ [[maybe_unused]] const auto __i3 = __to_intrin(__vv3);
+ [[maybe_unused]] const auto __i4 = __to_intrin(__vv4);
+ [[maybe_unused]] const auto __i5 = __to_intrin(__vv5);
+ [[maybe_unused]] const auto __i6 = __to_intrin(__vv6);
+ [[maybe_unused]] const auto __i7 = __to_intrin(__vv7);
+ using _U = typename _VectorTraits<_To>::value_type;
+ constexpr size_t _M = _VectorTraits<_To>::_S_width;
+
+ static_assert(8 * _N <= _M, "__v4-__v7 would be discarded; use the four/two/one-argument "
+ "__convert_x86 overload instead");
+
+ // [xyz]_to_[xyz] {{{2
+ [[maybe_unused]] constexpr bool __x_to_x = sizeof(__v0) == 16 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __x_to_y = sizeof(__v0) == 16 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __x_to_z = sizeof(__v0) == 16 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __y_to_x = sizeof(__v0) == 32 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __y_to_y = sizeof(__v0) == 32 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __y_to_z = sizeof(__v0) == 32 && sizeof(_To) == 64;
+ [[maybe_unused]] constexpr bool __z_to_x = sizeof(__v0) == 64 && sizeof(_To) == 16;
+ [[maybe_unused]] constexpr bool __z_to_y = sizeof(__v0) == 64 && sizeof(_To) == 32;
+ [[maybe_unused]] constexpr bool __z_to_z = sizeof(__v0) == 64 && sizeof(_To) == 64;
+
+ // [if]X_to_i8 {{{2
+ [[maybe_unused]] constexpr bool __i_to_i = std::is_integral_v<_U> && std::is_integral_v<_Tp>;
+ [[maybe_unused]] constexpr bool __i64_to_i8 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_U) == 1;
+ [[maybe_unused]] constexpr bool __f64_to_i8 = is_integral_v<_U> && sizeof(_U) == 1 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
+
+ if constexpr (__i_to_i) { // assert ISA {{{2
+ static_assert(__x_to_x || __have_avx2,
+ "integral conversions with ymm registers require AVX2");
+ static_assert(__have_avx512bw || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) &&
+ (sizeof(_U) >= 4 || sizeof(_To) < 64)),
+ "8/16-bit integers in zmm registers require AVX512BW");
+ static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
+ "integral conversions with ymm registers require AVX2");
+ }
+ // concat => use 4-arg __convert_x86 {{{2
+ if constexpr ((sizeof(__v0) == 16 && __have_avx2) ||
+ (sizeof(__v0) == 16 && __have_avx && std::is_floating_point_v<_Tp>) ||
+ (sizeof(__v0) == 32 && __have_avx512f)) {
+ // The ISA can handle wider input registers, so concat and use two-arg
+ // implementation. This reduces code duplication considerably.
+ return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3), __concat(__v4, __v5),
+ __concat(__v6, __v7));
+ } else { //{{{2
+ // conversion using bit reinterpretation (or no conversion at all) should all go
+ // through the concat branch above:
+ static_assert(!(std::is_floating_point_v<_Tp> == std::is_floating_point_v<_U> &&
+ sizeof(_Tp) == sizeof(_U)));
+ static_assert(!(8 * _N < _M && sizeof(_To) > 16),
+ "zero extension should be impossible");
+ if constexpr (__i64_to_i8) { //{{{2
+ if constexpr (__x_to_x && __have_ssse3) {
+ // unsure whether this is better than the variant below
+ return __intrin_bitcast<_To>(_mm_shuffle_epi8(
+ __to_intrin((((__vv0 & 0xff) | ((__vv1 & 0xff) << 8)) |
+ (((__vv2 & 0xff) << 16) | ((__vv3 & 0xff) << 24))) |
+ ((((__vv4 & 0xff) << 32) | ((__vv5 & 0xff) << 40)) |
+ (((__vv6 & 0xff) << 48) | (__vv7 << 56)))),
+ _mm_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15)));
+ } else if constexpr (__x_to_x) {
+ const auto __a = _mm_unpacklo_epi8(__i0, __i1); // ac
+ const auto __b = _mm_unpackhi_epi8(__i0, __i1); // bd
+ const auto __c = _mm_unpacklo_epi8(__i2, __i3); // eg
+ const auto __d = _mm_unpackhi_epi8(__i2, __i3); // fh
+ const auto __e = _mm_unpacklo_epi8(__i4, __i5); // ik
+ const auto __f = _mm_unpackhi_epi8(__i4, __i5); // jl
+ const auto __g = _mm_unpacklo_epi8(__i6, __i7); // mo
+ const auto __h = _mm_unpackhi_epi8(__i6, __i7); // np
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi64(
+ _mm_unpacklo_epi32(_mm_unpacklo_epi8(__a, __b), // abcd
+ _mm_unpacklo_epi8(__c, __d)), // efgh
+ _mm_unpacklo_epi32(_mm_unpacklo_epi8(__e, __f), // ijkl
+ _mm_unpacklo_epi8(__g, __h)) // mnop
+ ));
+ } else if constexpr (__y_to_y) {
+ auto __a = // 048C GKOS 159D HLPT 26AE IMQU 37BF JNRV
+ __to_intrin((((__vv0 & 0xff) | ((__vv1 & 0xff) << 8)) |
+ (((__vv2 & 0xff) << 16) | ((__vv3 & 0xff) << 24))) |
+ ((((__vv4 & 0xff) << 32) | ((__vv5 & 0xff) << 40)) |
+ (((__vv6 & 0xff) << 48) | ((__vv7 << 56)))));
+ /*
+ auto __b = _mm256_unpackhi_epi64(__a, __a); // 159D HLPT 159D HLPT 37BF JNRV 37BF JNRV
+ auto __c = _mm256_unpacklo_epi8(__a, __b); // 0145 89CD GHKL OPST 2367 ABEF IJMN QRUV
+ auto __d = __xzyw(__c); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV
+ return _mm256_shuffle_epi8(
+ __d, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14,
+ 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7,
+ 14, 15));
+ */
+ auto __b = _mm256_shuffle_epi8( // 0145 89CD GHKL OPST 2367 ABEF IJMN QRUV
+ __a, _mm256_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15,
+ 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15));
+ auto __c = __xzyw(__b); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV
+ return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
+ __c, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14,
+ 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7,
+ 14, 15)));
+ } else if constexpr(__z_to_z) {
+ return __concat(
+ __convert_x86<__vector_type_t<_U, _M / 2>>(__v0, __v1, __v2, __v3),
+ __convert_x86<__vector_type_t<_U, _M / 2>>(__v4, __v5, __v6, __v7));
+ }
+ } else if constexpr (__f64_to_i8) { //{{{2
+ return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _N * 2>>(__v0, __v1),
+ __convert_x86<__vector_type_t<int, _N * 2>>(__v2, __v3),
+ __convert_x86<__vector_type_t<int, _N * 2>>(__v4, __v5),
+ __convert_x86<__vector_type_t<int, _N * 2>>(__v6, __v7));
+ } else { // unreachable {{{2
+ __assert_unreachable<_Tp>();
+ } //}}}
+
+ // fallback: {{{2
+ if constexpr (sizeof(_To) >= 32) {
+ // if _To is ymm or zmm, then _SimdWrapper<_U, _M / 2> is xmm or ymm
+ return __concat(__convert_x86<__vector_type_t<_U, _M / 2>>(__v0, __v1, __v2, __v3),
+ __convert_x86<__vector_type_t<_U, _M / 2>>(__v4, __v5, __v6, __v7));
+ } else if constexpr (sizeof(_To) == 16) {
+ const auto __lo = __to_intrin(__convert_x86<_To>(__v0, __v1, __v2, __v3));
+ const auto __hi = __to_intrin(__convert_x86<_To>(__v4, __v5, __v6, __v7));
+ static_assert(sizeof(_U) == 1 && _N == 2);
+ return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
+ } else {
+ __assert_unreachable<_Tp>();
+ //return __vector_convert<_To>(__v0._M_data, __v1._M_data, __v2._M_data, __v3._M_data, __v4._M_data, __v5._M_data, __v6._M_data, __v7._M_data,
+ // make_index_sequence<_N>());
+ } //}}}2
+ }
+}//}}}
+
+#endif // __cplusplus >= 201703L
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/include/experimental/simd b/libstdc++-v3/include/experimental/simd
new file mode 100644
index 00000000000..70772a6af82
--- /dev/null
+++ b/libstdc++-v3/include/experimental/simd
@@ -0,0 +1,54 @@
+// Components for element-wise operations on data-parallel objects -*- C++ -*-
+
+// Copyright © 2015-2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+// Matthias Kretz <m.kretz@gsi.de>
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the names of contributing organizations nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/** @file experimental/simd
+ * This is a TS C++ Library header.
+ */
+
+//
+// N4773 §9 data-parallel types library
+//
+
+#ifndef _GLIBCXX_EXPERIMENTAL_SIMD
+#define _GLIBCXX_EXPERIMENTAL_SIMD
+
+#define __cpp_lib_experimental_parallel_simd 201803
+
+#pragma GCC diagnostic push
+// Many [[gnu::vector_size(N)]] types might lead to a -Wpsabi warning which is
+// irrelevant as those functions never appear on ABI borders
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+#include "bits/simd_detail.h"
+#include "bits/simd.h"
+#include "bits/simd_abis.h"
+#include "bits/simd_math.h"
+
+#pragma GCC diagnostic pop
+
+#endif // _GLIBCXX_EXPERIMENTAL_SIMD
+// vim: ft=cpp
diff --git a/libstdc++-v3/include/std/limits b/libstdc++-v3/include/std/limits
index dbea152d014..b14236e2dbe 100644
--- a/libstdc++-v3/include/std/limits
+++ b/libstdc++-v3/include/std/limits
@@ -1715,7 +1715,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
denorm_min() _GLIBCXX_USE_NOEXCEPT { return __FLT_DENORM_MIN__; }
static _GLIBCXX_USE_CONSTEXPR bool is_iec559
+#ifdef __STDC_IEC_559__
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
+#else
+ = false;
+#endif
static _GLIBCXX_USE_CONSTEXPR bool is_bounded = true;
static _GLIBCXX_USE_CONSTEXPR bool is_modulo = false;
@@ -1790,7 +1794,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
denorm_min() _GLIBCXX_USE_NOEXCEPT { return __DBL_DENORM_MIN__; }
static _GLIBCXX_USE_CONSTEXPR bool is_iec559
+#ifdef __STDC_IEC_559__
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
+#else
+ = false;
+#endif
static _GLIBCXX_USE_CONSTEXPR bool is_bounded = true;
static _GLIBCXX_USE_CONSTEXPR bool is_modulo = false;
@@ -1865,7 +1873,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
denorm_min() _GLIBCXX_USE_NOEXCEPT { return __LDBL_DENORM_MIN__; }
static _GLIBCXX_USE_CONSTEXPR bool is_iec559
+#ifdef __STDC_IEC_559__
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
+#else
+ = false;
+#endif
static _GLIBCXX_USE_CONSTEXPR bool is_bounded = true;
static _GLIBCXX_USE_CONSTEXPR bool is_modulo = false;
diff --git a/libstdc++-v3/testsuite/experimental/simd/generate_testcases.sh b/libstdc++-v3/testsuite/experimental/simd/generate_testcases.sh
new file mode 100755
index 00000000000..49c24c9493b
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/generate_testcases.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+floattypes=(
+"long double"
+"double"
+"float"
+)
+alltypes=(
+"${floattypes[@]}"
+"long long"
+"unsigned long long"
+"unsigned long"
+"long"
+"int"
+"unsigned int"
+"short"
+"unsigned short"
+"char"
+"signed char"
+"unsigned char"
+"char32_t"
+"char16_t"
+"wchar_t"
+)
+
+archlist=(
+sse sse2 sse3 ssse3 sse4
+avx avx2
+arch=knl
+arch=skylake-avx512
+arch=pentium3
+arch=armv7-a+simd
+arch=armv7ve+simd
+arch=armv8-a+simd
+arch=armv8.1-a+simd
+arch=armv8.2-a+simd
+arch=armv8.3-a+simd
+arch=armv8.4-a+simd
+arch=armv8-r+simd
+)
+
+cd ${0%/*}
+for testcase in tests/*.h; do
+ if grep -q "test only floattypes" "$testcase"; then
+ typelist=("${floattypes[@]}")
+ else
+ typelist=("${alltypes[@]}")
+ fi
+ testcase=${testcase%.h}
+ testcase=${testcase##*/}
+ for type in "${typelist[@]}"; do
+ for m in "${archlist[@]}"; do
+ target=$m
+ runtime=${m}_runtime
+ case "$m" in
+ *knl)
+ target=avx512f
+ runtime=avx512f_runtime
+ ;;
+ *skylake-avx512)
+ target='{ avx512bw && { avx512vl && avx512dq } }'
+ runtime='{ avx512bw_runtime && { avx512vl_runtime && avx512dq_runtime } }'
+ ;;
+ *pentium3)
+ target="{ia32 && sse}"
+ runtime=sse_runtime
+ ;;
+ *+neon*|*+simd*)
+ target=arm_neon
+ runtime=arm_neon_hw
+ ;;
+ esac
+ cat > "${testcase}-${type// /_}-${m// /_}.cc" <<EOF
+// { dg-options "-O2 -std=gnu++17 \"-DTESTTYPE=${type}\" -m${m}" }
+// { dg-do run { target { c++17 && ${target} } } }
+// { dg-xfail-run-if "expected SIGILL" { ! ${runtime} } }
+// { dg-require-effective-target run_expensive_tests }
+#include "tests/${testcase}.h"
+EOF
+ done
+ cat > "${testcase}-${type// /_}-native.cc" <<EOF
+// { dg-options "-O2 -std=gnu++17 \"-DTESTTYPE=${type}\" -march=native" }
+#include "tests/${testcase}.h"
+EOF
+ cat > "${testcase}-${type// /_}-native-fixed_size.cc" <<EOF
+// { dg-options "-O2 -std=gnu++17 \"-DTESTTYPE=${type}\" -DTESTFIXEDSIZE -march=native" }
+// { dg-require-effective-target run_expensive_tests }
+#include "tests/${testcase}.h"
+EOF
+ done
+done
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/abs.h b/libstdc++-v3/testsuite/experimental/simd/tests/abs.h
new file mode 100644
index 00000000000..0d35ab53148
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/abs.h
@@ -0,0 +1,23 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include <cmath> // abs & sqrt
+#include <cstdlib> // integer abs
+#include "bits/test_values.h"
+
+template <typename V> void test()
+{
+ if constexpr (std::is_signed_v<typename V::value_type>) {
+ using std::abs;
+ using T = typename V::value_type;
+ using L = std::numeric_limits<T>;
+ test_values<V>(
+ {L::max(), L::lowest(), L::min(), -L::max() / 2, T(), -T(), T(-1), T(-2)},
+ {100, L::lowest(), L::max()},
+ [](V input) {
+ const V expected([&](auto i) { return T(std::abs(T(input[i]))); });
+ COMPARE(abs(input), expected);
+ });
+ } else {
+ // VERIFY(!(sfinae_is_callable<V &, const int *>(call_memload())));
+ }
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/algorithms.h b/libstdc++-v3/testsuite/experimental/simd/tests/algorithms.h
new file mode 100644
index 00000000000..b3af5703dc3
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/algorithms.h
@@ -0,0 +1,13 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ V a{[](auto i) -> T { return i & 1u; }};
+ V b{[](auto i) -> T { return (i + 1u) & 1u; }};
+ COMPARE(min(a, b), V{0});
+ COMPARE(max(a, b), V{1});
+}
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/conversions.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/conversions.h
new file mode 100644
index 00000000000..f4e7b3b6f13
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/conversions.h
@@ -0,0 +1,145 @@
+#include <array>
+
+// is_conversion_undefined {{{1
+/* implementation-defined
+ * ======================
+ * §4.7 p3 (integral conversions)
+ * If the destination type is signed, the value is unchanged if it can be represented in the
+ * destination type (and bit-field width); otherwise, the value is implementation-defined.
+ *
+ * undefined
+ * =========
+ * §4.9/1 (floating-point conversions)
+ * If the source value is neither exactly represented in the destination type nor between
+ * two adjacent destination values the result is undefined.
+ *
+ * §4.10/1 (floating-integral conversions)
+ * floating point type can be converted to integer type.
+ * The behavior is undefined if the truncated value cannot be
+ * represented in the destination type.
+ *
+ * §4.10/2
+ * integer can be converted to floating point type.
+ * If the value being converted is outside the range of values that can be represented, the
+ * behavior is undefined.
+ */
+template <typename To, typename From>
+constexpr bool is_conversion_undefined_impl(From x, std::true_type)
+{
+ return x > static_cast<long double>(std::numeric_limits<To>::max()) ||
+ x < static_cast<long double>(std::numeric_limits<To>::min());
+}
+
+template <typename To, typename From>
+constexpr bool is_conversion_undefined_impl(From, std::false_type)
+{
+ return false;
+}
+
+template <typename To, typename From> constexpr bool is_conversion_undefined(From x)
+{
+ static_assert(std::is_arithmetic<From>::value,
+ "this overload is only meant for builtin arithmetic types");
+ return is_conversion_undefined_impl<To, From>(
+ x, std::integral_constant<bool, (std::is_floating_point<From>::value &&
+ (std::is_integral<To>::value ||
+ (std::is_floating_point<To>::value &&
+ sizeof(From) > sizeof(To))))>());
+}
+
+static_assert(is_conversion_undefined<uint>(float(0x100000000LL)),
+ "testing my expectations of is_conversion_undefined");
+static_assert(!is_conversion_undefined<float>(0x100000000LL),
+ "testing my expectations of is_conversion_undefined");
+
+template <typename To, typename T, typename A>
+inline std::experimental::simd_mask<T, A> is_conversion_undefined(const std::experimental::simd<T, A> &x)
+{
+ std::experimental::simd_mask<T, A> k = false;
+ for (std::size_t i = 0; i < x.size(); ++i) {
+ k[i] = is_conversion_undefined(x[i]);
+ }
+ return k;
+}
+
+//operators helpers //{{{1
+template <class T> constexpr T genHalfBits()
+{
+ return std::numeric_limits<T>::max() >> (std::numeric_limits<T>::digits / 2);
+}
+template <> constexpr long double genHalfBits<long double>() { return 0; }
+template <> constexpr double genHalfBits<double>() { return 0; }
+template <> constexpr float genHalfBits<float>() { return 0; }
+
+template <class U, class T, class UU> constexpr U avoid_ub(UU x)
+{
+ return is_conversion_undefined<T>(U(x)) ? U(0) : U(x);
+}
+
+template <class U, class T, class UU> constexpr U avoid_ub2(UU x)
+{
+ return is_conversion_undefined<U>(x) ? U(0) : avoid_ub<U, T>(x);
+}
+
+// conversion test input data //{{{1
+template <class U, class T>
+static const std::array<U, 53> cvt_input_data = {{
+ avoid_ub<U, T>(0xc0000080U),
+ avoid_ub<U, T>(0xc0000081U),
+ avoid_ub<U, T>(0xc0000082U),
+ avoid_ub<U, T>(0xc0000084U),
+ avoid_ub<U, T>(0xc0000088U),
+ avoid_ub<U, T>(0xc0000090U),
+ avoid_ub<U, T>(0xc00000A0U),
+ avoid_ub<U, T>(0xc00000C0U),
+ avoid_ub<U, T>(0xc000017fU),
+ avoid_ub<U, T>(0xc0000180U),
+ avoid_ub<U, T>(0x100000001LL),
+ avoid_ub<U, T>(0x100000011LL),
+ avoid_ub<U, T>(0x100000111LL),
+ avoid_ub<U, T>(0x100001111LL),
+ avoid_ub<U, T>(0x100011111LL),
+ avoid_ub<U, T>(0x100111111LL),
+ avoid_ub<U, T>(0x101111111LL),
+ avoid_ub<U, T>(-0x100000001LL),
+ avoid_ub<U, T>(-0x100000011LL),
+ avoid_ub<U, T>(-0x100000111LL),
+ avoid_ub<U, T>(-0x100001111LL),
+ avoid_ub<U, T>(-0x100011111LL),
+ avoid_ub<U, T>(-0x100111111LL),
+ avoid_ub<U, T>(-0x101111111LL),
+ avoid_ub<U, T>(std::numeric_limits<U>::min()),
+ avoid_ub<U, T>(std::numeric_limits<U>::min() + 1),
+ avoid_ub<U, T>(std::numeric_limits<U>::lowest()),
+ avoid_ub<U, T>(std::numeric_limits<U>::lowest() + 1),
+ avoid_ub<U, T>(-1),
+ avoid_ub<U, T>(-10),
+ avoid_ub<U, T>(-100),
+ avoid_ub<U, T>(-1000),
+ avoid_ub<U, T>(-10000),
+ avoid_ub<U, T>(0),
+ avoid_ub<U, T>(1),
+ avoid_ub<U, T>(genHalfBits<U>() - 1),
+ avoid_ub<U, T>(genHalfBits<U>()),
+ avoid_ub<U, T>(genHalfBits<U>() + 1),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() - 1),
+ avoid_ub<U, T>(std::numeric_limits<U>::max()),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() - 0xff),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() - 0xff),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() - 0x55),
+ avoid_ub<U, T>(-(std::numeric_limits<U>::min() + 1)),
+ avoid_ub<U, T>(-std::numeric_limits<U>::max()),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 6 - 1)),
+ avoid_ub2<U, T>(-std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 6 - 1)),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 4 - 1)),
+ avoid_ub2<U, T>(-std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 4 - 1)),
+ avoid_ub<U, T>(std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 2 - 1)),
+ avoid_ub2<U, T>(-std::numeric_limits<U>::max() / std::pow(2., sizeof(T) * 2 - 1)),
+ avoid_ub<U, T>(std::numeric_limits<T>::max() - 1),
+ avoid_ub<U, T>(std::numeric_limits<T>::max() * 0.75),
+}};
+
+template <class T, class U> struct cvt_inputs {
+ static constexpr size_t size() { return cvt_input_data<U, T>.size(); }
+ U operator[](size_t i) const { return cvt_input_data<U, T>[i]; }
+};
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/make_vec.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/make_vec.h
new file mode 100644
index 00000000000..931b36edb61
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/make_vec.h
@@ -0,0 +1,62 @@
+/* This file is part of the Vc library. {{{
+Copyright © 2017 Matthias Kretz <kretz@kde.org>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the names of contributing organizations nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+}}}*/
+
+#include <experimental/simd>
+
+template <class M> inline M make_mask(const std::initializer_list<bool> &init)
+{
+ std::size_t i = 0;
+ M r = {};
+ for (;;) {
+ for (bool x : init) {
+ r[i] = x;
+ if (++i == M::size()) {
+ return r;
+ }
+ }
+ }
+}
+
+template <class V>
+inline V make_vec(const std::initializer_list<typename V::value_type> &init,
+ typename V::value_type inc = 0)
+{
+ std::size_t i = 0;
+ V r = {};
+ typename V::value_type base = 0;
+ for (;;) {
+ for (auto x : init) {
+ r[i] = base + x;
+ if (++i == V::size()) {
+ return r;
+ }
+ }
+ base += inc;
+ }
+}
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/mathreference.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/mathreference.h
new file mode 100644
index 00000000000..149496f10a7
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/mathreference.h
@@ -0,0 +1,103 @@
+#include <tuple>
+#include <utility>
+#include <cstdio>
+
+template <typename T> struct SincosReference //{{{1
+{
+ T x, s, c;
+
+ std::tuple<const T &, const T &, const T &> as_tuple() const
+ {
+ return std::tie(x, s, c);
+ }
+};
+
+template <typename T> struct Reference {
+ T x, ref;
+
+ std::tuple<const T &, const T &> as_tuple() const { return std::tie(x, ref); }
+};
+
+template <typename T> struct Array
+{
+ std::size_t size_;
+ const T *data_;
+ Array() : size_(0), data_(nullptr) {}
+ Array(size_t s, const T *p) : size_(s), data_(p) {}
+ const T *begin() const { return data_; }
+ const T *end() const { return data_ + size_; }
+ std::size_t size() const { return size_; }
+};
+
+namespace function {
+struct sincos{ static constexpr const char *const str = "sincos"; };
+struct atan { static constexpr const char *const str = "atan"; };
+struct asin { static constexpr const char *const str = "asin"; };
+struct acos { static constexpr const char *const str = "acos"; };
+struct log { static constexpr const char *const str = "ln"; };
+struct log2 { static constexpr const char *const str = "log2"; };
+struct log10 { static constexpr const char *const str = "log10"; };
+}
+
+template <class F> struct testdatatype_for_function {
+ template <class T> using type = Reference<T>;
+};
+template <> struct testdatatype_for_function<function::sincos> {
+ template <class T> using type = SincosReference<T>;
+};
+template <class F, class T>
+using testdatatype_for_function_t =
+ typename testdatatype_for_function<F>::template type<T>;
+
+template<typename T> struct StaticDeleter
+{
+ const T *ptr;
+ StaticDeleter(const T *p) : ptr(p) {}
+ ~StaticDeleter() { delete[] ptr; }
+};
+
+template <class F, class T> inline std::string filename()
+{
+ static_assert(std::is_floating_point<T>::value, "");
+ static const auto cache =
+ std::string("reference-") + F::str +
+ (std::is_same<T, float>::value
+ ? "-sp"
+ : (std::is_same_v<T, double> ? "-dp" : "-dep")) +
+ ".dat";
+ return cache;
+}
+
+template <class Fun, class T, class Ref = testdatatype_for_function_t<Fun, T>>
+Array<Ref> referenceData()
+{
+ static Array<Ref> data;
+ if (data.data_ == nullptr)
+ {
+ FILE* file = std::fopen(filename<Fun, T>().c_str(), "rb");
+ if (file)
+ {
+ std::fseek(file, 0, SEEK_END);
+ const size_t size = std::ftell(file) / sizeof(Ref);
+ std::rewind(file);
+ auto mem = new Ref[size];
+ static StaticDeleter<Ref> _cleanup(data.data_);
+ data.size_ = std::fread(mem, sizeof(Ref), size, file);
+ data.data_ = mem;
+ std::fclose(file);
+ }
+ else
+ {
+ __builtin_fprintf(
+ stderr,
+ "%s:%d: the reference data %s does not exist in the current "
+ "working directory.\n",
+ __FILE__, __LINE__, filename<Fun, T>().c_str());
+ __builtin_abort();
+ }
+ }
+ return data;
+}
+
+//}}}1
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/metahelpers.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/metahelpers.h
new file mode 100644
index 00000000000..1eb1b0d1681
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/metahelpers.h
@@ -0,0 +1,170 @@
+#ifndef VC_TESTS_METAHELPERS_H_
+#define VC_TESTS_METAHELPERS_H_
+
+#include <functional>
+#include <limits>
+#include <type_traits>
+#include <utility>
+
+namespace vir
+{
+namespace test
+{
+// operator_is_substitution_failure {{{1
+template <class A, class B, class Op>
+constexpr bool operator_is_substitution_failure_impl(float)
+{
+ return true;
+}
+
+template <class A, class B, class Op>
+constexpr
+ typename std::conditional<true, bool,
+ decltype(Op()(std::declval<A>(), std::declval<B>()))>::type
+ operator_is_substitution_failure_impl(int)
+{
+ return false;
+}
+
+template <class... Ts> constexpr bool operator_is_substitution_failure()
+{
+ return operator_is_substitution_failure_impl<Ts...>(int());
+}
+
+// sfinae_is_callable{{{1
+#ifdef Vc_CLANG
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wundefined-inline"
+#endif
+template <class... Args, class F>
+constexpr auto sfinae_is_callable_impl(int, F &&f) -> typename std::conditional<
+ true, std::true_type, decltype(std::forward<F>(f)(std::declval<Args>()...))>::type;
+template <class... Args, class F> constexpr std::false_type sfinae_is_callable_impl(float, const F &);
+template <class... Args, class F> constexpr bool sfinae_is_callable(F &&)
+{
+ return decltype(sfinae_is_callable_impl<Args...>(int(), std::declval<F>()))::value;
+}
+template <class... Args, class F>
+constexpr auto sfinae_is_callable_t(F &&f)
+ -> decltype(sfinae_is_callable_impl<Args...>(int(), std::declval<F>()));
+
+#ifdef Vc_CLANG
+#pragma clang diagnostic pop
+#endif
+
+// traits {{{1
+template <class A, class B> constexpr bool has_less_bits()
+{
+ return std::numeric_limits<A>::digits < std::numeric_limits<B>::digits;
+}
+
+//}}}1
+} // namespace test
+} // namespace vir
+
+// more operator objects {{{1
+struct assignment {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() = std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) = std::forward<B>(b)))
+ {
+ return std::forward<A>(a) = std::forward<B>(b);
+ }
+};
+
+struct bit_shift_left {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() << std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) << std::forward<B>(b)))
+ {
+ return std::forward<A>(a) << std::forward<B>(b);
+ }
+};
+
+struct bit_shift_right {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() >> std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) >> std::forward<B>(b)))
+ {
+ return std::forward<A>(a) >> std::forward<B>(b);
+ }
+};
+
+struct assign_modulus {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() %= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) %= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) %= std::forward<B>(b);
+ }
+};
+
+struct assign_bit_and {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() &= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) &= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) &= std::forward<B>(b);
+ }
+};
+
+struct assign_bit_or {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() |= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) |= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) |= std::forward<B>(b);
+ }
+};
+
+struct assign_bit_xor {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() ^= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) ^= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) ^= std::forward<B>(b);
+ }
+};
+
+struct assign_bit_shift_left {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() <<= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) <<= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) <<= std::forward<B>(b);
+ }
+};
+
+struct assign_bit_shift_right {
+ template <class A, class B>
+ constexpr decltype(std::declval<A>() >>= std::declval<B>()) operator()(A &&a,
+ B &&b) const
+ noexcept(noexcept(std::forward<A>(a) >>= std::forward<B>(b)))
+ {
+ return std::forward<A>(a) >>= std::forward<B>(b);
+ }
+};
+
+// operator_is_substitution_failure {{{1
+template <class A, class B, class Op = std::plus<>>
+constexpr bool is_substitution_failure =
+ vir::test::operator_is_substitution_failure<A, B, Op>();
+
+// sfinae_is_callable{{{1
+using vir::test::sfinae_is_callable;
+
+// traits {{{1
+using vir::test::has_less_bits;
+
+//}}}1
+
+#endif // VC_TESTS_METAHELPERS_H_
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/simd_view.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/simd_view.h
new file mode 100644
index 00000000000..1b611c56b1d
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/simd_view.h
@@ -0,0 +1,112 @@
+/* This file is part of the Vc library. {{{
+Copyright © 2018 Matthias Kretz <kretz@kde.org>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the names of contributing organizations nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+}}}*/
+
+#ifndef VC_TESTS_SIMD_VIEW_H_
+#define VC_TESTS_SIMD_VIEW_H_
+
+#include <experimental/simd>
+
+_GLIBCXX_SIMD_BEGIN_NAMESPACE
+namespace experimental
+{
+namespace imported_begin_end
+{
+ using std::begin;
+ using std::end;
+ template <class T> using begin_type = decltype(begin(std::declval<T>()));
+ template <class T> using end_type = decltype(end(std::declval<T>()));
+} // namespace imported_begin_end
+
+template <class V, class It, class End> class viewer
+{
+ It it;
+ const End end;
+
+ template <class F> void for_each_impl(F &&fun, std::index_sequence<0, 1, 2>)
+ {
+ for (; it + V::size() <= end; it += V::size()) {
+ fun(V([&](auto i) { return std::get<0>(it[i].as_tuple()); }),
+ V([&](auto i) { return std::get<1>(it[i].as_tuple()); }),
+ V([&](auto i) { return std::get<2>(it[i].as_tuple()); }));
+ }
+ if (it != end) {
+ fun(V([&](auto i) {
+ auto ii = it + i < end ? i + 0 : 0;
+ return std::get<0>(it[ii].as_tuple());
+ }),
+ V([&](auto i) {
+ auto ii = it + i < end ? i + 0 : 0;
+ return std::get<1>(it[ii].as_tuple());
+ }),
+ V([&](auto i) {
+ auto ii = it + i < end ? i + 0 : 0;
+ return std::get<2>(it[ii].as_tuple());
+ }));
+ }
+ }
+
+ template <class F> void for_each_impl(F &&fun, std::index_sequence<0, 1>)
+ {
+ for (; it + V::size() <= end; it += V::size()) {
+ fun(V([&](auto i) { return std::get<0>(it[i].as_tuple()); }),
+ V([&](auto i) { return std::get<1>(it[i].as_tuple()); }));
+ }
+ if (it != end) {
+ fun(V([&](auto i) {
+ auto ii = it + i < end ? i + 0 : 0;
+ return std::get<0>(it[ii].as_tuple());
+ }),
+ V([&](auto i) {
+ auto ii = it + i < end ? i + 0 : 0;
+ return std::get<1>(it[ii].as_tuple());
+ }));
+ }
+ }
+
+public:
+ viewer(It _it, End _end) : it(_it), end(_end) {}
+
+ template <class F> void for_each(F &&fun) {
+ constexpr size_t N =
+ std::tuple_size<std::decay_t<decltype(it->as_tuple())>>::value;
+ for_each_impl(std::forward<F>(fun), std::make_index_sequence<N>());
+ }
+};
+
+template <class V, class Cont>
+viewer<V, imported_begin_end::begin_type<const Cont &>,
+ imported_begin_end::end_type<const Cont &>>
+simd_view(const Cont &data)
+{
+ using std::begin;
+ using std::end;
+ return {begin(data), end(data)};
+}
+} // namespace experimental
+_GLIBCXX_SIMD_END_NAMESPACE
+
+#endif // VC_TESTS_SIMD_VIEW_H_
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/test_values.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/test_values.h
new file mode 100644
index 00000000000..2c028ff1379
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/test_values.h
@@ -0,0 +1,176 @@
+#include <experimental/simd>
+#include <initializer_list>
+#include <random>
+
+template <class T, class A>
+std::experimental::simd<T, A> iif(std::experimental::simd_mask<T, A> k,
+ const typename std::experimental::simd_mask<T, A>::simd_type &t,
+ const std::experimental::simd<T, A> &f)
+{
+ auto r = f;
+ where(k, r) = t;
+ return r;
+}
+
+template <class V>
+V epilogue_load(const typename V::value_type *mem, const std::size_t size)
+{
+ const int rem = size % V::size();
+ return where(V([](int i) { return i; }) < rem, V(0))
+ .copy_from(mem + size / V::size() * V::size(), std::experimental::element_aligned);
+}
+
+template <class V, class... F>
+void test_values(const std::initializer_list<typename V::value_type> &inputs,
+ F &&... fun_pack)
+{
+ for (auto it = inputs.begin(); it + V::size() <= inputs.end(); it += V::size()) {
+ [](auto...) {}((fun_pack(V(&it[0], std::experimental::element_aligned)), 0)...);
+ }
+ [](auto...) {}((fun_pack(epilogue_load<V>(inputs.begin(), inputs.size())), 0)...);
+}
+
+template <class V> struct RandomValues {
+ const std::size_t count;
+ const typename V::value_type min;
+ const typename V::value_type max;
+};
+
+static std::mt19937 g_mt_gen{0};
+
+template <class V, class... F>
+void test_values(const std::initializer_list<typename V::value_type> &inputs,
+ const RandomValues<V> &random, F &&... fun_pack)
+{
+ test_values<V>(inputs, fun_pack...);
+
+ using T = typename V::value_type;
+ std::conditional_t<std::is_floating_point_v<T>, std::uniform_real_distribution<T>,
+ std::uniform_int_distribution<T>>
+ dist(random.min, random.max);
+ auto &&rnd_v = [&]() {
+ V tmp;
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ tmp[i] = dist(g_mt_gen);
+ }
+ return tmp;
+ };
+
+ for (size_t i = 0; i < (random.count + V::size() - 1) / V::size(); ++i) {
+ [](auto...) {}((fun_pack(rnd_v()), 0)...);
+ }
+}
+
+template <class V, class... F>
+void test_values_2arg(const std::initializer_list<typename V::value_type> &inputs,
+ F &&... fun_pack)
+{
+ for (auto scalar_it = inputs.begin(); scalar_it != inputs.end(); ++scalar_it) {
+ for (auto it = inputs.begin(); it + V::size() <= inputs.end(); it += V::size()) {
+ [](auto...) {
+ }((fun_pack(V(&it[0], std::experimental::element_aligned), V(*scalar_it)), 0)...);
+ }
+ [](auto...) {
+ }((fun_pack(epilogue_load<V>(inputs.begin(), inputs.size()), V(*scalar_it)),
+ 0)...);
+ }
+}
+
+template <class V, class... F>
+void test_values_2arg(const std::initializer_list<typename V::value_type> &inputs,
+ const RandomValues<V> &random, F &&... fun_pack)
+{
+ test_values_2arg<V>(inputs, fun_pack...);
+
+ using T = typename V::value_type;
+ std::conditional_t<std::is_floating_point_v<T>, std::uniform_real_distribution<T>,
+ std::uniform_int_distribution<T>>
+ dist(random.min, random.max);
+ auto &&rnd_v = [&]() {
+ V tmp;
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ tmp[i] = dist(g_mt_gen);
+ }
+ return tmp;
+ };
+
+ for (size_t i = 0; i < (random.count + V::size() - 1) / V::size(); ++i) {
+ [](auto...) {}((fun_pack(rnd_v(), rnd_v()), 0)...);
+ }
+}
+
+template <class V, class... F>
+void test_values_3arg(const std::initializer_list<typename V::value_type> &inputs,
+ F &&... fun_pack)
+{
+ for (auto scalar_it1 = inputs.begin(); scalar_it1 != inputs.end(); ++scalar_it1) {
+ for (auto scalar_it2 = inputs.begin(); scalar_it2 != inputs.end(); ++scalar_it2) {
+ for (auto it = inputs.begin(); it + V::size() <= inputs.end();
+ it += V::size()) {
+ [](auto...) {}((fun_pack(V(&it[0], std::experimental::element_aligned), V(*scalar_it1),
+ V(*scalar_it2)),
+ 0)...);
+ }
+ [](auto...) {}((fun_pack(epilogue_load<V>(inputs.begin(), inputs.size()),
+ V(*scalar_it1), V(*scalar_it2)),
+ 0)...);
+ }
+ }
+}
+
+template <class V, class... F>
+void test_values_3arg(const std::initializer_list<typename V::value_type> &inputs,
+ const RandomValues<V> &random, F &&... fun_pack)
+{
+ test_values_3arg<V>(inputs, fun_pack...);
+
+ using T = typename V::value_type;
+ std::conditional_t<std::is_floating_point_v<T>, std::uniform_real_distribution<T>,
+ std::uniform_int_distribution<T>>
+ dist(random.min, random.max);
+ auto &&rnd_v = [&]() {
+ V tmp;
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ tmp[i] = dist(g_mt_gen);
+ }
+ return tmp;
+ };
+
+ for (size_t i = 0; i < (random.count + V::size() - 1) / V::size(); ++i) {
+ [](auto...) {}((fun_pack(rnd_v(), rnd_v(), rnd_v()), 0)...);
+ }
+}
+
+#define MAKE_TESTER_2(name_, reference_) \
+ [&](const auto... inputs) { \
+ const auto totest = name_(inputs...); \
+ using R = std::remove_const_t<decltype(totest)>; \
+ auto&& expected = [&](const auto&... vs) -> const R { \
+ R tmp = {}; \
+ for (std::size_t i = 0; i < R::size(); ++i) \
+ { \
+ tmp[i] = reference_(vs[i]...); \
+ } \
+ return tmp; \
+ }; \
+ const R expect1 = expected(inputs...); \
+ if constexpr (std::is_floating_point_v<typename R::value_type>) \
+ { \
+ ((COMPARE(isnan(totest), isnan(expect1)) << #name_ "(") \
+ << ... << inputs) \
+ << ") = " << totest << " != " << expect1; \
+ const R expect2 = expected(iif(isnan(expect1), 0, inputs)...); \
+ ((FUZZY_COMPARE(name_(iif(isnan(expect1), 0, inputs)...), expect2) \
+ << "\nclean = ") \
+ << ... << iif(isnan(expect1), 0, inputs)); \
+ } \
+ else \
+ { \
+ ((COMPARE(name_(inputs...), expect1) << "\ninputs = ") \
+ << ... << inputs); \
+ } \
+ }
+
+#define MAKE_TESTER(name_) MAKE_TESTER_2(name_, std::name_)
+
+// vim: foldmethod=marker ts=8 sw=2 noet sts=2
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/ulp.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/ulp.h
new file mode 100644
index 00000000000..31d646bfb33
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/ulp.h
@@ -0,0 +1,94 @@
+/*{{{
+Copyright © 2011-2018 Matthias Kretz <kretz@kde.org>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the names of contributing organizations nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+}}}*/
+
+#ifndef ULP_H
+#define ULP_H
+
+#include <cmath>
+#include <experimental/simd>
+#include <limits>
+#include <type_traits>
+
+namespace vir
+{
+namespace test
+{
+ template <typename T, typename R = typename T::value_type>
+ R value_type_impl(int);
+ template <typename T>
+ T value_type_impl(float);
+ template <typename T>
+ using value_type_t = decltype(value_type_impl<T>(int()));
+
+ template <typename T,
+ typename = typename std::enable_if<
+ std::is_floating_point<value_type_t<T>>::value>::type>
+ inline T ulp_distance(const T& val_, const T& ref_)
+ {
+ using std::abs;
+ using std::fpclassify;
+ using std::frexp;
+ using std::isnan;
+ using std::ldexp;
+ using std::experimental::where;
+ using limits = std::numeric_limits<value_type_t<T>>;
+
+ T val = val_;
+ T ref = ref_;
+ T diff = T();
+
+ where(ref == 0, val) = abs(val);
+ where(ref == 0, diff) = 1;
+ where(ref == 0, ref) = limits::min();
+
+ where(val == 0, ref) = abs(ref);
+ where(val == 0, diff) += 1;
+ where(val == 0, val) = limits::min();
+
+ using I = decltype(fpclassify(std::declval<T>()));
+ I exp = {};
+ frexp(ref, &exp);
+ // lower bound for exp must be min_exponent to scale the resulting
+ // difference from a denormal correctly
+ exp = max(exp, I(limits::min_exponent));
+ diff += ldexp(abs(ref - val), limits::digits - exp);
+ where(val_ == ref_ || (isnan(val_) && isnan(ref_)), diff) = T();
+ return diff;
+ }
+
+ template <typename T>
+ inline T ulp_distance_signed(const T& _val, const T& _ref)
+ {
+ using std::copysign;
+ return copysign(ulp_distance(_val, _ref), _val - _ref);
+ }
+} // namespace test
+} // namespace vir
+
+#endif // ULP_H
+
+// vim: sw=2 et sts=2 foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/bits/verify.h b/libstdc++-v3/testsuite/experimental/simd/tests/bits/verify.h
new file mode 100644
index 00000000000..0f47111aab0
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/bits/verify.h
@@ -0,0 +1,253 @@
+#ifndef TESTS_BITS_VERIFY_H_
+#define TESTS_BITS_VERIFY_H_
+
+#include <experimental/simd>
+#include <sstream>
+#include <iomanip>
+#include "ulp.h"
+
+#ifdef _GLIBCXX_SIMD_HAVE_NEON
+// work around PR89357:
+#define alignas(...) __attribute__((aligned(__VA_ARGS__)))
+#endif
+
+using schar = signed char;
+using uchar = unsigned char;
+using ushort = unsigned short;
+using uint = unsigned int;
+using ulong = unsigned long;
+using llong = long long;
+using ullong = unsigned long long;
+using ldouble = long double;
+using wchar = wchar_t;
+using char16 = char16_t;
+using char32 = char32_t;
+
+class verify
+{
+ const bool m_failed = false;
+
+ template <typename T,
+ typename = decltype(std::declval<std::stringstream&>()
+ << std::declval<const T&>())>
+ void print(const T& x, int) const
+ {
+ std::stringstream ss;
+ ss << x;
+ __builtin_fprintf(stderr, "%s", ss.str().c_str());
+ }
+
+ template <typename T>
+ void print(const T& x, ...) const
+ {
+ if constexpr (std::experimental::is_simd_v<T>)
+ {
+ std::stringstream ss;
+ if constexpr (std::is_floating_point_v<typename T::value_type>)
+ {
+ ss << "\n(" << x[0] << " == " << std::hexfloat << x[0]
+ << std::defaultfloat << ')';
+ for (unsigned i = 1; i < x.size(); ++i)
+ {
+ ss << (i % 4 == 0 ? ",\n(" : ", (") << x[i]
+ << " == " << std::hexfloat << x[i] << std::defaultfloat
+ << ')';
+ }
+ }
+ else
+ {
+ ss << +x[0];
+ for (unsigned i = 1; i < x.size(); ++i)
+ {
+ ss << ", " << +x[i];
+ }
+ }
+ __builtin_fprintf(stderr, "%s", ss.str().c_str());
+ }
+ else if constexpr (std::experimental::is_simd_mask_v<T>)
+ {
+ __builtin_fprintf(stderr, (x[0] ? "[1" : "[0"));
+ for (unsigned i = 1; i < x.size(); ++i)
+ {
+ __builtin_fprintf(stderr, (x[i] ? "1" : "0"));
+ }
+ __builtin_fprintf(stderr, "]");
+ }
+ else
+ {
+ print_hex(&x, sizeof(T));
+ }
+ }
+
+ void print_hex(const void* x, std::size_t n) const
+ {
+ __builtin_fprintf(stderr, "0x");
+ const auto* bytes = static_cast<const unsigned char*>(x);
+ for (std::size_t i = 0; i < n; ++i)
+ {
+ __builtin_fprintf(stderr, (i && i % 4 == 0) ? "'%02x" : "%02x",
+ bytes[i]);
+ }
+ }
+
+public:
+ template <typename... Ts>
+ verify(bool ok,
+ const char* file,
+ const int line,
+ const char* func,
+ const char* cond,
+ const Ts&... extra_info)
+ : m_failed(!ok)
+ {
+ if (m_failed)
+ {
+ __builtin_fprintf(stderr, "%s:%d: (%s): Assertion '%s' failed.\n", file,
+ line, func, cond);
+ auto &&unused [[maybe_unused]] = {0, (print(extra_info, int()), 0)...};
+ }
+ }
+
+ ~verify()
+ {
+ if (m_failed)
+ {
+ __builtin_fprintf(stderr, "\n");
+ __builtin_abort();
+ }
+ }
+
+ template <typename T>
+ const verify& operator<<(const T& x) const
+ {
+ if (m_failed)
+ {
+ print(x, int());
+ }
+ return *this;
+ }
+};
+
+#define COMPARE(_a, _b) \
+ [&](auto&& _aa, auto&& _bb) { \
+ return verify(std::experimental::all_of(_aa == _bb), __FILE__, __LINE__, \
+ __PRETTY_FUNCTION__, "all_of(" #_a " == " #_b ")", \
+ #_a " = ", _aa, "\n" #_b " = ", _bb); \
+ }((_a), (_b))
+
+#define VERIFY(_test) \
+ verify(_test, __FILE__, __LINE__, __PRETTY_FUNCTION__, #_test)
+
+#define ULP_COMPARE(_a, _b, _allowed_distance) \
+ [&](auto&& _aa, auto&& _bb) { \
+ return verify(std::experimental::all_of( \
+ vir::test::ulp_distance(_aa, _bb) <= (_allowed_distance)), \
+ __FILE__, __LINE__, __PRETTY_FUNCTION__, \
+ "all_of(" #_a " ~~ " #_b ")", #_a " = ", _aa, \
+ "\n" #_b " = ", _bb, \
+ "\ndistance = ", vir::test::ulp_distance_signed(_aa, _bb)); \
+ }((_a), (_b))
+
+namespace vir
+{
+namespace test
+{
+ template <typename T>
+ inline T _S_fuzzyness = 0;
+ template <typename T>
+ void setFuzzyness(T x)
+ {
+ _S_fuzzyness<T> = x;
+ }
+} // namespace test
+} // namespace vir
+
+#define FUZZY_COMPARE(_a, _b) \
+ ULP_COMPARE( \
+ _a, _b, \
+ vir::test::_S_fuzzyness<vir::test::value_type_t<decltype((_a) + (_b))>>)
+
+template <typename V>
+void test();
+template <typename V>
+void invoke_test(...)
+{
+}
+template <typename V, typename = decltype(V())>
+void invoke_test(int)
+{
+ test<V>();
+ __builtin_fprintf(stderr, "PASS: %s\n", __PRETTY_FUNCTION__);
+}
+
+template <class T> void iterate_abis()/*{{{*/
+{
+ using namespace std::experimental::parallelism_v2;
+#ifndef TESTFIXEDSIZE
+ invoke_test<simd<T, simd_abi::__sse>>(int());
+ invoke_test<simd<T, simd_abi::__avx>>(int());
+ invoke_test<simd<T, simd_abi::__avx512>>(int());
+ invoke_test<simd<T, simd_abi::__neon>>(int());
+ invoke_test<simd<T, simd_abi::scalar>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<3>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<4>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<12>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<24>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<28>>>(int());
+#else
+ invoke_test<simd<T, simd_abi::fixed_size<1>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<2>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<5>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<6>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<7>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<8>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<9>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<10>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<11>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<13>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<14>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<15>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<16>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<17>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<18>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<19>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<20>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<21>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<22>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<23>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<25>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<26>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<27>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<29>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<30>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<31>>>(int());
+ invoke_test<simd<T, simd_abi::fixed_size<32>>>(int());
+#endif
+}/*}}}*/
+
+int main()/*{{{*/
+{
+ iterate_abis<TESTTYPE>();
+ /*
+ iterate_abis<long double>();
+ iterate_abis<double>();
+ iterate_abis<float>();
+ iterate_abis<long long>();
+ iterate_abis<unsigned long long>();
+ iterate_abis<unsigned long>();
+ iterate_abis<long>();
+ iterate_abis<int>();
+ iterate_abis<unsigned int>();
+ iterate_abis<short>();
+ iterate_abis<unsigned short>();
+ iterate_abis<char>();
+ iterate_abis<signed char>();
+ iterate_abis<unsigned char>();
+ iterate_abis<char32_t>();
+ iterate_abis<char16_t>();
+ iterate_abis<wchar_t>();
+ */
+ return 0;
+}/*}}}*/
+
+#endif // TESTS_BITS_VERIFY_H_
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/broadcast.h b/libstdc++-v3/testsuite/experimental/simd/tests/broadcast.h
new file mode 100644
index 00000000000..b19f4228bca
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/broadcast.h
@@ -0,0 +1,75 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+enum unscoped_enum { foo };
+enum class scoped_enum { bar };
+struct convertible { operator int(); operator float(); };
+
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ VERIFY(std::experimental::is_simd_v<V>);
+ VERIFY(std::experimental::is_abi_tag_v<typename V::abi_type>);
+
+ {
+ V x; // not initialized
+ x = V{}; // default broadcasts 0
+ COMPARE(x, V(0));
+ COMPARE(x, V());
+ COMPARE(x, V{});
+ x = V(); // default broadcasts 0
+ COMPARE(x, V(0));
+ COMPARE(x, V());
+ COMPARE(x, V{});
+ x = 0;
+ COMPARE(x, V(0));
+ COMPARE(x, V());
+ COMPARE(x, V{});
+
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(T(x[i]), T(0)) << "i = " << i;
+ COMPARE(x[i], T(0)) << "i = " << i;
+ }
+ }
+
+ V x = 3;
+ V y = T(0);
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(x[i], T(3)) << "i = " << i;
+ COMPARE(y[i], T(0)) << "i = " << i;
+ }
+ y = 3;
+ COMPARE(x, y);
+
+ VERIFY(!(is_substitution_failure<V &, unscoped_enum, assignment>));
+ VERIFY( (is_substitution_failure<V &, scoped_enum, assignment>));
+ COMPARE((is_substitution_failure<V &, convertible, assignment>),
+ (!std::is_convertible<convertible, T>::value));
+ COMPARE((is_substitution_failure<V &, long double, assignment>),
+ (sizeof(long double) > sizeof(T) || std::is_integral<T>::value));
+ COMPARE((is_substitution_failure<V &, double, assignment>),
+ (sizeof(double) > sizeof(T) || std::is_integral<T>::value));
+ COMPARE((is_substitution_failure<V &, float, assignment>),
+ (sizeof(float) > sizeof(T) || std::is_integral<T>::value));
+ COMPARE((is_substitution_failure<V &, long long, assignment>),
+ (has_less_bits<T, long long>() || std::is_unsigned<T>::value));
+ COMPARE((is_substitution_failure<V &, unsigned long long, assignment>),
+ (has_less_bits<T, unsigned long long>()));
+ COMPARE((is_substitution_failure<V &, long, assignment>),
+ (has_less_bits<T, long>() || std::is_unsigned<T>::value));
+ COMPARE((is_substitution_failure<V &, unsigned long, assignment>),
+ (has_less_bits<T, unsigned long>()));
+ // int broadcast *always* works:
+ VERIFY(!(is_substitution_failure<V &, int, assignment>));
+ // uint broadcast works for any unsigned T:
+ COMPARE((is_substitution_failure<V &, unsigned int, assignment>),
+ (!std::is_unsigned<T>::value && has_less_bits<T, unsigned int>()));
+ COMPARE((is_substitution_failure<V &, short, assignment>),
+ (has_less_bits<T, short>() || std::is_unsigned<T>::value));
+ COMPARE((is_substitution_failure<V &, unsigned short, assignment>),
+ (has_less_bits<T, unsigned short>()));
+ COMPARE((is_substitution_failure<V &, signed char, assignment>),
+ (has_less_bits<T, signed char>() || std::is_unsigned<T>::value));
+ COMPARE((is_substitution_failure<V &, unsigned char, assignment>),
+ (has_less_bits<T, unsigned char>()));
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/casts.h b/libstdc++-v3/testsuite/experimental/simd/tests/casts.h
new file mode 100644
index 00000000000..213522aee13
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/casts.h
@@ -0,0 +1,111 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/conversions.h"
+
+using std::experimental::simd_cast;
+using std::experimental::static_simd_cast;
+
+template <class T, size_t N> struct gen_cast {
+ std::array<T, N> data;
+ template <class V> gen_cast(const V &v)
+ {
+ for (size_t i = 0; i < V::size(); ++i) {
+ data[i] = static_cast<T>(v[i]);
+ }
+ }
+ template <class I> constexpr T operator()(I) { return data[I::value]; }
+};
+
+template <class V, class To> struct gen_seq_t {
+ using From = typename V::value_type;
+ const size_t N = cvt_input_data<From, To>.size();
+ size_t offset = 0;
+ constexpr void operator++() { offset += V::size(); }
+ explicit constexpr operator bool() const { return offset < N; }
+ template <class I>
+ constexpr From operator()(I) const
+ {
+ size_t i = I::value + offset;
+ return i < N ? cvt_input_data<From, To>[i] : From(i);
+ }
+};
+
+template <class To> struct foo {
+ template <class T> auto operator()(const T &v) -> decltype(simd_cast<To>(v));
+};
+
+template <typename V, typename To> void casts()
+{
+ using From = typename V::value_type;
+ constexpr auto N = V::size();
+ using W = std::experimental::fixed_size_simd<To, N>;
+
+ if constexpr (std::is_integral_v<From>) {
+ using A = typename V::abi_type;
+ using TU = std::make_unsigned_t<From>;
+ using TS = std::make_signed_t<From>;
+ COMPARE(typeid(static_simd_cast<TU>(V())), typeid(std::experimental::simd<TU, A>));
+ COMPARE(typeid(static_simd_cast<TS>(V())), typeid(std::experimental::simd<TS, A>));
+ }
+
+ using is_simd_cast_allowed =
+ decltype(vir::test::sfinae_is_callable_t<const V &>(foo<To>()));
+
+ COMPARE(is_simd_cast_allowed::value,
+ std::numeric_limits<From>::digits <= std::numeric_limits<To>::digits &&
+ std::numeric_limits<From>::max() <= std::numeric_limits<To>::max() &&
+ !(std::is_signed<From>::value && std::is_unsigned<To>::value));
+
+ if constexpr (is_simd_cast_allowed::value) {
+ for (gen_seq_t<V, To> gen_seq; gen_seq; ++gen_seq) {
+ const V seq(gen_seq);
+ COMPARE(simd_cast<V>(seq), seq);
+ COMPARE(simd_cast<W>(seq), W(gen_cast<To, N>(seq))) << "seq = " << seq;
+ auto test = simd_cast<To>(seq);
+ // decltype(test) is not W if
+ // a) V::abi_type is not fixed_size and
+ // b.1) V::value_type and To are integral and of equal rank or
+ // b.2) V::value_type and To are equal
+ COMPARE(test, decltype(test)(gen_cast<To, N>(seq)));
+ if (std::is_same<To, From>::value) {
+ COMPARE(typeid(decltype(test)), typeid(V));
+ }
+ }
+ }
+
+ for (gen_seq_t<V, To> gen_seq; gen_seq; ++gen_seq) {
+ const V seq(gen_seq);
+ COMPARE(static_simd_cast<V>(seq), seq);
+ COMPARE(static_simd_cast<W>(seq), W(gen_cast<To, N>(seq))) << '\n' << seq;
+ auto test = static_simd_cast<To>(seq);
+ // decltype(test) is not W if
+ // a) V::abi_type is not fixed_size and
+ // b.1) V::value_type and To are integral and of equal rank or
+ // b.2) V::value_type and To are equal
+ COMPARE(test, decltype(test)(gen_cast<To, N>(seq)));
+ if (std::is_same<To, From>::value) {
+ COMPARE(typeid(decltype(test)), typeid(V));
+ }
+ }
+}
+
+template <typename V> void test()
+{
+ casts<V, long double>();
+ casts<V, double>();
+ casts<V, float>();
+ casts<V, long long>();
+ casts<V, unsigned long long>();
+ casts<V, unsigned long>();
+ casts<V, long>();
+ casts<V, int>();
+ casts<V, unsigned int>();
+ casts<V, short>();
+ casts<V, unsigned short>();
+ casts<V, char>();
+ casts<V, signed char>();
+ casts<V, unsigned char>();
+ casts<V, char32_t>();
+ casts<V, char16_t>();
+ casts<V, wchar_t>();
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/fpclassify.h b/libstdc++-v3/testsuite/experimental/simd/tests/fpclassify.h
new file mode 100644
index 00000000000..474d4041ebe
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/fpclassify.h
@@ -0,0 +1,42 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values<V>(
+ {0., -0., 1., -1., limits::infinity(), -limits::infinity(), limits::max(),
+ -limits::max(), limits::min(), limits::min() * 0.9, -limits::min(),
+ -limits::min() * 0.9, limits::denorm_min(), -limits::denorm_min(),
+ limits::quiet_NaN(), limits::signaling_NaN()},
+ [](const V input) {
+ using intv = std::experimental::fixed_size_simd<int, V::size()>;
+ COMPARE(isfinite(input),
+ !V([&](auto i) { return std::isfinite(input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE(isinf(input),
+ !V([&](auto i) { return std::isinf(input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE(isnan(input),
+ !V([&](auto i) { return std::isnan(input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE(isnormal(input),
+ !V([&](auto i) { return std::isnormal(input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE(signbit(input),
+ !V([&](auto i) { return std::signbit(input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE((isunordered(input, V())),
+ !V([&](auto i) { return std::isunordered(input[i], 0) ? 0 : 1; }))
+ << input;
+ COMPARE((isunordered(V(), input)),
+ !V([&](auto i) { return std::isunordered(0, input[i]) ? 0 : 1; }))
+ << input;
+ COMPARE(fpclassify(input),
+ intv([&](auto i) { return std::fpclassify(input[i]); }))
+ << input;
+ });
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/frexp.h b/libstdc++-v3/testsuite/experimental/simd/tests/frexp.h
new file mode 100644
index 00000000000..f361a7ea8be
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/frexp.h
@@ -0,0 +1,143 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ using int_v = std::experimental::fixed_size_simd<int, V::size()>;
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values<V>({0,
+ 0.25,
+ 0.5,
+ 1,
+ 3,
+ 4,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14,
+ 15,
+ 16,
+ 17,
+ 18,
+ 19,
+ 20,
+ 21,
+ 22,
+ 23,
+ 24,
+ 25,
+ 26,
+ 27,
+ 28,
+ 29,
+ 32,
+ 31,
+ -0.,
+ -0.25,
+ -0.5,
+ -1,
+ -3,
+ -4,
+ -6,
+ -7,
+ -8,
+ -9,
+ -10,
+ -11,
+ -12,
+ -13,
+ -14,
+ -15,
+ -16,
+ -17,
+ -18,
+ -19,
+ -20,
+ -21,
+ -22,
+ -23,
+ -24,
+ -25,
+ -26,
+ -27,
+ -28,
+ -29,
+ -32,
+ -31,
+ limits::max(),
+ -limits::max(),
+ limits::max() * 0.123,
+ -limits::max() * 0.123,
+ limits::denorm_min(),
+ -limits::denorm_min(),
+ limits::min() / 2,
+ -limits::min() / 2},
+ [](const V input) {
+ V expectedFraction;
+ const int_v expectedExponent([&](auto i) {
+ int exp;
+ expectedFraction[i] = std::frexp(input[i], &exp);
+ return exp;
+ });
+ int_v exponent;
+ const V fraction = frexp(input, &exponent);
+ COMPARE(fraction, expectedFraction)
+ << ", input = " << input
+ << ", delta: " << fraction - expectedFraction;
+ COMPARE(exponent, expectedExponent)
+ << "\ninput: " << input << ", fraction: " << fraction;
+ });
+ test_values<V>(
+ // If x is a NaN, a NaN is returned, and the value of *exp is unspecified.
+ //
+ // If x is positive infinity (negative infinity), positive infinity
+ // (negative infinity) is returned, and the value of *exp is unspecified.
+ {limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::denorm_min(),
+ limits::denorm_min() * 1.72,
+ -limits::denorm_min(),
+ -limits::denorm_min() * 1.72,
+ 0.,
+ -0.,
+ 1,
+ -1},
+ [](const V input) {
+ const V expectedFraction([&](auto i) {
+ int exp;
+ return std::frexp(input[i], &exp);
+ });
+ int_v exponent;
+ const V fraction = frexp(input, &exponent);
+ COMPARE(isnan(fraction), isnan(expectedFraction))
+ << fraction << ", input = " << input
+ << ", delta: " << fraction - expectedFraction;
+ COMPARE(isinf(fraction), isinf(expectedFraction))
+ << fraction << ", input = " << input
+ << ", delta: " << fraction - expectedFraction;
+ COMPARE(signbit(fraction), signbit(expectedFraction))
+ << fraction << ", input = " << input
+ << ", delta: " << fraction - expectedFraction;
+ });
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/generator.h b/libstdc++-v3/testsuite/experimental/simd/tests/generator.h
new file mode 100644
index 00000000000..9514a3024a0
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/generator.h
@@ -0,0 +1,33 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+template <class V> struct call_generator {
+ template <class F> auto operator()(const F &f) -> decltype(V(f));
+};
+
+using schar = signed char;
+using uchar = unsigned char;
+using ullong = unsigned long long;
+
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ V x([](int) { return T(1); });
+ COMPARE(x, V(1));
+ x = V([](int) { return 1; }); // unconditionally returns int from generator lambda
+ COMPARE(x, V(1));
+ x = V([](auto i) { return T(i); });
+ COMPARE(x, V([](T i) { return i; }));
+
+ VERIFY((sfinae_is_callable<int (&)(int)>(call_generator<V>()))); // int always works
+ COMPARE(sfinae_is_callable<schar (&)(int)>(call_generator<V>()),
+ std::is_signed<T>::value);
+ COMPARE(sfinae_is_callable<uchar (&)(int)>(call_generator<V>()),
+ !(std::is_signed_v<T> && sizeof(T) <= sizeof(uchar)));
+ COMPARE(sfinae_is_callable<float (&)(int)>(call_generator<V>()),
+ (std::is_floating_point<T>::value));
+
+ COMPARE(sfinae_is_callable<ullong (&)(int)>(call_generator<V>()),
+ std::numeric_limits<T>::max() >= std::numeric_limits<ullong>::max() &&
+ std::numeric_limits<T>::digits >= std::numeric_limits<ullong>::digits);
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/hypot3_fma.h b/libstdc++-v3/testsuite/experimental/simd/tests/hypot3_fma.h
new file mode 100644
index 00000000000..730250d943a
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/hypot3_fma.h
@@ -0,0 +1,115 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ using T = typename V::value_type;
+ vir::test::setFuzzyness<float>(1);
+ vir::test::setFuzzyness<double>(1);
+ vir::test::setFuzzyness<long double>(2); // because of the bad reference
+
+ using limits = std::numeric_limits<T>;
+ // 3-arg std::hypot needs to be fixed, this is a better reference:
+ auto&& hypot3 = [](T x, T y, T z) -> T {
+ x = std::abs(x);
+ y = std::abs(y);
+ z = std::abs(z);
+ if (std::isinf(x) || std::isinf(y) || std::isinf(z))
+ {
+ return limits::infinity();
+ }
+ else if (std::isnan(x) || std::isnan(y) || std::isnan(z))
+ {
+ return limits::quiet_NaN();
+ }
+ else if (x == y && y == z)
+ {
+ return x * std::sqrt(T(3));
+ }
+ else if (z == 0 && y == 0)
+ return x;
+ else if (x == 0 && z == 0)
+ return y;
+ else if (x == 0 && y == 0)
+ return z;
+ else if (x == 0)
+ return std::hypot(y, z);
+ else if (y == 0)
+ return std::hypot(x, z);
+ else if (z == 0)
+ return std::hypot(x, y);
+ else
+ {
+ long double hi = std::max(std::max(x, y), z);
+ long double lo0 = std::min(std::max(x, y), z);
+ long double lo1 = std::min(x, y);
+ if (std::isinf(x * x + y * y + z * z) || 0 == (lo0 * lo0 + lo1 * lo1))
+ {
+ lo0 /= hi;
+ lo1 /= hi;
+ return std::abs(hi) * std::sqrt(1 + (lo0 * lo0 + lo1 * lo1));
+ }
+ else
+ {
+ return std::sqrt(hi * hi + (lo0 * lo0 + lo1 * lo1));
+ }
+ }
+ };
+ test_values_3arg<V>({limits::quiet_NaN(), limits::infinity(),
+ -limits::infinity(), +0., -0., limits::denorm_min(),
+ limits::min(), limits::max(), limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2},
+ MAKE_TESTER_2(hypot, hypot3));
+ VERIFY((sfinae_is_callable<V, V, V>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, T, V>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, T>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, V, T>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, V, V>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, V>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, V, T>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<int, int, V>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<int, V, int>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, int>(
+ [](auto a, auto b, auto c) -> decltype(hypot(a, b, c)) { return {}; })));
+
+ vir::test::setFuzzyness<float>(0);
+ vir::test::setFuzzyness<double>(0);
+ test_values_3arg<V>(
+ {limits::quiet_NaN(), limits::infinity(), -limits::infinity(), +0., -0.,
+ limits::denorm_min(), limits::min(), limits::max(), limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2}, MAKE_TESTER(fma));
+ VERIFY((sfinae_is_callable<V, V, V>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, T, V>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, T>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, V, T>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<T, V, V>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, V>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, V, T>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<int, int, V>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<int, V, int>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, T, int>(
+ [](auto a, auto b, auto c) -> decltype(fma(a, b, c)) { return {}; })));
+}
+
+// vim: ts=8 noet sw=2 sts=2
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/loadstore.h b/libstdc++-v3/testsuite/experimental/simd/tests/loadstore.h
new file mode 100644
index 00000000000..1a7aec9dc9d
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/loadstore.h
@@ -0,0 +1,197 @@
+#include "bits/verify.h"
+#include "bits/make_vec.h"
+#include "bits/conversions.h"
+
+template <typename V, typename U> void load_store()
+{
+ // types, tags, and constants {{{2
+ using T = typename V::value_type;
+ auto &&gen = make_vec<V>;
+ using std::experimental::element_aligned;
+ using std::experimental::vector_aligned;
+
+ // stride_alignment: consider V::size() == 6. The only reliable alignment is
+ // 2 * sizeof(U). I.e. if the first address is aligned to 8 * sizeof(U), then the next
+ // address is 6 * sizeof(U) larger, thus only aligned to 2 * sizeof(U).
+ // => the LSB determines the stride alignment
+ constexpr size_t stride_alignment =
+ V::size() & 1 ? 1 : V::size() & 2
+ ? 2
+ : V::size() & 4
+ ? 4
+ : V::size() & 8
+ ? 8
+ : V::size() & 16
+ ? 16
+ : V::size() & 32
+ ? 32
+ : V::size() & 64
+ ? 64
+ : V::size() & 128
+ ? 128
+ : V::size() & 256
+ ? 256
+ : 512;
+ using stride_aligned_t =
+ std::conditional_t<V::size() == stride_alignment, decltype(vector_aligned),
+ std::experimental::overaligned_tag<stride_alignment * sizeof(U)>>;
+ constexpr stride_aligned_t stride_aligned = {};
+ constexpr size_t alignment = 2 * std::experimental::memory_alignment_v<V, U>;
+ constexpr auto overaligned = std::experimental::overaligned<alignment>;
+ const V indexes_from_0([](auto i) { return i; });
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(indexes_from_0[i], T(i));
+ }
+
+ // loads {{{2
+ cvt_inputs<T, U> test_values;
+
+ constexpr auto mem_size =
+ test_values.size() > 3 * V::size() ? test_values.size() : 3 * V::size();
+ alignas(std::experimental::memory_alignment_v<V, U> * 2) U mem[mem_size] = {};
+ alignas(std::experimental::memory_alignment_v<V, T> * 2) T reference[mem_size] = {};
+ for (std::size_t i = 0; i < test_values.size(); ++i) {
+ const U value = test_values[i];
+ mem[i] = value;
+ reference[i] = static_cast<T>(value);
+ }
+ for (std::size_t i = test_values.size(); i < mem_size; ++i) {
+ mem[i] = U(i);
+ reference[i] = mem[i];
+ }
+
+ V x(&mem[V::size()], stride_aligned);
+ auto &&compare = [&](const std::size_t offset) {
+ static int n = 0;
+ const V ref(&reference[offset], element_aligned);
+ for (auto i = 0ul; i < V::size(); ++i) {
+ if (is_conversion_undefined<T>(mem[i + offset])) {
+ continue;
+ }
+ COMPARE(x[i], reference[i + offset])
+ << "\nbefore conversion: " << mem[i + offset]
+ << "\n offset = " << offset
+ << "\n x = " << x
+ << "\nreference = " << ref
+ << "\nx == ref = " << (x == ref)
+ << "\ncall no. " << n;
+ }
+ ++n;
+ };
+ compare(V::size());
+ x = V{mem, overaligned};
+ compare(0);
+ x = {&mem[1], element_aligned};
+ compare(1);
+
+ x.copy_from(&mem[V::size()], stride_aligned);
+ compare(V::size());
+ x.copy_from(&mem[1], element_aligned);
+ compare(1);
+ x.copy_from(mem, vector_aligned);
+ compare(0);
+
+ for (std::size_t i = 0; i < mem_size - V::size(); ++i) {
+ x.copy_from(&mem[i], element_aligned);
+ compare(i);
+ }
+
+ for (std::size_t i = 0; i < test_values.size(); ++i) {
+ mem[i] = U(i);
+ }
+ x = indexes_from_0;
+ using M = typename V::mask_type;
+ const M alternating_mask = make_mask<M>({0, 1});
+ where(alternating_mask, x).copy_from(&mem[V::size()], stride_aligned);
+
+ const V indexes_from_size = gen({T(V::size())}, 1);
+ COMPARE(x == indexes_from_size, alternating_mask)
+ << "x: " << x << "\nindexes_from_size: " << indexes_from_size;
+ COMPARE(x == indexes_from_0, !alternating_mask);
+ where(alternating_mask, x).copy_from(&mem[1], element_aligned);
+
+ const V indexes_from_1 = gen({1, 2, 3, 4}, 4);
+ COMPARE(x == indexes_from_1, alternating_mask);
+ COMPARE(x == indexes_from_0, !alternating_mask);
+ where(!alternating_mask, x).copy_from(mem, overaligned);
+ COMPARE(x == indexes_from_0, !alternating_mask);
+ COMPARE(x == indexes_from_1, alternating_mask);
+
+ x = where(alternating_mask, V()).copy_from(&mem[V::size()], stride_aligned);
+ COMPARE(x == indexes_from_size, alternating_mask);
+ COMPARE(x == 0, !alternating_mask);
+
+ x = where(!alternating_mask, V()).copy_from(&mem[1], element_aligned);
+ COMPARE(x == indexes_from_1, !alternating_mask);
+ COMPARE(x == 0, alternating_mask);
+
+ // stores {{{2
+ memset(mem, 0, sizeof(mem));
+ x = indexes_from_1;
+ x.copy_to(&mem[V::size()], stride_aligned);
+ std::size_t i = 0;
+ for (; i < V::size(); ++i) {
+ COMPARE(mem[i], U(0)) << "i: " << i;
+ }
+ for (; i < 2 * V::size(); ++i) {
+ COMPARE(mem[i], U(i - V::size() + 1)) << "i: " << i;
+ }
+ for (; i < 3 * V::size(); ++i) {
+ COMPARE(mem[i], U(0)) << "i: " << i;
+ }
+
+ memset(mem, 0, sizeof(mem));
+ x.copy_to(&mem[1], element_aligned);
+ COMPARE(mem[0], U(0));
+ for (i = 1; i <= V::size(); ++i) {
+ COMPARE(mem[i], U(i));
+ }
+ for (; i < 3 * V::size(); ++i) {
+ COMPARE(mem[i], U(0));
+ }
+
+ memset(mem, 0, sizeof(mem));
+ x.copy_to(mem, vector_aligned);
+ for (i = 0; i < V::size(); ++i) {
+ COMPARE(mem[i], U(i + 1));
+ }
+ for (; i < 3 * V::size(); ++i) {
+ COMPARE(mem[i], U(0));
+ }
+
+ memset(mem, 0, sizeof(mem));
+ where(alternating_mask, indexes_from_0).copy_to(&mem[V::size()], stride_aligned);
+ for (i = 0; i < V::size() + 1; ++i) {
+ COMPARE(mem[i], U(0));
+ }
+ for (; i < 2 * V::size(); i += 2) {
+ COMPARE(mem[i], U(i - V::size()));
+ }
+ for (i = V::size() + 2; i < 2 * V::size(); i += 2) {
+ COMPARE(mem[i], U(0));
+ }
+ for (; i < 3 * V::size(); ++i) {
+ COMPARE(mem[i], U(0));
+ }
+}
+
+template <typename V> void test()
+{
+ load_store<V, long double>();
+ load_store<V, double>();
+ load_store<V, float>();
+ load_store<V, long long>();
+ load_store<V, unsigned long long>();
+ load_store<V, unsigned long>();
+ load_store<V, long>();
+ load_store<V, int>();
+ load_store<V, unsigned int>();
+ load_store<V, short>();
+ load_store<V, unsigned short>();
+ load_store<V, char>();
+ load_store<V, signed char>();
+ load_store<V, unsigned char>();
+ load_store<V, char32_t>();
+ load_store<V, char16_t>();
+ load_store<V, wchar_t>();
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/logarithm.h b/libstdc++-v3/testsuite/experimental/simd/tests/logarithm.h
new file mode 100644
index 00000000000..2c5179e9c90
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/logarithm.h
@@ -0,0 +1,51 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/mathreference.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ vir::test::setFuzzyness<float>(1);
+ vir::test::setFuzzyness<double>(1);
+
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values<V>({1,
+ 2,
+ 4,
+ 8,
+ 16,
+ 32,
+ 64,
+ 128,
+ 256,
+ 512,
+ 1024,
+ 2048,
+ 3,
+ 5,
+ 7,
+ 15,
+ 17,
+ 31,
+ 33,
+ 63,
+ 65,
+ limits::quiet_NaN(),
+ limits::infinity(),
+ -limits::infinity(),
+ +0.,
+ -0.,
+ limits::denorm_min(),
+ limits::min(),
+ limits::max(),
+ limits::min() / 3,
+ -limits::denorm_min(),
+ -limits::min(),
+ -limits::max(),
+ -limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2},
+ MAKE_TESTER(log), MAKE_TESTER(log10), MAKE_TESTER(log1p),
+ MAKE_TESTER(log2), MAKE_TESTER(logb));
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_broadcast.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_broadcast.h
new file mode 100644
index 00000000000..61aed72c382
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_broadcast.h
@@ -0,0 +1,43 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ static_assert(std::is_convertible<typename M::reference, bool>::value,
+ "A smart_reference<simd_mask> must be convertible to bool.");
+ static_assert(std::is_same<bool, decltype(std::declval<const typename M::reference &>() == true)>::value,
+ "A smart_reference<simd_mask> must be comparable against bool.");
+ static_assert(vir::test::sfinae_is_callable<typename M::reference &&, bool>(
+ [](auto &&a, auto &&b) -> decltype(std::declval<decltype(a)>() ==
+ std::declval<decltype(b)>()) {
+ return {};
+ }),
+ "A smart_reference<simd_mask> must be comparable against bool.");
+ VERIFY(std::experimental::is_simd_mask_v<M>);
+
+ {
+ M x; // uninitialized
+ x = M{}; // default broadcasts 0
+ COMPARE(x, M(false));
+ COMPARE(x, M());
+ COMPARE(x, M{});
+ x = M(); // default broadcasts 0
+ COMPARE(x, M(false));
+ COMPARE(x, M());
+ COMPARE(x, M{});
+ x = x;
+ for (std::size_t i = 0; i < M::size(); ++i) {
+ COMPARE(x[i], false);
+ }
+ }
+
+ M x(true);
+ M y(false);
+ for (std::size_t i = 0; i < M::size(); ++i) {
+ COMPARE(x[i], true);
+ COMPARE(y[i], false);
+ }
+ y = M(true);
+ COMPARE(x, y);
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_conversions.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_conversions.h
new file mode 100644
index 00000000000..460fa3fe488
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_conversions.h
@@ -0,0 +1,70 @@
+#include "bits/verify.h"
+
+template <typename From, typename To>
+void conversions()
+{
+ using ToV = typename To::simd_type;
+
+ using std::experimental::simd_cast;
+ using std::experimental::static_simd_cast;
+ using std::experimental::__proposed::resizing_simd_cast;
+
+ auto x = resizing_simd_cast<To>(From());
+ COMPARE(typeid(x), typeid(To));
+ COMPARE(x, To());
+
+ x = resizing_simd_cast<To>(From(true));
+ const To ref = ToV([](auto i) { return i; }) < int(From::size());
+ COMPARE(x, ref) << "converted from: " << From(true);
+
+ const ullong all_bits = ~ullong() >> (64 - From::size());
+ for (ullong bit_pos = 1; bit_pos /*until overflow*/; bit_pos *= 2)
+ {
+ for (ullong bits : {bit_pos & all_bits, ~bit_pos & all_bits})
+ {
+ const auto from = From::__from_bitset(bits);
+ const auto to = resizing_simd_cast<To>(from);
+ COMPARE(to, To::__from_bitset(bits))
+ << "\nfrom: " << from << "\nbits: " << std::hex << bits << std::dec;
+ for (std::size_t i = 0; i < To::size(); ++i)
+ {
+ COMPARE(to[i], (bits >> i) & 1)
+ << "\nfrom: " << from << "\nto: " << to
+ << "\nbits: " << std::hex << bits << std::dec << "\ni: " << i;
+ }
+ }
+ }
+}
+
+template <typename From, typename To>
+void apply_abis()
+{
+ conversions<From, std::experimental::rebind_simd_t<To, From>>();
+ conversions<From, std::experimental::native_simd_mask<To>>();
+ conversions<From, std::experimental::simd_mask<To>>();
+ conversions<From, std::experimental::simd_mask<
+ To, std::experimental::simd_abi::scalar>>();
+}
+
+template <typename V>
+void test()
+{
+ using M = typename V::mask_type;
+ apply_abis<M, ldouble>();
+ apply_abis<M, double>();
+ apply_abis<M, float>();
+ apply_abis<M, ullong>();
+ apply_abis<M, llong>();
+ apply_abis<M, ulong>();
+ apply_abis<M, long>();
+ apply_abis<M, uint>();
+ apply_abis<M, int>();
+ apply_abis<M, ushort>();
+ apply_abis<M, short>();
+ apply_abis<M, uchar>();
+ apply_abis<M, schar>();
+ apply_abis<M, char>();
+ apply_abis<M, wchar_t>();
+ apply_abis<M, char16_t>();
+ apply_abis<M, char32_t>();
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_implicit_cvt.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_implicit_cvt.h
new file mode 100644
index 00000000000..f3e4442e5b7
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_implicit_cvt.h
@@ -0,0 +1,79 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+template <class M, class M2>
+constexpr bool assign_should_work =
+ std::is_same<M, M2>::value ||
+ (std::is_same<typename M::abi_type, std::experimental::simd_abi::fixed_size<M::size()>>::value &&
+ std::is_same<typename M::abi_type, typename M2::abi_type>::value);
+template <class M, class M2>
+constexpr bool assign_should_not_work = !assign_should_work<M, M2>;
+
+template <class L, class R>
+std::enable_if_t<assign_should_work<L, R>> implicit_conversions_test()
+{
+ L x = R(true);
+ COMPARE(x, L(true));
+ x = R(false);
+ COMPARE(x, L(false));
+ R y(false);
+ y[0] = true;
+ x = y;
+ L ref(false);
+ ref[0] = true;
+ COMPARE(x, ref);
+}
+
+template <class L, class R>
+std::enable_if_t<assign_should_not_work<L, R>> implicit_conversions_test()
+{
+ VERIFY((is_substitution_failure<L &, R, assignment>));
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ using std::experimental::simd_mask;
+ using std::experimental::native_simd_mask;
+ using std::experimental::fixed_size_simd_mask;
+
+ implicit_conversions_test<M, simd_mask<ldouble>>();
+ implicit_conversions_test<M, simd_mask<double>>();
+ implicit_conversions_test<M, simd_mask<float>>();
+ implicit_conversions_test<M, simd_mask<ullong>>();
+ implicit_conversions_test<M, simd_mask<llong>>();
+ implicit_conversions_test<M, simd_mask<ulong>>();
+ implicit_conversions_test<M, simd_mask<long>>();
+ implicit_conversions_test<M, simd_mask<uint>>();
+ implicit_conversions_test<M, simd_mask<int>>();
+ implicit_conversions_test<M, simd_mask<ushort>>();
+ implicit_conversions_test<M, simd_mask<short>>();
+ implicit_conversions_test<M, simd_mask<uchar>>();
+ implicit_conversions_test<M, simd_mask<schar>>();
+ implicit_conversions_test<M, native_simd_mask<ldouble>>();
+ implicit_conversions_test<M, native_simd_mask<double>>();
+ implicit_conversions_test<M, native_simd_mask<float>>();
+ implicit_conversions_test<M, native_simd_mask<ullong>>();
+ implicit_conversions_test<M, native_simd_mask<llong>>();
+ implicit_conversions_test<M, native_simd_mask<ulong>>();
+ implicit_conversions_test<M, native_simd_mask<long>>();
+ implicit_conversions_test<M, native_simd_mask<uint>>();
+ implicit_conversions_test<M, native_simd_mask<int>>();
+ implicit_conversions_test<M, native_simd_mask<ushort>>();
+ implicit_conversions_test<M, native_simd_mask<short>>();
+ implicit_conversions_test<M, native_simd_mask<uchar>>();
+ implicit_conversions_test<M, native_simd_mask<schar>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<ldouble, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<double, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<float, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<ullong, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<llong, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<ulong, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<long, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<uint, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<int, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<ushort, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<short, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<uchar, M::size()>>();
+ implicit_conversions_test<M, fixed_size_simd_mask<schar, M::size()>>();
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_loadstore.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_loadstore.h
new file mode 100644
index 00000000000..a384241e6a0
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_loadstore.h
@@ -0,0 +1,125 @@
+#include "bits/verify.h"
+
+// simd_mask generator functions {{{1
+template <class M> M make_mask(const std::initializer_list<bool> &init)
+{
+ std::size_t i = 0;
+ M r = {};
+ for (;;) {
+ for (bool x : init) {
+ r[i] = x;
+ if (++i == M::size()) {
+ return r;
+ }
+ }
+ }
+}
+
+template <class M> M make_alternating_mask()
+{
+ return make_mask<M>({false, true});
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ // loads {{{2
+ constexpr size_t alignment = 2 * std::experimental::memory_alignment_v<M>;
+ alignas(alignment) bool mem[3 * M::size()];
+ std::memset(mem, 0, sizeof(mem));
+ for (std::size_t i = 1; i < sizeof(mem) / sizeof(*mem); i += 2) {
+ COMPARE(mem[i - 1], false);
+ mem[i] = true;
+ }
+ using std::experimental::element_aligned;
+ using std::experimental::vector_aligned;
+ constexpr size_t stride_alignment =
+ M::size() & 1 ? 1 : M::size() & 2
+ ? 2
+ : M::size() & 4
+ ? 4
+ : M::size() & 8
+ ? 8
+ : M::size() & 16
+ ? 16
+ : M::size() & 32
+ ? 32
+ : M::size() & 64
+ ? 64
+ : M::size() & 128
+ ? 128
+ : M::size() & 256
+ ? 256
+ : 512;
+ using stride_aligned_t =
+ std::conditional_t<M::size() == stride_alignment, decltype(vector_aligned),
+ std::experimental::overaligned_tag<stride_alignment * sizeof(bool)>>;
+ constexpr stride_aligned_t stride_aligned = {};
+ constexpr auto overaligned = std::experimental::overaligned<alignment>;
+
+ const M alternating_mask = make_alternating_mask<M>();
+
+ M x(&mem[M::size()], stride_aligned);
+ COMPARE(x, M::size() % 2 == 1 ? !alternating_mask : alternating_mask)
+ << x.__to_bitset() << ", alternating_mask: " << alternating_mask.__to_bitset();
+ x = {&mem[1], element_aligned};
+ COMPARE(x, !alternating_mask);
+ x = M{mem, overaligned};
+ COMPARE(x, alternating_mask);
+
+ x.copy_from(&mem[M::size()], stride_aligned);
+ COMPARE(x, M::size() % 2 == 1 ? !alternating_mask : alternating_mask);
+ x.copy_from(&mem[1], element_aligned);
+ COMPARE(x, !alternating_mask);
+ x.copy_from(mem, vector_aligned);
+ COMPARE(x, alternating_mask);
+
+ x = !alternating_mask;
+ where(alternating_mask, x).copy_from(&mem[M::size()], stride_aligned);
+ COMPARE(x, M::size() % 2 == 1 ? !alternating_mask : M{true});
+ x = M(true); // 1111
+ where(alternating_mask, x).copy_from(&mem[1], element_aligned); // load .0.0
+ COMPARE(x, !alternating_mask); // 1010
+ where(alternating_mask, x).copy_from(mem, overaligned); // load .1.1
+ COMPARE(x, M{true}); // 1111
+
+ // stores {{{2
+ memset(mem, 0, sizeof(mem));
+ x = M(true);
+ x.copy_to(&mem[M::size()], stride_aligned);
+ std::size_t i = 0;
+ for (; i < M::size(); ++i) {
+ COMPARE(mem[i], false);
+ }
+ for (; i < 2 * M::size(); ++i) {
+ COMPARE(mem[i], true) << "i: " << i << ", x: " << x;
+ }
+ for (; i < 3 * M::size(); ++i) {
+ COMPARE(mem[i], false);
+ }
+ memset(mem, 0, sizeof(mem));
+ x.copy_to(&mem[1], element_aligned);
+ COMPARE(mem[0], false);
+ for (i = 1; i <= M::size(); ++i) {
+ COMPARE(mem[i], true);
+ }
+ for (; i < 3 * M::size(); ++i) {
+ COMPARE(mem[i], false);
+ }
+ memset(mem, 0, sizeof(mem));
+ alternating_mask.copy_to(mem, overaligned);
+ for (i = 0; i < M::size(); ++i) {
+ COMPARE(mem[i], (i & 1) == 1);
+ }
+ for (; i < 3 * M::size(); ++i) {
+ COMPARE(mem[i], false);
+ }
+ x.copy_to(mem, vector_aligned);
+ where(alternating_mask, !x).copy_to(mem, overaligned);
+ for (i = 0; i < M::size(); ++i) {
+ COMPARE(mem[i], i % 2 == 0);
+ }
+ for (; i < 3 * M::size(); ++i) {
+ COMPARE(mem[i], false);
+ }
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_operator_cvt.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_operator_cvt.h
new file mode 100644
index 00000000000..8fd9f8b60cc
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_operator_cvt.h
@@ -0,0 +1,88 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+using schar = signed char;
+using uchar = unsigned char;
+using ushort = unsigned short;
+using uint = unsigned int;
+using ulong = unsigned long;
+using llong = long long;
+using ullong = unsigned long long;
+using ldouble = long double;
+using wchar = wchar_t;
+using char16 = char16_t;
+using char32 = char32_t;
+
+template <typename M0, typename M1> constexpr bool bit_and_is_illformed()
+{
+ return is_substitution_failure<M0, M1, std::bit_and<>>;
+}
+
+template <typename M0, typename M1> void test_binary_op_cvt()
+{
+ COMPARE((bit_and_is_illformed<M0, M1>()), !(std::is_same_v<M0, M1>));
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ // binary ops without conversions work
+ COMPARE(typeid(M() & M()), typeid(M));
+
+ // nothing else works: no implicit conv. or ambiguous
+ using std::experimental::simd_mask;
+ using std::experimental::native_simd_mask;
+ using std::experimental::fixed_size_simd_mask;
+ test_binary_op_cvt<M, bool>();
+
+ test_binary_op_cvt<M, simd_mask<ldouble>>();
+ test_binary_op_cvt<M, simd_mask<double >>();
+ test_binary_op_cvt<M, simd_mask<float >>();
+ test_binary_op_cvt<M, simd_mask<ullong >>();
+ test_binary_op_cvt<M, simd_mask<llong >>();
+ test_binary_op_cvt<M, simd_mask<ulong >>();
+ test_binary_op_cvt<M, simd_mask<long >>();
+ test_binary_op_cvt<M, simd_mask<uint >>();
+ test_binary_op_cvt<M, simd_mask<int >>();
+ test_binary_op_cvt<M, simd_mask<ushort >>();
+ test_binary_op_cvt<M, simd_mask<short >>();
+ test_binary_op_cvt<M, simd_mask<uchar >>();
+ test_binary_op_cvt<M, simd_mask<schar >>();
+ test_binary_op_cvt<M, simd_mask<wchar >>();
+ test_binary_op_cvt<M, simd_mask<char16 >>();
+ test_binary_op_cvt<M, simd_mask<char32 >>();
+
+ test_binary_op_cvt<M, native_simd_mask<ldouble>>();
+ test_binary_op_cvt<M, native_simd_mask<double >>();
+ test_binary_op_cvt<M, native_simd_mask<float >>();
+ test_binary_op_cvt<M, native_simd_mask<ullong >>();
+ test_binary_op_cvt<M, native_simd_mask<llong >>();
+ test_binary_op_cvt<M, native_simd_mask<ulong >>();
+ test_binary_op_cvt<M, native_simd_mask<long >>();
+ test_binary_op_cvt<M, native_simd_mask<uint >>();
+ test_binary_op_cvt<M, native_simd_mask<int >>();
+ test_binary_op_cvt<M, native_simd_mask<ushort >>();
+ test_binary_op_cvt<M, native_simd_mask<short >>();
+ test_binary_op_cvt<M, native_simd_mask<uchar >>();
+ test_binary_op_cvt<M, native_simd_mask<schar >>();
+ test_binary_op_cvt<M, native_simd_mask<wchar >>();
+ test_binary_op_cvt<M, native_simd_mask<char16 >>();
+ test_binary_op_cvt<M, native_simd_mask<char32 >>();
+
+ test_binary_op_cvt<M, fixed_size_simd_mask<ldouble, 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<double , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<float , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<ullong , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<llong , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<ulong , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<long , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<uint , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<int , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<ushort , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<short , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<uchar , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<schar , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<wchar , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<char16 , 2>>();
+ test_binary_op_cvt<M, fixed_size_simd_mask<char32 , 2>>();
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_operators.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_operators.h
new file mode 100644
index 00000000000..fbe846a1751
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_operators.h
@@ -0,0 +1,36 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ { // compares{{{2
+ M x(true), y(false);
+ VERIFY(all_of(x == x));
+ VERIFY(all_of(x != y));
+ VERIFY(all_of(y != x));
+ VERIFY(!all_of(x != x));
+ VERIFY(!all_of(x == y));
+ VERIFY(!all_of(y == x));
+ }
+ { // subscripting{{{2
+ M x(true);
+ for (std::size_t i = 0; i < M::size(); ++i) {
+ COMPARE(x[i], true) << "\nx: " << x << ", i: " << i;
+ x[i] = !x[i];
+ }
+ COMPARE(x, M{false});
+ for (std::size_t i = 0; i < M::size(); ++i) {
+ COMPARE(x[i], false) << "\nx: " << x << ", i: " << i;
+ x[i] = !x[i];
+ }
+ COMPARE(x, M{true});
+ }
+ { // negation{{{2
+ M x(false);
+ M y = !x;
+ COMPARE(y, M{true});
+ COMPARE(!y, x);
+ }
+}
+
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/mask_reductions.h b/libstdc++-v3/testsuite/experimental/simd/tests/mask_reductions.h
new file mode 100644
index 00000000000..36949d123d4
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/mask_reductions.h
@@ -0,0 +1,148 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+// simd_mask generator functions {{{1
+template <class M> M make_mask(const std::initializer_list<bool> &init)
+{
+ std::size_t i = 0;
+ M r = {};
+ for (;;) {
+ for (bool x : init) {
+ r[i] = x;
+ if (++i == M::size()) {
+ return r;
+ }
+ }
+ }
+}
+
+template <class M> M make_alternating_mask()
+{
+ return make_mask<M>({false, true});
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ const M alternating_mask = make_alternating_mask<M>();
+ COMPARE(alternating_mask[0], false); // assumption below
+ auto &&gen = make_mask<M>;
+
+ // all_of
+ VERIFY( all_of(M{true}));
+ VERIFY(!all_of(alternating_mask));
+ VERIFY(!all_of(M{false}));
+ using std::experimental::all_of;
+ VERIFY( all_of(true));
+ VERIFY(!all_of(false));
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::all_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::all_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::all_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::all_of(x)) { return {}; }));
+
+ // any_of
+ VERIFY( any_of(M{true}));
+ COMPARE(any_of(alternating_mask), M::size() > 1);
+ VERIFY(!any_of(M{false}));
+ using std::experimental::any_of;
+ VERIFY( any_of(true));
+ VERIFY(!any_of(false));
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::any_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::any_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::any_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::any_of(x)) { return {}; }));
+
+ // none_of
+ VERIFY(!none_of(M{true}));
+ COMPARE(none_of(alternating_mask), M::size() == 1);
+ VERIFY( none_of(M{false}));
+ using std::experimental::none_of;
+ VERIFY(!none_of(true));
+ VERIFY( none_of(false));
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::none_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::none_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::none_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::none_of(x)) { return {}; }));
+
+ // some_of
+ VERIFY(!some_of(M{true}));
+ VERIFY(!some_of(M{false}));
+ if (M::size() > 1) {
+ VERIFY(some_of(gen({true, false})));
+ VERIFY(some_of(gen({false, true})));
+ if (M::size() > 3) {
+ VERIFY(some_of(gen({0, 0, 0, 1})));
+ }
+ }
+ using std::experimental::some_of;
+ VERIFY(!some_of(true));
+ VERIFY(!some_of(false));
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::some_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::some_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::some_of(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::some_of(x)) { return {}; }));
+
+ // popcount
+ COMPARE(popcount(M{true}), int(M::size()));
+ COMPARE(popcount(alternating_mask), int(M::size()) / 2);
+ COMPARE(popcount(M{false}), 0);
+ COMPARE(popcount(gen({0, 0, 1})), int(M::size()) / 3);
+ COMPARE(popcount(gen({0, 0, 0, 1})), int(M::size()) / 4);
+ COMPARE(popcount(gen({0, 0, 0, 0, 1})), int(M::size()) / 5);
+ COMPARE(std::experimental::popcount(true), 1);
+ COMPARE(std::experimental::popcount(false), 0);
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::popcount(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::popcount(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::popcount(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::popcount(x)) { return {}; }));
+
+ // find_first_set
+ {
+ M x(false);
+ for (int i = int(M::size() / 2 - 1); i >= 0; --i) {
+ x[i] = true;
+ COMPARE(find_first_set(x), i) << x;
+ }
+ x = M(false);
+ for (int i = int(M::size() - 1); i >= 0; --i) {
+ x[i] = true;
+ COMPARE(find_first_set(x), i) << x;
+ }
+ }
+ COMPARE(find_first_set(M{true}), 0);
+ if (M::size() > 1) {
+ COMPARE(find_first_set(gen({0, 1})), 1);
+ }
+ if (M::size() > 2) {
+ COMPARE(find_first_set(gen({0, 0, 1})), 2);
+ }
+ COMPARE(std::experimental::find_first_set(true), 0);
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::find_first_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::find_first_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::find_first_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::find_first_set(x)) { return {}; }));
+
+ // find_last_set
+ {
+ M x(false);
+ for (int i = 0; i < int(M::size()); ++i) {
+ x[i] = true;
+ COMPARE(find_last_set(x), i) << x;
+ }
+ }
+ COMPARE(find_last_set(M{true}), int(M::size()) - 1);
+ if (M::size() > 1) {
+ COMPARE(find_last_set(gen({1, 0})), int(M::size()) - 2 + int(M::size() & 1));
+ }
+ if (M::size() > 3 && (M::size() & 3) == 0) {
+ COMPARE(find_last_set(gen({1, 0, 0, 0})), int(M::size()) - 4 - int(M::size() & 3));
+ }
+ COMPARE(std::experimental::find_last_set(true), 0);
+ VERIFY( sfinae_is_callable< bool>([](auto x) -> decltype(std::experimental::find_last_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< int>([](auto x) -> decltype(std::experimental::find_last_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable<float>([](auto x) -> decltype(std::experimental::find_last_set(x)) { return {}; }));
+ VERIFY(!sfinae_is_callable< char>([](auto x) -> decltype(std::experimental::find_last_set(x)) { return {}; }));
+}
+
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/math_2arg.h b/libstdc++-v3/testsuite/experimental/simd/tests/math_2arg.h
new file mode 100644
index 00000000000..3c63f358d23
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/math_2arg.h
@@ -0,0 +1,39 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V> void test()
+{
+ using limits = std::numeric_limits<typename V::value_type>;
+
+ vir::test::setFuzzyness<float>(1);
+ vir::test::setFuzzyness<double>(1);
+ vir::test::setFuzzyness<long double>(1);
+ test_values_2arg<V>(
+ {limits::quiet_NaN(), limits::infinity(), -limits::infinity(), +0., -0.,
+ limits::denorm_min(), limits::min(), limits::max(), limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2}, MAKE_TESTER(hypot));
+ VERIFY((sfinae_is_callable<V, V>(
+ [](auto a, auto b) -> decltype(hypot(a, b)) { return {}; })));
+ VERIFY((sfinae_is_callable<typename V::value_type, V>(
+ [](auto a, auto b) -> decltype(hypot(a, b)) { return {}; })));
+ VERIFY((sfinae_is_callable<V, typename V::value_type>(
+ [](auto a, auto b) -> decltype(hypot(a, b)) { return {}; })));
+
+ vir::test::setFuzzyness<float>(0);
+ vir::test::setFuzzyness<double>(0);
+ vir::test::setFuzzyness<long double>(0);
+ test_values_2arg<V>(
+ {limits::quiet_NaN(), limits::infinity(), -limits::infinity(), +0., -0.,
+ limits::denorm_min(), limits::min(), limits::max(), limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2}, MAKE_TESTER(pow),
+ MAKE_TESTER(fmod), MAKE_TESTER(remainder), MAKE_TESTER(copysign),
+ MAKE_TESTER(nextafter), // MAKE_TESTER(nexttoward),
+ MAKE_TESTER(fdim), MAKE_TESTER(fmax), MAKE_TESTER(fmin), MAKE_TESTER(fdim),
+ MAKE_TESTER(isgreater), MAKE_TESTER(isgreaterequal), MAKE_TESTER(isless),
+ MAKE_TESTER(islessequal), MAKE_TESTER(islessgreater),
+ MAKE_TESTER(isunordered));
+}
+
+// vim: ts=8 et sw=2 sts=2
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/operator_cvt.h b/libstdc++-v3/testsuite/experimental/simd/tests/operator_cvt.h
new file mode 100644
index 00000000000..d8c51a1f6f6
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/operator_cvt.h
@@ -0,0 +1,970 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+
+// type with sizeof(char) but different signedness
+using xchar = std::conditional_t<std::is_unsigned_v<char>, schar, uchar>;
+
+// vT {{{
+using vschar = std::experimental::native_simd<schar>;
+using vuchar = std::experimental::native_simd<uchar>;
+using vshort = std::experimental::native_simd<short>;
+using vushort = std::experimental::native_simd<ushort>;
+using vint = std::experimental::native_simd<int>;
+using vuint = std::experimental::native_simd<uint>;
+using vlong = std::experimental::native_simd<long>;
+using vulong = std::experimental::native_simd<ulong>;
+using vllong = std::experimental::native_simd<llong>;
+using vullong = std::experimental::native_simd<ullong>;
+using vfloat = std::experimental::native_simd<float>;
+using vdouble = std::experimental::native_simd<double>;
+using vldouble = std::experimental::native_simd<long double>;
+using vchar = std::experimental::native_simd<char>;
+using vxchar = std::experimental::native_simd<xchar>;
+// }}}
+// viN/vfN {{{
+template <typename T> using vi8 = std::experimental::fixed_size_simd<T, vschar::size()>;
+template <typename T> using vi16 = std::experimental::fixed_size_simd<T, vshort::size()>;
+template <typename T> using vf32 = std::experimental::fixed_size_simd<T, vfloat::size()>;
+template <typename T> using vi32 = std::experimental::fixed_size_simd<T, vint::size()>;
+template <typename T> using vf64 = std::experimental::fixed_size_simd<T, vdouble::size()>;
+template <typename T> using vi64 = std::experimental::fixed_size_simd<T, vllong::size()>;
+template <typename T>
+using vl = typename std::conditional<sizeof(long) == sizeof(llong), vi64<T>, vi32<T>>::type;
+// }}}
+
+template <class A, class B, class Expected = A> void binary_op_return_type()
+{
+ using namespace vir::test;
+ static_assert(std::is_same<A, Expected>::value, "");
+ using AC = std::add_const_t<A>;
+ using BC = std::add_const_t<B>;
+ COMPARE(typeid(A() + B()), typeid(Expected));
+ COMPARE(typeid(B() + A()), typeid(Expected));
+ COMPARE(typeid(AC() + BC()), typeid(Expected));
+ COMPARE(typeid(BC() + AC()), typeid(Expected));
+}
+
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ binary_op_return_type<V, V, V>();
+ binary_op_return_type<V, T, V>();
+ binary_op_return_type<V, int, V>();
+
+ if constexpr (std::is_same_v<V, vfloat>) { //{{{2
+ binary_op_return_type<vfloat, schar>();
+ binary_op_return_type<vfloat, uchar>();
+ binary_op_return_type<vfloat, short>();
+ binary_op_return_type<vfloat, ushort>();
+
+ binary_op_return_type<vf32<float>, schar>();
+ binary_op_return_type<vf32<float>, uchar>();
+ binary_op_return_type<vf32<float>, short>();
+ binary_op_return_type<vf32<float>, ushort>();
+ binary_op_return_type<vf32<float>, int>();
+ binary_op_return_type<vf32<float>, float>();
+
+ binary_op_return_type<vf32<float>, vf32<schar>>();
+ binary_op_return_type<vf32<float>, vf32<uchar>>();
+ binary_op_return_type<vf32<float>, vf32<short>>();
+ binary_op_return_type<vf32<float>, vf32<ushort>>();
+ binary_op_return_type<vf32<float>, vf32<float>>();
+
+ VERIFY((is_substitution_failure<vfloat, uint>));
+ VERIFY((is_substitution_failure<vfloat, long>));
+ VERIFY((is_substitution_failure<vfloat, ulong>));
+ VERIFY((is_substitution_failure<vfloat, llong>));
+ VERIFY((is_substitution_failure<vfloat, ullong>));
+ VERIFY((is_substitution_failure<vfloat, double>));
+ VERIFY((is_substitution_failure<vfloat, vf32<schar>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<uchar>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<short>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<ushort>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<int>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<uint>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<long>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<ulong>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<llong>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<ullong>>));
+ VERIFY((is_substitution_failure<vfloat, vf32<float>>));
+
+ VERIFY((is_substitution_failure<vf32<float>, vfloat>));
+ VERIFY((is_substitution_failure<vf32<float>, uint>));
+ VERIFY((is_substitution_failure<vf32<float>, long>));
+ VERIFY((is_substitution_failure<vf32<float>, ulong>));
+ VERIFY((is_substitution_failure<vf32<float>, llong>));
+ VERIFY((is_substitution_failure<vf32<float>, ullong>));
+ VERIFY((is_substitution_failure<vf32<float>, double>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<int>>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<uint>>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<long>>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<ulong>>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<llong>>));
+ VERIFY((is_substitution_failure<vf32<float>, vf32<ullong>>));
+
+ VERIFY((is_substitution_failure<vfloat, vf32<double>>));
+ } else if constexpr (std::is_same_v<V, vdouble>) { //{{{2
+ binary_op_return_type<vdouble, float, vdouble>();
+ binary_op_return_type<vdouble, schar>();
+ binary_op_return_type<vdouble, uchar>();
+ binary_op_return_type<vdouble, short>();
+ binary_op_return_type<vdouble, ushort>();
+ binary_op_return_type<vdouble, uint>();
+
+ binary_op_return_type<vf64<double>, schar>();
+ binary_op_return_type<vf64<double>, uchar>();
+ binary_op_return_type<vf64<double>, short>();
+ binary_op_return_type<vf64<double>, ushort>();
+ binary_op_return_type<vf64<double>, uint>();
+ binary_op_return_type<vf64<double>, int, vf64<double>>();
+ binary_op_return_type<vf64<double>, float, vf64<double>>();
+ binary_op_return_type<vf64<double>, double, vf64<double>>();
+ binary_op_return_type<vf64<double>, vf64<double>, vf64<double>>();
+ binary_op_return_type<vf32<double>, schar>();
+ binary_op_return_type<vf32<double>, uchar>();
+ binary_op_return_type<vf32<double>, short>();
+ binary_op_return_type<vf32<double>, ushort>();
+ binary_op_return_type<vf32<double>, uint>();
+ binary_op_return_type<vf32<double>, int, vf32<double>>();
+ binary_op_return_type<vf32<double>, float, vf32<double>>();
+ binary_op_return_type<vf32<double>, double, vf32<double>>();
+ binary_op_return_type<vf64<double>, vf64<schar>>();
+ binary_op_return_type<vf64<double>, vf64<uchar>>();
+ binary_op_return_type<vf64<double>, vf64<short>>();
+ binary_op_return_type<vf64<double>, vf64<ushort>>();
+ binary_op_return_type<vf64<double>, vf64<int>>();
+ binary_op_return_type<vf64<double>, vf64<uint>>();
+ binary_op_return_type<vf64<double>, vf64<float>>();
+
+ VERIFY((is_substitution_failure<vdouble, llong>));
+ VERIFY((is_substitution_failure<vdouble, ullong>));
+ VERIFY((is_substitution_failure<vdouble, vf64<schar>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<uchar>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<short>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<ushort>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<int>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<uint>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<long>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<ulong>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<llong>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<ullong>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<float>>));
+ VERIFY((is_substitution_failure<vdouble, vf64<double>>));
+
+ VERIFY((is_substitution_failure<vf64<double>, vdouble>));
+ VERIFY((is_substitution_failure<vf64<double>, llong>));
+ VERIFY((is_substitution_failure<vf64<double>, ullong>));
+ VERIFY((is_substitution_failure<vf64<double>, vf64<llong>>));
+ VERIFY((is_substitution_failure<vf64<double>, vf64<ullong>>));
+
+ VERIFY((is_substitution_failure<vf32<double>, llong>));
+ VERIFY((is_substitution_failure<vf32<double>, ullong>));
+
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ VERIFY((is_substitution_failure<vdouble, long>));
+ VERIFY((is_substitution_failure<vdouble, ulong>));
+ VERIFY((is_substitution_failure<vf64<double>, long>));
+ VERIFY((is_substitution_failure<vf64<double>, ulong>));
+ VERIFY((is_substitution_failure<vf64<double>, vf64<long>>));
+ VERIFY((is_substitution_failure<vf64<double>, vf64<ulong>>));
+ VERIFY((is_substitution_failure<vf32<double>, long>));
+ VERIFY((is_substitution_failure<vf32<double>, ulong>));
+ } else {
+ binary_op_return_type<vdouble, long>();
+ binary_op_return_type<vdouble, ulong>();
+ binary_op_return_type<vf64<double>, long>();
+ binary_op_return_type<vf64<double>, ulong>();
+ binary_op_return_type<vf64<double>, vf64<long>>();
+ binary_op_return_type<vf64<double>, vf64<ulong>>();
+ binary_op_return_type<vf32<double>, long>();
+ binary_op_return_type<vf32<double>, ulong>();
+ }
+ } else if constexpr (std::is_same_v<V, vldouble>) { //{{{2
+ binary_op_return_type<vldouble, schar>();
+ binary_op_return_type<vldouble, uchar>();
+ binary_op_return_type<vldouble, short>();
+ binary_op_return_type<vldouble, ushort>();
+ binary_op_return_type<vldouble, uint>();
+ binary_op_return_type<vldouble, long>();
+ binary_op_return_type<vldouble, ulong>();
+ binary_op_return_type<vldouble, float>();
+ binary_op_return_type<vldouble, double>();
+
+ binary_op_return_type<vf64<long double>, schar>();
+ binary_op_return_type<vf64<long double>, uchar>();
+ binary_op_return_type<vf64<long double>, short>();
+ binary_op_return_type<vf64<long double>, ushort>();
+ binary_op_return_type<vf64<long double>, int>();
+ binary_op_return_type<vf64<long double>, uint>();
+ binary_op_return_type<vf64<long double>, long>();
+ binary_op_return_type<vf64<long double>, ulong>();
+ binary_op_return_type<vf64<long double>, float>();
+ binary_op_return_type<vf64<long double>, double>();
+ binary_op_return_type<vf64<long double>, vf64<long double>>();
+
+ using std::experimental::simd;
+ using A = std::experimental::simd_abi::fixed_size<vldouble::size()>;
+ binary_op_return_type<simd<long double, A>, schar>();
+ binary_op_return_type<simd<long double, A>, uchar>();
+ binary_op_return_type<simd<long double, A>, short>();
+ binary_op_return_type<simd<long double, A>, ushort>();
+ binary_op_return_type<simd<long double, A>, int>();
+ binary_op_return_type<simd<long double, A>, uint>();
+ binary_op_return_type<simd<long double, A>, long>();
+ binary_op_return_type<simd<long double, A>, ulong>();
+ binary_op_return_type<simd<long double, A>, float>();
+ binary_op_return_type<simd<long double, A>, double>();
+
+ if constexpr (sizeof(ldouble) == sizeof(double)) {
+ VERIFY((is_substitution_failure<vldouble, llong>));
+ VERIFY((is_substitution_failure<vldouble, ullong>));
+ VERIFY((is_substitution_failure<vf64<ldouble>, llong>));
+ VERIFY((is_substitution_failure<vf64<ldouble>, ullong>));
+ VERIFY((is_substitution_failure<simd<ldouble, A>, llong>));
+ VERIFY((is_substitution_failure<simd<ldouble, A>, ullong>));
+ } else {
+ binary_op_return_type<vldouble, llong>();
+ binary_op_return_type<vldouble, ullong>();
+ binary_op_return_type<vf64<long double>, llong>();
+ binary_op_return_type<vf64<long double>, ullong>();
+ binary_op_return_type<simd<long double, A>, llong>();
+ binary_op_return_type<simd<long double, A>, ullong>();
+ }
+
+ VERIFY((is_substitution_failure<vf64<long double>, vldouble>));
+ COMPARE((is_substitution_failure<simd<long double, A>, vldouble>),
+ (!std::is_same<A, vldouble::abi_type>::value));
+ } else if constexpr (std::is_same_v<V, vlong>) { //{{{2
+ VERIFY((is_substitution_failure<vi32<long>, double>));
+ VERIFY((is_substitution_failure<vi32<long>, float>));
+ VERIFY((is_substitution_failure<vi32<long>, vi32<float>>));
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ binary_op_return_type<vlong, uint>();
+ binary_op_return_type<vlong, llong>();
+ binary_op_return_type<vi32<long>, uint>();
+ binary_op_return_type<vi32<long>, llong>();
+ binary_op_return_type<vi64<long>, uint>();
+ binary_op_return_type<vi64<long>, llong>();
+ binary_op_return_type<vi32<long>, vi32<uint>>();
+ binary_op_return_type<vi64<long>, vi64<uint>>();
+ VERIFY((is_substitution_failure<vi32<long>, vi32<double>>));
+ VERIFY((is_substitution_failure<vi64<long>, vi64<double>>));
+ } else {
+ VERIFY((is_substitution_failure<vlong, uint>));
+ VERIFY((is_substitution_failure<vlong, llong>));
+ VERIFY((is_substitution_failure<vi32<long>, uint>));
+ VERIFY((is_substitution_failure<vi32<long>, llong>));
+ VERIFY((is_substitution_failure<vi64<long>, uint>));
+ VERIFY((is_substitution_failure<vi64<long>, llong>));
+ VERIFY((is_substitution_failure<vi32<long>, vi32<uint>>));
+ VERIFY((is_substitution_failure<vi64<long>, vi64<uint>>));
+ binary_op_return_type<vi32<double>, vi32<long>>();
+ binary_op_return_type<vi64<double>, vi64<long>>();
+ }
+
+ binary_op_return_type<vlong, schar, vlong>();
+ binary_op_return_type<vlong, uchar, vlong>();
+ binary_op_return_type<vlong, short, vlong>();
+ binary_op_return_type<vlong, ushort, vlong>();
+
+ binary_op_return_type<vi32<long>, schar, vi32<long>>();
+ binary_op_return_type<vi32<long>, uchar, vi32<long>>();
+ binary_op_return_type<vi32<long>, short, vi32<long>>();
+ binary_op_return_type<vi32<long>, ushort, vi32<long>>();
+ binary_op_return_type<vi32<long>, int, vi32<long>>();
+ binary_op_return_type<vi32<long>, long, vi32<long>>();
+ binary_op_return_type<vi32<long>, vi32<long>, vi32<long>>();
+ binary_op_return_type<vi64<long>, schar, vi64<long>>();
+ binary_op_return_type<vi64<long>, uchar, vi64<long>>();
+ binary_op_return_type<vi64<long>, short, vi64<long>>();
+ binary_op_return_type<vi64<long>, ushort, vi64<long>>();
+ binary_op_return_type<vi64<long>, int, vi64<long>>();
+ binary_op_return_type<vi64<long>, long, vi64<long>>();
+ binary_op_return_type<vi64<long>, vi64<long>, vi64<long>>();
+
+ VERIFY((is_substitution_failure<vlong, vulong>));
+ VERIFY((is_substitution_failure<vlong, ulong>));
+ VERIFY((is_substitution_failure<vlong, ullong>));
+ VERIFY((is_substitution_failure<vlong, float>));
+ VERIFY((is_substitution_failure<vlong, double>));
+ VERIFY((is_substitution_failure<vlong, vl<schar>>));
+ VERIFY((is_substitution_failure<vlong, vl<uchar>>));
+ VERIFY((is_substitution_failure<vlong, vl<short>>));
+ VERIFY((is_substitution_failure<vlong, vl<ushort>>));
+ VERIFY((is_substitution_failure<vlong, vl<int>>));
+ VERIFY((is_substitution_failure<vlong, vl<uint>>));
+ VERIFY((is_substitution_failure<vlong, vl<long>>));
+ VERIFY((is_substitution_failure<vlong, vl<ulong>>));
+ VERIFY((is_substitution_failure<vlong, vl<llong>>));
+ VERIFY((is_substitution_failure<vlong, vl<ullong>>));
+ VERIFY((is_substitution_failure<vlong, vl<float>>));
+ VERIFY((is_substitution_failure<vlong, vl<double>>));
+ VERIFY((is_substitution_failure<vl<long>, vlong>));
+ VERIFY((is_substitution_failure<vl<long>, vulong>));
+ VERIFY((is_substitution_failure<vi32<long>, ulong>));
+ VERIFY((is_substitution_failure<vi32<long>, ullong>));
+ binary_op_return_type<vi32<long>, vi32<schar>>();
+ binary_op_return_type<vi32<long>, vi32<uchar>>();
+ binary_op_return_type<vi32<long>, vi32<short>>();
+ binary_op_return_type<vi32<long>, vi32<ushort>>();
+ binary_op_return_type<vi32<long>, vi32<int>>();
+ VERIFY((is_substitution_failure<vi32<long>, vi32<ulong>>));
+ VERIFY((is_substitution_failure<vi32<long>, vi32<ullong>>));
+ VERIFY((is_substitution_failure<vi64<long>, ulong>));
+ VERIFY((is_substitution_failure<vi64<long>, ullong>));
+ VERIFY((is_substitution_failure<vi64<long>, float>));
+ VERIFY((is_substitution_failure<vi64<long>, double>));
+ binary_op_return_type<vi64<long>, vi64<schar>>();
+ binary_op_return_type<vi64<long>, vi64<uchar>>();
+ binary_op_return_type<vi64<long>, vi64<short>>();
+ binary_op_return_type<vi64<long>, vi64<ushort>>();
+ binary_op_return_type<vi64<long>, vi64<int>>();
+ VERIFY((is_substitution_failure<vi64<long>, vi64<ulong>>));
+ VERIFY((is_substitution_failure<vi64<long>, vi64<ullong>>));
+ VERIFY((is_substitution_failure<vi64<long>, vi64<float>>));
+
+ binary_op_return_type<vi32<llong>, vi32<long>>();
+ binary_op_return_type<vi64<llong>, vi64<long>>();
+ } else if constexpr (std::is_same_v<V, vulong>) { //{{{2
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ binary_op_return_type<vulong, ullong, vulong>();
+ binary_op_return_type<vi32<ulong>, ullong, vi32<ulong>>();
+ binary_op_return_type<vi64<ulong>, ullong, vi64<ulong>>();
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<llong>>));
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<double>>));
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<llong>>));
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<double>>));
+ } else {
+ VERIFY((is_substitution_failure<vulong, ullong>));
+ VERIFY((is_substitution_failure<vi32<ulong>, ullong>));
+ VERIFY((is_substitution_failure<vi64<ulong>, ullong>));
+ binary_op_return_type<vi32<llong>, vi32<ulong>>();
+ binary_op_return_type<vi32<double>, vi32<ulong>>();
+ binary_op_return_type<vi64<llong>, vi64<ulong>>();
+ binary_op_return_type<vi64<double>, vi64<ulong>>();
+ }
+
+ binary_op_return_type<vulong, uchar, vulong>();
+ binary_op_return_type<vulong, ushort, vulong>();
+ binary_op_return_type<vulong, uint, vulong>();
+ binary_op_return_type<vi32<ulong>, uchar, vi32<ulong>>();
+ binary_op_return_type<vi32<ulong>, ushort, vi32<ulong>>();
+ binary_op_return_type<vi32<ulong>, int, vi32<ulong>>();
+ binary_op_return_type<vi32<ulong>, uint, vi32<ulong>>();
+ binary_op_return_type<vi32<ulong>, ulong, vi32<ulong>>();
+ binary_op_return_type<vi32<ulong>, vi32<ulong>, vi32<ulong>>();
+ binary_op_return_type<vi64<ulong>, uchar, vi64<ulong>>();
+ binary_op_return_type<vi64<ulong>, ushort, vi64<ulong>>();
+ binary_op_return_type<vi64<ulong>, int, vi64<ulong>>();
+ binary_op_return_type<vi64<ulong>, uint, vi64<ulong>>();
+ binary_op_return_type<vi64<ulong>, ulong, vi64<ulong>>();
+ binary_op_return_type<vi64<ulong>, vi64<ulong>, vi64<ulong>>();
+
+ VERIFY((is_substitution_failure<vi32<ulong>, llong>));
+ VERIFY((is_substitution_failure<vi32<ulong>, float>));
+ VERIFY((is_substitution_failure<vi32<ulong>, double>));
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<float>>));
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<float>>));
+ VERIFY((is_substitution_failure<vulong, schar>));
+ VERIFY((is_substitution_failure<vulong, short>));
+ VERIFY((is_substitution_failure<vulong, vlong>));
+ VERIFY((is_substitution_failure<vulong, long>));
+ VERIFY((is_substitution_failure<vulong, llong>));
+ VERIFY((is_substitution_failure<vulong, float>));
+ VERIFY((is_substitution_failure<vulong, double>));
+ VERIFY((is_substitution_failure<vulong, vl<schar>>));
+ VERIFY((is_substitution_failure<vulong, vl<uchar>>));
+ VERIFY((is_substitution_failure<vulong, vl<short>>));
+ VERIFY((is_substitution_failure<vulong, vl<ushort>>));
+ VERIFY((is_substitution_failure<vulong, vl<int>>));
+ VERIFY((is_substitution_failure<vulong, vl<uint>>));
+ VERIFY((is_substitution_failure<vulong, vl<long>>));
+ VERIFY((is_substitution_failure<vulong, vl<ulong>>));
+ VERIFY((is_substitution_failure<vulong, vl<llong>>));
+ VERIFY((is_substitution_failure<vulong, vl<ullong>>));
+ VERIFY((is_substitution_failure<vulong, vl<float>>));
+ VERIFY((is_substitution_failure<vulong, vl<double>>));
+ VERIFY((is_substitution_failure<vl<ulong>, vlong>));
+ VERIFY((is_substitution_failure<vl<ulong>, vulong>));
+ VERIFY((is_substitution_failure<vi32<ulong>, schar>));
+ VERIFY((is_substitution_failure<vi32<ulong>, short>));
+ VERIFY((is_substitution_failure<vi32<ulong>, long>));
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<schar>>));
+ binary_op_return_type<vi32<ulong>, vi32<uchar>>();
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<short>>));
+ binary_op_return_type<vi32<ulong>, vi32<ushort>>();
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<int>>));
+ binary_op_return_type<vi32<ulong>, vi32<uint>>();
+ VERIFY((is_substitution_failure<vi32<ulong>, vi32<long>>));
+ binary_op_return_type<vi32<ullong>, vi32<ulong>>();
+ VERIFY((is_substitution_failure<vi64<ulong>, schar>));
+ VERIFY((is_substitution_failure<vi64<ulong>, short>));
+ VERIFY((is_substitution_failure<vi64<ulong>, long>));
+ VERIFY((is_substitution_failure<vi64<ulong>, llong>));
+ VERIFY((is_substitution_failure<vi64<ulong>, float>));
+ VERIFY((is_substitution_failure<vi64<ulong>, double>));
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<schar>>));
+ binary_op_return_type<vi64<ulong>, vi64<uchar>>();
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<short>>));
+ binary_op_return_type<vi64<ulong>, vi64<ushort>>();
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<int>>));
+ binary_op_return_type<vi64<ulong>, vi64<uint>>();
+ VERIFY((is_substitution_failure<vi64<ulong>, vi64<long>>));
+ binary_op_return_type<vi64<ullong>, vi64<ulong>>();
+ } else if constexpr (std::is_same_v<V, vllong>) { //{{{2
+ binary_op_return_type<vllong, schar, vllong>();
+ binary_op_return_type<vllong, uchar, vllong>();
+ binary_op_return_type<vllong, short, vllong>();
+ binary_op_return_type<vllong, ushort, vllong>();
+ binary_op_return_type<vllong, uint, vllong>();
+ binary_op_return_type<vllong, long, vllong>();
+ binary_op_return_type<vi32<llong>, schar, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, uchar, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, short, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, ushort, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, int, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, uint, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, long, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, llong, vi32<llong>>();
+ binary_op_return_type<vi32<llong>, vi32<llong>, vi32<llong>>();
+ binary_op_return_type<vi64<llong>, schar, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, uchar, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, short, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, ushort, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, int, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, uint, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, long, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, llong, vi64<llong>>();
+ binary_op_return_type<vi64<llong>, vi64<llong>>();
+ binary_op_return_type<vi32<llong>, vi32<schar>>();
+ binary_op_return_type<vi32<llong>, vi32<uchar>>();
+ binary_op_return_type<vi32<llong>, vi32<short>>();
+ binary_op_return_type<vi32<llong>, vi32<ushort>>();
+ binary_op_return_type<vi32<llong>, vi32<int>>();
+ binary_op_return_type<vi32<llong>, vi32<uint>>();
+ binary_op_return_type<vi32<llong>, vi32<long>>();
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ VERIFY((is_substitution_failure<vi32<llong>, vi32<ulong>>));
+ VERIFY((is_substitution_failure<vi32<llong>, ulong>));
+ VERIFY((is_substitution_failure<vi64<llong>, ulong>));
+ VERIFY((is_substitution_failure<vllong, ulong>));
+ } else {
+ binary_op_return_type<vi32<llong>, vi32<ulong>>();
+ binary_op_return_type<vi32<llong>, ulong>();
+ binary_op_return_type<vi64<llong>, ulong>();
+ binary_op_return_type<vllong, ulong>();
+ }
+
+ VERIFY((is_substitution_failure<vllong, vullong>));
+ VERIFY((is_substitution_failure<vllong, ullong>));
+ VERIFY((is_substitution_failure<vllong, float>));
+ VERIFY((is_substitution_failure<vllong, double>));
+ VERIFY((is_substitution_failure<vllong, vi64<schar>>));
+ VERIFY((is_substitution_failure<vllong, vi64<uchar>>));
+ VERIFY((is_substitution_failure<vllong, vi64<short>>));
+ VERIFY((is_substitution_failure<vllong, vi64<ushort>>));
+ VERIFY((is_substitution_failure<vllong, vi64<int>>));
+ VERIFY((is_substitution_failure<vllong, vi64<uint>>));
+ VERIFY((is_substitution_failure<vllong, vi64<long>>));
+ VERIFY((is_substitution_failure<vllong, vi64<ulong>>));
+ VERIFY((is_substitution_failure<vllong, vi64<llong>>));
+ VERIFY((is_substitution_failure<vllong, vi64<ullong>>));
+ VERIFY((is_substitution_failure<vllong, vi64<float>>));
+ VERIFY((is_substitution_failure<vllong, vi64<double>>));
+ VERIFY((is_substitution_failure<vi32<llong>, ullong>));
+ VERIFY((is_substitution_failure<vi32<llong>, float>));
+ VERIFY((is_substitution_failure<vi32<llong>, double>));
+ VERIFY((is_substitution_failure<vi32<llong>, vi32<ullong>>));
+ VERIFY((is_substitution_failure<vi32<llong>, vi32<float>>));
+ VERIFY((is_substitution_failure<vi32<llong>, vi32<double>>));
+ VERIFY((is_substitution_failure<vi64<llong>, vllong>));
+ VERIFY((is_substitution_failure<vi64<llong>, vullong>));
+ VERIFY((is_substitution_failure<vi64<llong>, ullong>));
+ VERIFY((is_substitution_failure<vi64<llong>, float>));
+ VERIFY((is_substitution_failure<vi64<llong>, double>));
+ binary_op_return_type<vi64<llong>, vi64<schar>>();
+ binary_op_return_type<vi64<llong>, vi64<uchar>>();
+ binary_op_return_type<vi64<llong>, vi64<short>>();
+ binary_op_return_type<vi64<llong>, vi64<ushort>>();
+ binary_op_return_type<vi64<llong>, vi64<int>>();
+ binary_op_return_type<vi64<llong>, vi64<uint>>();
+ binary_op_return_type<vi64<llong>, vi64<long>>();
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ VERIFY((is_substitution_failure<vi64<llong>, vi64<ulong>>));
+ } else {
+ binary_op_return_type<vi64<llong>, vi64<ulong>>();
+ }
+ VERIFY((is_substitution_failure<vi64<llong>, vi64<ullong>>));
+ VERIFY((is_substitution_failure<vi64<llong>, vi64<float>>));
+ VERIFY((is_substitution_failure<vi64<llong>, vi64<double>>));
+ } else if constexpr (std::is_same_v<V, vullong>) { //{{{2
+ binary_op_return_type<vullong, uchar, vullong>();
+ binary_op_return_type<vullong, ushort, vullong>();
+ binary_op_return_type<vullong, uint, vullong>();
+ binary_op_return_type<vullong, ulong, vullong>();
+ binary_op_return_type<vi32<ullong>, uchar, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, ushort, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, int, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, uint, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, ulong, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, ullong, vi32<ullong>>();
+ binary_op_return_type<vi32<ullong>, vi32<ullong>, vi32<ullong>>();
+ binary_op_return_type<vi64<ullong>, uchar, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, ushort, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, int, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, uint, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, ulong, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, ullong, vi64<ullong>>();
+ binary_op_return_type<vi64<ullong>, vi64<ullong>, vi64<ullong>>();
+
+ VERIFY((is_substitution_failure<vullong, schar>));
+ VERIFY((is_substitution_failure<vullong, short>));
+ VERIFY((is_substitution_failure<vullong, long>));
+ VERIFY((is_substitution_failure<vullong, llong>));
+ VERIFY((is_substitution_failure<vullong, vllong>));
+ VERIFY((is_substitution_failure<vullong, float>));
+ VERIFY((is_substitution_failure<vullong, double>));
+ VERIFY((is_substitution_failure<vullong, vi64<schar>>));
+ VERIFY((is_substitution_failure<vullong, vi64<uchar>>));
+ VERIFY((is_substitution_failure<vullong, vi64<short>>));
+ VERIFY((is_substitution_failure<vullong, vi64<ushort>>));
+ VERIFY((is_substitution_failure<vullong, vi64<int>>));
+ VERIFY((is_substitution_failure<vullong, vi64<uint>>));
+ VERIFY((is_substitution_failure<vullong, vi64<long>>));
+ VERIFY((is_substitution_failure<vullong, vi64<ulong>>));
+ VERIFY((is_substitution_failure<vullong, vi64<llong>>));
+ VERIFY((is_substitution_failure<vullong, vi64<ullong>>));
+ VERIFY((is_substitution_failure<vullong, vi64<float>>));
+ VERIFY((is_substitution_failure<vullong, vi64<double>>));
+ VERIFY((is_substitution_failure<vi32<ullong>, schar>));
+ VERIFY((is_substitution_failure<vi32<ullong>, short>));
+ VERIFY((is_substitution_failure<vi32<ullong>, long>));
+ VERIFY((is_substitution_failure<vi32<ullong>, llong>));
+ VERIFY((is_substitution_failure<vi32<ullong>, float>));
+ VERIFY((is_substitution_failure<vi32<ullong>, double>));
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<schar>>));
+ binary_op_return_type<vi32<ullong>, vi32<uchar>>();
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<short>>));
+ binary_op_return_type<vi32<ullong>, vi32<ushort>>();
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<int>>));
+ binary_op_return_type<vi32<ullong>, vi32<uint>>();
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<long>>));
+ binary_op_return_type<vi32<ullong>, vi32<ulong>>();
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<llong>>));
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<float>>));
+ VERIFY((is_substitution_failure<vi32<ullong>, vi32<double>>));
+ VERIFY((is_substitution_failure<vi64<ullong>, schar>));
+ VERIFY((is_substitution_failure<vi64<ullong>, short>));
+ VERIFY((is_substitution_failure<vi64<ullong>, long>));
+ VERIFY((is_substitution_failure<vi64<ullong>, llong>));
+ VERIFY((is_substitution_failure<vi64<ullong>, vllong>));
+ VERIFY((is_substitution_failure<vi64<ullong>, vullong>));
+ VERIFY((is_substitution_failure<vi64<ullong>, float>));
+ VERIFY((is_substitution_failure<vi64<ullong>, double>));
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<schar>>));
+ binary_op_return_type<vi64<ullong>, vi64<uchar>>();
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<short>>));
+ binary_op_return_type<vi64<ullong>, vi64<ushort>>();
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<int>>));
+ binary_op_return_type<vi64<ullong>, vi64<uint>>();
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<long>>));
+ binary_op_return_type<vi64<ullong>, vi64<ulong>>();
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<llong>>));
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<float>>));
+ VERIFY((is_substitution_failure<vi64<ullong>, vi64<double>>));
+ } else if constexpr (std::is_same_v<V, vint>) { //{{{2
+ binary_op_return_type<vint, schar, vint>();
+ binary_op_return_type<vint, uchar, vint>();
+ binary_op_return_type<vint, short, vint>();
+ binary_op_return_type<vint, ushort, vint>();
+ binary_op_return_type<vi32<int>, schar, vi32<int>>();
+ binary_op_return_type<vi32<int>, uchar, vi32<int>>();
+ binary_op_return_type<vi32<int>, short, vi32<int>>();
+ binary_op_return_type<vi32<int>, ushort, vi32<int>>();
+ binary_op_return_type<vi32<int>, int, vi32<int>>();
+ binary_op_return_type<vi32<int>, vi32<int>, vi32<int>>();
+ binary_op_return_type<vi32<int>, vi32<schar>>();
+ binary_op_return_type<vi32<int>, vi32<uchar>>();
+ binary_op_return_type<vi32<int>, vi32<short>>();
+ binary_op_return_type<vi32<int>, vi32<ushort>>();
+
+ binary_op_return_type<vi32<llong>, vi32<int>>();
+ binary_op_return_type<vi32<double>, vi32<int>>();
+
+ // order is important for MSVC. This compiler is just crazy: It considers
+ // operators from unrelated simd template instantiations as candidates - but only
+ // after they have been tested. So e.g. vi32<int> + llong will produce a
+ // vi32<llong> if a vi32<llong> operator test is done before the vi32<int> + llong
+ // test.
+ VERIFY((is_substitution_failure<vi32<int>, double>));
+ VERIFY((is_substitution_failure<vi32<int>, float>));
+ VERIFY((is_substitution_failure<vi32<int>, llong>));
+ VERIFY((is_substitution_failure<vi32<int>, vi32<float>>));
+ VERIFY((is_substitution_failure<vint, vuint>));
+ VERIFY((is_substitution_failure<vint, uint>));
+ VERIFY((is_substitution_failure<vint, ulong>));
+ VERIFY((is_substitution_failure<vint, llong>));
+ VERIFY((is_substitution_failure<vint, ullong>));
+ VERIFY((is_substitution_failure<vint, float>));
+ VERIFY((is_substitution_failure<vint, double>));
+ VERIFY((is_substitution_failure<vint, vi32<schar>>));
+ VERIFY((is_substitution_failure<vint, vi32<uchar>>));
+ VERIFY((is_substitution_failure<vint, vi32<short>>));
+ VERIFY((is_substitution_failure<vint, vi32<ushort>>));
+ VERIFY((is_substitution_failure<vint, vi32<int>>));
+ VERIFY((is_substitution_failure<vint, vi32<uint>>));
+ VERIFY((is_substitution_failure<vint, vi32<long>>));
+ VERIFY((is_substitution_failure<vint, vi32<ulong>>));
+ VERIFY((is_substitution_failure<vint, vi32<llong>>));
+ VERIFY((is_substitution_failure<vint, vi32<ullong>>));
+ VERIFY((is_substitution_failure<vint, vi32<float>>));
+ VERIFY((is_substitution_failure<vint, vi32<double>>));
+ VERIFY((is_substitution_failure<vi32<int>, vint>));
+ VERIFY((is_substitution_failure<vi32<int>, vuint>));
+ VERIFY((is_substitution_failure<vi32<int>, uint>));
+ VERIFY((is_substitution_failure<vi32<int>, ulong>));
+ VERIFY((is_substitution_failure<vi32<int>, ullong>));
+ VERIFY((is_substitution_failure<vi32<int>, vi32<uint>>));
+ VERIFY((is_substitution_failure<vi32<int>, vi32<ulong>>));
+ VERIFY((is_substitution_failure<vi32<int>, vi32<ullong>>));
+
+ binary_op_return_type<vi32<long>, vi32<int>>();
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ VERIFY((is_substitution_failure<vint, long>));
+ VERIFY((is_substitution_failure<vi32<int>, long>));
+ } else {
+ binary_op_return_type<vint, long>();
+ binary_op_return_type<vi32<int>, long>();
+ }
+ } else if constexpr (std::is_same_v<V, vuint>) { //{{{2
+ VERIFY((is_substitution_failure<vi32<uint>, llong>));
+ VERIFY((is_substitution_failure<vi32<uint>, ullong>));
+ VERIFY((is_substitution_failure<vi32<uint>, float>));
+ VERIFY((is_substitution_failure<vi32<uint>, double>));
+ VERIFY((is_substitution_failure<vi32<uint>, vi32<float>>));
+
+ binary_op_return_type<vuint, uchar, vuint>();
+ binary_op_return_type<vuint, ushort, vuint>();
+ binary_op_return_type<vi32<uint>, uchar, vi32<uint>>();
+ binary_op_return_type<vi32<uint>, ushort, vi32<uint>>();
+ binary_op_return_type<vi32<uint>, int, vi32<uint>>();
+ binary_op_return_type<vi32<uint>, uint, vi32<uint>>();
+ binary_op_return_type<vi32<uint>, vi32<uint>, vi32<uint>>();
+ binary_op_return_type<vi32<uint>, vi32<uchar>>();
+ binary_op_return_type<vi32<uint>, vi32<ushort>>();
+
+ binary_op_return_type<vi32<llong>, vi32<uint>>();
+ binary_op_return_type<vi32<ullong>, vi32<uint>>();
+ binary_op_return_type<vi32<double>, vi32<uint>>();
+
+ VERIFY((is_substitution_failure<vuint, schar>));
+ VERIFY((is_substitution_failure<vuint, short>));
+ VERIFY((is_substitution_failure<vuint, vint>));
+ VERIFY((is_substitution_failure<vuint, long>));
+ VERIFY((is_substitution_failure<vuint, llong>));
+ VERIFY((is_substitution_failure<vuint, ullong>));
+ VERIFY((is_substitution_failure<vuint, float>));
+ VERIFY((is_substitution_failure<vuint, double>));
+ VERIFY((is_substitution_failure<vuint, vi32<schar>>));
+ VERIFY((is_substitution_failure<vuint, vi32<uchar>>));
+ VERIFY((is_substitution_failure<vuint, vi32<short>>));
+ VERIFY((is_substitution_failure<vuint, vi32<ushort>>));
+ VERIFY((is_substitution_failure<vuint, vi32<int>>));
+ VERIFY((is_substitution_failure<vuint, vi32<uint>>));
+ VERIFY((is_substitution_failure<vuint, vi32<long>>));
+ VERIFY((is_substitution_failure<vuint, vi32<ulong>>));
+ VERIFY((is_substitution_failure<vuint, vi32<llong>>));
+ VERIFY((is_substitution_failure<vuint, vi32<ullong>>));
+ VERIFY((is_substitution_failure<vuint, vi32<float>>));
+ VERIFY((is_substitution_failure<vuint, vi32<double>>));
+ VERIFY((is_substitution_failure<vi32<uint>, schar>));
+ VERIFY((is_substitution_failure<vi32<uint>, short>));
+ VERIFY((is_substitution_failure<vi32<uint>, vint>));
+ VERIFY((is_substitution_failure<vi32<uint>, vuint>));
+ VERIFY((is_substitution_failure<vi32<uint>, long>));
+ VERIFY((is_substitution_failure<vi32<uint>, vi32<schar>>));
+ VERIFY((is_substitution_failure<vi32<uint>, vi32<short>>));
+ VERIFY((is_substitution_failure<vi32<uint>, vi32<int>>));
+
+ binary_op_return_type<vi32<ulong>, vi32<uint>>();
+ if constexpr (sizeof(long) == sizeof(llong)) {
+ VERIFY((is_substitution_failure<vuint, ulong>));
+ VERIFY((is_substitution_failure<vi32<uint>, ulong>));
+ binary_op_return_type<vi32<long>, vi32<uint>>();
+ } else {
+ binary_op_return_type<vuint, ulong>();
+ binary_op_return_type<vi32<uint>, ulong>();
+ VERIFY((is_substitution_failure<vi32<uint>, vi32<long>>));
+ }
+ } else if constexpr (std::is_same_v<V, vshort>) { //{{{2
+ binary_op_return_type<vshort, schar, vshort>();
+ binary_op_return_type<vshort, uchar, vshort>();
+ binary_op_return_type<vi16<short>, schar, vi16<short>>();
+ binary_op_return_type<vi16<short>, uchar, vi16<short>>();
+ binary_op_return_type<vi16<short>, short, vi16<short>>();
+ binary_op_return_type<vi16<short>, int, vi16<short>>();
+ binary_op_return_type<vi16<short>, vi16<schar>>();
+ binary_op_return_type<vi16<short>, vi16<uchar>>();
+ binary_op_return_type<vi16<short>, vi16<short>>();
+
+ binary_op_return_type<vi16<int>, vi16<short>>();
+ binary_op_return_type<vi16<long>, vi16<short>>();
+ binary_op_return_type<vi16<llong>, vi16<short>>();
+ binary_op_return_type<vi16<float>, vi16<short>>();
+ binary_op_return_type<vi16<double>, vi16<short>>();
+
+ VERIFY((is_substitution_failure<vi16<short>, double>));
+ VERIFY((is_substitution_failure<vi16<short>, llong>));
+ VERIFY((is_substitution_failure<vshort, vushort>));
+ VERIFY((is_substitution_failure<vshort, ushort>));
+ VERIFY((is_substitution_failure<vshort, uint>));
+ VERIFY((is_substitution_failure<vshort, long>));
+ VERIFY((is_substitution_failure<vshort, ulong>));
+ VERIFY((is_substitution_failure<vshort, llong>));
+ VERIFY((is_substitution_failure<vshort, ullong>));
+ VERIFY((is_substitution_failure<vshort, float>));
+ VERIFY((is_substitution_failure<vshort, double>));
+ VERIFY((is_substitution_failure<vshort, vi16<schar>>));
+ VERIFY((is_substitution_failure<vshort, vi16<uchar>>));
+ VERIFY((is_substitution_failure<vshort, vi16<short>>));
+ VERIFY((is_substitution_failure<vshort, vi16<ushort>>));
+ VERIFY((is_substitution_failure<vshort, vi16<int>>));
+ VERIFY((is_substitution_failure<vshort, vi16<uint>>));
+ VERIFY((is_substitution_failure<vshort, vi16<long>>));
+ VERIFY((is_substitution_failure<vshort, vi16<ulong>>));
+ VERIFY((is_substitution_failure<vshort, vi16<llong>>));
+ VERIFY((is_substitution_failure<vshort, vi16<ullong>>));
+ VERIFY((is_substitution_failure<vshort, vi16<float>>));
+ VERIFY((is_substitution_failure<vshort, vi16<double>>));
+ VERIFY((is_substitution_failure<vi16<short>, vshort>));
+ VERIFY((is_substitution_failure<vi16<short>, vushort>));
+ VERIFY((is_substitution_failure<vi16<short>, ushort>));
+ VERIFY((is_substitution_failure<vi16<short>, uint>));
+ VERIFY((is_substitution_failure<vi16<short>, long>));
+ VERIFY((is_substitution_failure<vi16<short>, ulong>));
+ VERIFY((is_substitution_failure<vi16<short>, ullong>));
+ VERIFY((is_substitution_failure<vi16<short>, float>));
+ VERIFY((is_substitution_failure<vi16<short>, vi16<ushort>>));
+ VERIFY((is_substitution_failure<vi16<short>, vi16<uint>>));
+ VERIFY((is_substitution_failure<vi16<short>, vi16<ulong>>));
+ VERIFY((is_substitution_failure<vi16<short>, vi16<ullong>>));
+ } else if constexpr (std::is_same_v<V, vushort>) { //{{{2
+ binary_op_return_type<vushort, uchar, vushort>();
+ binary_op_return_type<vushort, uint, vushort>();
+ binary_op_return_type<vi16<ushort>, uchar, vi16<ushort>>();
+ binary_op_return_type<vi16<ushort>, ushort, vi16<ushort>>();
+ binary_op_return_type<vi16<ushort>, int, vi16<ushort>>();
+ binary_op_return_type<vi16<ushort>, uint, vi16<ushort>>();
+ binary_op_return_type<vi16<ushort>, vi16<uchar>>();
+ binary_op_return_type<vi16<ushort>, vi16<ushort>>();
+
+ binary_op_return_type<vi16<int>, vi16<ushort>>();
+ binary_op_return_type<vi16<long>, vi16<ushort>>();
+ binary_op_return_type<vi16<llong>, vi16<ushort>>();
+ binary_op_return_type<vi16<uint>, vi16<ushort>>();
+ binary_op_return_type<vi16<ulong>, vi16<ushort>>();
+ binary_op_return_type<vi16<ullong>, vi16<ushort>>();
+ binary_op_return_type<vi16<float>, vi16<ushort>>();
+ binary_op_return_type<vi16<double>, vi16<ushort>>();
+
+ VERIFY((is_substitution_failure<vi16<ushort>, llong>));
+ VERIFY((is_substitution_failure<vi16<ushort>, ullong>));
+ VERIFY((is_substitution_failure<vi16<ushort>, double>));
+ VERIFY((is_substitution_failure<vushort, schar>));
+ VERIFY((is_substitution_failure<vushort, short>));
+ VERIFY((is_substitution_failure<vushort, vshort>));
+ VERIFY((is_substitution_failure<vushort, long>));
+ VERIFY((is_substitution_failure<vushort, ulong>));
+ VERIFY((is_substitution_failure<vushort, llong>));
+ VERIFY((is_substitution_failure<vushort, ullong>));
+ VERIFY((is_substitution_failure<vushort, float>));
+ VERIFY((is_substitution_failure<vushort, double>));
+ VERIFY((is_substitution_failure<vushort, vi16<schar>>));
+ VERIFY((is_substitution_failure<vushort, vi16<uchar>>));
+ VERIFY((is_substitution_failure<vushort, vi16<short>>));
+ VERIFY((is_substitution_failure<vushort, vi16<ushort>>));
+ VERIFY((is_substitution_failure<vushort, vi16<int>>));
+ VERIFY((is_substitution_failure<vushort, vi16<uint>>));
+ VERIFY((is_substitution_failure<vushort, vi16<long>>));
+ VERIFY((is_substitution_failure<vushort, vi16<ulong>>));
+ VERIFY((is_substitution_failure<vushort, vi16<llong>>));
+ VERIFY((is_substitution_failure<vushort, vi16<ullong>>));
+ VERIFY((is_substitution_failure<vushort, vi16<float>>));
+ VERIFY((is_substitution_failure<vushort, vi16<double>>));
+ VERIFY((is_substitution_failure<vi16<ushort>, schar>));
+ VERIFY((is_substitution_failure<vi16<ushort>, short>));
+ VERIFY((is_substitution_failure<vi16<ushort>, vshort>));
+ VERIFY((is_substitution_failure<vi16<ushort>, vushort>));
+ VERIFY((is_substitution_failure<vi16<ushort>, long>));
+ VERIFY((is_substitution_failure<vi16<ushort>, ulong>));
+ VERIFY((is_substitution_failure<vi16<ushort>, float>));
+ VERIFY((is_substitution_failure<vi16<ushort>, vi16<schar>>));
+ VERIFY((is_substitution_failure<vi16<ushort>, vi16<short>>));
+ } else if constexpr (std::is_same_v<V, vchar>) { //{{{2
+ binary_op_return_type<vi8<char>, char, vi8<char>>();
+ binary_op_return_type<vi8<char>, int, vi8<char>>();
+ binary_op_return_type<vi8<char>, vi8<char>, vi8<char>>();
+
+ binary_op_return_type<vi8<short>, vi8<char>>();
+ binary_op_return_type<vi8<int>, vi8<char>>();
+ binary_op_return_type<vi8<long>, vi8<char>>();
+ binary_op_return_type<vi8<llong>, vi8<char>>();
+ binary_op_return_type<vi8<float>, vi8<char>>();
+ binary_op_return_type<vi8<double>, vi8<char>>();
+
+ VERIFY((is_substitution_failure<vi8<char>, llong>));
+ VERIFY((is_substitution_failure<vi8<char>, double>));
+ VERIFY((is_substitution_failure<vchar, vxchar>));
+ VERIFY((is_substitution_failure<vchar, xchar>));
+ VERIFY((is_substitution_failure<vchar, short>));
+ VERIFY((is_substitution_failure<vchar, ushort>));
+ COMPARE((is_substitution_failure<vchar, uint>), std::is_signed_v<char>);
+ VERIFY((is_substitution_failure<vchar, long>));
+ VERIFY((is_substitution_failure<vchar, ulong>));
+ VERIFY((is_substitution_failure<vchar, llong>));
+ VERIFY((is_substitution_failure<vchar, ullong>));
+ VERIFY((is_substitution_failure<vchar, float>));
+ VERIFY((is_substitution_failure<vchar, double>));
+ VERIFY((is_substitution_failure<vchar, vi8<char>>));
+ VERIFY((is_substitution_failure<vchar, vi8<uchar>>));
+ VERIFY((is_substitution_failure<vchar, vi8<schar>>));
+ VERIFY((is_substitution_failure<vchar, vi8<short>>));
+ VERIFY((is_substitution_failure<vchar, vi8<ushort>>));
+ VERIFY((is_substitution_failure<vchar, vi8<int>>));
+ VERIFY((is_substitution_failure<vchar, vi8<uint>>));
+ VERIFY((is_substitution_failure<vchar, vi8<long>>));
+ VERIFY((is_substitution_failure<vchar, vi8<ulong>>));
+ VERIFY((is_substitution_failure<vchar, vi8<llong>>));
+ VERIFY((is_substitution_failure<vchar, vi8<ullong>>));
+ VERIFY((is_substitution_failure<vchar, vi8<float>>));
+ VERIFY((is_substitution_failure<vchar, vi8<double>>));
+ VERIFY((is_substitution_failure<vi8<char>, vchar>));
+ VERIFY((is_substitution_failure<vi8<char>, vuchar>));
+ VERIFY((is_substitution_failure<vi8<char>, vschar>));
+ VERIFY((is_substitution_failure<vi8<char>, xchar>));
+ VERIFY((is_substitution_failure<vi8<char>, short>));
+ VERIFY((is_substitution_failure<vi8<char>, ushort>));
+ COMPARE((is_substitution_failure<vi8<char>, uint>), std::is_signed_v<char>);
+ VERIFY((is_substitution_failure<vi8<char>, long>));
+ VERIFY((is_substitution_failure<vi8<char>, ulong>));
+ VERIFY((is_substitution_failure<vi8<char>, ullong>));
+ VERIFY((is_substitution_failure<vi8<char>, float>));
+ VERIFY((is_substitution_failure<vi8<char>, vi8<uchar>>));
+ COMPARE((is_substitution_failure<vi8<char>, vi8<ushort>>), std::is_signed_v<char>);
+ COMPARE((is_substitution_failure<vi8<char>, vi8<uint>>), std::is_signed_v<char>);
+ COMPARE((is_substitution_failure<vi8<char>, vi8<ulong>>), std::is_signed_v<char>);
+ COMPARE((is_substitution_failure<vi8<char>, vi8<ullong>>), std::is_signed_v<char>);
+ VERIFY((is_substitution_failure<vi8<char>, vi8<schar>>));
+ VERIFY(!(is_substitution_failure<vi8<char>, vi8<short>>));
+ VERIFY(!(is_substitution_failure<vi8<char>, vi8<int>>));
+ VERIFY(!(is_substitution_failure<vi8<char>, vi8<long>>));
+ VERIFY(!(is_substitution_failure<vi8<char>, vi8<llong>>));
+ } else if constexpr (std::is_same_v<V, vschar>) { //{{{2
+ binary_op_return_type<vi8<schar>, schar, vi8<schar>>();
+ binary_op_return_type<vi8<schar>, int, vi8<schar>>();
+ binary_op_return_type<vi8<schar>, vi8<schar>, vi8<schar>>();
+
+ binary_op_return_type<vi8<short>, vi8<schar>>();
+ binary_op_return_type<vi8<int>, vi8<schar>>();
+ binary_op_return_type<vi8<long>, vi8<schar>>();
+ binary_op_return_type<vi8<llong>, vi8<schar>>();
+ binary_op_return_type<vi8<float>, vi8<schar>>();
+ binary_op_return_type<vi8<double>, vi8<schar>>();
+
+ VERIFY((is_substitution_failure<vi8<schar>, llong>));
+ VERIFY((is_substitution_failure<vi8<schar>, double>));
+ VERIFY((is_substitution_failure<vschar, vuchar>));
+ VERIFY((is_substitution_failure<vschar, uchar>));
+ VERIFY((is_substitution_failure<vschar, short>));
+ VERIFY((is_substitution_failure<vschar, ushort>));
+ VERIFY((is_substitution_failure<vschar, uint>));
+ VERIFY((is_substitution_failure<vschar, long>));
+ VERIFY((is_substitution_failure<vschar, ulong>));
+ VERIFY((is_substitution_failure<vschar, llong>));
+ VERIFY((is_substitution_failure<vschar, ullong>));
+ VERIFY((is_substitution_failure<vschar, float>));
+ VERIFY((is_substitution_failure<vschar, double>));
+ VERIFY((is_substitution_failure<vschar, vi8<schar>>));
+ VERIFY((is_substitution_failure<vschar, vi8<uchar>>));
+ VERIFY((is_substitution_failure<vschar, vi8<short>>));
+ VERIFY((is_substitution_failure<vschar, vi8<ushort>>));
+ VERIFY((is_substitution_failure<vschar, vi8<int>>));
+ VERIFY((is_substitution_failure<vschar, vi8<uint>>));
+ VERIFY((is_substitution_failure<vschar, vi8<long>>));
+ VERIFY((is_substitution_failure<vschar, vi8<ulong>>));
+ VERIFY((is_substitution_failure<vschar, vi8<llong>>));
+ VERIFY((is_substitution_failure<vschar, vi8<ullong>>));
+ VERIFY((is_substitution_failure<vschar, vi8<float>>));
+ VERIFY((is_substitution_failure<vschar, vi8<double>>));
+ VERIFY((is_substitution_failure<vi8<schar>, vschar>));
+ VERIFY((is_substitution_failure<vi8<schar>, vuchar>));
+ VERIFY((is_substitution_failure<vi8<schar>, uchar>));
+ VERIFY((is_substitution_failure<vi8<schar>, short>));
+ VERIFY((is_substitution_failure<vi8<schar>, ushort>));
+ VERIFY((is_substitution_failure<vi8<schar>, uint>));
+ VERIFY((is_substitution_failure<vi8<schar>, long>));
+ VERIFY((is_substitution_failure<vi8<schar>, ulong>));
+ VERIFY((is_substitution_failure<vi8<schar>, ullong>));
+ VERIFY((is_substitution_failure<vi8<schar>, float>));
+ VERIFY((is_substitution_failure<vi8<schar>, vi8<uchar>>));
+ VERIFY((is_substitution_failure<vi8<schar>, vi8<ushort>>));
+ VERIFY((is_substitution_failure<vi8<schar>, vi8<uint>>));
+ VERIFY((is_substitution_failure<vi8<schar>, vi8<ulong>>));
+ VERIFY((is_substitution_failure<vi8<schar>, vi8<ullong>>));
+ } else if constexpr (std::is_same_v<V, vuchar>) { //{{{2
+ VERIFY((is_substitution_failure<vi8<uchar>, llong>));
+
+ binary_op_return_type<vuchar, uint, vuchar>();
+ binary_op_return_type<vi8<uchar>, uchar, vi8<uchar>>();
+ binary_op_return_type<vi8<uchar>, int, vi8<uchar>>();
+ binary_op_return_type<vi8<uchar>, uint, vi8<uchar>>();
+ binary_op_return_type<vi8<uchar>, vi8<uchar>, vi8<uchar>>();
+
+ binary_op_return_type<vi8<short>, vi8<uchar>>();
+ binary_op_return_type<vi8<ushort>, vi8<uchar>>();
+ binary_op_return_type<vi8<int>, vi8<uchar>>();
+ binary_op_return_type<vi8<uint>, vi8<uchar>>();
+ binary_op_return_type<vi8<long>, vi8<uchar>>();
+ binary_op_return_type<vi8<ulong>, vi8<uchar>>();
+ binary_op_return_type<vi8<llong>, vi8<uchar>>();
+ binary_op_return_type<vi8<ullong>, vi8<uchar>>();
+ binary_op_return_type<vi8<float>, vi8<uchar>>();
+ binary_op_return_type<vi8<double>, vi8<uchar>>();
+
+ VERIFY((is_substitution_failure<vi8<uchar>, ullong>));
+ VERIFY((is_substitution_failure<vi8<uchar>, double>));
+ VERIFY((is_substitution_failure<vuchar, schar>));
+ VERIFY((is_substitution_failure<vuchar, vschar>));
+ VERIFY((is_substitution_failure<vuchar, short>));
+ VERIFY((is_substitution_failure<vuchar, ushort>));
+ VERIFY((is_substitution_failure<vuchar, long>));
+ VERIFY((is_substitution_failure<vuchar, ulong>));
+ VERIFY((is_substitution_failure<vuchar, llong>));
+ VERIFY((is_substitution_failure<vuchar, ullong>));
+ VERIFY((is_substitution_failure<vuchar, float>));
+ VERIFY((is_substitution_failure<vuchar, double>));
+ VERIFY((is_substitution_failure<vuchar, vi8<schar>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<uchar>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<short>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<ushort>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<int>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<uint>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<long>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<ulong>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<llong>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<ullong>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<float>>));
+ VERIFY((is_substitution_failure<vuchar, vi8<double>>));
+ VERIFY((is_substitution_failure<vi8<uchar>, schar>));
+ VERIFY((is_substitution_failure<vi8<uchar>, vschar>));
+ VERIFY((is_substitution_failure<vi8<uchar>, vuchar>));
+ VERIFY((is_substitution_failure<vi8<uchar>, short>));
+ VERIFY((is_substitution_failure<vi8<uchar>, ushort>));
+ VERIFY((is_substitution_failure<vi8<uchar>, long>));
+ VERIFY((is_substitution_failure<vi8<uchar>, ulong>));
+ VERIFY((is_substitution_failure<vi8<uchar>, float>));
+ VERIFY((is_substitution_failure<vi8<uchar>, vi8<schar>>));
+ } //}}}2
+}
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/operators.h b/libstdc++-v3/testsuite/experimental/simd/tests/operators.h
new file mode 100644
index 00000000000..5ae7f5b6647
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/operators.h
@@ -0,0 +1,391 @@
+// Copyright (C) 2018 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library. This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3. If not see
+// <http://www.gnu.org/licenses/>.
+
+#include "bits/verify.h"
+#include "bits/make_vec.h"
+#include "bits/metahelpers.h"
+
+//operators helpers //{{{1
+template <class T> constexpr T genHalfBits()
+{
+ return std::numeric_limits<T>::max() >> (std::numeric_limits<T>::digits / 2);
+}
+template <> constexpr long double genHalfBits<long double>() { return 0; }
+template <> constexpr double genHalfBits<double>() { return 0; }
+template <> constexpr float genHalfBits<float>() { return 0; }
+
+// integral_operators {{{1
+template <class V>
+std::enable_if_t<std::is_integral<typename V::value_type>::value, void>
+integral_operators()
+{
+ using T = typename V::value_type;
+ { // complement{{{2
+ COMPARE(~V(), V(~T()));
+ COMPARE(~V(~T()), V());
+ }
+
+ { // modulus{{{2
+ V x = make_vec<V>({3, 4}, 2);
+ COMPARE(x % x, V(0));
+ V y = x - 1;
+ COMPARE(x % y, V(1));
+ y = x + 1;
+ COMPARE(x % y, x);
+ if (std::is_signed<T>::value) {
+ x = -x;
+ COMPARE(x % y, x);
+ x = -y;
+ COMPARE(x % y, V(0));
+ x = x - 1;
+ COMPARE(x % y, V(-1));
+ x %= y;
+ COMPARE(x, V(-1));
+ }
+ }
+
+ { // bit_and{{{2
+ V x = make_vec<V>({3, 4, 5}, 8);
+ COMPARE(x & x, x);
+ COMPARE(x & ~x, V());
+ COMPARE(x & V(), V());
+ COMPARE(V() & x, V());
+ V y = make_vec<V>({1, 5, 3}, 8);
+ COMPARE(x & y, make_vec<V>({1, 4, 1}, 8));
+ x &= y;
+ COMPARE(x, make_vec<V>({1, 4, 1}, 8));
+ }
+
+ { // bit_or{{{2
+ V x = make_vec<V>({3, 4, 5}, 8);
+ COMPARE(x | x, x);
+ COMPARE(x | ~x, ~V());
+ COMPARE(x | V(), x);
+ COMPARE(V() | x, x);
+ V y = make_vec<V>({1, 5, 3}, 8);
+ COMPARE(x | y, make_vec<V>({3, 5, 7}, 8));
+ x |= y;
+ COMPARE(x, make_vec<V>({3, 5, 7}, 8));
+ }
+
+ { // bit_xor{{{2
+ V x = make_vec<V>({3, 4, 5}, 8);
+ COMPARE(x ^ x, V());
+ COMPARE(x ^ ~x, ~V());
+ COMPARE(x ^ V(), x);
+ COMPARE(V() ^ x, x);
+ V y = make_vec<V>({1, 5, 3}, 8);
+ COMPARE(x ^ y, make_vec<V>({2, 1, 6}, 0));
+ x ^= y;
+ COMPARE(x, make_vec<V>({2, 1, 6}, 0));
+ }
+
+ { // bit_shift_left{{{2
+ COMPARE(V() << 1, V());
+ // Note:
+ // - negative RHS or RHS >= #bits is UB
+ // - negative LHS is UB
+ // - shifting into (or over) the sign bit is UB
+ // - unsigned LHS overflow is modulo arithmetic
+ constexpr int nbits(sizeof(T) * CHAR_BIT);
+ {
+ V seq = make_vec<V>({0, 1}, nbits - 2);
+ seq %= nbits - 1;
+ COMPARE(make_vec<V>({0, 1}, 0) << seq,
+ V([&](auto i) { return T(T(i & 1) << seq[i]); }))
+ << "seq = " << seq;
+ COMPARE(make_vec<V>({1, 0}, 0) << seq,
+ V([&](auto i) { return T(T(~i & 1) << seq[i]); }));
+ COMPARE(V(1) << seq, V([&](auto i) { return T(T(1) << seq[i]); }));
+ }
+ for (int i = 0; i < nbits - 1; ++i) {
+ COMPARE(V(1) << i, V(T(1) << i));
+ }
+ if (std::is_unsigned<T>::value) {
+ constexpr int shift_count = nbits - 1;
+ COMPARE(V(1) << shift_count, V(T(1) << shift_count));
+ constexpr T max = // avoid overflow warning in the last COMPARE
+ std::is_unsigned<T>::value ? std::numeric_limits<T>::max() : T(1);
+ COMPARE(V(max) << shift_count, V(max << shift_count)) << "shift_count: " << shift_count;
+ }
+ }
+
+ { // bit_shift_right{{{2
+ constexpr int nbits(sizeof(T) * CHAR_BIT);
+ // Note:
+ // - negative LHS is implementation defined
+ // - negative RHS or RHS >= #bits is UB
+ // - no other UB
+ COMPARE(V(~T()) >> V(0), V(~T()));
+ for (int s = 1; s < nbits; ++s) {
+ COMPARE(V(~T()) >> V(s), V(T(~T()) >> s)) << "s: " << s;
+ }
+ for (int s = 1; s < nbits; ++s) {
+ COMPARE(V(~T(1)) >> V(s), V(T(~T(1)) >> s)) << "s: " << s;
+ }
+ COMPARE(V(0) >> V(1), V(0));
+ COMPARE(V(1) >> V(1), V(0));
+ COMPARE(V(2) >> V(1), V(1));
+ COMPARE(V(3) >> V(1), V(1));
+ COMPARE(V(7) >> V(2), V(1));
+ {
+ V seq = make_vec<V>({0, 1}, 2);
+ seq %= nbits - 1;
+ COMPARE(V(1) >> seq, V([&](auto i) { return T(T(1) >> seq[i]); }));
+ }
+ }
+
+ //}}}2
+}
+
+template <class V>
+std::enable_if_t<!std::is_integral<typename V::value_type>::value, void>
+integral_operators()
+{
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ using T = typename V::value_type;
+ constexpr auto min = std::numeric_limits<T>::min();
+ constexpr auto max = std::numeric_limits<T>::max();
+ { // compares{{{2
+ COMPARE(V(0) == make_vec<V>({0, 1}, 0), make_mask<M>({1, 0}));
+ COMPARE(V(0) == make_vec<V>({0, 1, 2}, 0), make_mask<M>({1, 0, 0}));
+ COMPARE(V(1) == make_vec<V>({0, 1, 2}, 0), make_mask<M>({0, 1, 0}));
+ COMPARE(V(2) == make_vec<V>({0, 1, 2}, 0), make_mask<M>({0, 0, 1}));
+ COMPARE(V(0) < make_vec<V>({0, 1, 2}, 0), make_mask<M>({0, 1, 1}));
+
+ constexpr T half = genHalfBits<T>();
+ for (T lo_ : {min, T(min + 1), T(-1), T(0), T(1), T(half - 1), half, T(half + 1),
+ T(max - 1)}) {
+ for (T hi_ : {T(min + 1), T(-1), T(0), T(1), T(half - 1), half, T(half + 1),
+ T(max - 1), max}) {
+ if (hi_ <= lo_) {
+ continue;
+ }
+ for (std::size_t pos = 0; pos < V::size(); ++pos) {
+ V lo = lo_;
+ V hi = hi_;
+ lo[pos] = 0; // have a different value in the vector in case
+ hi[pos] = 1; // this affects neighbors
+ COMPARE(hi, hi);
+ VERIFY(all_of(hi != lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(lo != hi)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(none_of(hi != hi)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(none_of(hi == lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(none_of(lo == hi)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(lo < hi))
+ << "hi: " << hi << ", lo: " << lo << ", lo < hi: " << (lo < hi);
+ VERIFY(none_of(hi < lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(none_of(hi <= lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(hi <= hi)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(hi > lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(none_of(lo > hi)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(hi >= lo)) << "hi: " << hi << ", lo: " << lo;
+ VERIFY(all_of(hi >= hi)) << "hi: " << hi << ", lo: " << lo;
+ }
+ }
+ }
+ }
+ { // subscripting{{{2
+ V x = max;
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(x[i], max);
+ x[i] = 0;
+ }
+ COMPARE(x, V{0});
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(x[i], T(0));
+ x[i] = max;
+ }
+ COMPARE(x, V{max});
+ COMPARE(typeid(x[0] * x[0]), typeid(T() * T()));
+ COMPARE(typeid(x[0] * T()), typeid(T() * T()));
+ COMPARE(typeid(T() * x[0]), typeid(T() * T()));
+ COMPARE(typeid(x * x[0]), typeid(x));
+ COMPARE(typeid(x[0] * x), typeid(x));
+
+ x = V([](auto i) -> T { return i; });
+ for (std::size_t i = 0; i < V::size(); ++i) {
+ COMPARE(x[i], T(i));
+ }
+ for (std::size_t i = 0; i + 1 < V::size(); i += 2) {
+ using std::swap;
+ swap(x[i], x[i + 1]);
+ }
+ for (std::size_t i = 0; i + 1 < V::size(); i += 2) {
+ COMPARE(x[i], T(i + 1));
+ COMPARE(x[i + 1], T(i));
+ }
+ x = 1;
+ V y = 0;
+ COMPARE(x[0], T(1));
+ x[0] = y[0]; // make sure non-const smart_reference assignment works
+ COMPARE(x[0], T(0));
+ x = 1;
+ x[0] = x[0]; // self-assignment on smart_reference
+ COMPARE(x[0], T(1));
+
+ std::experimental::simd<typename V::value_type, std::experimental::simd_abi::scalar> z = 2;
+ x[0] = z[0];
+ COMPARE(x[0], T(2));
+ x = 3;
+ z[0] = x[0];
+ COMPARE(z[0], T(3));
+
+ //TODO: check that only value-preserving conversions happen on subscript
+ //assignment
+ }
+ { // not{{{2
+ V x = 0;
+ COMPARE(!x, M{true});
+ V y = 1;
+ COMPARE(!y, M{false});
+ }
+
+ { // unary minus{{{2
+ V x = 0;
+ COMPARE(-x, V(T(-T(0))));
+ V y = 1;
+ COMPARE(-y, V(T(-T(1))));
+ }
+
+ { // plus{{{2
+ V x = 0;
+ V y = 0;
+ COMPARE(x + y, x);
+ COMPARE(x = x + T(1), V(1));
+ COMPARE(x + x, V(2));
+ y = make_vec<V>({1, 2, 3, 4, 5, 6, 7});
+ COMPARE(x = x + y, make_vec<V>({2, 3, 4, 5, 6, 7, 8}));
+ COMPARE(x = x + -y, V(1));
+ COMPARE(x += y, make_vec<V>({2, 3, 4, 5, 6, 7, 8}));
+ COMPARE(x, make_vec<V>({2, 3, 4, 5, 6, 7, 8}));
+ COMPARE(x += -y, V(1));
+ COMPARE(x, V(1));
+ }
+
+ { // minus{{{2
+ V x = 1;
+ V y = 0;
+ COMPARE(x - y, x);
+ COMPARE(x - T(1), y);
+ COMPARE(y, x - T(1));
+ COMPARE(x - x, y);
+ y = make_vec<V>({1, 2, 3, 4, 5, 6, 7});
+ COMPARE(x = y - x, make_vec<V>({0, 1, 2, 3, 4, 5, 6}));
+ COMPARE(x = y - x, V(1));
+ COMPARE(y -= x, make_vec<V>({0, 1, 2, 3, 4, 5, 6}));
+ COMPARE(y, make_vec<V>({0, 1, 2, 3, 4, 5, 6}));
+ COMPARE(y -= y, V(0));
+ COMPARE(y, V(0));
+ }
+
+ { // multiplies{{{2
+ V x = 1;
+ V y = 0;
+ COMPARE(x * y, y);
+ COMPARE(x = x * T(2), V(2));
+ COMPARE(x * x, V(4));
+ y = make_vec<V>({1, 2, 3, 4, 5, 6, 7});
+ COMPARE(x = x * y, make_vec<V>({2, 4, 6, 8, 10, 12, 14}));
+ y = 2;
+ for (T n : {T(std::numeric_limits<T>::max() - 1), std::numeric_limits<T>::min()}) {
+ x = n / 2;
+ COMPARE(x * y, V(n));
+ }
+ if (std::is_integral<T>::value && std::is_unsigned<T>::value) {
+ // test modulo arithmetics
+ T n = std::numeric_limits<T>::max();
+ x = n;
+ for (T m : {T(2), T(7), T(std::numeric_limits<T>::max() / 127), std::numeric_limits<T>::max()}) {
+ y = m;
+ // if T is of lower rank than int, `n * m` will promote to int before executing the
+ // multiplication. In this case an overflow will be UB (and ubsan will
+ // warn about it). The solution is to cast to uint in that case.
+ using U = std::conditional_t<(sizeof(T) < sizeof(int)), unsigned, T>;
+ COMPARE(x * y, V(T(U(n) * U(m))));
+ }
+ }
+ x = 2;
+ COMPARE(x *= make_vec<V>({1, 2, 3}), make_vec<V>({2, 4, 6}));
+ COMPARE(x, make_vec<V>({2, 4, 6}));
+ }
+
+ { // divides{{{2
+ V x = 2;
+ COMPARE(x / x, V(1));
+ COMPARE(T(3) / x, V(T(3) / T(2)));
+ COMPARE(x / T(3), V(T(2) / T(3)));
+ V y = make_vec<V>({1, 2, 3, 4, 5, 6, 7});
+ COMPARE(y / x, make_vec<V>({T(.5), T(1), T(1.5), T(2), T(2.5), T(3), T(3.5)}));
+
+ y = make_vec<V>({std::numeric_limits<T>::max(), std::numeric_limits<T>::min()});
+ V ref = make_vec<V>(
+ {T(std::numeric_limits<T>::max() / 2), T(std::numeric_limits<T>::min() / 2)});
+ COMPARE(y / x, ref);
+
+ y = make_vec<V>({std::numeric_limits<T>::min(), std::numeric_limits<T>::max()});
+ ref = make_vec<V>(
+ {T(std::numeric_limits<T>::min() / 2), T(std::numeric_limits<T>::max() / 2)});
+ COMPARE(y / x, ref);
+
+ y = make_vec<V>(
+ {std::numeric_limits<T>::max(), T(std::numeric_limits<T>::min() + 1)});
+ COMPARE(y / y, V(1));
+
+ ref = make_vec<V>({T(2 / std::numeric_limits<T>::max()),
+ T(2 / (std::numeric_limits<T>::min() + 1))});
+ COMPARE(x / y, ref);
+ COMPARE(x /= y, ref);
+ COMPARE(x, ref);
+ }
+
+ { // increment & decrement {{{2
+ const V from0 = make_vec<V>({0, 1, 2, 3}, 4);
+ V x = from0;
+ COMPARE(x++, from0);
+ COMPARE(x, from0 + 1);
+ COMPARE(++x, from0 + 2);
+ COMPARE(x, from0 + 2);
+
+ COMPARE(x--, from0 + 2);
+ COMPARE(x, from0 + 1);
+ COMPARE(--x, from0);
+ COMPARE(x, from0);
+ }
+
+ integral_operators<V>();
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, std::modulus<>>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, std::bit_and<>>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, std::bit_or<>>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, std::bit_xor<>>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, bit_shift_left>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V, V, bit_shift_right>));
+
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_modulus>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_bit_and>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_bit_or>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_bit_xor>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_bit_shift_left>));
+ COMPARE(!std::is_integral<T>::value, (is_substitution_failure<V &, V, assign_bit_shift_right>));
+ // }}}2
+}
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/reductions.h b/libstdc++-v3/testsuite/experimental/simd/tests/reductions.h
new file mode 100644
index 00000000000..de2073cf808
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/reductions.h
@@ -0,0 +1,71 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include <random>
+
+static std::mt19937 g_mt_gen{0};
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ COMPARE(reduce(V(1)), T(V::size()));
+ COMPARE(std::experimental::reduce(V(1), std::multiplies<>()), T(1));
+ COMPARE(reduce(V([](int i) { return i & 1; })), T(V::size() / 2));
+ COMPARE(reduce(V([](int i) { return i % 3; })),
+ T(3 * (V::size() / 3) // 0+1+2 for every complete 3 elements in V
+ + (V::size() % 3) / 2 // 0->0, 1->0, 2->1 adjustment
+ ));
+ if ((1 + V::size()) * V::size() / 2 <= std::numeric_limits<T>::max()) {
+ COMPARE(reduce(V([](int i) { return i + 1; })),
+ T((1 + V::size()) * V::size() / 2));
+ }
+
+ {
+ const V y = 2;
+ COMPARE(reduce(y), T(2 * V::size()));
+ COMPARE(reduce(where(y > 2, y)), T(0));
+ COMPARE(reduce(where(y == 2, y)), T(2 * V::size()));
+ }
+
+ {
+ const V z([](T i) { return i + 1; });
+ COMPARE(std::experimental::reduce(z,
+ [](auto a, auto b) {
+ using std::min;
+ return min(a, b);
+ }),
+ T(1))
+ << "z: " << z;
+ COMPARE(std::experimental::reduce(z,
+ [](auto a, auto b) {
+ using std::max;
+ return max(a, b);
+ }),
+ T(V::size()))
+ << "z: " << z;
+ COMPARE(std::experimental::reduce(where(z > 1, z), 117,
+ [](auto a, auto b) {
+ using std::min;
+ return min(a, b);
+ }),
+ T(V::size() == 1 ? 117 : 2))
+ << "z: " << z;
+ }
+
+ {
+ std::conditional_t<std::is_floating_point_v<T>, std::uniform_real_distribution<T>,
+ std::uniform_int_distribution<T>>
+ dist(std::numeric_limits<T>::lowest(), std::numeric_limits<T>::max());
+ for (int repeat = 0; repeat < 100; ++repeat) {
+ const V x([&](int) { return dist(g_mt_gen); });
+ // TODO: don't fail for floating point rounding differences
+ COMPARE(reduce(x), [x]() {
+ T acc = x[0];
+ for (size_t i = 1; i < V::size(); ++i) {
+ acc += x[i];
+ }
+ return acc;
+ }());
+ }
+ }
+}
+
+// vim: foldmethod=marker
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/remqo.h b/libstdc++-v3/testsuite/experimental/simd/tests/remqo.h
new file mode 100644
index 00000000000..372e6d5576e
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/remqo.h
@@ -0,0 +1,43 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ vir::test::setFuzzyness<float>(0);
+ vir::test::setFuzzyness<double>(0);
+
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values_2arg<V>(
+ {limits::quiet_NaN(), limits::infinity(), -limits::infinity(), +0., -0.,
+ limits::denorm_min(), limits::min(), limits::max(), limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2}, [](const V a, const V b) {
+ using IV = std::experimental::fixed_size_simd<int, V::size()>;
+ IV quo = {}; // the type is wrong, this should fail
+ const V totest = remquo(a, b, &quo);
+ auto&& expected = [&](const auto& v,
+ const auto& w) -> std::pair<const V, const IV> {
+ std::pair<V, IV> tmp = {};
+ using std::remquo;
+ for (std::size_t i = 0; i < V::size(); ++i)
+ {
+ int tmp2;
+ tmp.first[i] = remquo(v[i], w[i], &tmp2);
+ tmp.second[i] = tmp2;
+ }
+ return tmp;
+ };
+ const auto expect1 = expected(a, b);
+ COMPARE(isnan(totest), isnan(expect1.first))
+ << "remquo(" << a << ", " << b << ", quo) = " << totest
+ << " != " << expect1.first;
+ const V clean_a = iif(isnan(totest), 0, a);
+ const V clean_b = iif(isnan(totest), 1, b);
+ const auto expect2 = expected(clean_a, clean_b);
+ COMPARE(remquo(clean_a, clean_b, &quo), expect2.first)
+ << "\nclean_a/b = " << clean_a << ", " << clean_b;
+ COMPARE(quo, expect2.second);
+ });
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/simd.h b/libstdc++-v3/testsuite/experimental/simd/tests/simd.h
new file mode 100644
index 00000000000..99cbac66a8a
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/simd.h
@@ -0,0 +1,18 @@
+#include "bits/verify.h"
+
+template <typename V> void test() {
+ using T = typename V::value_type;
+
+ // V must store V::size() values of type T giving us the lower bound on the sizeof
+ VERIFY(sizeof(V) >= sizeof(T) * V::size());
+
+ // V should not pad more than to the next-power-of-2 of V::size() values of
+ // type T giving us the upper bound on the sizeof
+ auto n = V::size();
+ n = ((n << 1) & ~n) & ~((n >> 1) | (n >> 3));
+ while (n & (n - 1))
+ {
+ n &= n - 1;
+ }
+ VERIFY(sizeof(V) <= sizeof(T) * n);
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/sincos.h b/libstdc++-v3/testsuite/experimental/simd/tests/sincos.h
new file mode 100644
index 00000000000..6aee3ec5484
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/sincos.h
@@ -0,0 +1,25 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/mathreference.h"
+#include "bits/simd_view.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ using std::cos;
+ using std::sin;
+ using T = typename V::value_type;
+
+ const auto& testdata = referenceData<function::sincos, T>();
+ std::experimental::experimental::simd_view<V>(testdata).for_each(
+ [&](const V input, const V expected_sin, const V expected_cos) {
+ ULP_COMPARE(sin(input), expected_sin, 2) << " input = " << input;
+ ULP_COMPARE(sin(-input), -expected_sin, 2) << " input = " << input;
+ ULP_COMPARE(cos(input), expected_cos, 2) << " input = " << input;
+ ULP_COMPARE(cos(-input), expected_cos, 2) << " input = " << input;
+ });
+}
+
+// vim: sw=2 sts=2 noet ts=8
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/split_concat.h b/libstdc++-v3/testsuite/experimental/simd/tests/split_concat.h
new file mode 100644
index 00000000000..202fc43645a
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/split_concat.h
@@ -0,0 +1,122 @@
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/conversions.h"
+
+using std::experimental::simd_cast;
+
+template <typename V> void test()
+{
+ using T = typename V::value_type;
+ if constexpr (V::size() * 3 <= std::experimental::simd_abi::max_fixed_size<T>) {
+ V a(0), b(1), c(2);
+ auto x = concat(a, b, c);
+ COMPARE(x.size(), a.size() * 3);
+ std::size_t i = 0;
+ for (; i < a.size(); ++i) {
+ COMPARE(x[i], T(0));
+ }
+ for (; i < 2 * a.size(); ++i) {
+ COMPARE(x[i], T(1));
+ }
+ for (; i < 3 * a.size(); ++i) {
+ COMPARE(x[i], T(2));
+ }
+ }
+
+ if constexpr (V::size() >= 4) {
+ const V a([](auto i) -> T { return i; });
+ constexpr auto N0 = V::size() / 4u;
+ constexpr auto N1 = V::size() - 2 * N0;
+ using V0 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, N0>>;
+ using V1 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, N1>>;
+ {
+ auto x = std::experimental::split<N0, N0, N1>(a);
+ COMPARE(std::tuple_size<decltype(x)>::value, 3u);
+ COMPARE(std::get<0>(x), V0([](auto i) -> T { return i; }));
+ COMPARE(std::get<1>(x), V0([](auto i) -> T { return i + N0; }));
+ COMPARE(std::get<2>(x), V1([](auto i) -> T { return i + 2 * N0; }));
+ auto b = concat(std::get<1>(x), std::get<2>(x), std::get<0>(x));
+ // a and b may have different types if a was fixed_size<N> such that another
+ // ABI tag exists with equal N, then b will have the non-fixed-size ABI tag.
+ COMPARE(a.size(), b.size());
+ COMPARE(b, decltype(b)([](auto i) -> T { return (N0 + i) % V::size(); }));
+ }
+ {
+ auto x = std::experimental::split<N0, N1, N0>(a);
+ COMPARE(std::tuple_size<decltype(x)>::value, 3u);
+ COMPARE(std::get<0>(x), V0([](auto i) -> T { return i; }));
+ COMPARE(std::get<1>(x), V1([](auto i) -> T { return i + N0; }));
+ COMPARE(std::get<2>(x), V0([](auto i) -> T { return i + N0 + N1; }));
+ auto b = concat(std::get<1>(x), std::get<2>(x), std::get<0>(x));
+ // a and b may have different types if a was fixed_size<N> such that another
+ // ABI tag exists with equal N, then b will have the non-fixed-size ABI tag.
+ COMPARE(a.size(), b.size());
+ COMPARE(b, decltype(b)([](auto i) -> T { return (N0 + i) % V::size(); }));
+ }
+ {
+ auto x = std::experimental::split<N1, N0, N0>(a);
+ COMPARE(std::tuple_size<decltype(x)>::value, 3u);
+ COMPARE(std::get<0>(x), V1([](auto i) -> T { return i; }));
+ COMPARE(std::get<1>(x), V0([](auto i) -> T { return i + N1; }));
+ COMPARE(std::get<2>(x), V0([](auto i) -> T { return i + N0 + N1; }));
+ auto b = concat(std::get<1>(x), std::get<2>(x), std::get<0>(x));
+ // a and b may have different types if a was fixed_size<N> such that another
+ // ABI tag exists with equal N, then b will have the non-fixed-size ABI tag.
+ COMPARE(a.size(), b.size());
+ COMPARE(b, decltype(b)([](auto i) -> T { return (N1 + i) % V::size(); }));
+ }
+ }
+
+ if constexpr (V::size() % 3 == 0) {
+ const V a([](auto i) -> T { return i; });
+ constexpr auto N0 = V::size() / 3;
+ using V0 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, N0>>;
+ using V1 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, 2 * N0>>;
+ {
+ auto [x, y, z] = std::experimental::split<N0, N0, N0>(a);
+ COMPARE(x, V0([](auto i) -> T { return i; }));
+ COMPARE(y, V0([](auto i) -> T { return i + N0; }));
+ COMPARE(z, V0([](auto i) -> T { return i + N0 * 2; }));
+ auto b = concat(x, y, z);
+ COMPARE(a.size(), b.size());
+ COMPARE(b, simd_cast<decltype(b)>(a));
+ COMPARE(simd_cast<V>(b), a);
+ }
+ {
+ auto [x, y] = std::experimental::split<N0, 2 * N0>(a);
+ COMPARE(x, V0([](auto i) -> T { return i; }));
+ COMPARE(y, V1([](auto i) -> T { return i + N0; }));
+ auto b = concat(x, y);
+ COMPARE(a.size(), b.size());
+ COMPARE(b, simd_cast<decltype(b)>(a));
+ COMPARE(simd_cast<V>(b), a);
+ }
+ {
+ auto [x, y] = std::experimental::split<2 * N0, N0>(a);
+ COMPARE(x, V1([](auto i) -> T { return i; }));
+ COMPARE(y, V0([](auto i) -> T { return i + 2 * N0; }));
+ auto b = concat(x, y);
+ COMPARE(a.size(), b.size());
+ COMPARE(b, simd_cast<decltype(b)>(a));
+ COMPARE(simd_cast<V>(b), a);
+ }
+ }
+
+ if constexpr ((V::size() & 1) == 0) {
+ using V2 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, 2>>;
+ using V3 = std::experimental::simd<T, std::experimental::simd_abi::deduce_t<T, V::size() / 2>>;
+
+ V a([](auto i) -> T { return i; });
+
+ std::array<V2, V::size() / 2> v2s = std::experimental::split<V2>(a);
+ int offset = 0;
+ for (V2 test : v2s) {
+ COMPARE(test, V2([&](auto i) -> T { return i + offset; }));
+ offset += 2;
+ }
+
+ std::array<V3, 2> v3s = std::experimental::split<V3>(a);
+ COMPARE(v3s[0], V3([](auto i) -> T { return i; }));
+ COMPARE(v3s[1], V3([](auto i) -> T { return i + V3::size(); }));
+ }
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/splits.h b/libstdc++-v3/testsuite/experimental/simd/tests/splits.h
new file mode 100644
index 00000000000..bc80ca6e985
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/splits.h
@@ -0,0 +1,19 @@
+#include "bits/verify.h"
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ using namespace std::experimental::parallelism_v2;
+ using T = typename V::value_type;
+ if constexpr (V::size() / simd_size_v<T> * simd_size_v<T> == V::size())
+ {
+ M k(true);
+ VERIFY(all_of(k)) << k;
+ const auto parts = split<simd_mask<T>>(k);
+ for (auto k2 : parts)
+ {
+ VERIFY(all_of(k2)) << k2;
+ COMPARE(typeid(k2), typeid(simd_mask<T>));
+ }
+ }
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/trigonometric.h b/libstdc++-v3/testsuite/experimental/simd/tests/trigonometric.h
new file mode 100644
index 00000000000..838a6c62aa0
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/trigonometric.h
@@ -0,0 +1,20 @@
+// test only floattypes
+#include "bits/verify.h"
+#include "bits/metahelpers.h"
+#include "bits/test_values.h"
+
+template <typename V>
+void test()
+{
+ vir::test::setFuzzyness<float>(1);
+ vir::test::setFuzzyness<double>(1);
+
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values<V>({limits::quiet_NaN(), limits::infinity(), -limits::infinity(),
+ +0., -0., limits::denorm_min(), limits::min(), limits::max(),
+ limits::min() / 3},
+ {10000, -limits::max() / 2, limits::max() / 2},
+ MAKE_TESTER(acos), MAKE_TESTER(tan), MAKE_TESTER(acosh),
+ MAKE_TESTER(asinh), MAKE_TESTER(atanh), MAKE_TESTER(cosh),
+ MAKE_TESTER(sinh), MAKE_TESTER(tanh));
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/trunc_ceil_floor.h b/libstdc++-v3/testsuite/experimental/simd/tests/trunc_ceil_floor.h
new file mode 100644
index 00000000000..357a3404b22
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/trunc_ceil_floor.h
@@ -0,0 +1,79 @@
+// test only floattypes
+#include "bits/test_values.h"
+#include "bits/verify.h"
+
+template <typename V>
+void test()
+{
+ using limits = std::numeric_limits<typename V::value_type>;
+ test_values<V>(
+ {2.1,
+ 2.0,
+ 2.9,
+ 2.5,
+ 2.499,
+ 1.5,
+ 1.499,
+ 1.99,
+ 0.99,
+ 0.5,
+ 0.499,
+ 0.,
+ -2.1,
+ -2.0,
+ -2.9,
+ -2.5,
+ -2.499,
+ -1.5,
+ -1.499,
+ -1.99,
+ -0.99,
+ -0.5,
+ -0.499,
+ -0.,
+ 3 << 21,
+ 3 << 22,
+ 3 << 23,
+ -(3 << 21),
+ -(3 << 22),
+ -(3 << 23),
+ limits::infinity(),
+ -limits::infinity(),
+ limits::denorm_min(),
+ limits::max(),
+ limits::min(),
+ limits::min() * 0.9,
+ limits::lowest(),
+ -limits::denorm_min(),
+ -limits::max(),
+ -limits::min(),
+ -limits::min() * 0.9,
+ -limits::lowest()},
+ [](const V input) {
+ const V expected([&](auto i) { return std::trunc(input[i]); });
+ COMPARE(trunc(input), expected) << input;
+ },
+ [](const V input) {
+ const V expected([&](auto i) { return std::ceil(input[i]); });
+ COMPARE(ceil(input), expected) << input;
+ },
+ [](const V input) {
+ const V expected([&](auto i) { return std::floor(input[i]); });
+ COMPARE(floor(input), expected) << input;
+ });
+
+ test_values<V>(
+ {limits::quiet_NaN(), limits::signaling_NaN()},
+ [](const V input) {
+ const V expected([&](auto i) { return std::trunc(input[i]); });
+ COMPARE(isnan(trunc(input)), isnan(expected)) << input;
+ },
+ [](const V input) {
+ const V expected([&](auto i) { return std::ceil(input[i]); });
+ COMPARE(isnan(ceil(input)), isnan(expected)) << input;
+ },
+ [](const V input) {
+ const V expected([&](auto i) { return std::floor(input[i]); });
+ COMPARE(isnan(floor(input)), isnan(expected)) << input;
+ });
+}
diff --git a/libstdc++-v3/testsuite/experimental/simd/tests/where.h b/libstdc++-v3/testsuite/experimental/simd/tests/where.h
new file mode 100644
index 00000000000..748678b8881
--- /dev/null
+++ b/libstdc++-v3/testsuite/experimental/simd/tests/where.h
@@ -0,0 +1,98 @@
+#include "bits/verify.h"
+#include "bits/make_vec.h"
+#include "bits/metahelpers.h"
+
+template <class V> struct Convertible {
+ operator V() const { return V(4); }
+};
+
+template <class M, class T> constexpr bool where_is_ill_formed_impl(M, const T &, float)
+{
+ return true;
+}
+template <class M, class T>
+constexpr auto where_is_ill_formed_impl(M m, const T &v, int)
+ -> std::conditional_t<true, bool, decltype(std::experimental::where(m, v))>
+{
+ return false;
+}
+
+template <class M, class T> constexpr bool where_is_ill_formed(M m, const T &v)
+{
+ return where_is_ill_formed_impl(m, v, int());
+}
+
+template <typename T> void where_fundamental()
+{
+ using std::experimental::where;
+ T x = T();
+ where(true, x) = x + 1;
+ COMPARE(x, T(1));
+ where(false, x) = x - 1;
+ COMPARE(x, T(1));
+ where(true, x) += T(1);
+ COMPARE(x, T(2));
+}
+
+template <typename V> void test()
+{
+ using M = typename V::mask_type;
+ using T = typename V::value_type;
+ where_fundamental<T>();
+ VERIFY(!(sfinae_is_callable<V>(
+ [](auto x) -> decltype(where(true, x))* { return nullptr; })));
+
+ const V indexes([](int i) { return i + 1; });
+ const M alternating_mask = make_mask<M>({true, false});
+ V x = 0;
+ where(alternating_mask, x) = indexes;
+ COMPARE(alternating_mask, x == indexes);
+
+ where(!alternating_mask, x) = T(2);
+ COMPARE(!alternating_mask, x == T(2)) << x;
+
+ where(!alternating_mask, x) = Convertible<V>();
+ COMPARE(!alternating_mask, x == T(4));
+
+ x = 0;
+ COMPARE(x, T(0));
+ where(alternating_mask, x) += indexes;
+ COMPARE(alternating_mask, x == indexes);
+
+ x = 10;
+ COMPARE(x, T(10));
+ where(!alternating_mask, x) += T(1);
+ COMPARE(!alternating_mask, x == T(11));
+ where(alternating_mask, x) -= Convertible<V>();
+ COMPARE(alternating_mask, x == T(6));
+ where(alternating_mask, x) /= T(2);
+ COMPARE(alternating_mask, x == T(3));
+ where(alternating_mask, x) *= T(3);
+ COMPARE(alternating_mask, x == T(9));
+
+ x = 10;
+ where(alternating_mask, x)++;
+ COMPARE(alternating_mask, x == T(11));
+ ++where(alternating_mask, x);
+ COMPARE(alternating_mask, x == T(12));
+ where(alternating_mask, x)--;
+ COMPARE(alternating_mask, x == T(11));
+ --where(alternating_mask, x);
+ --where(alternating_mask, x);
+ COMPARE(alternating_mask, x == T(9));
+ COMPARE(alternating_mask, -where(alternating_mask, x) == T(-T(9)));
+
+ const auto y = x;
+ VERIFY(where_is_ill_formed(true, y));
+ VERIFY(where_is_ill_formed(true, x));
+ VERIFY(where_is_ill_formed(true, V(x)));
+
+ M test = alternating_mask;
+ where(alternating_mask, test) = M(true);
+ COMPARE(test, alternating_mask);
+ where(alternating_mask, test) = M(false);
+ COMPARE(test, M(false));
+ where(alternating_mask, test) = M(true);
+ COMPARE(test, alternating_mask);
+}
+