[AVX]: Initial AVX patch
H.J. Lu
hjl.tools@gmail.com
Wed Apr 9 22:04:00 GMT 2008
This is the initial AVX patch.
H.J.
---
2008-04-09 H.J. Lu <hongjiu.lu@intel.com>
* config.gcc (extra_headers): Add gmmintrin.h for x86 and x86-64.
* config/i386/cpuid.h (bit_FMA): New.
(bit_XSAVE): Likewise.
(bit_OSXSAVE): Likewise.
(bit_AVX): Likewise.
* config/i386/gmmintrin.h: New.
* config/i386/i386.c (OPTION_MASK_ISA_AVX_SET): New.
(OPTION_MASK_ISA_FMA_SET): Likewise.
(OPTION_MASK_ISA_AVX_UNSET): Likewise.
(OPTION_MASK_ISA_FMA_SET): Likewise.
(OPTION_MASK_ISA_SSE4_2_UNSET): Updated.
(ix86_handle_option): Handle OPT_mavx and OPT_mfma.
(pta_flags): Add PTA_AVX and PTA_FMA.
(override_options): Handle PTA_AVX and PTA_FMA.
(init_cumulative_args): Handle warn_avx.
(classify_argument): Handle V8SFmode, V8SImode, V32QImode,
V16HImode, V4DFmode and V4DImode.
(function_arg_advance_32): Likewise.
(function_arg_64): Likewise.
(standard_sse_constant_opcode): Handle AVX modes.
(print_reg): Handle 'x' and AVX registers.
(print_operand): Handle 'x' and AVX vector compare instructions.
(output_387_binary_op): Support AVX.
(ix86_builtins): Add IX86_BUILTIN_ADDPD256 and
IX86_BUILTIN_ADDPS256, IX86_BUILTIN_ADDSUBPD256,
IX86_BUILTIN_ADDSUBPS256, IX86_BUILTIN_ANDPD256,
IX86_BUILTIN_ANDPS256, IX86_BUILTIN_ANDNPD256,
IX86_BUILTIN_ANDNPS256, IX86_BUILTIN_BLENDPD256,
IX86_BUILTIN_BLENDPS256, IX86_BUILTIN_BLENDVPD256,
IX86_BUILTIN_BLENDVPS256, IX86_BUILTIN_DPPS256.
IX86_BUILTIN_DIVPD256, IX86_BUILTIN_DIVPS256,
IX86_BUILTIN_HADDPD256, IX86_BUILTIN_HADDPS256,
IX86_BUILTIN_HSUBPD256, IX86_BUILTIN_HSUBPS256,
IX86_BUILTIN_MAXPD256, IX86_BUILTIN_MAXPS256,
IX86_BUILTIN_MINPD256, IX86_BUILTIN_MINPS256,
IX86_BUILTIN_MULPD256, IX86_BUILTIN_MULPS256,
IX86_BUILTIN_ORPD256, IX86_BUILTIN_ORPS256,
IX86_BUILTIN_SHUFPD256, IX86_BUILTIN_SHUFPS256,
IX86_BUILTIN_SUBPD256, IX86_BUILTIN_SUBPS256,
IX86_BUILTIN_XORPD256, IX86_BUILTIN_XORPS256,
IX86_BUILTIN_CMPSD, IX86_BUILTIN_CMPSS, IX86_BUILTIN_CMPPD,
IX86_BUILTIN_CMPPS, IX86_BUILTIN_CMPPD256, IX86_BUILTIN_CMPPS256,
IX86_BUILTIN_CVTPD2PS256, IX86_BUILTIN_CVTPS2DQ256,
IX86_BUILTIN_CVTPS2PD256, IX86_BUILTIN_CVTTPD2DQ256,
IX86_BUILTIN_CVTPD2DQ256 and IX86_BUILTIN_CVTTPS2DQ256.
(bdesc_sse_3arg): Add __builtin_ia32_blendpd256,
__builtin_ia32_blendps256, __builtin_ia32_blendvpd256,
__builtin_ia32_blendvps256, __builtin_ia32_dpps256,
__builtin_ia32_shufpd256, __builtin_ia32_shufps256,
__builtin_ia32_cmpsd, __builtin_ia32_cmpss,
__builtin_ia32_cmppd, __builtin_ia32_cmpps,
__builtin_ia32_cmppd256 and __builtin_ia32_cmpps256.
(bdesc_2arg): Add __builtin_ia32_addpd256,
__builtin_ia32_addps256, __builtin_ia32_addsubpd256,
__builtin_ia32_addsubps256, __builtin_ia32_andpd256,
__builtin_ia32_andps256, __builtin_ia32_andnpd256,
__builtin_ia32_andnps256, __builtin_ia32_divpd256,
__builtin_ia32_divps256, __builtin_ia32_haddpd256,
__builtin_ia32_hsubps256, __builtin_ia32_hsubpd256,
__builtin_ia32_haddps256, __builtin_ia32_maxpd256,
__builtin_ia32_maxps256, __builtin_ia32_minpd256,
__builtin_ia32_minps256, __builtin_ia32_mulpd256,
__builtin_ia32_mulps256, __builtin_ia32_orpd256,
__builtin_ia32_orps256, __builtin_ia32_subpd256,
__builtin_ia32_subps256, __builtin_ia32_xorpd256,
__builtin_ia32_xorps256.
(bdesc_1arg): Add entries for IX86_BUILTIN_CVTDQ2PD256,
IX86_BUILTIN_CVTDQ2PS256, IX86_BUILTIN_CVTPD2PS256,
IX86_BUILTIN_CVTPS2DQ256, IX86_BUILTIN_CVTPS2PD256,
IX86_BUILTIN_CVTTPD2DQ256, IX86_BUILTIN_CVTPD2DQ256 and
IX86_BUILTIN_CVTTPS2DQ256.
(ix86_init_mmx_sse_builtins): Handle __builtin_ia32_blendpd256,
__builtin_ia32_blendps256, __builtin_ia32_blendvpd256,
__builtin_ia32_blendvps256, __builtin_ia32_addpd256 and
__builtin_ia32_addps256. Define __builtin_ia32_cvtdq2pd256,
__builtin_ia32_cvtdq2ps256, __builtin_ia32_cvtpd2ps256,
__builtin_ia32_cvtps2dq256, __builtin_ia32_cvtps2pd256,
__builtin_ia32_cvttpd2dq256, __builtin_ia32_cvtpd2dq256 and
__builtin_ia32_cvttps2dq256.
(ix86_expand_sse_4_operands_builtin): Handle
CODE_FOR_avx_blendpd256, CODE_FOR_avx_blendvpd256,
CODE_FOR_avx_blendvps256, CODE_FOR_avx_cmpsdv2df3,
CODE_FOR_avx_cmpssv4sf3, CODE_FOR_avx_cmppdv2df3,
CODE_FOR_avx_cmppsv4sf3, CODE_FOR_avx_cmppsv8sf3 and
CODE_FOR_avx_cmppdv4df3.
(ix86_expand_builtin): Use switch instead of if.
(ix86_hard_regno_mode_ok): Handle AVX mode.
(ix86_vector_mode_supported_p): Likewise.
* config/i386/i386.h (TARGET_AVX): New.
(TARGET_FMA): Likewise.
(TARGET_CPU_CPP_BUILTINS): Handle TARGET_AVX and TARGET_FMA.
(BIGGEST_ALIGNMENT): Set to 256 for TARGET_AVX.
(VALID_AVX_REG_MODE): New.
(AVX_VEC_FLOAT_MODE_P): Likewise.
(UNITS_PER_SIMD_WORD): Set to 32 for TARGET_AVX.
(SSE_REG_MODE_P): Allow AVX modes.
(ix86_args): Add a warn_avx field.
* config/i386/i386-modes.def (VECTOR_MODES (INT, 32)): New.
(VECTOR_MODES (FLOAT, 32)): Likewise.
(VECTOR_MODE (INT, DI, 8)): Likewise.
(VECTOR_MODE (INT, HI, 32)): Likewise.
(VECTOR_MODE (INT, QI, 64)): Likewise.
(VECTOR_MODE (FLOAT, DF, 8)): Likewise.
(VECTOR_MODE (FLOAT, SF, 16)): Likewise.
(VECTOR_MODE (INT, DI, 4)): Removed.
(VECTOR_MODE (INT, SI, 8)): Likewise.
(VECTOR_MODE (INT, HI, 16)): Likewise.
(VECTOR_MODE (INT, QI, 32)): Likewise.
(VECTOR_MODE (FLOAT, SF, 8)): Likewise.
(INT_MODE (OI, 32)): Likewise.
* config/i386/i386.opt (mavx): New.
(mfma): Likewise.
Index: gcc/config.gcc
===================================================================
--- gcc/config.gcc (.../fsf/trunk) (revision 2097)
+++ gcc/config.gcc (.../branches/avx) (revision 2097)
@@ -310,14 +310,14 @@ i[34567]86-*-*)
extra_headers="cpuid.h mmintrin.h mm3dnow.h xmmintrin.h emmintrin.h
pmmintrin.h tmmintrin.h ammintrin.h smmintrin.h
nmmintrin.h bmmintrin.h mmintrin-common.h
- wmmintrin.h"
+ wmmintrin.h gmmintrin.h"
;;
x86_64-*-*)
cpu_type=i386
extra_headers="cpuid.h mmintrin.h mm3dnow.h xmmintrin.h emmintrin.h
pmmintrin.h tmmintrin.h ammintrin.h smmintrin.h
nmmintrin.h bmmintrin.h mmintrin-common.h
- wmmintrin.h"
+ wmmintrin.h gmmintrin.h"
need_64bit_hwint=yes
;;
ia64-*-*)
Index: gcc/config/i386/i386.h
===================================================================
--- gcc/config/i386/i386.h (.../fsf/trunk) (revision 2097)
+++ gcc/config/i386/i386.h (.../branches/avx) (revision 2097)
@@ -46,6 +46,8 @@ along with GCC; see the file COPYING3.
#define TARGET_SSSE3 OPTION_ISA_SSSE3
#define TARGET_SSE4_1 OPTION_ISA_SSE4_1
#define TARGET_SSE4_2 OPTION_ISA_SSE4_2
+#define TARGET_AVX OPTION_ISA_AVX
+#define TARGET_FMA OPTION_ISA_FMA
#define TARGET_SSE4A OPTION_ISA_SSE4A
#define TARGET_SSE5 OPTION_ISA_SSE5
#define TARGET_ROUND OPTION_ISA_ROUND
@@ -689,6 +691,10 @@ extern const char *host_detect_local_cpu
builtin_define ("__AES__"); \
if (TARGET_PCLMUL) \
builtin_define ("__PCLMUL__"); \
+ if (TARGET_AVX) \
+ builtin_define ("__AVX__"); \
+ if (TARGET_FMA) \
+ builtin_define ("__FMA_"); \
if (TARGET_SSE4A) \
builtin_define ("__SSE4A__"); \
if (TARGET_SSE5) \
@@ -840,7 +846,7 @@ enum target_cpu_default
Pentium+ prefers DFmode values to be aligned to 64 bit boundary
and Pentium Pro XFmode values at 128 bit boundaries. */
-#define BIGGEST_ALIGNMENT 128
+#define BIGGEST_ALIGNMENT (TARGET_AVX ? 256: 128)
/* Decide whether a variable of mode MODE should be 128 bit aligned. */
#define ALIGN_MODE_128(MODE) \
@@ -1114,6 +1120,10 @@ do { \
#define HARD_REGNO_NREGS_WITH_PADDING(REGNO, MODE) ((MODE) == XFmode ? 4 : 8)
+#define VALID_AVX_REG_MODE(MODE) \
+ ((MODE) == V32QImode || (MODE) == V16HImode || (MODE) == V8SImode \
+ || (MODE) == V4DImode || (MODE) == V8SFmode || (MODE) == V4DFmode)
+
#define VALID_SSE2_REG_MODE(MODE) \
((MODE) == V16QImode || (MODE) == V8HImode || (MODE) == V2DFmode \
|| (MODE) == V2DImode || (MODE) == DFmode)
@@ -1132,7 +1142,8 @@ do { \
/* ??? No autovectorization into MMX or 3DNOW until we can reliably
place emms and femms instructions. */
-#define UNITS_PER_SIMD_WORD (TARGET_SSE ? 16 : UNITS_PER_WORD)
+#define UNITS_PER_SIMD_WORD \
+ (TARGET_AVX ? 32 : (TARGET_SSE ? 16 : UNITS_PER_WORD))
#define VALID_DFP_MODE_P(MODE) \
((MODE) == SDmode || (MODE) == DDmode || (MODE) == TDmode)
@@ -1153,7 +1164,9 @@ do { \
#define SSE_REG_MODE_P(MODE) \
((MODE) == TImode || (MODE) == V16QImode || (MODE) == TFmode \
|| (MODE) == V8HImode || (MODE) == V2DFmode || (MODE) == V2DImode \
- || (MODE) == V4SFmode || (MODE) == V4SImode)
+ || (MODE) == V4SFmode || (MODE) == V4SImode || (MODE) == V32QImode \
+ || (MODE) == V16HImode || (MODE) == V8SImode || (MODE) == V4DImode \
+ || (MODE) == V8SFmode || (MODE) == V4DFmode)
/* Value is 1 if hard register REGNO can hold a value of machine-mode MODE. */
@@ -1464,6 +1477,9 @@ enum reg_class
#define SSE_VEC_FLOAT_MODE_P(MODE) \
((TARGET_SSE && (MODE) == V4SFmode) || (TARGET_SSE2 && (MODE) == V2DFmode))
+#define AVX_VEC_FLOAT_MODE_P(MODE) \
+ (TARGET_AVX && ((MODE) == V8SFmode || (MODE) == V4DFmode))
+
#define MMX_REG_P(XOP) (REG_P (XOP) && MMX_REGNO_P (REGNO (XOP)))
#define MMX_REGNO_P(N) IN_RANGE ((N), FIRST_MMX_REG, LAST_MMX_REG)
@@ -1683,6 +1699,7 @@ typedef struct ix86_args {
int fastcall; /* fastcall calling convention is used */
int sse_words; /* # sse words passed so far */
int sse_nregs; /* # sse registers available for passing */
+ int warn_avx; /* True when we want to warn about AVX ABI. */
int warn_sse; /* True when we want to warn about SSE ABI. */
int warn_mmx; /* True when we want to warn about MMX ABI. */
int sse_regno; /* next available sse register number */
Index: gcc/config/i386/gmmintrin.h
===================================================================
--- gcc/config/i386/gmmintrin.h (.../fsf/trunk) (revision 0)
+++ gcc/config/i386/gmmintrin.h (.../branches/avx) (revision 2097)
@@ -0,0 +1,1193 @@
+/* Copyright (C) 2008 Free Software Foundation, Inc.
+
+ This file is part of GCC.
+
+ GCC is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+
+ GCC is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GCC; see the file COPYING. If not, write to
+ the Free Software Foundation, 59 Temple Place - Suite 330,
+ Boston, MA 02111-1307, USA. */
+
+/* As a special exception, if you include this header file into source
+ files compiled by GCC, this header file does not by itself cause
+ the resulting executable to be covered by the GNU General Public
+ License. This exception does not however invalidate any other
+ reasons why the executable file might be covered by the GNU General
+ Public License. */
+
+/* Implemented from the specification included in the Intel C++ Compiler
+ User Guide and Reference, version 11.0. */
+
+#ifndef _GMMINTRIN_H_INCLUDED
+#define _GMMINTRIN_H_INCLUDED
+
+#ifndef __AVX__
+# error "AVX instruction set not enabled"
+#else
+
+/* We need definitions from the SSE4, SSSE3, SSE3, SSE2 and SSE header
+ files. */
+#include <smmintrin.h>
+
+/* Internal data types for implementing the intrinsics. */
+typedef double __v4df __attribute__ ((__vector_size__ (32)));
+typedef float __v8sf __attribute__ ((__vector_size__ (32)));
+typedef long long __v4di __attribute__ ((__vector_size__ (32)));
+typedef int __v8si __attribute__ ((__vector_size__ (32)));
+typedef short __v16hi __attribute__ ((__vector_size__ (32)));
+typedef char __v32qi __attribute__ ((__vector_size__ (32)));
+
+/* The Intel API is flexible enough that we must allow aliasing with other
+ vector types, and their scalar components. */
+typedef float __m256 __attribute__ ((__vector_size__ (32),
+ __may_alias__));
+typedef long long __m256i __attribute__ ((__vector_size__ (32),
+ __may_alias__));
+typedef double __m256d __attribute__ ((__vector_size__ (32),
+ __may_alias__));
+
+/* Compare predicates for scalar and packed compare intrinsics. */
+
+/* Equal (ordered, non-signaling) */
+#define _CMP_EQ_OQ 0x00
+/* Less-than (ordered, signaling) */
+#define _CMP_LT_OS 0x01
+/* Less-than-or-equal (ordered, signaling) */
+#define _CMP_LE_OS 0x02
+/* Unordered (non-signaling) */
+#define _CMP_UNORD_Q 0x03
+/* Not-equal (unordered, non-signaling) */
+#define _CMP_NEQ_UQ 0x04
+/* Not-less-than (unordered, signaling) */
+#define _CMP_NLT_US 0x05
+/* Not-less-than-or-equal (unordered, signaling) */
+#define _CMP_NLE_US 0x06
+/* Ordered (nonsignaling) */
+#define _CMP_ORD_Q 0x07
+/* Equal (unordered, non-signaling) */
+#define _CMP_EQ_UQ 0x08
+/* Not-greater-than-or-equal (unordered, signaling) */
+#define _CMP_NGE_US 0x09
+/* Not-greater-than (unordered, signaling) */
+#define _CMP_NGT_US 0x0a
+/* False (ordered, non-signaling) */
+#define _CMP_FALSE_OQ 0x0b
+/* Not-equal (ordered, non-signaling) */
+#define _CMP_NEQ_OQ 0x0c
+/* Greater-than-or-equal (ordered, signaling) */
+#define _CMP_GE_OS 0x0d
+/* Greater-than (ordered, signaling) */
+#define _CMP_GT_OS 0x0e
+/* True (unordered, non-signaling) */
+#define _CMP_TRUE_UQ 0x0f
+/* Equal (ordered, signaling) */
+#define _CMP_EQ_OS 0x10
+/* Less-than (ordered, non-signaling) */
+#define _CMP_LT_OQ 0x11
+/* Less-than-or-equal (ordered, non-signaling) */
+#define _CMP_LE_OQ 0x12
+/* Unordered (signaling) */
+#define _CMP_UNORD_S 0x13
+/* Not-equal (unordered, signaling) */
+#define _CMP_NEQ_US 0x14
+/* Not-less-than (unordered, non-signaling) */
+#define _CMP_NLT_UQ 0x15
+/* Not-less-than-or-equal (unordered, non-signaling) */
+#define _CMP_NLE_UQ 0x16
+/* Ordered (signaling) */
+#define _CMP_ORD_S 0x17
+/* Equal (unordered, signaling) */
+#define _CMP_EQ_US 0x18
+/* Not-greater-than-or-equal (unordered, non-signaling) */
+#define _CMP_NGE_UQ 0x19
+/* Not-greater-than (unordered, non-signaling) */
+#define _CMP_NGT_UQ 0x1a
+/* False (ordered, signaling) */
+#define _CMP_FALSE_OS 0x1b
+/* Not-equal (ordered, signaling) */
+#define _CMP_NEQ_OS 0x1c
+/* Greater-than-or-equal (ordered, non-signaling) */
+#define _CMP_GE_OQ 0x1d
+/* Greater-than (ordered, non-signaling) */
+#define _CMP_GT_OQ 0x1e
+/* True (unordered, signaling) */
+#define _CMP_TRUE_US 0x1f
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_add_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_addpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_add_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_addps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_addsub_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_addsubpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_addsub_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_addsubps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_and_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_andpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_and_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_andps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_andnot_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_andnpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_andnot_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_andnps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+/* Double/single precision floating point blend instructions - select
+ data from 2 sources using constant/variable mask. */
+
+#ifdef __OPTIMIZE__
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_blend_pd (__m256d __X, __m256d __Y, const int __M)
+{
+ return (__m256d) __builtin_ia32_blendpd256 ((__v4df)__X,
+ (__v4df)__Y,
+ __M);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_blend_ps (__m256 __X, __m256 __Y, const int __M)
+{
+ return (__m256) __builtin_ia32_blendps256 ((__v8sf)__X,
+ (__v8sf)__Y,
+ __M);
+}
+#else
+#define _mm256_blend_pd(X, Y, M) \
+ ((__m256d) __builtin_ia32_blendpd256 ((__v4df)(__m256d)(X), \
+ (__v4df)(__m256d)(Y), (int)(M)))
+
+#define _mm256_blend_ps(X, Y, M) \
+ ((__m256) __builtin_ia32_blendps256 ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), (int)(M)))
+#endif
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_blendv_pd (__m256d __X, __m256d __Y, __m256d __M)
+{
+ return (__m256d) __builtin_ia32_blendvpd256 ((__v4df)__X,
+ (__v4df)__Y,
+ (__v4df)__M);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_blendv_ps (__m256 __X, __m256 __Y, __m256 __M)
+{
+ return (__m256) __builtin_ia32_blendvps256 ((__v8sf)__X,
+ (__v8sf)__Y,
+ (__v8sf)__M);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_div_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_divpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_div_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_divps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+/* Dot product instructions with mask-defined summing and zeroing parts
+ of result. */
+
+#ifdef __OPTIMIZE__
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_dp_ps (__m256 __X, __m256 __Y, const int __M)
+{
+ return (__m256) __builtin_ia32_dpps256 ((__v8sf)__X,
+ (__v8sf)__Y,
+ __M);
+}
+#else
+#define _mm256_dp_ps(X, Y, M) \
+ ((__m256) __builtin_ia32_dpps256 ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), (int)(M)))
+#endif
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_hadd_pd (__m256d __X, __m256d __Y)
+{
+ return (__m256d) __builtin_ia32_haddpd256 ((__v4df)__X, (__v4df)__Y);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_hadd_ps (__m256 __X, __m256 __Y)
+{
+ return (__m256) __builtin_ia32_haddps256 ((__v8sf)__X, (__v8sf)__Y);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_hsub_pd (__m256d __X, __m256d __Y)
+{
+ return (__m256d) __builtin_ia32_hsubpd256 ((__v4df)__X, (__v4df)__Y);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_hsub_ps (__m256 __X, __m256 __Y)
+{
+ return (__m256) __builtin_ia32_hsubps256 ((__v8sf)__X, (__v8sf)__Y);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_max_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_maxpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_max_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_maxps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_min_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_minpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_min_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_minps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mul_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_mulpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mul_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_mulps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_or_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_orpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_or_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_orps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+#ifdef __OPTIMIZE__
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_shuffle_pd (__m256d __A, __m256d __B, const int __mask)
+{
+ return (__m256d) __builtin_ia32_shufpd256 ((__v4df)__A, (__v4df)__B,
+ __mask);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_shuffle_ps (__m256 __A, __m256 __B, const int __mask)
+{
+ return (__m256) __builtin_ia32_shufps256 ((__v8sf)__A, (__v8sf)__B,
+ __mask);
+}
+#else
+#define _mm256_shuffle_pd(A, B, N) \
+ ((__m256d)__builtin_ia32_shufpd256 ((__v4df)(__m256d)(A), \
+ (__v4df)(__m256d)(B), (int)(N)))
+
+#define _mm256_shuffle_ps(A, B, N) \
+ ((__m256) __builtin_ia32_shufps256 ((__v8sf)(__m256)(A), \
+ (__v8sf)(__m256)(B), (int)(N)))
+#endif
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_sub_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_subpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_sub_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_subps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_xor_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_xorpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_xor_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_xorps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+#ifdef __OPTIMIZE__
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmp_pd (__m128d __X, __m128d __Y, const int __P)
+{
+ return (__m128d) __builtin_ia32_cmppd ((__v2df)__X, (__v2df)__Y, __P);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmp_ps (__m128 __X, __m128 __Y, const int __P)
+{
+ return (__m128) __builtin_ia32_cmpps ((__v4sf)__X, (__v4sf)__Y, __P);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cmp_pd (__m256d __X, __m256d __Y, const int __P)
+{
+ return (__m256d) __builtin_ia32_cmppd256 ((__v4df)__X, (__v4df)__Y,
+ __P);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cmp_ps (__m256 __X, __m256 __Y, const int __P)
+{
+ return (__m256) __builtin_ia32_cmpps256 ((__v8sf)__X, (__v8sf)__Y,
+ __P);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmp_sd (__m128d __X, __m128d __Y, const int __P)
+{
+ return (__m128d) __builtin_ia32_cmpsd ((__v2df)__X, (__v2df)__Y, __P);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_cmp_ss (__m128 __X, __m128 __Y, const int __P)
+{
+ return (__m128) __builtin_ia32_cmpss ((__v4sf)__X, (__v4sf)__Y, __P);
+}
+#else
+#define _mm_cmp_pd(X, Y, P) \
+ ((__m128d) __builtin_ia32_cmppd ((__v2df)(__m128d)(X), \
+ (__v2df)(__m128d)(Y), (int)(P)))
+
+#define _mm_cmp_ps(X, Y, P) \
+ ((__m128) __builtin_ia32_cmpps ((__v4sf)(__m128)(X), \
+ (__v4sf)(__m128)(Y), (int)(P)))
+
+#define _mm256_cmp_pd(X, Y, P) \
+ ((__m256d) __builtin_ia32_cmppd256 ((__v4df)(__m256d)(X), \
+ (__v4df)(__m256d)(Y), (int)(P)))
+
+#define _mm256_cmp_ps(X, Y, P) \
+ ((__m256) __builtin_ia32_cmpps256 ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), (int)(P)))
+
+#define _mm_cmp_sd(X, Y, P) \
+ ((__m128d) __builtin_ia32_cmpsd ((__v2df)(__m128d)(X), \
+ (__v2df)(__m128d)(Y), (int)(P)))
+
+#define _mm_cmp_ss(X, Y, P) \
+ ((__m128) __builtin_ia32_cmpss ((__v4sf)(__m128)(X), \
+ (__v4sf)(__m128)(Y), (int)(P)))
+#endif
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtepi32_pd (__m128i __A)
+{
+ return (__m256d)__builtin_ia32_cvtdq2pd256 ((__v4si) __A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtepi32_ps (__m256i __A)
+{
+ return (__m256)__builtin_ia32_cvtdq2ps256 ((__v8si) __A);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtpd_ps (__m256d __A)
+{
+ return (__m128)__builtin_ia32_cvtpd2ps256 ((__v4df) __A);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtps_epi32 (__m256 __A)
+{
+ return (__m256i)__builtin_ia32_cvtps2dq256 ((__v8sf) __A);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtps_pd (__m128 __A)
+{
+ return (__m256d)__builtin_ia32_cvtps2pd256 ((__v4sf) __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvttpd_epi32 (__m256d __A)
+{
+ return (__m128i)__builtin_ia32_cvttpd2dq256 ((__v4df) __A);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvtpd_epi32 (__m256d __A)
+{
+ return (__m128i)__builtin_ia32_cvtpd2dq256 ((__v4df) __A);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_cvttps_epi32 (__m256 __A)
+{
+ return (__m256i)__builtin_ia32_cvttps2dq256 ((__v8sf) __A);
+}
+
+#if 0
+#ifdef __OPTIMIZE__
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_extractf128_pd (__m256d __X, const int __N)
+{
+ return (__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)__X, __N);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_extractf128_ps (__m256 __X, const int __N)
+{
+ return (__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)__X, __N);
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_extractf128_si256 (__m256i __X, const int __N)
+{
+ return (__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)__X, __N);
+}
+#else
+#define _mm256_extractf128_pd(X, N) \
+ ((__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)(__m256d)(X), \
+ (int)(M)))
+
+#define _mm256_extractf128_ps(X, N) \
+ ((__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)(__m256)(X), \
+ (int)(M)))
+
+#define _mm256_extractf128_si256(X, N) \
+ ((__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)(__m256i)(X), \
+ (int)(M)))
+#endif
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_zeroall (void)
+{
+ __builtin_ia32_vzeroall ();
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_zeroupper (void)
+{
+ __builtin_ia32_vzeroupper ();
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permutevar_pd (__m128d __A, __m128d __B)
+{
+ return (__m128d) __builtin_ia32_vpermilpd128 ((__v2df)__A, (__v2df)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permutevar_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_vpermilpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permutevar_ps (__m128 __A, __m128 __B)
+{
+ return (__m128) __builtin_ia32_vpermilps128 ((__v4sf)__A, (__v4sf)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permutevar_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+#ifdef __OPTIMIZE__
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permute_pd (__m128d __X, const int __C)
+{
+ return (__m128d) __builtin_ia32_vpermilpd128 ((__v2df)__X, __C);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute_pd (__m256d __X, const int __C)
+{
+ return (__m256d) __builtin_ia32_vpermilps256 ((__v4df)__X, __C);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permute_ps (__m128 __X, const int __C)
+{
+ return (__m128) __builtin_ia32_vpermilps128 ((__v4sf)__X, __C);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute_ps (__m256 __X, const int __C)
+{
+ return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__X, __C);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permute2_pd (__m128d __X, __m128d __Y, __m128d __C, const int __I)
+{
+ return (__m128d) __builtin_ia32_vpermil2pd128 ((__v2df)__X,
+ (__v2df)__Y,
+ (__v2df)__C,
+ __I);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute2_pd (__m256d __X, __m256d __Y, __m256d __C, const int __I)
+{
+ return (__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)__X,
+ (__v4df)__Y,
+ (__v4df)__C,
+ __I);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_permute2_ps (__m128 __X, __m128 __Y, __m128 __C, const int __I)
+{
+ return (__m128) __builtin_ia32_vpermil2ps128 ((__v4sf)__X,
+ (__v4sf)__Y,
+ (__v4sf)__C,
+ __I);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute2_ps (__m256 __X, __m256 __Y, __m256 __C, const int __I)
+{
+ return (__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)__X,
+ (__v8sf)__Y,
+ (__v8sf)__C,
+ __I);
+}
+#else
+#define _mm_permute_pd(X, C) \
+ ((__m128d) __builtin_ia32_vpermilpd128 ((__v2df)(__m128d)(X), \
+ (int)(C)))
+
+#define _mm256_permute_pd(X, C) \
+ ((__m256d) __builtin_ia32_vpermilpd256 ((__v4df)(__m256d)(X), \
+ (int)(C)))
+
+#define _mm_permute_ps(X, C) \
+ ((__m128) __builtin_ia32_vpermilps128 ((__v4sf)(__m128)(X), \
+ (int)(C)))
+
+#define _mm256_permute_ps(X, C) \
+ ((__m256) __builtin_ia32_vpermilps256 ((__v8sf)(__m256)(X), \
+ (int)(C)))
+
+#define _mm_permute2_pd(X, Y, C, I) \
+ ((__m128d) __builtin_ia32_vpermil2pd128 ((__v2df)(__m128d)(X),\
+ (__v2df)(__m128d)(Y),\
+ (__v2df)(__m128d)(C),\
+ (int)(I)))
+
+#define _mm256_permute2_pd(X, Y, C, I) \
+ ((__m256d) __builtin_ia32_vpermil2pd256 ((__v4sf)(__m256d)(X),\
+ (__v4df)(__m256d)(Y),\
+ (__v4df)(__m256d)(C),\
+ (int)(I)))
+
+#define _mm_permute2_ps(X, Y, C, I) \
+ ((__m128) __builtin_ia32_vpermil2ps128 ((__v4sf)(__m128)(X), \
+ (__v4sf)(__m128)(Y), \
+ (__v4sf)(__m128)(C), \
+ (int)(I)))
+
+#define _mm256_permute2_ps(X, Y, C, I) \
+ ((__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), \
+ (__v8sf)(__m256)(C), \
+ (int)(I)))
+#endif
+
+#ifdef __OPTIMIZE__
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute2f128_pd (__m256d __X, __m256d __Y, const int __C)
+{
+ return (__m256d) __builtin_ia32_vperm2f128_pd ((__v4df)__X,
+ (__v4df)__y,
+ __C);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute2f128_ps (__m256 __X, __m256 __Y, const int __C)
+{
+ return (__m256) __builtin_ia32_vperm2f128_ps ((__v8sf)__X,
+ (__v8sf)__y,
+ __C);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_permute2f128_si256 (__m256i __X, __m256i __Y, const int __C)
+{
+ return (__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)__X,
+ (__v8si)__y,
+ __C);
+}
+#else
+#define _mm256_permute2f128_pd(X, Y, C) \
+ ((__m256d) __builtin_ia32_vperm2f128_pd ((__v4df)(__m256d)(X),\
+ (__v4df)(__m256d)(Y),\
+ (int)(C)))
+
+#define _mm256_permute2f128_ps(X, Y, C) \
+ ((__m256) __builtin_ia32_vperm2f128_ps ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), \
+ (int)(C)))
+
+#define _mm256_permute2f128_si256(X, Y, C) \
+ ((__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)(__m256i)(X),\
+ (__v8si)(__m256i)(Y),\
+ (int)(C)))
+#endif
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_broadcast_ss (float const *__X)
+{
+ return (__m128) __builtin_ia32_vbroadcastss128 (*(const float *)__X);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_broadcast_ss (float const *__X)
+{
+ return (__m256) __builtin_ia32_vbroadcastss256 (*(const float *)__X);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_broadcast_sd (double const *__X)
+{
+ return (__m256d) __builtin_ia32_vbroadcastsd256 (*(const double *)__X);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_broadcast_pd (__m128d const *__X)
+{
+ return (__m256d) __builtin_ia32_vbroadcastf128_pd256 (*(const __m128d *)__X);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_broadcast_ps (__m128 const *__X)
+{
+ return (__m256) __builtin_ia32_vbroadcastf128_ps256 (*(const __m128 *)__X);
+}
+
+#ifdef __OPTIMIZE__
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_insertf128_pd (__m256d __X, __m256d __Y, const int __O)
+{
+ return (__m256d) __builtin_ia32_vinsertf128_pd ((__v4df)__X,
+ (__v4df)__y,
+ __O);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_insertf128_ps (__m256 __X, __m256 __Y, const int __O)
+{
+ return (__m256) __builtin_ia32_vinsertf128_ps ((__v8sf)__X,
+ (__v8sf)__y,
+ __O);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_insertf128_si256 (__m256i __X, __m256i __Y, const int __O)
+{
+ return (__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)__X,
+ (__v8si)__y,
+ __O);
+}
+#else
+#define _mm256_insertf128_pd (X, Y, O) \
+ ((__m256d) __builtin_ia32_vinsertf128_pd ((__v4df)(__m256d)(X),\
+ (__v4df)(__m256d)(Y),\
+ (int)(O)))
+
+#define _mm256_insertf128_ps(X, Y, O) \
+ ((__m256) __builtin_ia32_vinsertf128_ps ((__v8sf)(__m256)(X), \
+ (__v8sf)(__m256)(Y), \
+ (int)(O)))
+
+#define _mm256_insertf128_si256(X, Y, O) \
+ ((__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)(__m256i)(X),\
+ (__v8si)(__m256i)(Y),\
+ (int)(O)))
+#endif
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_load_pd (double const *__P)
+{
+ return *(__m256d *)__P;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_store_pd (double *__P, __m256d __A)
+{
+ *(__m256d *)__P = __A;
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_load_ps (float const *__P)
+{
+ return *(__m256 *)__P;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_store_ps (float *__P, __m256 __A)
+{
+ *(__m256 *)__P = __A;
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_loadu_pd (double const *__P)
+{
+ return (__m256d) __builtin_ia32_loadupd256 (__P);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_storeu_pd (double *__P, __m256d __A)
+{
+ __builtin_ia32_storeupd256 (__P, (__v4df)__A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_loadu_ps (float const *__P)
+{
+ return (__m256) __builtin_ia32_loadups256 (__P);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_storeu_ps (float *__P, __m256 __A)
+{
+ __builtin_ia32_storeups256 (__P, (__v8sf)__A);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_load_si256 (__m128i const *__P)
+{
+ return *__P;
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_store_si256 (__m256i *__P, __m256i __A)
+{
+ *__P = __A;
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_loadu_si256 (__m256i *__P)
+{
+ return (__m256i) __builtin_ia32_loaddqu256 ((char const *)__P);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_storeu_si256 (__m256i *__P, __m256i __A)
+{
+ __builtin_ia32_storedqu256 ((char *)__P, (__v32qi)__A);
+}
+
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskload_pd (double const *__P, __m128d __M)
+{
+ return (__m128d) __builtin_ia32_maskloadpd128 ((__v2df)__P,
+ (__v2df)__M);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskstore_pd (double *__P, __m128d __M, __m128d __A)
+{
+ __builtin_ia32_maskstorepd128 (__P, (__v2df)__M, (__v2df)__A);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskload_pd (double const *__P, __m256d __M)
+{
+ return (__m256d) __builtin_ia32_maskloadpd256 ((__v4df)__P,
+ (__v4df)__M);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskstore_pd (double *__P, __m256d __M, __m256d __A)
+{
+ __builtin_ia32_maskstorepd256 (__P, (__v4df)__M, (__v4df)__A);
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskload_ps (float const *__P, __m128 __M)
+{
+ return (__m128) __builtin_ia32_maskloadps128 ((__v4sf)__P,
+ (__v4sf)__M);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskstore_ps (float *__P, __m128 __M, __m128 __A)
+{
+ __builtin_ia32_maskstoreps128 (__P, (__v4sf)__M, (__v4sf)__A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskload_ps (float const *__P, __m256 __M)
+{
+ return (__m256) __builtin_ia32_maskloadps256 ((__v8sf)__P,
+ (__v8sf)__M);
+}
+
+extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskstore_ps (float *__P, __m256 __M, __m256 __A)
+{
+ __builtin_ia32_maskstoreps256 (__P, (__v8sf)__M, (__v8sf)__A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_movehdup_ps (__m256 __X)
+{
+ return (__m256) __builtin_ia32_movshdup256 ((__v8sf)__X);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_moveldup_ps (__m256 __X)
+{
+ return (__m256) __builtin_ia32_movsldup256 ((__v8sf)__X);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_movedup_pd (__m256d __X)
+{
+ return (__m256) __builtin_ia32_movddup256 ((__v8sf)__X);
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_lddqu_si256 (__m256i const *__P)
+{
+ return (__m256i) __builtin_ia32_lddqu256 ((char const *)__P);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_rcp_ps (__m256 __A)
+{
+ return (__m256) __builtin_ia32_rcpps256 ((__v8sf)__A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_rsqrt_ps (__m256 __A)
+{
+ return (__m256) __builtin_ia32_rsqrtps256 ((__v8sf)__A);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_sqrt_pd (__m256d __A)
+{
+ return (__m256d) __builtin_ia32_sqrtpd256 ((__v4df)__A);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_sqrt_ps (__m256 __A)
+{
+ return (__m256) __builtin_ia32_sqrtps256 ((__v8sf)__A);
+}
+
+#ifdef __OPTIMIZE__
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_round_pd (__m256d __V, const int __M)
+{
+ return (__m128d) __builtin_ia32_roundpd256 ((__v4df)__V, __M);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_round_ps (__m256 __V, const int __M)
+{
+ return (__m128) __builtin_ia32_roundps256 ((__v8sf)__V, __M);
+}
+#else
+#define _mm256_round_pd(V, M) \
+ ((__m256d) __builtin_ia32_roundpd256 ((__v4df)(__m256d)(V), (int)(M)))
+
+#define _mm256_round_ps(V, M) \
+ ((__m256) __builtin_ia32_roundps256 ((__v8sf)(__m256)(V), (int)(M)))
+#endif
+
+#define _mm256_ceil_pd(V) _mm256_round_pd ((V), _MM_FROUND_CEIL)
+#define _mm256_floor_pd(V) _mm256_round_pd ((V), _MM_FROUND_FLOOR)
+#define _mm256_ceil_ps(V) _mm256_round_ps ((V), _MM_FROUND_CEIL)
+#define _mm256_floor_ps(V) _mm256_round_ps ((V), _MM_FROUND_FLOOR)
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_unpackhi_pd (__m256d __A, __m256d __B)
+{
+ return (__m256) __builtin_ia32_unpckhpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_unpacklo_pd (__m256d __A, __m256d __B)
+{
+ return (__m256d) __builtin_ia32_unpcklpd256 ((__v4df)__A, (__v4df)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_unpackhi_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_unpckhps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_unpacklo_ps (__m256 __A, __m256 __B)
+{
+ return (__m256) __builtin_ia32_unpcklps256 ((__v8sf)__A, (__v8sf)__B);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testz_pd (__m256d __M, __m256d __V)
+{
+ return __builtin_ia32_ptestzpd256 ((__v4df)__M, (__v4df)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testc_pd (__m256d __M, __m256d __V)
+{
+ return __builtin_ia32_ptestcpd256 ((__v4df)__M, (__v4df)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testnzc_pd (__m256d __M, __m256d __V)
+{
+ return __builtin_ia32_ptestnzcpd256 ((__v4df)__M, (__v4df)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testz_ps (__m256 __M, __m256 __V)
+{
+ return __builtin_ia32_ptestzps256 ((__v8sf)__M, (__v8sf)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testc_ps (__m256 __M, __m256 __V)
+{
+ return __builtin_ia32_ptestcps256 ((__v8sf)__M, (__v8sf)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testnzc_ps (__m256 __M, __m256 __V)
+{
+ return __builtin_ia32_ptestnzcps256 ((__v8sf)__M, (__v8sf)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testz_si256 (__m256i __M, __m256i __V)
+{
+ return __builtin_ia32_ptestz256 ((__v4di)__M, (__v4di)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testc_si256 (__m256i __M, __m256i __V)
+{
+ return __builtin_ia32_ptestc256 ((__v4di)__M, (__v4di)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_testnzc_si256 (__m256i __M, __m256i __V)
+{
+ return __builtin_ia32_ptestnzc256 ((__v4di)__M, (__v4di)__V);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_movemask_pd (__m256d __A)
+{
+ return __builtin_ia32_movmskpd256 ((__v4df)__A);
+}
+
+extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_movemask_ps (__m256 __A)
+{
+ return __builtin_ia32_movmskps256 ((__v8sf)__A);
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_setzero_pd (void)
+{
+ return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 };
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_setzero_ps (void)
+{
+ return __extension__ (__m256){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_setzero_si256 (void)
+{
+ return __extension__ (__m256i){ 0, 0, 0, 0 };
+}
+
+/* Create the vector [A B C D]. */
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set_pd (double __A, double __B, double __C, double __D)
+{
+ return __extension__ (__m256d){ __D, __C, __B, __A };
+}
+
+/* Create the vector [A B C D E F G H]. */
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set_ps (float __A, float __B, float __C, float __D,
+ float __E, float __F, float __G, float __H)
+{
+ return __extension__ (__m256){ __H, __G, __F, __E, __D, __C, __B, __A };
+}
+
+/* Create the vector [A B C D E F G H]. */
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set_epi32 (int __A, int __B, int __C, int __D,
+ int __E, int __F, int __G, int __H)
+{
+ return __extension__ (__m256i){ __H, __G, __F, __E, __D, __C, __B, __A };
+}
+
+/* Create a vector with all elements equal to A. */
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set1_pd (double __A)
+{
+ return __extension__ (__m256d){ __A, __A, __A, __A };
+}
+
+/* Create a vector with all elements equal to A. */
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set1_ps (float __A)
+{
+ return __extension__ (__m256){ __A, __A, __A, __A, __A, __A, __A, __A };
+}
+
+/* Create a vector with all elements equal to A. */
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_set1_epi32 (int __A)
+{
+ return __extension__ (__m256i){ __A, __A, __A, __A, __A, __A, __A, __A };
+}
+
+/* Casts between various SP, DP, INT vector types. Note that these do no
+ conversion of values, they just change the type. */
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castpd_ps (__m256d __A)
+{
+ return (__m256) __A;
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castpd_si256 (__m256d __A)
+{
+ return (__m256i) __A;
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castps_pd (__m256 __A)
+{
+ return (__m256d) __A;
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castps_si256(__m256 __A)
+{
+ return (__m256i) __A;
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castsi256_ps (__m256i __A)
+{
+ return (__m256) __A;
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castsi256_pd (__m256i __A)
+{
+ return (__m256d) __A;
+}
+
+/* When cast is done from a 128 to 256-bit type, the low 128 bits of
+ the 256-bit result contain source parameter value and the upper 128
+ bits of the result are undefined. FIXME: Those intrinsics shouldn't
+ generate any extra moves. */
+extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castpd256_pd128 (__m256d __A)
+{
+ return (__m128d) __A;
+}
+
+extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castps256_ps128 (__m256 __A)
+{
+ return (__m128) __A;
+}
+
+extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castsi256_si128 (__m256i __A)
+{
+ return (__m128i) __A;
+}
+
+extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castpd128_pd256 (__m128d __A)
+{
+ return (__m256d) __A;
+}
+
+extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castps128_ps256 (__m128 __A)
+{
+ return (__m256) __A;
+}
+
+extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_castsi128_si256 (__m128i __A)
+{
+ return (__m256i) __A;
+}
+#endif
+
+#endif /* __AVX__ */
+
+#endif /* _GMMINTRIN_H_INCLUDED */
Index: gcc/config/i386/cpuid.h
===================================================================
--- gcc/config/i386/cpuid.h (.../fsf/trunk) (revision 2097)
+++ gcc/config/i386/cpuid.h (.../branches/avx) (revision 2097)
@@ -35,11 +35,15 @@
#define bit_SSE3 (1 << 0)
#define bit_PCLMUL (1 << 1)
#define bit_SSSE3 (1 << 9)
+#define bit_FMA (1 << 12)
#define bit_CMPXCHG16B (1 << 13)
#define bit_SSE4_1 (1 << 19)
#define bit_SSE4_2 (1 << 20)
#define bit_POPCNT (1 << 23)
#define bit_AES (1 << 25)
+#define bit_XSAVE (1 << 26)
+#define bit_OSXSAVE (1 << 27)
+#define bit_AVX (1 << 28)
/* %edx */
#define bit_CMPXCHG8B (1 << 8)
Index: gcc/config/i386/i386-modes.def
===================================================================
--- gcc/config/i386/i386-modes.def (.../fsf/trunk) (revision 2097)
+++ gcc/config/i386/i386-modes.def (.../branches/avx) (revision 2097)
@@ -73,17 +73,20 @@ CC_MODE (CCFPU);
VECTOR_MODES (INT, 4); /* V4QI V2HI */
VECTOR_MODES (INT, 8); /* V8QI V4HI V2SI */
VECTOR_MODES (INT, 16); /* V16QI V8HI V4SI V2DI */
+VECTOR_MODES (INT, 32); /* V32QI V16HI V8SI V4DI */
VECTOR_MODES (FLOAT, 8); /* V4HF V2SF */
VECTOR_MODES (FLOAT, 16); /* V8HF V4SF V2DF */
+VECTOR_MODES (FLOAT, 32); /* V16HF V8SF V4DF */
VECTOR_MODE (INT, DI, 1); /* V1DI */
VECTOR_MODE (INT, SI, 1); /* V1SI */
VECTOR_MODE (INT, QI, 2); /* V2QI */
-VECTOR_MODE (INT, DI, 4); /* V4DI */
-VECTOR_MODE (INT, SI, 8); /* V8SI */
-VECTOR_MODE (INT, HI, 16); /* V16HI */
-VECTOR_MODE (INT, QI, 32); /* V32QI */
-VECTOR_MODE (FLOAT, DF, 4); /* V4DF */
-VECTOR_MODE (FLOAT, SF, 8); /* V8SF */
+VECTOR_MODE (INT, DI, 8); /* V8DI */
+VECTOR_MODE (INT, HI, 32); /* V32HI */
+VECTOR_MODE (INT, QI, 64); /* V64QI */
+VECTOR_MODE (FLOAT, DF, 8); /* V8DF */
+VECTOR_MODE (FLOAT, SF, 16); /* V16SF */
+
+INT_MODE (OI, 32);
/* The symbol Pmode stands for one of the above machine modes (usually SImode).
The tm.h file specifies which one. It is not a distinct mode. */
Index: gcc/config/i386/i386.opt
===================================================================
--- gcc/config/i386/i386.opt (.../fsf/trunk) (revision 2097)
+++ gcc/config/i386/i386.opt (.../branches/avx) (revision 2097)
@@ -240,6 +240,14 @@ mno-sse4
Target RejectNegative Report InverseMask(ISA_SSE4_1) MaskExists Var(ix86_isa_flags) VarExists
Do not support SSE4.1 and SSE4.2 built-in functions and code generation
+mavx
+Target Report Mask(ISA_AVX) Var(ix86_isa_flags) VarExists
+Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2 and AVX built-in functions and code generation
+
+mfma
+Target Report Mask(ISA_FMA) Var(ix86_isa_flags) VarExists
+Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX and FMA built-in functions and code generation
+
msse4a
Target Report Mask(ISA_SSE4A) Var(ix86_isa_flags) VarExists
Support MMX, SSE, SSE2, SSE3 and SSE4A built-in functions and code generation
Index: gcc/config/i386/i386.c
===================================================================
--- gcc/config/i386/i386.c (.../fsf/trunk) (revision 2097)
+++ gcc/config/i386/i386.c (.../branches/avx) (revision 2097)
@@ -1787,6 +1787,10 @@ static int ix86_isa_flags_explicit;
(OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSSE3_SET)
#define OPTION_MASK_ISA_SSE4_2_SET \
(OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_SSE4_1_SET)
+#define OPTION_MASK_ISA_AVX_SET \
+ (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_SSE4_2_SET)
+#define OPTION_MASK_ISA_FMA_SET \
+ (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_AVX_SET)
/* SSE4 includes both SSE4.1 and SSE4.2. -msse4 should be the same
as -msse4.2. */
@@ -1818,7 +1822,11 @@ static int ix86_isa_flags_explicit;
(OPTION_MASK_ISA_SSSE3 | OPTION_MASK_ISA_SSE4_1_UNSET)
#define OPTION_MASK_ISA_SSE4_1_UNSET \
(OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSE4_2_UNSET)
-#define OPTION_MASK_ISA_SSE4_2_UNSET OPTION_MASK_ISA_SSE4_2
+#define OPTION_MASK_ISA_SSE4_2_UNSET \
+ (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_AVX_UNSET )
+#define OPTION_MASK_ISA_AVX_UNSET \
+ (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_FMA_UNSET)
+#define OPTION_MASK_ISA_FMA_UNSET OPTION_MASK_ISA_FMA
/* SSE4 includes both SSE4.1 and SSE4.2. -mno-sse4 should the same
as -mno-sse4.1. */
@@ -1948,6 +1956,32 @@ ix86_handle_option (size_t code, const c
}
return true;
+ case OPT_mavx:
+ if (value)
+ {
+ ix86_isa_flags |= OPTION_MASK_ISA_AVX_SET;
+ ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_SET;
+ }
+ else
+ {
+ ix86_isa_flags &= ~OPTION_MASK_ISA_AVX_UNSET;
+ ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_UNSET;
+ }
+ return true;
+
+ case OPT_mfma:
+ if (value)
+ {
+ ix86_isa_flags |= OPTION_MASK_ISA_FMA_SET;
+ ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_SET;
+ }
+ else
+ {
+ ix86_isa_flags &= ~OPTION_MASK_ISA_FMA_UNSET;
+ ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_UNSET;
+ }
+ return true;
+
case OPT_msse4:
ix86_isa_flags |= OPTION_MASK_ISA_SSE4_SET;
ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_SET;
@@ -2081,7 +2115,9 @@ override_options (void)
PTA_SSE4_2 = 1 << 15,
PTA_SSE5 = 1 << 16,
PTA_AES = 1 << 17,
- PTA_PCLMUL = 1 << 18
+ PTA_PCLMUL = 1 << 18,
+ PTA_AVX = 1 << 19,
+ PTA_FMA = 1 << 20
};
static struct pta
@@ -2371,6 +2407,12 @@ override_options (void)
if (processor_alias_table[i].flags & PTA_SSE4_2
&& !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
+ if (processor_alias_table[i].flags & PTA_AVX
+ && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
+ ix86_isa_flags |= OPTION_MASK_ISA_AVX;
+ if (processor_alias_table[i].flags & PTA_FMA
+ && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
+ ix86_isa_flags |= OPTION_MASK_ISA_FMA;
if (processor_alias_table[i].flags & PTA_SSE4A
&& !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
@@ -3533,6 +3575,7 @@ init_cumulative_args (CUMULATIVE_ARGS *c
cum->sse_nregs = SSE_REGPARM_MAX;
if (TARGET_MMX)
cum->mmx_nregs = MMX_REGPARM_MAX;
+ cum->warn_avx = true;
cum->warn_sse = true;
cum->warn_mmx = true;
@@ -3557,6 +3600,7 @@ init_cumulative_args (CUMULATIVE_ARGS *c
cum->nregs = 0;
cum->sse_nregs = 0;
cum->mmx_nregs = 0;
+ cum->warn_avx = 0;
cum->warn_sse = 0;
cum->warn_mmx = 0;
return;
@@ -3940,6 +3984,12 @@ classify_argument (enum machine_mode mod
case TCmode:
/* This modes is larger than 16 bytes. */
return 0;
+ case V8SFmode:
+ case V8SImode:
+ case V32QImode:
+ case V16HImode:
+ case V4DFmode:
+ case V4DImode:
case V4SFmode:
case V4SImode:
case V16QImode:
@@ -4228,6 +4278,12 @@ function_arg_advance_32 (CUMULATIVE_ARGS
/* FALLTHRU */
case TImode:
+ case V8SFmode:
+ case V8SImode:
+ case V32QImode:
+ case V16HImode:
+ case V4DFmode:
+ case V4DImode:
case V16QImode:
case V8HImode:
case V4SImode:
@@ -4342,7 +4398,7 @@ function_arg_32 (CUMULATIVE_ARGS *cum, e
enum machine_mode orig_mode, tree type,
HOST_WIDE_INT bytes, HOST_WIDE_INT words)
{
- static bool warnedsse, warnedmmx;
+ static bool warnedavx, warnedsse, warnedmmx;
/* Avoid the AL settings for the Unix64 ABI. */
if (mode == VOIDmode)
@@ -4411,6 +4467,27 @@ function_arg_32 (CUMULATIVE_ARGS *cum, e
}
break;
+ case OImode:
+ case V8SFmode:
+ case V8SImode:
+ case V32QImode:
+ case V16HImode:
+ case V4DFmode:
+ case V4DImode:
+ if (!type || !AGGREGATE_TYPE_P (type))
+ {
+ if (!TARGET_AVX && !warnedavx && cum->warn_avx)
+ {
+ warnedavx = true;
+ warning (0, "AVX vector argument without AVX enabled "
+ "changes the ABI");
+ }
+ if (cum->sse_nregs)
+ return gen_reg_or_parallel (mode, orig_mode,
+ cum->sse_regno + FIRST_SSE_REG);
+ }
+ break;
+
case V8QImode:
case V4HImode:
case V2SImode:
@@ -4438,6 +4515,8 @@ static rtx
function_arg_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
enum machine_mode orig_mode, tree type)
{
+ static bool warnedavx;
+
/* Handle a hidden AL argument containing number of registers
for varargs x86-64 functions. */
if (mode == VOIDmode)
@@ -4447,6 +4526,30 @@ function_arg_64 (CUMULATIVE_ARGS *cum, e
: cum->sse_regno)
: -1);
+ switch (mode)
+ {
+ default:
+ break;
+
+ case OImode:
+ case V8SFmode:
+ case V8SImode:
+ case V32QImode:
+ case V16HImode:
+ case V4DFmode:
+ case V4DImode:
+ if (!type || !AGGREGATE_TYPE_P (type))
+ {
+ if (!TARGET_AVX && !warnedavx && cum->warn_avx)
+ {
+ warnedavx = true;
+ warning (0, "AVX vector argument without AVX enabled "
+ "changes the ABI");
+ }
+ }
+ break;
+ }
+
return construct_container (mode, orig_mode, type, 0, cum->nregs,
cum->sse_nregs,
&x86_64_int_parameter_registers [cum->regno],
@@ -5648,7 +5751,8 @@ standard_80387_constant_rtx (int idx)
XFmode);
}
-/* Return 1 if mode is a valid mode for sse. */
+/* Return 1 if mode is a valid mode for sse. 256bit AVX modes aren't
+ supported since we can't generate all ones without using memory. */
static int
standard_sse_mode_p (enum machine_mode mode)
{
@@ -5692,12 +5796,23 @@ standard_sse_constant_opcode (rtx insn,
switch (standard_sse_constant_p (x))
{
case 1:
- if (get_attr_mode (insn) == MODE_V4SF)
- return "xorps\t%0, %0";
- else if (get_attr_mode (insn) == MODE_V2DF)
- return "xorpd\t%0, %0";
- else
- return "pxor\t%0, %0";
+ switch (get_attr_mode (insn))
+ {
+ case MODE_V4SF:
+ return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0";
+ case MODE_V2DF:
+ return TARGET_AVX ? "vxorpd\t%0, %0, %0" : "xorpd\t%0, %0";
+ case MODE_TI:
+ return TARGET_AVX ? "vpxor\t%0, %0, %0" : "pxor\t%0, %0";
+ case MODE_V8SF:
+ return "vxorps\t%x0, %x0, %x0";
+ case MODE_V4DF:
+ return "vxorpd\t%x0, %x0, %x0";
+ case MODE_OI:
+ return "vpxor\t%x0, %x0, %x0";
+ default:
+ break;
+ }
case 2:
return "pcmpeqd\t%0, %0";
}
@@ -8714,6 +8829,7 @@ put_condition_code (enum rtx_code code,
If CODE is 'b', pretend the mode is QImode.
If CODE is 'k', pretend the mode is SImode.
If CODE is 'q', pretend the mode is DImode.
+ If CODE is 'x', pretend the mode is V4SFmode.
If CODE is 'h', pretend the reg is the 'high' byte register.
If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op. */
@@ -8749,6 +8865,8 @@ print_reg (rtx x, int code, FILE *file)
code = 3;
else if (code == 'h')
code = 0;
+ else if (code == 'x')
+ code = 16;
else
code = GET_MODE_SIZE (GET_MODE (x));
@@ -8810,6 +8928,13 @@ print_reg (rtx x, int code, FILE *file)
goto normal;
fputs (qi_high_reg_name[REGNO (x)], file);
break;
+ case 32:
+ if (SSE_REG_P (x))
+ {
+ putc ('y', file);
+ fputs (hi_reg_name[REGNO (x)] + 1, file);
+ }
+ break;
default:
gcc_unreachable ();
}
@@ -8869,6 +8994,7 @@ get_some_local_dynamic_name (void)
w -- likewise, print the HImode name of the register.
k -- likewise, print the SImode name of the register.
q -- likewise, print the DImode name of the register.
+ x -- likewise, print the V4SFmode name of the register.
h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
y -- print "st(0)" instead of "st" as a register.
D -- print condition for SSE cmp instruction.
@@ -9022,6 +9148,7 @@ print_operand (FILE *file, rtx x, int co
case 'q':
case 'h':
case 'y':
+ case 'x':
case 'X':
case 'P':
break;
@@ -9038,40 +9165,93 @@ print_operand (FILE *file, rtx x, int co
/* Little bit of braindamage here. The SSE compare instructions
does use completely different names for the comparisons that the
fp conditional moves. */
- switch (GET_CODE (x))
+ if (TARGET_AVX)
{
- case EQ:
- case UNEQ:
- fputs ("eq", file);
- break;
- case LT:
- case UNLT:
- fputs ("lt", file);
- break;
- case LE:
- case UNLE:
- fputs ("le", file);
- break;
- case UNORDERED:
- fputs ("unord", file);
- break;
- case NE:
- case LTGT:
- fputs ("neq", file);
- break;
- case UNGE:
- case GE:
- fputs ("nlt", file);
- break;
- case UNGT:
- case GT:
- fputs ("nle", file);
- break;
- case ORDERED:
- fputs ("ord", file);
- break;
- default:
- gcc_unreachable ();
+ switch (GET_CODE (x))
+ {
+ case EQ:
+ fputs ("eq", file);
+ break;
+ case UNEQ:
+ fputs ("eq_us", file);
+ break;
+ case LT:
+ fputs ("lt", file);
+ break;
+ case UNLT:
+ fputs ("nge", file);
+ break;
+ case LE:
+ fputs ("le", file);
+ break;
+ case UNLE:
+ fputs ("ngt", file);
+ break;
+ case UNORDERED:
+ fputs ("unord", file);
+ break;
+ case NE:
+ fputs ("neq", file);
+ break;
+ case LTGT:
+ fputs ("neq_oq", file);
+ break;
+ case GE:
+ fputs ("ge", file);
+ break;
+ case UNGE:
+ fputs ("nlt", file);
+ break;
+ case GT:
+ fputs ("gt", file);
+ break;
+ case UNGT:
+ fputs ("nle", file);
+ break;
+ case ORDERED:
+ fputs ("ord", file);
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+ else
+ {
+ switch (GET_CODE (x))
+ {
+ case EQ:
+ case UNEQ:
+ fputs ("eq", file);
+ break;
+ case LT:
+ case UNLT:
+ fputs ("lt", file);
+ break;
+ case LE:
+ case UNLE:
+ fputs ("le", file);
+ break;
+ case UNORDERED:
+ fputs ("unord", file);
+ break;
+ case NE:
+ case LTGT:
+ fputs ("neq", file);
+ break;
+ case UNGE:
+ case GE:
+ fputs ("nlt", file);
+ break;
+ case UNGT:
+ case GT:
+ fputs ("nle", file);
+ break;
+ case ORDERED:
+ fputs ("ord", file);
+ break;
+ default:
+ gcc_unreachable ();
+ }
}
return;
case 'O':
@@ -9616,7 +9796,7 @@ split_ti (rtx operands[], int num, rtx l
const char *
output_387_binary_op (rtx insn, rtx *operands)
{
- static char buf[30];
+ static char buf[40];
const char *p;
const char *ssep;
int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
@@ -9681,11 +9861,23 @@ output_387_binary_op (rtx insn, rtx *ope
if (is_sse)
{
- strcpy (buf, ssep);
- if (GET_MODE (operands[0]) == SFmode)
- strcat (buf, "ss\t{%2, %0|%0, %2}");
- else
- strcat (buf, "sd\t{%2, %0|%0, %2}");
+ if (TARGET_AVX)
+ {
+ buf[0] = 'v';
+ strcpy (buf + 1, ssep);
+ if (GET_MODE (operands[0]) == SFmode)
+ strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
+ else
+ strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
+ }
+ else
+ {
+ strcpy (buf, ssep);
+ if (GET_MODE (operands[0]) == SFmode)
+ strcat (buf, "ss\t{%2, %0|%0, %2}");
+ else
+ strcat (buf, "sd\t{%2, %0|%0, %2}");
+ }
return buf;
}
strcpy (buf, p);
@@ -17645,6 +17837,55 @@ enum ix86_builtins
/* PCLMUL instruction */
IX86_BUILTIN_PCLMULQDQ128,
+ /* AVX */
+ IX86_BUILTIN_ADDPD256,
+ IX86_BUILTIN_ADDPS256,
+ IX86_BUILTIN_ADDSUBPD256,
+ IX86_BUILTIN_ADDSUBPS256,
+ IX86_BUILTIN_ANDPD256,
+ IX86_BUILTIN_ANDPS256,
+ IX86_BUILTIN_ANDNPD256,
+ IX86_BUILTIN_ANDNPS256,
+ IX86_BUILTIN_BLENDPD256,
+ IX86_BUILTIN_BLENDPS256,
+ IX86_BUILTIN_BLENDVPD256,
+ IX86_BUILTIN_BLENDVPS256,
+ IX86_BUILTIN_DIVPD256,
+ IX86_BUILTIN_DIVPS256,
+ IX86_BUILTIN_DPPS256,
+ IX86_BUILTIN_HADDPD256,
+ IX86_BUILTIN_HADDPS256,
+ IX86_BUILTIN_HSUBPD256,
+ IX86_BUILTIN_HSUBPS256,
+ IX86_BUILTIN_MAXPD256,
+ IX86_BUILTIN_MAXPS256,
+ IX86_BUILTIN_MINPD256,
+ IX86_BUILTIN_MINPS256,
+ IX86_BUILTIN_MULPD256,
+ IX86_BUILTIN_MULPS256,
+ IX86_BUILTIN_ORPD256,
+ IX86_BUILTIN_ORPS256,
+ IX86_BUILTIN_SHUFPD256,
+ IX86_BUILTIN_SHUFPS256,
+ IX86_BUILTIN_SUBPD256,
+ IX86_BUILTIN_SUBPS256,
+ IX86_BUILTIN_XORPD256,
+ IX86_BUILTIN_XORPS256,
+ IX86_BUILTIN_CMPSD,
+ IX86_BUILTIN_CMPSS,
+ IX86_BUILTIN_CMPPD,
+ IX86_BUILTIN_CMPPS,
+ IX86_BUILTIN_CMPPD256,
+ IX86_BUILTIN_CMPPS256,
+ IX86_BUILTIN_CVTDQ2PD256,
+ IX86_BUILTIN_CVTDQ2PS256,
+ IX86_BUILTIN_CVTPD2PS256,
+ IX86_BUILTIN_CVTPS2DQ256,
+ IX86_BUILTIN_CVTPS2PD256,
+ IX86_BUILTIN_CVTTPD2DQ256,
+ IX86_BUILTIN_CVTPD2DQ256,
+ IX86_BUILTIN_CVTTPS2DQ256,
+
/* TFmode support builtins. */
IX86_BUILTIN_INFQ,
IX86_BUILTIN_FABSQ,
@@ -18009,6 +18250,21 @@ static const struct builtin_description
/* PCLMUL */
{ OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, 0 },
+
+ /* AVX */
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpsdv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpssv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, 0 },
};
static const struct builtin_description bdesc_2arg[] =
@@ -18296,6 +18552,34 @@ static const struct builtin_description
{ OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, 0 },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, 0 },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, 0 },
+
+ /* AVX */
+ { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_nandv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_nandv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, 0 },
};
static const struct builtin_description bdesc_1arg[] =
@@ -18376,6 +18660,16 @@ static const struct builtin_description
/* AES */
{ OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, 0 },
+
+ /* AVX */
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2pd256, 0, IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2ps256, 0, IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, 0, IX86_BUILTIN_CVTPD2PS256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, 0, IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, 0, IX86_BUILTIN_CVTPS2PD256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttpd2dq256, 0, IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, 0, IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, 0 },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttps2dq256, 0, IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, 0 },
};
/* SSE5 */
@@ -18684,11 +18978,14 @@ ix86_init_mmx_sse_builtins (void)
tree V2DI_type_node
= build_vector_type_for_mode (long_long_integer_type_node, V2DImode);
tree V2DF_type_node = build_vector_type_for_mode (double_type_node, V2DFmode);
+ tree V4DF_type_node = build_vector_type_for_mode (double_type_node, V4DFmode);
tree V4SF_type_node = build_vector_type_for_mode (float_type_node, V4SFmode);
tree V4SI_type_node = build_vector_type_for_mode (intSI_type_node, V4SImode);
tree V4HI_type_node = build_vector_type_for_mode (intHI_type_node, V4HImode);
+ tree V8SF_type_node = build_vector_type_for_mode (float_type_node, V8SFmode);
tree V8QI_type_node = build_vector_type_for_mode (char_type_node, V8QImode);
tree V8HI_type_node = build_vector_type_for_mode (intHI_type_node, V8HImode);
+ tree V8SI_type_node = build_vector_type_for_mode (intSI_type_node, V8SImode);
tree pchar_type_node = build_pointer_type (char_type_node);
tree pcchar_type_node = build_pointer_type (
@@ -18923,6 +19220,32 @@ ix86_init_mmx_sse_builtins (void)
tree v2di_ftype_v2di_v2di
= build_function_type_list (V2DI_type_node,
V2DI_type_node, V2DI_type_node, NULL_TREE);
+ tree v8sf_ftype_v8sf_v8sf
+ = build_function_type_list (V8SF_type_node,
+ V8SF_type_node, V8SF_type_node, NULL_TREE);
+ tree v8sf_ftype_v8sf_v8sf_int
+ = build_function_type_list (V8SF_type_node,
+ V8SF_type_node, V8SF_type_node,
+ integer_type_node,
+ NULL_TREE);
+ tree v8sf_ftype_v8sf_v8sf_v8sf
+ = build_function_type_list (V8SF_type_node,
+ V8SF_type_node, V8SF_type_node,
+ V8SF_type_node,
+ NULL_TREE);
+ tree v4df_ftype_v4df_v4df
+ = build_function_type_list (V4DF_type_node,
+ V4DF_type_node, V4DF_type_node, NULL_TREE);
+ tree v4df_ftype_v4df_v4df_int
+ = build_function_type_list (V4DF_type_node,
+ V4DF_type_node, V4DF_type_node,
+ integer_type_node,
+ NULL_TREE);
+ tree v4df_ftype_v4df_v4df_v4df
+ = build_function_type_list (V4DF_type_node,
+ V4DF_type_node, V4DF_type_node,
+ V4DF_type_node,
+ NULL_TREE);
tree v2di_ftype_v2df_v2df
= build_function_type_list (V2DI_type_node,
V2DF_type_node, V2DF_type_node, NULL_TREE);
@@ -19132,6 +19455,19 @@ ix86_init_mmx_sse_builtins (void)
tree v2di_ftype_v2di
= build_function_type_list (V2DI_type_node, V2DI_type_node, NULL_TREE);
+ tree v8sf_ftype_v8si
+ = build_function_type_list (V8SF_type_node, V8SI_type_node, NULL_TREE);
+ tree v8si_ftype_v8sf
+ = build_function_type_list (V8SI_type_node, V8SF_type_node, NULL_TREE);
+ tree v4sf_ftype_v4df
+ = build_function_type_list (V4SF_type_node, V4DF_type_node, NULL_TREE);
+ tree v4df_ftype_v4sf
+ = build_function_type_list (V4DF_type_node, V4SF_type_node, NULL_TREE);
+ tree v4df_ftype_v4si
+ = build_function_type_list (V4DF_type_node, V4SI_type_node, NULL_TREE);
+ tree v4si_ftype_v4df
+ = build_function_type_list (V4SI_type_node, V4DF_type_node, NULL_TREE);
+
tree ftype;
/* The __float80 type. */
@@ -19210,6 +19546,12 @@ ix86_init_mmx_sse_builtins (void)
case V4SFmode:
type = v4sf_ftype_v4sf_v4sf_int;
break;
+ case V4DFmode:
+ type = v4df_ftype_v4df_v4df_int;
+ break;
+ case V8SFmode:
+ type = v8sf_ftype_v8sf_v8sf_int;
+ break;
default:
gcc_unreachable ();
}
@@ -19226,6 +19568,12 @@ ix86_init_mmx_sse_builtins (void)
case CODE_FOR_sse4_1_pblendvb:
type = v16qi_ftype_v16qi_v16qi_v16qi;
break;
+ case CODE_FOR_avx_blendvpd256:
+ type = v4df_ftype_v4df_v4df_v4df;
+ break;
+ case CODE_FOR_avx_blendvps256:
+ type = v8sf_ftype_v8sf_v8sf_v8sf;
+ break;
default:
break;
}
@@ -19248,6 +19596,12 @@ ix86_init_mmx_sse_builtins (void)
switch (mode)
{
+ case V8SFmode:
+ type = v8sf_ftype_v8sf_v8sf;
+ break;
+ case V4DFmode:
+ type = v4df_ftype_v4df_v4df;
+ break;
case V16QImode:
type = v16qi_ftype_v16qi_v16qi;
break;
@@ -19319,6 +19673,9 @@ ix86_init_mmx_sse_builtins (void)
case V4SImode:
type = v4si_ftype_v4si;
break;
+ case V2DImode:
+ type = v2di_ftype_v2di;
+ break;
case V2DFmode:
type = v2df_ftype_v2df;
break;
@@ -19631,6 +19988,16 @@ ix86_init_mmx_sse_builtins (void)
def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_pclmulqdq128", v2di_ftype_v2di_v2di_int, IX86_BUILTIN_PCLMULQDQ128);
}
+ /* AVX */
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtdq2pd256", v4df_ftype_v4si, IX86_BUILTIN_CVTDQ2PD256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtdq2ps256", v8sf_ftype_v8si, IX86_BUILTIN_CVTDQ2PS256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtpd2ps256", v4sf_ftype_v4df, IX86_BUILTIN_CVTPD2PS256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtps2dq256", v8si_ftype_v8sf, IX86_BUILTIN_CVTPS2DQ256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtps2pd256", v4df_ftype_v4sf, IX86_BUILTIN_CVTPS2PD256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvttpd2dq256", v4si_ftype_v4df, IX86_BUILTIN_CVTTPD2DQ256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvtpd2dq256", v4si_ftype_v4df, IX86_BUILTIN_CVTPD2DQ256);
+ def_builtin_const (OPTION_MASK_ISA_AVX, "__builtin_ia32_cvttps2dq256", v8si_ftype_v8sf, IX86_BUILTIN_CVTTPS2DQ256);
+
/* AMDFAM10 SSE4A New built-ins */
def_builtin (OPTION_MASK_ISA_SSE4A, "__builtin_ia32_movntsd", void_ftype_pdouble_v2df, IX86_BUILTIN_MOVNTSD);
def_builtin (OPTION_MASK_ISA_SSE4A, "__builtin_ia32_movntss", void_ftype_pfloat_v4sf, IX86_BUILTIN_MOVNTSS);
@@ -19857,6 +20224,7 @@ ix86_expand_sse_4_operands_builtin (enum
case CODE_FOR_sse4_1_roundsd:
case CODE_FOR_sse4_1_roundss:
case CODE_FOR_sse4_1_blendps:
+ case CODE_FOR_avx_blendpd256:
error ("the third argument must be a 4-bit immediate");
return const0_rtx;
@@ -19864,6 +20232,20 @@ ix86_expand_sse_4_operands_builtin (enum
error ("the third argument must be a 2-bit immediate");
return const0_rtx;
+ case CODE_FOR_avx_cmpsdv2df3:
+ case CODE_FOR_avx_cmpssv4sf3:
+ case CODE_FOR_avx_cmppdv2df3:
+ case CODE_FOR_avx_cmppsv4sf3:
+ case CODE_FOR_avx_cmppsv8sf3:
+ case CODE_FOR_avx_cmppdv4df3:
+ error ("the third argument must be a 5-bit immediate");
+ return const0_rtx;
+
+ case CODE_FOR_avx_blendvpd256:
+ case CODE_FOR_avx_blendvps256:
+ op2 = copy_to_mode_reg (mode3, op2);
+ break;
+
default:
error ("the third argument must be an 8-bit immediate");
return const0_rtx;
@@ -21422,14 +21804,17 @@ ix86_expand_builtin (tree exp, rtx targe
for (i = 0, d = bdesc_2arg; i < ARRAY_SIZE (bdesc_2arg); i++, d++)
if (d->code == fcode)
{
- /* Compares are treated specially. */
- if (d->icode == CODE_FOR_sse_maskcmpv4sf3
- || d->icode == CODE_FOR_sse_vmmaskcmpv4sf3
- || d->icode == CODE_FOR_sse2_maskcmpv2df3
- || d->icode == CODE_FOR_sse2_vmmaskcmpv2df3)
- return ix86_expand_sse_compare (d, exp, target);
-
- return ix86_expand_binop_builtin (d->icode, exp, target);
+ switch (d->icode)
+ {
+ case CODE_FOR_sse_maskcmpv4sf3:
+ case CODE_FOR_sse_vmmaskcmpv4sf3:
+ case CODE_FOR_sse2_maskcmpv2df3:
+ case CODE_FOR_sse2_vmmaskcmpv2df3:
+ /* Compares are treated specially. */
+ return ix86_expand_sse_compare (d, exp, target);
+ default:
+ return ix86_expand_binop_builtin (d->icode, exp, target);
+ }
}
for (i = 0, d = bdesc_1arg; i < ARRAY_SIZE (bdesc_1arg); i++, d++)
@@ -22301,7 +22686,8 @@ ix86_hard_regno_mode_ok (int regno, enum
/* We implement the move patterns for all vector modes into and
out of SSE registers, even when no operation instructions
are available. */
- return (VALID_SSE_REG_MODE (mode)
+ return (VALID_AVX_REG_MODE (mode)
+ || VALID_SSE_REG_MODE (mode)
|| VALID_SSE2_REG_MODE (mode)
|| VALID_MMX_REG_MODE (mode)
|| VALID_MMX_REG_MODE_3DNOW (mode));
@@ -24318,6 +24704,8 @@ ix86_vector_mode_supported_p (enum machi
return true;
if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
return true;
+ if (TARGET_AVX && VALID_AVX_REG_MODE (mode))
+ return true;
if (TARGET_MMX && VALID_MMX_REG_MODE (mode))
return true;
if (TARGET_3DNOW && VALID_MMX_REG_MODE_3DNOW (mode))
More information about the Gcc-patches
mailing list