ref: b4e0986391819a33c71795b2dd76710104761b52
dir: /vpx_ports/x86.h/
/* * Copyright (c) 2010 The WebM project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef VPX_VPX_PORTS_X86_H_ #define VPX_VPX_PORTS_X86_H_ #include <stdlib.h> #if defined(_MSC_VER) #include <intrin.h> /* For __cpuidex, __rdtsc */ #endif #include "vpx_config.h" #include "vpx/vpx_integer.h" #ifdef __cplusplus extern "C" { #endif typedef enum { VPX_CPU_UNKNOWN = -1, VPX_CPU_AMD, VPX_CPU_AMD_OLD, VPX_CPU_CENTAUR, VPX_CPU_CYRIX, VPX_CPU_INTEL, VPX_CPU_NEXGEN, VPX_CPU_NSC, VPX_CPU_RISE, VPX_CPU_SIS, VPX_CPU_TRANSMETA, VPX_CPU_TRANSMETA_OLD, VPX_CPU_UMC, VPX_CPU_VIA, VPX_CPU_LAST } vpx_cpu_t; #if defined(__GNUC__) && __GNUC__ || defined(__ANDROID__) #if ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__("cpuid \n\t" \ : "=a"(ax), "=b"(bx), "=c"(cx), "=d"(dx) \ : "a"(func), "c"(func2)); #else #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__( \ "mov %%ebx, %%edi \n\t" \ "cpuid \n\t" \ "xchg %%edi, %%ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ : "a"(func), "c"(func2)); #endif #elif defined(__SUNPRO_C) || \ defined(__SUNPRO_CC) /* end __GNUC__ or __ANDROID__*/ #if ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ asm volatile( \ "xchg %rsi, %rbx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "xchg %rsi, %rbx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ : "a"(func), "c"(func2)); #else #define cpuid(func, func2, ax, bx, cx, dx) \ asm volatile( \ "pushl %ebx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "popl %ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ : "a"(func), "c"(func2)); #endif #else /* end __SUNPRO__ */ #if ARCH_X86_64 #if defined(_MSC_VER) && _MSC_VER > 1500 #define cpuid(func, func2, a, b, c, d) \ do { \ int regs[4]; \ __cpuidex(regs, func, func2); \ a = regs[0]; \ b = regs[1]; \ c = regs[2]; \ d = regs[3]; \ } while (0) #else #define cpuid(func, func2, a, b, c, d) \ do { \ int regs[4]; \ __cpuid(regs, func); \ a = regs[0]; \ b = regs[1]; \ c = regs[2]; \ d = regs[3]; \ } while (0) #endif #else #define cpuid(func, func2, a, b, c, d) \ __asm mov eax, func __asm mov ecx, func2 __asm cpuid __asm mov a, \ eax __asm mov b, ebx __asm mov c, ecx __asm mov d, edx #endif #endif /* end others */ // NaCl has no support for xgetbv or the raw opcode. #if !defined(__native_client__) && (defined(__i386__) || defined(__x86_64__)) static INLINE uint64_t xgetbv(void) { const uint32_t ecx = 0; uint32_t eax, edx; // Use the raw opcode for xgetbv for compatibility with older toolchains. __asm__ volatile(".byte 0x0f, 0x01, 0xd0\n" : "=a"(eax), "=d"(edx) : "c"(ecx)); return ((uint64_t)edx << 32) | eax; } #elif (defined(_M_X64) || defined(_M_IX86)) && defined(_MSC_FULL_VER) && \ _MSC_FULL_VER >= 160040219 // >= VS2010 SP1 #include <immintrin.h> #define xgetbv() _xgetbv(0) #elif defined(_MSC_VER) && defined(_M_IX86) static INLINE uint64_t xgetbv(void) { uint32_t eax_, edx_; __asm { xor ecx, ecx // ecx = 0 // Use the raw opcode for xgetbv for compatibility with older toolchains. __asm _emit 0x0f __asm _emit 0x01 __asm _emit 0xd0 mov eax_, eax mov edx_, edx } return ((uint64_t)edx_ << 32) | eax_; } #else #define xgetbv() 0U // no AVX for older x64 or unrecognized toolchains. #endif #if defined(_MSC_VER) && _MSC_VER >= 1700 #undef NOMINMAX #define NOMINMAX #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include <windows.h> #if WINAPI_FAMILY_PARTITION(WINAPI_FAMILY_APP) #define getenv(x) NULL #endif #endif #define HAS_MMX 0x001 #define HAS_SSE 0x002 #define HAS_SSE2 0x004 #define HAS_SSE3 0x008 #define HAS_SSSE3 0x010 #define HAS_SSE4_1 0x020 #define HAS_AVX 0x040 #define HAS_AVX2 0x080 #define HAS_AVX512 0x100 #ifndef BIT #define BIT(n) (1u << (n)) #endif static INLINE int x86_simd_caps(void) { unsigned int flags = 0; unsigned int mask = ~0; unsigned int max_cpuid_val, reg_eax, reg_ebx, reg_ecx, reg_edx; char *env; (void)reg_ebx; /* See if the CPU capabilities are being overridden by the environment */ env = getenv("VPX_SIMD_CAPS"); if (env && *env) return (int)strtol(env, NULL, 0); env = getenv("VPX_SIMD_CAPS_MASK"); if (env && *env) mask = (unsigned int)strtoul(env, NULL, 0); /* Ensure that the CPUID instruction supports extended features */ cpuid(0, 0, max_cpuid_val, reg_ebx, reg_ecx, reg_edx); if (max_cpuid_val < 1) return 0; /* Get the standard feature flags */ cpuid(1, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); if (reg_edx & BIT(23)) flags |= HAS_MMX; if (reg_edx & BIT(25)) flags |= HAS_SSE; /* aka xmm */ if (reg_edx & BIT(26)) flags |= HAS_SSE2; /* aka wmt */ if (reg_ecx & BIT(0)) flags |= HAS_SSE3; if (reg_ecx & BIT(9)) flags |= HAS_SSSE3; if (reg_ecx & BIT(19)) flags |= HAS_SSE4_1; // bits 27 (OSXSAVE) & 28 (256-bit AVX) if ((reg_ecx & (BIT(27) | BIT(28))) == (BIT(27) | BIT(28))) { if ((xgetbv() & 0x6) == 0x6) { flags |= HAS_AVX; if (max_cpuid_val >= 7) { /* Get the leaf 7 feature flags. Needed to check for AVX2 support */ cpuid(7, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); if (reg_ebx & BIT(5)) flags |= HAS_AVX2; // bits 16 (AVX-512F) & 17 (AVX-512DQ) & 28 (AVX-512CD) & // 30 (AVX-512BW) & 32 (AVX-512VL) if ((reg_ebx & (BIT(16) | BIT(17) | BIT(28) | BIT(30) | BIT(31))) == (BIT(16) | BIT(17) | BIT(28) | BIT(30) | BIT(31))) flags |= HAS_AVX512; } } } return flags & mask; } // Fine-Grain Measurement Functions // // If you are timing a small region of code, access the timestamp counter // (TSC) via: // // unsigned int start = x86_tsc_start(); // ... // unsigned int end = x86_tsc_end(); // unsigned int diff = end - start; // // The start/end functions introduce a few more instructions than using // x86_readtsc directly, but prevent the CPU's out-of-order execution from // affecting the measurement (by having earlier/later instructions be evaluated // in the time interval). See the white paper, "How to Benchmark Code // Execution Times on Intel® IA-32 and IA-64 Instruction Set Architectures" by // Gabriele Paoloni for more information. // // If you are timing a large function (CPU time > a couple of seconds), use // x86_readtsc64 to read the timestamp counter in a 64-bit integer. The // out-of-order leakage that can occur is minimal compared to total runtime. static INLINE unsigned int x86_readtsc(void) { #if defined(__GNUC__) && __GNUC__ unsigned int tsc; __asm__ __volatile__("rdtsc\n\t" : "=a"(tsc) :); return tsc; #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) unsigned int tsc; asm volatile("rdtsc\n\t" : "=a"(tsc) :); return tsc; #else #if ARCH_X86_64 return (unsigned int)__rdtsc(); #else __asm rdtsc; #endif #endif } // 64-bit CPU cycle counter static INLINE uint64_t x86_readtsc64(void) { #if defined(__GNUC__) && __GNUC__ uint32_t hi, lo; __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); return ((uint64_t)hi << 32) | lo; #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) uint_t hi, lo; asm volatile("rdtsc\n\t" : "=a"(lo), "=d"(hi)); return ((uint64_t)hi << 32) | lo; #else #if ARCH_X86_64 return (uint64_t)__rdtsc(); #else __asm rdtsc; #endif #endif } // 32-bit CPU cycle counter with a partial fence against out-of-order execution. static INLINE unsigned int x86_readtscp(void) { #if defined(__GNUC__) && __GNUC__ unsigned int tscp; __asm__ __volatile__("rdtscp\n\t" : "=a"(tscp) :); return tscp; #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) unsigned int tscp; asm volatile("rdtscp\n\t" : "=a"(tscp) :); return tscp; #elif defined(_MSC_VER) unsigned int ui; return (unsigned int)__rdtscp(&ui); #else #if ARCH_X86_64 return (unsigned int)__rdtscp(); #else __asm rdtscp; #endif #endif } static INLINE unsigned int x86_tsc_start(void) { unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx; cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); return x86_readtsc(); } static INLINE unsigned int x86_tsc_end(void) { uint32_t v = x86_readtscp(); unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx; cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); return v; } #if defined(__GNUC__) && __GNUC__ #define x86_pause_hint() __asm__ __volatile__("pause \n\t") #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) #define x86_pause_hint() asm volatile("pause \n\t") #else #if ARCH_X86_64 #define x86_pause_hint() _mm_pause(); #else #define x86_pause_hint() __asm pause #endif #endif #if defined(__GNUC__) && __GNUC__ static void x87_set_control_word(unsigned short mode) { __asm__ __volatile__("fldcw %0" : : "m"(*&mode)); } static unsigned short x87_get_control_word(void) { unsigned short mode; __asm__ __volatile__("fstcw %0\n\t" : "=m"(*&mode) :); return mode; } #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) static void x87_set_control_word(unsigned short mode) { asm volatile("fldcw %0" : : "m"(*&mode)); } static unsigned short x87_get_control_word(void) { unsigned short mode; asm volatile("fstcw %0\n\t" : "=m"(*&mode) :); return mode; } #elif ARCH_X86_64 /* No fldcw intrinsics on Windows x64, punt to external asm */ extern void vpx_winx64_fldcw(unsigned short mode); extern unsigned short vpx_winx64_fstcw(void); #define x87_set_control_word vpx_winx64_fldcw #define x87_get_control_word vpx_winx64_fstcw #else static void x87_set_control_word(unsigned short mode) { __asm { fldcw mode } } static unsigned short x87_get_control_word(void) { unsigned short mode; __asm { fstcw mode } return mode; } #endif static INLINE unsigned int x87_set_double_precision(void) { unsigned int mode = x87_get_control_word(); // Intel 64 and IA-32 Architectures Developer's Manual: Vol. 1 // https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-vol-1-manual.pdf // 8.1.5.2 Precision Control Field // Bits 8 and 9 (0x300) of the x87 FPU Control Word ("Precision Control") // determine the number of bits used in floating point calculations. To match // later SSE instructions restrict x87 operations to Double Precision (0x200). // Precision PC Field // Single Precision (24-Bits) 00B // Reserved 01B // Double Precision (53-Bits) 10B // Extended Precision (64-Bits) 11B x87_set_control_word((mode & ~0x300) | 0x200); return mode; } #ifdef __cplusplus } // extern "C" #endif #endif // VPX_VPX_PORTS_X86_H_