Add MSVC support for pg_leftmost_one_pos32() and friends

To allow testing for general support for fast bitscan intrinsics,
add symbols HAVE_BITSCAN_REVERSE and HAVE_BITSCAN_FORWARD.

Also do related cleanup in AllocSetFreeIndex(): Previously, we
tested for HAVE__BUILTIN_CLZ and copied the relevant internals of
pg_leftmost_one_pos32(), with a special fallback that does less
work than the general fallback for that function. Now that we have
a more general test, we just call pg_leftmost_one_pos32() directly
for platforms with intrinsic support. On gcc at least, there is no
difference in the binary for non-assert builds.

Discussion: https://www.postgresql.org/message-id/CAFBsxsEPc%2BFnX_0vmmQ5DHv60sk4rL_RZJ%2BMD6ei%3D76L0kFMvA%40mail.gmail.com
This commit is contained in:
John Naylor 2023-02-08 12:05:58 +07:00
parent 204b0cbecb
commit 6773197464
2 changed files with 70 additions and 11 deletions

View File

@ -289,7 +289,7 @@ AllocSetFreeIndex(Size size)
* or equivalently * or equivalently
* pg_leftmost_one_pos32(size - 1) - ALLOC_MINBITS + 1 * pg_leftmost_one_pos32(size - 1) - ALLOC_MINBITS + 1
* *
* However, rather than just calling that function, we duplicate the * However, for platforms without intrinsic support, we duplicate the
* logic here, allowing an additional optimization. It's reasonable * logic here, allowing an additional optimization. It's reasonable
* to assume that ALLOC_CHUNK_LIMIT fits in 16 bits, so we can unroll * to assume that ALLOC_CHUNK_LIMIT fits in 16 bits, so we can unroll
* the byte-at-a-time loop in pg_leftmost_one_pos32 and just handle * the byte-at-a-time loop in pg_leftmost_one_pos32 and just handle
@ -299,8 +299,8 @@ AllocSetFreeIndex(Size size)
* much trouble. * much trouble.
*---------- *----------
*/ */
#ifdef HAVE__BUILTIN_CLZ #ifdef HAVE_BITSCAN_REVERSE
idx = 31 - __builtin_clz((uint32) size - 1) - ALLOC_MINBITS + 1; idx = pg_leftmost_one_pos32((uint32) size - 1) - ALLOC_MINBITS + 1;
#else #else
uint32 t, uint32 t,
tsize; tsize;

View File

@ -13,6 +13,21 @@
#ifndef PG_BITUTILS_H #ifndef PG_BITUTILS_H
#define PG_BITUTILS_H #define PG_BITUTILS_H
#ifdef _MSC_VER
#include <intrin.h>
#define HAVE_BITSCAN_FORWARD
#define HAVE_BITSCAN_REVERSE
#else
#if defined(HAVE__BUILTIN_CTZ)
#define HAVE_BITSCAN_FORWARD
#endif
#if defined(HAVE__BUILTIN_CLZ)
#define HAVE_BITSCAN_REVERSE
#endif
#endif /* _MSC_VER */
extern PGDLLIMPORT const uint8 pg_leftmost_one_pos[256]; extern PGDLLIMPORT const uint8 pg_leftmost_one_pos[256];
extern PGDLLIMPORT const uint8 pg_rightmost_one_pos[256]; extern PGDLLIMPORT const uint8 pg_rightmost_one_pos[256];
extern PGDLLIMPORT const uint8 pg_number_of_ones[256]; extern PGDLLIMPORT const uint8 pg_number_of_ones[256];
@ -27,9 +42,12 @@ pg_leftmost_one_pos32(uint32 word)
{ {
#ifdef HAVE__BUILTIN_CLZ #ifdef HAVE__BUILTIN_CLZ
int bitscan_result; int bitscan_result;
#elif defined(_MSC_VER)
unsigned long bitscan_result;
bool non_zero;
#endif #endif
#if !defined(HAVE__BUILTIN_CLZ) || defined(USE_ASSERT_CHECKING) #if !defined(HAVE_BITSCAN_REVERSE) || defined(USE_ASSERT_CHECKING)
int result; int result;
int shift = 32 - 8; int shift = 32 - 8;
@ -41,13 +59,20 @@ pg_leftmost_one_pos32(uint32 word)
result = shift + pg_leftmost_one_pos[(word >> shift) & 255]; result = shift + pg_leftmost_one_pos[(word >> shift) & 255];
#endif #endif
#ifdef HAVE_BITSCAN_REVERSE
#if defined(HAVE__BUILTIN_CLZ) #if defined(HAVE__BUILTIN_CLZ)
bitscan_result = 31 - __builtin_clz(word); bitscan_result = 31 - __builtin_clz(word);
#elif defined(_MSC_VER)
non_zero = _BitScanReverse(&bitscan_result, word);
Assert(non_zero);
#endif
Assert(bitscan_result == result); Assert(bitscan_result == result);
return bitscan_result; return bitscan_result;
#else #else
return result; return result;
#endif /* HAVE__BUILTIN_CLZ */ #endif /* HAVE_BITSCAN_REVERSE */
} }
/* /*
@ -59,9 +84,12 @@ pg_leftmost_one_pos64(uint64 word)
{ {
#ifdef HAVE__BUILTIN_CLZ #ifdef HAVE__BUILTIN_CLZ
int bitscan_result; int bitscan_result;
#elif defined(_MSC_VER)
unsigned long bitscan_result;
bool non_zero;
#endif #endif
#if !defined(HAVE__BUILTIN_CLZ) || defined(USE_ASSERT_CHECKING) #if !defined(HAVE_BITSCAN_REVERSE) || defined(USE_ASSERT_CHECKING)
int result; int result;
int shift = 64 - 8; int shift = 64 - 8;
@ -73,6 +101,8 @@ pg_leftmost_one_pos64(uint64 word)
result = shift + pg_leftmost_one_pos[(word >> shift) & 255]; result = shift + pg_leftmost_one_pos[(word >> shift) & 255];
#endif #endif
#ifdef HAVE_BITSCAN_REVERSE
#if defined(HAVE__BUILTIN_CLZ) #if defined(HAVE__BUILTIN_CLZ)
#if defined(HAVE_LONG_INT_64) #if defined(HAVE_LONG_INT_64)
bitscan_result = 63 - __builtin_clzl(word); bitscan_result = 63 - __builtin_clzl(word);
@ -81,11 +111,17 @@ pg_leftmost_one_pos64(uint64 word)
#else #else
#error must have a working 64-bit integer datatype #error must have a working 64-bit integer datatype
#endif /* HAVE_LONG_INT_64 */ #endif /* HAVE_LONG_INT_64 */
#elif defined(_MSC_VER)
non_zero = _BitScanReverse64(&bitscan_result, word);
Assert(non_zero);
#endif /* HAVE__BUILTIN_CLZ */
Assert(bitscan_result == result); Assert(bitscan_result == result);
return bitscan_result; return bitscan_result;
#else #else
return result; return result;
#endif /* HAVE__BUILTIN_CLZ */ #endif /* HAVE_BITSCAN_REVERSE */
} }
/* /*
@ -99,9 +135,13 @@ pg_rightmost_one_pos32(uint32 word)
#ifdef HAVE__BUILTIN_CTZ #ifdef HAVE__BUILTIN_CTZ
const uint32 orig_word = word; const uint32 orig_word = word;
int bitscan_result; int bitscan_result;
#elif defined(_MSC_VER)
const unsigned long orig_word = word;
unsigned long bitscan_result;
bool non_zero;
#endif #endif
#if !defined(HAVE__BUILTIN_CTZ) || defined(USE_ASSERT_CHECKING) #if !defined(HAVE_BITSCAN_FORWARD) || defined(USE_ASSERT_CHECKING)
int result = 0; int result = 0;
Assert(word != 0); Assert(word != 0);
@ -114,13 +154,20 @@ pg_rightmost_one_pos32(uint32 word)
result += pg_rightmost_one_pos[word & 255]; result += pg_rightmost_one_pos[word & 255];
#endif #endif
#ifdef HAVE_BITSCAN_FORWARD
#if defined(HAVE__BUILTIN_CTZ) #if defined(HAVE__BUILTIN_CTZ)
bitscan_result = __builtin_ctz(orig_word); bitscan_result = __builtin_ctz(orig_word);
#elif defined(_MSC_VER)
non_zero = _BitScanForward(&bitscan_result, orig_word);
Assert(non_zero);
#endif
Assert(bitscan_result == result); Assert(bitscan_result == result);
return bitscan_result; return bitscan_result;
#else #else
return result; return result;
#endif /* HAVE__BUILTIN_CTZ */ #endif /* HAVE_BITSCAN_FORWARD */
} }
/* /*
@ -133,9 +180,13 @@ pg_rightmost_one_pos64(uint64 word)
#ifdef HAVE__BUILTIN_CTZ #ifdef HAVE__BUILTIN_CTZ
const uint64 orig_word = word; const uint64 orig_word = word;
int bitscan_result; int bitscan_result;
#elif defined(_MSC_VER)
const unsigned __int64 orig_word = word;
unsigned long bitscan_result;
bool non_zero;
#endif #endif
#if !defined(HAVE__BUILTIN_CTZ) || defined(USE_ASSERT_CHECKING) #if !defined(HAVE_BITSCAN_FORWARD) || defined(USE_ASSERT_CHECKING)
int result = 0; int result = 0;
Assert(word != 0); Assert(word != 0);
@ -148,6 +199,8 @@ pg_rightmost_one_pos64(uint64 word)
result += pg_rightmost_one_pos[word & 255]; result += pg_rightmost_one_pos[word & 255];
#endif #endif
#ifdef HAVE_BITSCAN_FORWARD
#if defined(HAVE__BUILTIN_CTZ) #if defined(HAVE__BUILTIN_CTZ)
#if defined(HAVE_LONG_INT_64) #if defined(HAVE_LONG_INT_64)
bitscan_result = __builtin_ctzl(orig_word); bitscan_result = __builtin_ctzl(orig_word);
@ -156,11 +209,17 @@ pg_rightmost_one_pos64(uint64 word)
#else #else
#error must have a working 64-bit integer datatype #error must have a working 64-bit integer datatype
#endif /* HAVE_LONG_INT_64 */ #endif /* HAVE_LONG_INT_64 */
#elif defined(_MSC_VER)
non_zero = _BitScanForward64(&bitscan_result, orig_word);
Assert(non_zero);
#endif /* HAVE__BUILTIN_CTZ */
Assert(bitscan_result == result); Assert(bitscan_result == result);
return bitscan_result; return bitscan_result;
#else #else
return result; return result;
#endif /* HAVE__BUILTIN_CTZ */ #endif /* HAVE_BITSCAN_FORWARD */
} }
/* /*