MSVC <bit>

stl/inc/bit

@@ -12,6 +12,8 @@
 #pragma message("The contents of <bit> are available only with C++20 or later.")
 #else // ^^^ !_HAS_CXX20 / _HAS_CXX20 vvv
 
+#include <intrin0.h>
+#include <isa_availability.h>
 #include <limits>
 #include <type_traits>
 
@@ -23,6 +25,7 @@ _STL_DISABLE_CLANG_WARNINGS
 #undef new
 
 _STD_BEGIN
+
 template <class _To, class _From,
     enable_if_t<conjunction_v<bool_constant<sizeof(_To) == sizeof(_From)>, is_trivially_copyable<_To>,
                     is_trivially_copyable<_From>>,
@@ -31,7 +34,6 @@ _NODISCARD constexpr _To bit_cast(const _From& _Val) noexcept {
     return __builtin_bit_cast(_To, _Val);
 }
 
-#ifdef __cpp_lib_bitops // TRANSITION, VSO-1020212
 template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> = 0>
 _NODISCARD constexpr int countl_zero(_Ty _Val) noexcept;
 
@@ -94,17 +96,126 @@ _NODISCARD constexpr _Ty rotr(const _Ty _Val, const int _Rotation) noexcept {
     }
 }
 
-template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> _Enabled>
-_NODISCARD constexpr int countl_zero(const _Ty _Val) noexcept {
+// Implementation of popcount without using specialized CPU instructions.
+// Used at compile time and when said instructions are not supported.
+template <class _Ty>
+_NODISCARD constexpr int _Popcount_fallback(_Ty _Val) noexcept {
+    constexpr int _Digits = numeric_limits<_Ty>::digits;
+    // we static_cast these bit patterns in order to truncate them to the correct size
+    _Val = static_cast<_Ty>(_Val - ((_Val >> 1) & static_cast<_Ty>(0x5555'5555'5555'5555ull)));
+    _Val = static_cast<_Ty>((_Val & static_cast<_Ty>(0x3333'3333'3333'3333ull))
+                            + ((_Val >> 2) & static_cast<_Ty>(0x3333'3333'3333'3333ull)));
+    _Val = static_cast<_Ty>((_Val + (_Val >> 4)) & static_cast<_Ty>(0x0F0F'0F0F'0F0F'0F0Full));
+    for (int _Shift_digits = 8; _Shift_digits < _Digits; _Shift_digits <<= 1) {
+        _Val = static_cast<_Ty>(_Val + static_cast<_Ty>(_Val >> _Shift_digits));
+    }
+    // we want the bottom "slot" that's big enough to store _Digits
+    return static_cast<int>(_Val & static_cast<_Ty>(_Digits + _Digits - 1));
+}
+
+#if defined(_M_IX86) || defined(_M_X64)
+
+// TRANSITION, VS 2019 16.8 Preview 1, intrin0.h will declare __lzcnt* and __popcnt*
+extern "C" {
+__MACHINEX86_X64(unsigned int __lzcnt(unsigned int))
+__MACHINEX86_X64(unsigned short __lzcnt16(unsigned short))
+__MACHINEX64(unsigned __int64 __lzcnt64(unsigned __int64))
+__MACHINEX86_X64(unsigned int __popcnt(unsigned int))
+__MACHINEX86_X64(unsigned short __popcnt16(unsigned short))
+__MACHINEX64(unsigned __int64 __popcnt64(unsigned __int64))
+extern int __isa_available;
+}
+
+template <class _Ty>
+_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept {
+    constexpr int _Digits  = numeric_limits<_Ty>::digits;
+
+#ifndef __AVX2__
+    const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2;
+    // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero
+    // bsr has undefined output for zero
+    if (!_Have_lzcnt && _Val == 0) {
+        return _Digits;
+    }
+#endif // __AVX2__
+
+    // We use lzcnt (actually bsr if lzcnt is not supported) now that we know
+    // we're not zero. We can do this because lzcnt and bsr share the same instruction
+    // encoding.
+    if constexpr (_Digits <= 16) {
+        return static_cast<int>(__lzcnt16(_Val) - (16 - _Digits));
+    } else if constexpr (_Digits == 32) {
+        return static_cast<int>(__lzcnt(_Val));
+    } else {
+#ifdef _M_IX86
+        const unsigned int _High = _Val >> 32;
+        const auto _Low          = static_cast<unsigned int>(_Val);
+        if (_High == 0) {
+            return 32 + _Checked_x86_x64_countl_zero(_Low);
+        } else {
+            return _Checked_x86_x64_countl_zero(_High);
+        }
+#else // ^^^ _M_IX86 / !_M_IX86 vvv
+        return static_cast<int>(__lzcnt64(_Val));
+#endif // _M_IX86
+    }
+    // note: we don't need to call a fallback here because
+    // all supported x86 processors at least have bsr/bsf
+}
+
+template <class _Ty>
+_NODISCARD int _Checked_x86_x64_popcount(const _Ty _Val) noexcept {
+    constexpr int _Digits   = numeric_limits<_Ty>::digits;
+#ifndef __AVX__
+    const bool _Have_popcnt = __isa_available >= __ISA_AVAILABLE_SSE42;
+    if (!_Have_popcnt) {
+        return _Popcount_fallback(_Val);
+    }
+#endif // !defined(__AVX__)
+
+    if constexpr (_Digits <= 16) {
+        return static_cast<int>(__popcnt16(_Val));
+    } else if constexpr (_Digits == 32) {
+        return static_cast<int>(__popcnt(_Val));
+    } else {
+#ifdef _M_IX86
+        return static_cast<int>(__popcnt(_Val >> 32) + __popcnt(static_cast<unsigned int>(_Val)));
+#else // ^^^ _M_IX86 / !_M_IX86 vvv
+        return static_cast<int>(__popcnt64(_Val));
+#endif // _M_IX86
+    }
+}
+#endif // defined(_M_IX86) || defined(_M_X64)
+
+
+#if defined(_M_ARM) || defined(_M_ARM64)
+template <class _Ty>
+_NODISCARD int _Checked_arm_arm64_countl_zero(const _Ty _Val) noexcept {
     constexpr int _Digits = numeric_limits<_Ty>::digits;
     if (_Val == 0) {
         return _Digits;
     }
 
-    if constexpr (sizeof(_Ty) <= sizeof(unsigned int)) {
-        return __builtin_clz(_Val) - (numeric_limits<unsigned int>::digits - _Digits);
+    if constexpr (_Digits <= 32) {
+        return _CountLeadingZeros(_Val);
     } else {
-        return __builtin_clzll(_Val) - (numeric_limits<unsigned long long>::digits - _Digits);
+        return _CountLeadingZeros64(_Val);
+    }
+}
+#endif // defined(_M_ARM) || defined(_M_ARM64)
+
+template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> _Enabled>
+_NODISCARD constexpr int countl_zero(const _Ty _Val) noexcept {
+    if (_STD is_constant_evaluated()) {
+        return _Countl_zero_fallback(_Val);
+    } else {
+#if defined(_M_IX86) || defined(_M_X64)
+        return _Checked_x86_x64_countl_zero(_Val);
+#elif defined(_M_ARM) || defined(_M_ARM64)
+        return _Checked_arm_arm64_countl_zero(_Val);
+#else
+#error Unsupported Hardware
+#endif
     }
 }
 
@@ -125,17 +236,17 @@ _NODISCARD constexpr int countr_one(const _Ty _Val) noexcept {
 
 template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> _Enabled = 0>
 _NODISCARD constexpr int popcount(const _Ty _Val) noexcept {
-    if constexpr (sizeof(_Ty) <= sizeof(unsigned int)) {
-        return __builtin_popcount(_Val);
-    } else {
-        return __builtin_popcountll(_Val);
+#if defined(_M_IX86) || defined(_M_X64)
+    if (!_STD is_constant_evaluated()) {
+        return _Checked_x86_x64_popcount(_Val);
     }
+#endif // defined(_M_IX86) || defined(_M_X64)
+    return _Popcount_fallback(_Val);
 }
-#endif // __cpp_lib_bitops
 
 enum class endian { little = 0, big = 1, native = little };
-_STD_END
 
+_STD_END
 #pragma pop_macro("new")
 _STL_RESTORE_CLANG_WARNINGS
 #pragma warning(pop)

stl/inc/limits

@@ -11,6 +11,8 @@
 #include <cfloat>
 #include <climits>
 #include <cwchar>
+#include <intrin0.h>
+#include <isa_availability.h>
 #include <xstddef>
 
 #pragma pack(push, _CRT_PACKING)
@@ -1009,24 +1011,102 @@ public:
     static constexpr int min_exponent10 = LDBL_MIN_10_EXP;
 };
 
-#ifdef __cpp_lib_bitops // TRANSITION, VSO-1020212
+// Implementation of countl_zero without using specialized CPU instructions.
+// Used at compile time and when said instructions are not supported.
+// see "Hacker's Delight" section 5-3
 template <class _Ty>
-inline constexpr bool _Is_standard_unsigned_integer =
-    _Is_any_of_v<remove_cv_t<_Ty>, unsigned char, unsigned short, unsigned int, unsigned long, unsigned long long>;
+_NODISCARD constexpr int _Countl_zero_fallback(_Ty _Val) noexcept {
+    _Ty _Yy = 0;
+
+    unsigned int _Nn = numeric_limits<_Ty>::digits;
+    unsigned int _Cc = numeric_limits<_Ty>::digits / 2;
+    do {
+        _Yy = static_cast<_Ty>(_Val >> _Cc);
+        if (_Yy != 0) {
+            _Nn -= _Cc;
+            _Val = _Yy;
+        }
+        _Cc >>= 1;
+    } while (_Cc != 0);
+    return static_cast<int>(_Nn) - static_cast<int>(_Val);
+}
 
-template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> = 0>
-_NODISCARD constexpr int _Countr_zero(const _Ty _Val) noexcept {
-    if (_Val == 0) {
-        return numeric_limits<_Ty>::digits;
-    }
+// Implementation of countr_zero without using specialized CPU instructions.
+// Used at compile time and when said instructions are not supported.
+// see "Hacker's Delight" section 5-4
+template <class _Ty>
+_NODISCARD constexpr int _Countr_zero_fallback(const _Ty _Val) noexcept {
+    constexpr int _Digits = std::numeric_limits<_Ty>::digits;
+    return _Digits - _Countl_zero_fallback(static_cast<_Ty>(static_cast<_Ty>(~_Val) & static_cast<_Ty>(_Val - 1)));
+}
+
+#if defined(_M_IX86) || defined(_M_X64)
+
+// TRANSITION, VS 2019 16.8 Preview 1, intrin0.h will declare _tzcnt*
+extern "C" {
+extern int __isa_available;
+#ifdef __clang__
+#define _TZCNT_U32 __builtin_ia32_tzcnt_u32
+#define _TZCNT_U64 __builtin_ia32_tzcnt_u64
+#else // ^^^ __clang__ / !__clang__ vvv
+__MACHINEX86_X64(unsigned int _tzcnt_u32(unsigned int))
+__MACHINEX64(unsigned __int64 _tzcnt_u64(unsigned __int64));
+#define _TZCNT_U32 _tzcnt_u32
+#define _TZCNT_U64 _tzcnt_u64
+#endif // __clang__
+}
 
-    if constexpr (sizeof(_Ty) <= sizeof(unsigned int)) {
-        return __builtin_ctz(_Val);
+template <class _Ty>
+_NODISCARD int _Checked_x86_x64_countr_zero(const _Ty _Val) noexcept {
+    constexpr int _Digits = numeric_limits<_Ty>::digits;
+    constexpr _Ty _Max    = (numeric_limits<_Ty>::max)();
+
+#ifndef __AVX2__
+    const bool _Have_tzcnt = __isa_available >= __ISA_AVAILABLE_AVX2;
+    if (!_Have_tzcnt && _Val == 0) {
+        return _Digits;
+    }
+#endif // __AVX2__
+
+    if constexpr (_Digits <= 32) {
+        // Intended widening to int. This operation means that a narrow 0 will widen
+        // to 0xFFFF....FFFF0... instead of 0. We need this to avoid counting all the zeros
+        // of the wider type.
+        return static_cast<int>(_TZCNT_U32(static_cast<unsigned int>(~_Max | _Val)));
     } else {
-        return __builtin_ctzll(_Val);
+#ifdef _M_IX86
+        const unsigned int _High = _Val >> 32;
+        const unsigned int _Low  = static_cast<unsigned int>(_Val);
+        if (_Low == 0) {
+            return 32 + _Checked_x86_x64_countr_zero(_High);
+        } else {
+            return _Checked_x86_x64_countr_zero(_Low);
+        }
+#else // ^^^ _M_IX86 / !_M_IX86 vvv
+        return static_cast<int>(_TZCNT_U64(_Val));
+#endif // _M_IX86
     }
 }
-#endif // __cpp_lib_bitops
+#undef _TZCNT_U32
+#undef _TZCNT_U64
+#endif // defined(_M_IX86) || defined(_M_X64)
+
+template <class _Ty>
+constexpr bool _Is_standard_unsigned_integer =
+    _Is_any_of_v<remove_cv_t<_Ty>, unsigned char, unsigned short, unsigned int, unsigned long, unsigned long long>;
+
+template <class _Ty, enable_if_t<_Is_standard_unsigned_integer<_Ty>, int> = 0>
+_NODISCARD constexpr int _Countr_zero(const _Ty _Val) noexcept {
+#if defined(_M_IX86) || defined(_M_X64)
+#ifdef __cpp_lib_is_constant_evaluated
+    if (!_STD is_constant_evaluated()) {
+        return _Checked_x86_x64_countr_zero(_Val);
+    }
+#endif // defined(__cpp_lib_is_constant_evaluated)
+#endif // defined(_M_IX86) || defined(_M_X64)
+    // C++17 constexpr gcd() calls this function, so it should be constexpr unless we detect runtime evaluation.
+    return _Countr_zero_fallback(_Val);
+}
 
 _STD_END
 #pragma pop_macro("new")

stl/inc/numeric

@@ -560,26 +560,6 @@ _NODISCARD constexpr auto _Abs_u(const _Arithmetic _Val) noexcept {
     }
 }
 
-// FUNCTION TEMPLATE _Stl_bitscan_forward
-template <class _Unsigned>
-_NODISCARD constexpr unsigned long _Stl_bitscan_forward(_Unsigned _Mask) noexcept {
-#ifdef __cpp_lib_bitops // TRANSITION, VSO-1020212
-    return static_cast<unsigned long>(_Countr_zero(_Mask));
-#else // ^^^ __cpp_lib_bitops / !__cpp_lib_bitops vvv
-    // find the index of the least significant set bit (_BitScanForward isn't constexpr... yet :))
-    static_assert(is_unsigned_v<_Unsigned>, "Bitscan only works on bits");
-    unsigned long _Count = 0;
-    if (_Mask != 0) {
-        while ((_Mask & 1U) == 0) {
-            _Mask >>= 1;
-            ++_Count;
-        }
-    }
-
-    return _Count;
-#endif // !__cpp_lib_bitops
-}
-
 // FUNCTION TEMPLATE gcd
 template <class _Mt, class _Nt>
 _NODISCARD constexpr common_type_t<_Mt, _Nt> gcd(const _Mt _Mx, const _Nt _Nx) noexcept /* strengthened */ {
@@ -598,12 +578,13 @@ _NODISCARD constexpr common_type_t<_Mt, _Nt> gcd(const _Mt _Mx, const _Nt _Nx) n
         return static_cast<_Common>(_Mx_magnitude);
     }
 
-    const auto _Mx_trailing_zeroes  = _Stl_bitscan_forward(_Mx_magnitude);
-    const auto _Common_factors_of_2 = (_STD min)(_Mx_trailing_zeroes, _Stl_bitscan_forward(_Nx_magnitude));
+    const auto _Mx_trailing_zeroes = static_cast<unsigned long>(_Countr_zero(_Mx_magnitude));
+    const auto _Common_factors_of_2 =
+        (_STD min)(_Mx_trailing_zeroes, static_cast<unsigned long>(_Countr_zero(_Nx_magnitude)));
     _Nx_magnitude >>= _Common_factors_of_2;
     _Mx_magnitude >>= _Mx_trailing_zeroes;
     do {
-        _Nx_magnitude >>= _Stl_bitscan_forward(_Nx_magnitude);
+        _Nx_magnitude >>= static_cast<unsigned long>(_Countr_zero(_Nx_magnitude));
         if (_Mx_magnitude > _Nx_magnitude) {
             _Common_unsigned _Temp = _Mx_magnitude;
             _Mx_magnitude          = _Nx_magnitude;

stl/inc/type_traits

@@ -1842,13 +1842,6 @@ inline constexpr bool is_nothrow_invocable_r_v =
     _Select_invoke_traits<_Callable, _Args...>::template _Is_nothrow_invocable_r<_Rx>::value;
 #endif // _HAS_CXX17
 
-#if _HAS_CXX20
-// FUNCTION is_constant_evaluated
-_NODISCARD constexpr bool is_constant_evaluated() noexcept {
-    return __builtin_is_constant_evaluated();
-}
-#endif // _HAS_CXX20
-
 // STRUCT TEMPLATE _Weak_types
 template <class _Ty, class = void>
 struct _Weak_result_type {}; // default definition

stl/inc/xtr1common

@@ -175,6 +175,13 @@ template <class _Ty, class... _Types>
 _INLINE_VAR constexpr bool _Is_any_of_v = // true if and only if _Ty is in _Types
     disjunction_v<is_same<_Ty, _Types>...>;
 
+#if _HAS_CXX20
+// FUNCTION is_constant_evaluated
+_NODISCARD constexpr bool is_constant_evaluated() noexcept {
+    return __builtin_is_constant_evaluated();
+}
+#endif // _HAS_CXX20
+
 // STRUCT TEMPLATE is_integral
 template <class _Ty>
 _INLINE_VAR constexpr bool is_integral_v = _Is_any_of_v<remove_cv_t<_Ty>, bool, char, signed char, unsigned char,