Fix missed vectorization of int128_t bitwise ops

include/boost/int128/detail/int128_imp.hpp

@@ -972,13 +972,65 @@ BOOST_INT128_HOST_DEVICE constexpr std::strong_ordering operator<=>(const Unsign
 
 #endif
 
+//=====================================
+// Bitwise Operator Helpers
+//=====================================
+
+#if !defined(BOOST_INT128_NO_CONSTEVAL_DETECTION) && !(defined(__CUDACC__) && defined(BOOST_INT128_ENABLE_CUDA))
+
+#define BOOST_INT128_HAS_BITWISE_WORD_PATH
+
+namespace detail {
+
+struct bitwise_words
+{
+    std::uint64_t first;
+    std::uint64_t second;
+};
+
+// These must be force inlined. Clang leaves the plain inline form out of line for some of
+// the operators, and the call boundary is enough to stop the loop vectorizer.
+BOOST_INT128_FORCE_INLINE bitwise_words to_bitwise_words(const int128_t& value) noexcept
+{
+    bitwise_words words {};
+    std::memcpy(&words, &value, sizeof(words));
+    return words;
+}
+
+BOOST_INT128_FORCE_INLINE int128_t from_bitwise_words(const bitwise_words words) noexcept
+{
+    int128_t value {};
+    std::memcpy(&value, &words, sizeof(value));
+    return value;
+}
+
+} // namespace detail
+
+#endif // BOOST_INT128_HAS_BITWISE_WORD_PATH
+
 //=====================================
 // Not Operator
 //=====================================
 
 BOOST_INT128_EXPORT BOOST_INT128_HOST_DEVICE constexpr int128_t operator~(const int128_t rhs) noexcept
 {
+    #ifdef BOOST_INT128_HAS_BITWISE_WORD_PATH
+
+    if (BOOST_INT128_IS_CONSTANT_EVALUATED(rhs))
+    {
+        return {~rhs.high, ~rhs.low};
+    }
+    else
+    {
+        const auto words {detail::to_bitwise_words(rhs)};
+        return detail::from_bitwise_words({~words.first, ~words.second});
+    }
+
+    #else
+
     return {~rhs.high, ~rhs.low};
+
+    #endif
 }
 
 //=====================================
@@ -987,7 +1039,24 @@ BOOST_INT128_EXPORT BOOST_INT128_HOST_DEVICE constexpr int128_t operator~(const
 
 BOOST_INT128_EXPORT BOOST_INT128_HOST_DEVICE constexpr int128_t operator|(const int128_t lhs, const int128_t rhs) noexcept
 {
+    #ifdef BOOST_INT128_HAS_BITWISE_WORD_PATH
+
+    if (BOOST_INT128_IS_CONSTANT_EVALUATED(lhs))
+    {
+        return {lhs.high | rhs.high, lhs.low | rhs.low};
+    }
+    else
+    {
+        const auto l {detail::to_bitwise_words(lhs)};
+        const auto r {detail::to_bitwise_words(rhs)};
+        return detail::from_bitwise_words({l.first | r.first, l.second | r.second});
+    }
+
+    #else
+
     return {lhs.high | rhs.high, lhs.low | rhs.low};
+
+    #endif
 }
 
 BOOST_INT128_EXPORT template <BOOST_INT128_DEFAULTED_SIGNED_INTEGER_CONCEPT>
@@ -1063,7 +1132,24 @@ BOOST_INT128_HOST_DEVICE inline int128_t& int128_t::operator|=(const Integer rhs
 
 BOOST_INT128_EXPORT BOOST_INT128_HOST_DEVICE constexpr int128_t operator&(const int128_t lhs, const int128_t rhs) noexcept
 {
+    #ifdef BOOST_INT128_HAS_BITWISE_WORD_PATH
+
+    if (BOOST_INT128_IS_CONSTANT_EVALUATED(lhs))
+    {
+        return {lhs.high & rhs.high, lhs.low & rhs.low};
+    }
+    else
+    {
+        const auto l {detail::to_bitwise_words(lhs)};
+        const auto r {detail::to_bitwise_words(rhs)};
+        return detail::from_bitwise_words({l.first & r.first, l.second & r.second});
+    }
+
+    #else
+
     return {lhs.high & rhs.high, lhs.low & rhs.low};
+
+    #endif
 }
 
 BOOST_INT128_EXPORT template <BOOST_INT128_DEFAULTED_SIGNED_INTEGER_CONCEPT>
@@ -1139,7 +1225,24 @@ BOOST_INT128_HOST_DEVICE constexpr int128_t& int128_t::operator&=(const int128_t
 
 BOOST_INT128_EXPORT BOOST_INT128_HOST_DEVICE constexpr int128_t operator^(const int128_t lhs, const int128_t rhs) noexcept
 {
+    #ifdef BOOST_INT128_HAS_BITWISE_WORD_PATH
+
+    if (BOOST_INT128_IS_CONSTANT_EVALUATED(lhs))
+    {
+        return {lhs.high ^ rhs.high, lhs.low ^ rhs.low};
+    }
+    else
+    {
+        const auto l {detail::to_bitwise_words(lhs)};
+        const auto r {detail::to_bitwise_words(rhs)};
+        return detail::from_bitwise_words({l.first ^ r.first, l.second ^ r.second});
+    }
+
+    #else
+
     return {lhs.high ^ rhs.high, lhs.low ^ rhs.low};
+
+    #endif
 }
 
 BOOST_INT128_EXPORT template <BOOST_INT128_DEFAULTED_SIGNED_INTEGER_CONCEPT>