Subject: [Boost-commit] svn:boost r84695 - in trunk: boost/atomic/detail libs/atomic/test
From: andrey.semashev_at_[hidden]
Date: 2013-06-08 13:36:38
Author: andysem
Date: 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)
New Revision: 84695
URL: http://svn.boost.org/trac/boost/changeset/84695
Log:
Added implementation of 64-bit atomic operations on 32-bit Windows. Pointer specializations of 64-bit CAS-based atomic implementation extracted to a separate header, since these specializations are not needed on 32-bit platforms.
Added:
trunk/boost/atomic/detail/cas64strong-ptr.hpp (contents, props changed)
Text files modified:
trunk/boost/atomic/detail/cas64strong-ptr.hpp | 232 ++++++++++++++++++++
trunk/boost/atomic/detail/cas64strong.hpp | 198 -----------------
trunk/boost/atomic/detail/gcc-x86.hpp | 21 +
trunk/boost/atomic/detail/interlocked.hpp | 7
trunk/boost/atomic/detail/windows.hpp | 447 +++++++++++++++++++++++++--------------
trunk/libs/atomic/test/lockfree.cpp | 2
6 files changed, 540 insertions(+), 367 deletions(-)
Added: trunk/boost/atomic/detail/cas64strong-ptr.hpp
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ trunk/boost/atomic/detail/cas64strong-ptr.hpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -0,0 +1,232 @@
+#ifndef BOOST_ATOMIC_DETAIL_CAS64STRONG_PTR_HPP
+#define BOOST_ATOMIC_DETAIL_CAS64STRONG_PTR_HPP
+
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+//
+// Copyright (c) 2011 Helge Bahmann
+// Copyright (c) 2013 Tim Blechmann
+
+// Build 64-bit atomic operation on pointers from platform_cmpxchg64_strong
+// primitive. It is assumed that 64-bit loads/stores are not
+// atomic, so they are implemented through platform_load64/platform_store64.
+//
+// The reason for extracting pointer specializations to a separate header is
+// that 64-bit CAS is available on some 32-bit platforms (notably, x86).
+// On these platforms there is no need for 64-bit pointer specializations,
+// since they will never be used.
+
+#include <cstddef>
+#include <boost/cstdint.hpp>
+#include <boost/memory_order.hpp>
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/base.hpp>
+
+#ifdef BOOST_ATOMIC_HAS_PRAGMA_ONCE
+#pragma once
+#endif
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+/* pointer types */
+
+template<bool Sign>
+class base_atomic<void *, void *, 8, Sign>
+{
+ typedef base_atomic this_type;
+ typedef void * value_type;
+ typedef ptrdiff_t difference_type;
+public:
+ BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
+ base_atomic(void) {}
+
+ void
+ store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before_store(order);
+ platform_store64(v, &v_);
+ platform_fence_after_store(order);
+ }
+
+ value_type
+ load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+ {
+ value_type v = platform_load64(&v_);
+ platform_fence_after_load(order);
+ return v;
+ }
+
+ value_type
+ exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
+ return original;
+ }
+
+ bool
+ compare_exchange_weak(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ return compare_exchange_strong(expected, desired, success_order, failure_order);
+ }
+
+ bool
+ compare_exchange_strong(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before(success_order);
+
+ bool success = platform_cmpxchg64_strong(expected, desired, &v_);
+
+ if (success) {
+ platform_fence_after(success_order);
+ } else {
+ platform_fence_after(failure_order);
+ }
+
+ return success;
+ }
+
+ value_type
+ fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, (char*)original + v, order, memory_order_relaxed));
+ return original;
+ }
+
+ value_type
+ fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, (char*)original - v, order, memory_order_relaxed));
+ return original;
+ }
+
+ bool
+ is_lock_free(void) const volatile BOOST_NOEXCEPT
+ {
+ return true;
+ }
+
+ BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
+private:
+ base_atomic(const base_atomic &) /* = delete */ ;
+ void operator=(const base_atomic &) /* = delete */ ;
+ value_type v_;
+};
+
+template<typename T, bool Sign>
+class base_atomic<T *, void *, 8, Sign>
+{
+ typedef base_atomic this_type;
+ typedef T * value_type;
+ typedef ptrdiff_t difference_type;
+public:
+ BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
+ base_atomic(void) {}
+
+ void
+ store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before_store(order);
+ platform_store64(v, &v_);
+ platform_fence_after_store(order);
+ }
+
+ value_type
+ load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+ {
+ value_type v = platform_load64(&v_);
+ platform_fence_after_load(order);
+ return v;
+ }
+
+ value_type
+ exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
+ return original;
+ }
+
+ bool
+ compare_exchange_weak(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ return compare_exchange_strong(expected, desired, success_order, failure_order);
+ }
+
+ bool
+ compare_exchange_strong(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before(success_order);
+
+ bool success = platform_cmpxchg64_strong(expected, desired, &v_);
+
+ if (success) {
+ platform_fence_after(success_order);
+ } else {
+ platform_fence_after(failure_order);
+ }
+
+ return success;
+ }
+
+ value_type
+ fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, original + v, order, memory_order_relaxed));
+ return original;
+ }
+
+ value_type
+ fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ value_type original = load(memory_order_relaxed);
+ do {
+ } while (!compare_exchange_weak(original, original - v, order, memory_order_relaxed));
+ return original;
+ }
+
+ bool
+ is_lock_free(void) const volatile BOOST_NOEXCEPT
+ {
+ return true;
+ }
+
+ BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
+private:
+ base_atomic(const base_atomic &) /* = delete */ ;
+ void operator=(const base_atomic &) /* = delete */ ;
+ value_type v_;
+};
+
+}
+}
+}
+
+#endif
Modified: trunk/boost/atomic/detail/cas64strong.hpp
==============================================================================
--- trunk/boost/atomic/detail/cas64strong.hpp Sat Jun 8 13:24:33 2013 (r84694)
+++ trunk/boost/atomic/detail/cas64strong.hpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -8,9 +8,9 @@
// Copyright (c) 2011 Helge Bahmann
// Copyright (c) 2013 Tim Blechmann
-// Build 64-bit atomic operation from platform_cmpxchg64_strong
+// Build 64-bit atomic operation on integers/UDTs from platform_cmpxchg64_strong
// primitive. It is assumed that 64-bit loads/stores are not
-// atomic, so they are funnelled through cmpxchg as well.
+// atomic, so they are implemented through platform_load64/platform_store64.
#include <cstddef>
#include <boost/cstdint.hpp>
@@ -151,200 +151,6 @@
value_type v_;
};
-/* pointer types */
-
-template<bool Sign>
-class base_atomic<void *, void *, 8, Sign>
-{
- typedef base_atomic this_type;
- typedef void * value_type;
- typedef ptrdiff_t difference_type;
-public:
- BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
- base_atomic(void) {}
-
- void
- store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- platform_fence_before_store(order);
- platform_store64(v, &v_);
- platform_fence_after_store(order);
- }
-
- value_type
- load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
- {
- value_type v = platform_load64(&v_);
- platform_fence_after_load(order);
- return v;
- }
-
- value_type
- exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
- return original;
- }
-
- bool
- compare_exchange_weak(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- return compare_exchange_strong(expected, desired, success_order, failure_order);
- }
-
- bool
- compare_exchange_strong(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- platform_fence_before(success_order);
-
- bool success = platform_cmpxchg64_strong(expected, desired, &v_);
-
- if (success) {
- platform_fence_after(success_order);
- } else {
- platform_fence_after(failure_order);
- }
-
- return success;
- }
-
- value_type
- fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, (char*)original + v, order, memory_order_relaxed));
- return original;
- }
-
- value_type
- fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, (char*)original - v, order, memory_order_relaxed));
- return original;
- }
-
- bool
- is_lock_free(void) const volatile BOOST_NOEXCEPT
- {
- return true;
- }
-
- BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
-private:
- base_atomic(const base_atomic &) /* = delete */ ;
- void operator=(const base_atomic &) /* = delete */ ;
- value_type v_;
-};
-
-template<typename T, bool Sign>
-class base_atomic<T *, void *, 8, Sign>
-{
- typedef base_atomic this_type;
- typedef T * value_type;
- typedef ptrdiff_t difference_type;
-public:
- BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
- base_atomic(void) {}
-
- void
- store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- platform_fence_before_store(order);
- platform_store64(v, &v_);
- platform_fence_after_store(order);
- }
-
- value_type
- load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
- {
- value_type v = platform_load64(&v_);
- platform_fence_after_load(order);
- return v;
- }
-
- value_type
- exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
- return original;
- }
-
- bool
- compare_exchange_weak(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- return compare_exchange_strong(expected, desired, success_order, failure_order);
- }
-
- bool
- compare_exchange_strong(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- platform_fence_before(success_order);
-
- bool success = platform_cmpxchg64_strong(expected, desired, &v_);
-
- if (success) {
- platform_fence_after(success_order);
- } else {
- platform_fence_after(failure_order);
- }
-
- return success;
- }
-
- value_type
- fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, original + v, order, memory_order_relaxed));
- return original;
- }
-
- value_type
- fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- value_type original = load(memory_order_relaxed);
- do {
- } while (!compare_exchange_weak(original, original - v, order, memory_order_relaxed));
- return original;
- }
-
- bool
- is_lock_free(void) const volatile BOOST_NOEXCEPT
- {
- return true;
- }
-
- BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
-private:
- base_atomic(const base_atomic &) /* = delete */ ;
- void operator=(const base_atomic &) /* = delete */ ;
- value_type v_;
-};
-
/* generic types */
template<typename T, bool Sign>
Modified: trunk/boost/atomic/detail/gcc-x86.hpp
==============================================================================
--- trunk/boost/atomic/detail/gcc-x86.hpp Sat Jun 8 13:24:33 2013 (r84694)
+++ trunk/boost/atomic/detail/gcc-x86.hpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -1655,14 +1655,16 @@
In theory, could push/pop ebx onto/off the stack, but movs
to a prepared stack slot turn out to be faster. */
- __asm__ __volatile__ (
+ __asm__ __volatile__
+ (
"movl %%ebx, %1\n"
"movl %2, %%ebx\n"
"lock; cmpxchg8b 0(%4)\n"
"movl %1, %%ebx\n"
: "=A" (prev), "=m" (scratch)
: "D" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)), "S" (ptr), "0" (prev)
- : "memory");
+ : "memory"
+ );
bool success = (prev == expected);
expected = prev;
return success;
@@ -1686,11 +1688,11 @@
#if defined(__SSE2__)
__asm__ __volatile__
(
- "movq %1, %%xmm0\n\t"
- "movq %%xmm0, %0\n\t"
+ "movq %1, %%xmm4\n\t"
+ "movq %%xmm4, %0\n\t"
: "=m" (*ptr)
: "m" (value)
- : "memory", "xmm0"
+ : "memory", "xmm4"
);
#else
__asm__ __volatile__
@@ -1717,18 +1719,18 @@
inline T
platform_load64(const volatile T * ptr) BOOST_NOEXCEPT
{
- T value = T();
+ T value;
if (((uint32_t)ptr & 0x00000007) == 0)
{
#if defined(__SSE2__)
__asm__ __volatile__
(
- "movq %1, %%xmm0\n\t"
- "movq %%xmm0, %0\n\t"
+ "movq %1, %%xmm4\n\t"
+ "movq %%xmm4, %0\n\t"
: "=m" (value)
: "m" (*ptr)
- : "memory", "xmm0"
+ : "memory", "xmm4"
);
#else
__asm__ __volatile__
@@ -1743,6 +1745,7 @@
}
else
{
+ value = T();
// We don't care for comparison result here; the previous value will be stored into value anyway.
platform_cmpxchg64_strong(value, value, const_cast<volatile T*>(ptr));
}
Modified: trunk/boost/atomic/detail/interlocked.hpp
==============================================================================
--- trunk/boost/atomic/detail/interlocked.hpp Sat Jun 8 13:24:33 2013 (r84694)
+++ trunk/boost/atomic/detail/interlocked.hpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -43,6 +43,11 @@
#define BOOST_ATOMIC_INTERLOCKED_OR(dest, arg) _InterlockedOr((long*)(dest), (long)(arg))
#define BOOST_ATOMIC_INTERLOCKED_XOR(dest, arg) _InterlockedXor((long*)(dest), (long)(arg))
+#if (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
+#pragma intrinsic(_InterlockedCompareExchange64)
+#define BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(dest, exchange, compare) _InterlockedCompareExchange64((__int64*)(dest), (__int64)(exchange), (__int64)(compare))
+#endif
+
#if _MSC_VER >= 1600
// MSVC 2010 and later provide intrinsics for 8 and 16 bit integers.
@@ -81,14 +86,12 @@
#if defined(_M_AMD64) || defined(_M_IA64)
-#pragma intrinsic(_InterlockedCompareExchange64)
#pragma intrinsic(_InterlockedExchangeAdd64)
#pragma intrinsic(_InterlockedExchange64)
#pragma intrinsic(_InterlockedAnd64)
#pragma intrinsic(_InterlockedOr64)
#pragma intrinsic(_InterlockedXor64)
-#define BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(dest, exchange, compare) _InterlockedCompareExchange64((__int64*)(dest), (__int64)(exchange), (__int64)(compare))
#define BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(dest, addend) _InterlockedExchangeAdd64((__int64*)(dest), (__int64)(addend))
#define BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(dest, newval) _InterlockedExchange64((__int64*)(dest), (__int64)(newval))
#define BOOST_ATOMIC_INTERLOCKED_AND64(dest, arg) _InterlockedAnd64((__int64*)(dest), (__int64)(arg))
Modified: trunk/boost/atomic/detail/windows.hpp
==============================================================================
--- trunk/boost/atomic/detail/windows.hpp Sat Jun 8 13:24:33 2013 (r84694)
+++ trunk/boost/atomic/detail/windows.hpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -179,7 +179,7 @@
#define BOOST_ATOMIC_SHORT_LOCK_FREE 2
#define BOOST_ATOMIC_INT_LOCK_FREE 2
#define BOOST_ATOMIC_LONG_LOCK_FREE 2
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#if (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
#define BOOST_ATOMIC_LLONG_LOCK_FREE 2
#else
#define BOOST_ATOMIC_LLONG_LOCK_FREE 0
@@ -716,161 +716,6 @@
storage_type v_;
};
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
-
-template<typename T, bool Sign>
-class base_atomic<T, int, 8, Sign>
-{
- typedef base_atomic this_type;
- typedef T value_type;
- typedef value_type storage_type;
- typedef T difference_type;
-public:
- BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT: v_(v) {}
- base_atomic(void) {}
-
- void
- store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- if (order != memory_order_seq_cst) {
- platform_fence_before(order);
- v_ = static_cast< storage_type >(v);
- } else {
- exchange(v, order);
- }
- }
-
- value_type
- load(memory_order order = memory_order_seq_cst)const volatile BOOST_NOEXCEPT
- {
- value_type v = static_cast< value_type >(v_);
- platform_fence_after_load(order);
- return v;
- }
-
- value_type
- fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- platform_fence_before(order);
- v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&v_, v));
- platform_fence_after(order);
- return v;
- }
-
- value_type
- fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- typedef typename make_signed< value_type >::type signed_value_type;
- return fetch_add(static_cast< value_type >(-static_cast< signed_value_type >(v)), order);
- }
-
- value_type
- exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
- platform_fence_before(order);
- v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&v_, v));
- platform_fence_after(order);
- return v;
- }
-
- bool
- compare_exchange_strong(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- value_type previous = expected;
- platform_fence_before(success_order);
- value_type oldval = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&v_, desired, previous));
- bool success = (previous == oldval);
- if (success)
- platform_fence_after(success_order);
- else
- platform_fence_after(failure_order);
- expected = oldval;
- return success;
- }
-
- bool
- compare_exchange_weak(
- value_type & expected,
- value_type desired,
- memory_order success_order,
- memory_order failure_order) volatile BOOST_NOEXCEPT
- {
- return compare_exchange_strong(expected, desired, success_order, failure_order);
- }
-
- value_type
- fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
-#if defined(BOOST_ATOMIC_INTERLOCKED_AND64)
- platform_fence_before(order);
- v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&v_, v));
- platform_fence_after(order);
- return v;
-#else
- value_type tmp = load(memory_order_relaxed);
- for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
- {
- BOOST_ATOMIC_X86_PAUSE();
- }
- return tmp;
-#endif
- }
-
- value_type
- fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
-#if defined(BOOST_ATOMIC_INTERLOCKED_OR64)
- platform_fence_before(order);
- v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&v_, v));
- platform_fence_after(order);
- return v;
-#else
- value_type tmp = load(memory_order_relaxed);
- for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
- {
- BOOST_ATOMIC_X86_PAUSE();
- }
- return tmp;
-#endif
- }
-
- value_type
- fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
- {
-#if defined(BOOST_ATOMIC_INTERLOCKED_XOR64)
- platform_fence_before(order);
- v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&v_, v));
- platform_fence_after(order);
- return v;
-#else
- value_type tmp = load(memory_order_relaxed);
- for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
- {
- BOOST_ATOMIC_X86_PAUSE();
- }
- return tmp;
-#endif
- }
-
- bool
- is_lock_free(void)const volatile BOOST_NOEXCEPT
- {
- return true;
- }
-
- BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
-private:
- base_atomic(const base_atomic &) /* = delete */ ;
- void operator=(const base_atomic &) /* = delete */ ;
- storage_type v_;
-};
-
-#endif // defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
-
// MSVC 2012 fails to recognize sizeof(T) as a constant expression in template specializations
enum msvc_sizeof_pointer_workaround { sizeof_pointer = sizeof(void*) };
@@ -1362,7 +1207,7 @@
}
bool
- is_lock_free(void)const volatile BOOST_NOEXCEPT
+ is_lock_free(void) const volatile BOOST_NOEXCEPT
{
return true;
}
@@ -1374,7 +1219,158 @@
storage_type v_;
};
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#if defined(_M_AMD64) || defined(_M_IA64)
+
+template<typename T, bool Sign>
+class base_atomic<T, int, 8, Sign>
+{
+ typedef base_atomic this_type;
+ typedef T value_type;
+ typedef value_type storage_type;
+ typedef T difference_type;
+public:
+ BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT: v_(v) {}
+ base_atomic(void) {}
+
+ void
+ store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ if (order != memory_order_seq_cst) {
+ platform_fence_before(order);
+ v_ = static_cast< storage_type >(v);
+ } else {
+ exchange(v, order);
+ }
+ }
+
+ value_type
+ load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+ {
+ value_type v = static_cast< value_type >(v_);
+ platform_fence_after_load(order);
+ return v;
+ }
+
+ value_type
+ fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before(order);
+ v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&v_, v));
+ platform_fence_after(order);
+ return v;
+ }
+
+ value_type
+ fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ typedef typename make_signed< value_type >::type signed_value_type;
+ return fetch_add(static_cast< value_type >(-static_cast< signed_value_type >(v)), order);
+ }
+
+ value_type
+ exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+ platform_fence_before(order);
+ v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&v_, v));
+ platform_fence_after(order);
+ return v;
+ }
+
+ bool
+ compare_exchange_strong(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ value_type previous = expected;
+ platform_fence_before(success_order);
+ value_type oldval = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&v_, desired, previous));
+ bool success = (previous == oldval);
+ if (success)
+ platform_fence_after(success_order);
+ else
+ platform_fence_after(failure_order);
+ expected = oldval;
+ return success;
+ }
+
+ bool
+ compare_exchange_weak(
+ value_type & expected,
+ value_type desired,
+ memory_order success_order,
+ memory_order failure_order) volatile BOOST_NOEXCEPT
+ {
+ return compare_exchange_strong(expected, desired, success_order, failure_order);
+ }
+
+ value_type
+ fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+#if defined(BOOST_ATOMIC_INTERLOCKED_AND64)
+ platform_fence_before(order);
+ v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&v_, v));
+ platform_fence_after(order);
+ return v;
+#else
+ value_type tmp = load(memory_order_relaxed);
+ for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+ {
+ BOOST_ATOMIC_X86_PAUSE();
+ }
+ return tmp;
+#endif
+ }
+
+ value_type
+ fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+#if defined(BOOST_ATOMIC_INTERLOCKED_OR64)
+ platform_fence_before(order);
+ v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&v_, v));
+ platform_fence_after(order);
+ return v;
+#else
+ value_type tmp = load(memory_order_relaxed);
+ for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+ {
+ BOOST_ATOMIC_X86_PAUSE();
+ }
+ return tmp;
+#endif
+ }
+
+ value_type
+ fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+ {
+#if defined(BOOST_ATOMIC_INTERLOCKED_XOR64)
+ platform_fence_before(order);
+ v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&v_, v));
+ platform_fence_after(order);
+ return v;
+#else
+ value_type tmp = load(memory_order_relaxed);
+ for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+ {
+ BOOST_ATOMIC_X86_PAUSE();
+ }
+ return tmp;
+#endif
+ }
+
+ bool
+ is_lock_free(void)const volatile BOOST_NOEXCEPT
+ {
+ return true;
+ }
+
+ BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
+private:
+ base_atomic(const base_atomic &) /* = delete */ ;
+ void operator=(const base_atomic &) /* = delete */ ;
+ storage_type v_;
+};
template<typename T, bool Sign>
class base_atomic<T, void, 8, Sign>
@@ -1469,12 +1465,145 @@
storage_type v_;
};
-#endif // defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#elif defined(_M_IX86) && _M_IX86 >= 500
+
+template<typename T>
+inline bool
+platform_cmpxchg64_strong(T & expected, T desired, volatile T * p) BOOST_NOEXCEPT
+{
+#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+ const T oldval = BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(p, desired, expected);
+ const bool result = (oldval == expected);
+ expected = oldval;
+ return result;
+#else
+ bool result;
+ __asm
+ {
+ mov edi, p
+ mov esi, expected
+ mov ebx, dword ptr [desired]
+ mov ecx, dword ptr [desired + 4]
+ mov eax, dword ptr [esi]
+ mov edx, dword ptr [esi + 4]
+ lock cmpxchg8b qword ptr [edi]
+ mov dword ptr [esi], eax
+ mov dword ptr [esi + 4], edx
+#if _M_IX86 >= 600
+ sete result
+#else
+ mov al, 0
+ jne not_equal_label
+ mov al, 1
+not_equal_label:
+ mov result, al
+#endif
+ };
+ return result;
+#endif
+}
+
+// Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations:
+//
+// The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically:
+// * Reading or writing a quadword aligned on a 64-bit boundary
+//
+// Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations
+// have at least 8 byte alignment. The only unfortunate case is when atomic is placeod on the stack and it is not 8-byte aligned (like on 32 bit Windows).
+
+template<typename T>
+inline void
+platform_store64(T value, volatile T * p) BOOST_NOEXCEPT
+{
+ if (((uint32_t)p & 0x00000007) == 0)
+ {
+#if defined(_M_IX86_FP) && _M_IX86_FP >= 2
+ __asm
+ {
+ mov edx, p
+ movq xmm4, value
+ movq qword ptr [edx], xmm4
+ };
+#else
+ __asm
+ {
+ mov edx, p
+ fild value
+ fistp qword ptr [edx]
+ };
+#endif
+ }
+ else
+ {
+ __asm
+ {
+ mov edi, p
+ mov ebx, dword ptr [value]
+ mov ecx, dword ptr [value + 4]
+ mov eax, dword ptr [edi]
+ mov edx, dword ptr [edi + 4]
+ align 16
+again:
+ lock cmpxchg8b qword ptr [edi]
+ jne again
+ };
+ }
+}
+
+template<typename T>
+inline T
+platform_load64(const volatile T * p) BOOST_NOEXCEPT
+{
+ T value;
+
+ if (((uint32_t)p & 0x00000007) == 0)
+ {
+#if defined(_M_IX86_FP) && _M_IX86_FP >= 2
+ __asm
+ {
+ mov edx, p
+ movq xmm4, qword ptr [edx]
+ movq value, xmm4
+ };
+#else
+ __asm
+ {
+ mov edx, p
+ fild qword ptr [edx]
+ fistp value
+ };
+#endif
+ }
+ else
+ {
+ // We don't care for comparison result here; the previous value will be stored into value anyway.
+ __asm
+ {
+ mov edi, p
+ xor ebx, ebx
+ xor ecx, ecx
+ xor eax, eax
+ xor edx, edx
+ lock cmpxchg8b qword ptr [edi]
+ mov dword ptr [value], eax
+ mov dword ptr [value + 4], edx
+ };
+ }
+
+ return value;
+}
+
+#endif
} // namespace detail
} // namespace atomics
} // namespace boost
+/* pull in 64-bit atomic type using cmpxchg8b above */
+#if defined(_M_IX86) && _M_IX86 >= 500
+#include <boost/atomic/detail/cas64strong.hpp>
+#endif
+
#endif /* !defined(BOOST_ATOMIC_FORCE_FALLBACK) */
#ifdef _MSC_VER
Modified: trunk/libs/atomic/test/lockfree.cpp
==============================================================================
--- trunk/libs/atomic/test/lockfree.cpp Sat Jun 8 13:24:33 2013 (r84694)
+++ trunk/libs/atomic/test/lockfree.cpp 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013) (r84695)
@@ -129,7 +129,7 @@
#define EXPECT_SHORT_LOCK_FREE 2
#define EXPECT_INT_LOCK_FREE 2
#define EXPECT_LONG_LOCK_FREE 2
-#if defined(_WIN64)
+#if defined(_WIN64) || (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
#define EXPECT_LLONG_LOCK_FREE 2
#else
#define EXPECT_LLONG_LOCK_FREE 0