libitm: Conversion to c++11 atomics.

* local_atomic: New file.
        * libitm_i.h: Include it.
        (gtm_thread::shared_state): Use atomic template.
        * beginend.cc (GTM::gtm_clock): Use atomic template.
        (global_tid): Use atomic template if 64-bit atomics available.
        (gtm_thread::gtm_thread): Update shared_state access.
        (gtm_thread::trycommit): Likewise.
        (choose_code_path): Update global_tid access.
        * method-gl.cc (gl_mg::orec): Use atomic template.  Update all users.
        * stmlock.h (GTM::gtm_clock): Use atomic template.
        (gtm_get_clock, gtm_inc_clock): Update accesses.
        * config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
        redundant __sync_synchronize after atomic shared_state access.
        * config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
        (gtm_rwlock::write_lock_generic): Likewise.
        (gtm_rwlock::read_unlock): Likewise.
        * config/alpha/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.
        * config/x86/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.

From-SVN: r182294

libitm/ChangeLog

@@ -1,3 +1,26 @@
+2011-12-13  Richard Henderson  <rth@redhat.com>
+
+	* local_atomic: New file.
+	* libitm_i.h: Include it.
+	(gtm_thread::shared_state): Use atomic template.
+	* beginend.cc (GTM::gtm_clock): Use atomic template.
+	(global_tid): Use atomic template if 64-bit atomics available.
+	(gtm_thread::gtm_thread): Update shared_state access.
+	(gtm_thread::trycommit): Likewise.
+	(choose_code_path): Update global_tid access.
+	* method-gl.cc (gl_mg::orec): Use atomic template.  Update all users.
+	* stmlock.h (GTM::gtm_clock): Use atomic template.
+	(gtm_get_clock, gtm_inc_clock): Update accesses.
+	* config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
+	redundant __sync_synchronize after atomic shared_state access.
+	* config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
+	(gtm_rwlock::write_lock_generic): Likewise.
+	(gtm_rwlock::read_unlock): Likewise.
+	* config/alpha/target.h (atomic_read_barrier): Remove.
+	(atomic_write_barrier): Remove.
+	* config/x86/target.h (atomic_read_barrier): Remove.
+	(atomic_write_barrier): Remove.
+
 2011-11-30  Richard Henderson  <rth@redhat.com>
 
 	* libitm_i.h (GTM_longjmp): Swap first and second arguments.

libitm/beginend.cc

@@ -37,12 +37,18 @@ gtm_thread *GTM::gtm_thread::list_of_threads = 0;
 unsigned GTM::gtm_thread::number_of_threads = 0;
 
 gtm_stmlock GTM::gtm_stmlock_array[LOCK_ARRAY_SIZE];
-gtm_version GTM::gtm_clock;
+atomic<gtm_version> GTM::gtm_clock;
 
 /* ??? Move elsewhere when we figure out library initialization.  */
 uint64_t GTM::gtm_spin_count_var = 1000;
 
+#ifdef HAVE_64BIT_SYNC_BUILTINS
+static atomic<_ITM_transactionId_t> global_tid;
+#else
 static _ITM_transactionId_t global_tid;
+static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
 
 // Provides a on-thread-exit callback used to release per-thread data.
 static pthread_key_t thr_release_key;
@@ -114,7 +120,7 @@ GTM::gtm_thread::gtm_thread ()
   // This object's memory has been set to zero by operator new, so no need
   // to initialize any of the other primitive-type members that do not have
   // constructors.
-  shared_state = ~(typeof shared_state)0;
+  shared_state.store(-1, memory_order_relaxed);
 
   // Register this transaction with the list of all threads' transactions.
   serial_lock.write_lock ();
@@ -132,13 +138,8 @@ GTM::gtm_thread::gtm_thread ()
     GTM_fatal("Setting thread release TLS key failed.");
 }
 
-
+static inline uint32_t
-
+choose_code_path(uint32_t prop, abi_dispatch *disp)
-#ifndef HAVE_64BIT_SYNC_BUILTINS
-static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
-#endif
-
-static inline uint32_t choose_code_path(uint32_t prop, abi_dispatch *disp)
 {
   if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
     return a_runUninstrumentedCode;
@@ -258,7 +259,7 @@ GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
   else
     {
 #ifdef HAVE_64BIT_SYNC_BUILTINS
-      tx->id = __sync_add_and_fetch (&global_tid, tid_block_size);
+      tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
       tx->local_tid = tx->id + 1;
 #else
       pthread_mutex_lock (&global_tid_lock);
@@ -480,7 +481,7 @@ GTM::gtm_thread::trycommit ()
 	      it = it->next_thread)
 	    {
 	      if (it == this) continue;
-	      while (it->shared_state < priv_time)
+	      while (it->shared_state.load(memory_order_relaxed) < priv_time)
 		cpu_relax();
 	    }
 	}

libitm/config/alpha/target.h

@@ -45,16 +45,4 @@ cpu_relax (void)
   __asm volatile ("" : : : "memory");
 }
 
-static inline void
-atomic_read_barrier (void)
-{
-  __sync_synchronize ();
-}
-
-static inline void
-atomic_write_barrier (void)
-{
-  __asm volatile ("wmb" : : : "memory");
-}
-
 } // namespace GTM

libitm/config/linux/rwlock.cc

@@ -36,10 +36,9 @@ gtm_rwlock::read_lock (gtm_thread *tx)
   for (;;)
     {
       // Fast path: first announce our intent to read, then check for
-      // conflicting intents to write. The barrier makes sure that this
+      // conflicting intents to write.  Note that direct assignment to
-      // happens in exactly this order.
+      // an atomic object is memory_order_seq_cst.
       tx->shared_state = 0;
-      __sync_synchronize();
       if (likely(writers == 0))
 	return;
 
@@ -51,8 +50,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
       // We need the barrier here for the same reason that we need it in
       // read_unlock().
       // TODO Potentially too many wake-ups. See comments in read_unlock().
-      tx->shared_state = ~(typeof tx->shared_state)0;
+      tx->shared_state = -1;
-      __sync_synchronize();
       if (writer_readers > 0)
 	{
 	  writer_readers = 0;
@@ -71,7 +69,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
 	  // are no writers anymore after the barrier because this pending
 	  // store could then lead to lost wake-ups at other readers.
 	  readers = 1;
-	  __sync_synchronize();
+	  atomic_thread_fence(memory_order_acq_rel);
 	  if (writers)
 	    futex_wait(&readers, 1);
 	}

libitm/config/posix/rwlock.cc

@@ -53,10 +53,9 @@ void
 gtm_rwlock::read_lock (gtm_thread *tx)
 {
   // Fast path: first announce our intent to read, then check for conflicting
-  // intents to write. The barrier makes sure that this happens in exactly
+  // intents to write.  Note that direct assignment to an atomic object
-  // this order.
+  // is memory_order_seq_cst.
   tx->shared_state = 0;
-  __sync_synchronize();
   unsigned int sum = this->summary;
   if (likely(!(sum & (a_writer | w_writer))))
     return;
@@ -69,7 +68,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
   // to happen before we leave the slow path and before we wait for any
   // writer).
   // ??? Add a barrier to enforce early visibility of this?
-  tx->shared_state = ~(typeof tx->shared_state)0;
+  tx->shared_state.store(-1, memory_order_relaxed);
 
   pthread_mutex_lock (&this->mutex);
 
@@ -101,7 +100,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
     }
 
   // Otherwise we can acquire the lock for read.
-  tx->shared_state = 0;
+  tx->shared_state.store(0, memory_order_relaxed);
 
   pthread_mutex_unlock(&this->mutex);
 }
@@ -153,11 +152,11 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
   // sure that we first set our write intent and check for active readers
   // after that, in strictly this order (similar to the barrier in the fast
   // path of read_lock()).
-  __sync_synchronize();
+  atomic_thread_fence(memory_order_acq_rel);
 
   // If this is an upgrade, we are not a reader anymore.
   if (tx != 0)
-    tx->shared_state = ~(typeof tx->shared_state)0;
+    tx->shared_state.store(-1, memory_order_relaxed);
 
   // Count the number of active readers to be able to decrease the number of
   // wake-ups and wait calls that are necessary.
@@ -194,7 +193,7 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
 	  it = it->next_thread)
 	{
 	  // Don't count ourself if this is an upgrade.
-	  if (it->shared_state != ~(typeof it->shared_state)0)
+	  if (it->shared_state.load(memory_order_relaxed) != -1)
 	    readers++;
 	}
 
@@ -236,8 +235,7 @@ gtm_rwlock::write_upgrade (gtm_thread *tx)
 void
 gtm_rwlock::read_unlock (gtm_thread *tx)
 {
-  tx->shared_state = ~(typeof tx->shared_state)0;
+  tx->shared_state = -1;
-  __sync_synchronize();
   unsigned int sum = this->summary;
   if (likely(!(sum & (a_writer | w_writer))))
     return;

libitm/config/x86/target.h

@@ -66,20 +66,6 @@ cpu_relax (void)
   __asm volatile ("rep; nop" : : : "memory");
 }
 
-static inline void
-atomic_read_barrier (void)
-{
-  /* x86 is a strong memory ordering machine.  */
-  __asm volatile ("" : : : "memory");
-}
-
-static inline void
-atomic_write_barrier (void)
-{
-  /* x86 is a strong memory ordering machine.  */
-  __asm volatile ("" : : : "memory");
-}
-
 } // namespace GTM
 
 // We'll be using some of the cpu builtins, and their associated types.

libitm/libitm_i.h

@@ -37,6 +37,7 @@
 #include <string.h>
 #include <unwind.h>
 #include "local_type_traits"
+#include "local_atomic"
 
 #include "common.h"
 
@@ -206,7 +207,7 @@ struct gtm_thread
 
   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
   // an active or serial transaction.
-  gtm_word shared_state;
+  atomic<gtm_word> shared_state;
 
   // The lock that provides access to serial mode.  Non-serialized
   // transactions acquire read locks; a serialized transaction aquires

libitm/method-gl.cc

@@ -41,10 +41,11 @@ struct gl_mg : public method_group
   static gtm_word clear_locked(gtm_word l) { return l & ~LOCK_BIT; }
 
   // The global ownership record.
-  gtm_word orec;
+  atomic<gtm_word> orec;
+
   virtual void init()
   {
-    orec = 0;
+    orec.store(0, memory_order_relaxed);
   }
   virtual void fini() { }
 };
@@ -84,28 +85,25 @@ protected:
   static void pre_write(const void *addr, size_t len)
   {
     gtm_thread *tx = gtm_thr();
-    if (unlikely(!gl_mg::is_locked(tx->shared_state)))
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
+    if (unlikely(!gl_mg::is_locked(v)))
       {
 	// Check for and handle version number overflow.
-	if (unlikely(tx->shared_state >= gl_mg::VERSION_MAX))
+	if (unlikely(v >= gl_mg::VERSION_MAX))
 	  tx->restart(RESTART_INIT_METHOD_GROUP);
 
 	// CAS global orec from our snapshot time to the locked state.
 	// This validates that we have a consistent snapshot, which is also
 	// for making privatization safety work (see the class' comments).
-	gtm_word now = o_gl_mg.orec;
+	gtm_word now = o_gl_mg.orec.load(memory_order_relaxed);
-	if (now != tx->shared_state)
+	if (now != v)
 	  tx->restart(RESTART_VALIDATE_WRITE);
-	if (__sync_val_compare_and_swap(&o_gl_mg.orec, now,
+	if (!o_gl_mg.orec.compare_exchange_strong (now, gl_mg::set_locked(now),
-	    gl_mg::set_locked(now)) != now)
+						   memory_order_acquire))
 	  tx->restart(RESTART_LOCKED_WRITE);
 
-	// Set shared_state to new value. The CAS is a full barrier, so the
+	// Set shared_state to new value.
-	// acquisition of the global orec is visible before this store here,
+	tx->shared_state.store(gl_mg::set_locked(now), memory_order_release);
-	// and the store will not be visible before earlier data loads, which
-	// is required to correctly ensure privatization safety (see
-	// begin_and_restart() and release_orec() for further comments).
-	tx->shared_state = gl_mg::set_locked(now);
       }
 
     // TODO Ensure that this gets inlined: Use internal log interface and LTO.
@@ -115,11 +113,12 @@ protected:
   static void validate()
   {
     // Check that snapshot is consistent. The barrier ensures that this
-    // happens after previous data loads.
+    // happens after previous data loads.  Recall that load cannot itself
-    atomic_read_barrier();
+    // have memory_order_release.
     gtm_thread *tx = gtm_thr();
-    gtm_word l = o_gl_mg.orec;
+    atomic_thread_fence(memory_order_release);
-    if (l != tx->shared_state)
+    gtm_word l = o_gl_mg.orec.load(memory_order_relaxed);
+    if (l != tx->shared_state.load(memory_order_relaxed))
       tx->restart(RESTART_VALIDATE_READ);
   }
 
@@ -180,17 +179,18 @@ public:
 
     // Spin until global orec is not locked.
     // TODO This is not necessary if there are no pure loads (check txn props).
-    gtm_word v;
     unsigned i = 0;
-    while (gl_mg::is_locked(v = o_gl_mg.orec))
+    gtm_word v;
+    while (1)
       {
+        v = o_gl_mg.orec.load(memory_order_acquire);
+        if (!gl_mg::is_locked(v))
+	  break;
 	// TODO need method-specific max spin count
-	if (++i > gtm_spin_count_var) return RESTART_VALIDATE_READ;
+	if (++i > gtm_spin_count_var)
+	  return RESTART_VALIDATE_READ;
 	cpu_relax();
       }
-    // This barrier ensures that we have read the global orec before later
-    // data loads.
-    atomic_read_barrier();
 
     // Everything is okay, we have a snapshot time.
     // We don't need to enforce any ordering for the following store. There
@@ -202,14 +202,14 @@ public:
     // marking the transaction as active, and restarts enforce immediate
     // visibility of a smaller or equal value with a barrier (see
     // release_orec()).
-    tx->shared_state = v;
+    tx->shared_state.store(v, memory_order_relaxed);
     return NO_RESTART;
   }
 
   virtual bool trycommit(gtm_word& priv_time)
   {
     gtm_thread* tx = gtm_thr();
-    gtm_word v = tx->shared_state;
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
 
     // Special case: If shared_state is ~0, then we have acquired the
     // serial lock (tx->state is not updated yet). In this case, the previous
@@ -218,7 +218,7 @@ public:
     // anymore. In particular, if it is locked, then we are an update
     // transaction, which is all we care about for commit.
     if (v == ~(typeof v)0)
-      v = o_gl_mg.orec;
+      v = o_gl_mg.orec.load(memory_order_relaxed);
 
     // Release the orec but do not reset shared_state, which will be modified
     // by the serial lock right after our commit anyway. Also, resetting
@@ -227,10 +227,8 @@ public:
     if (gl_mg::is_locked(v))
       {
 	// Release the global orec, increasing its version number / timestamp.
-	// TODO replace with C++0x-style atomics (a release in this case)
-	atomic_write_barrier();
 	v = gl_mg::clear_locked(v) + 1;
-	o_gl_mg.orec = v;
+	o_gl_mg.orec.store(v, memory_order_release);
 
 	// Need to ensure privatization safety. Every other transaction must
 	// have a snapshot time that is at least as high as our commit time
@@ -247,15 +245,16 @@ public:
       return;
 
     gtm_thread *tx = gtm_thr();
-    gtm_word v = tx->shared_state;
+    gtm_word v = tx->shared_state.load(memory_order_acquire);
     // Special case: If shared_state is ~0, then we have acquired the
     // serial lock (tx->state is not updated yet). In this case, the previous
     // value isn't available anymore, so grab it from the global lock, which
     // must have a meaningful value because no other transactions are active
     // anymore. In particular, if it is locked, then we are an update
     // transaction, which is all we care about for rollback.
-    if (v == ~(typeof v)0)
+    bool is_serial = v == ~(typeof v)0;
-      v = o_gl_mg.orec;
+    if (is_serial)
+      v = o_gl_mg.orec.load(memory_order_relaxed);
 
     // Release lock and increment version number to prevent dirty reads.
     // Also reset shared state here, so that begin_or_restart() can expect a
@@ -263,16 +262,14 @@ public:
     if (gl_mg::is_locked(v))
       {
 	// Release the global orec, increasing its version number / timestamp.
-	// TODO replace with C++0x-style atomics (a release in this case)
-	atomic_write_barrier();
 	v = gl_mg::clear_locked(v) + 1;
-	o_gl_mg.orec = v;
+	o_gl_mg.orec.store(v, memory_order_release);
 
 	// Also reset the timestamp published via shared_state.
 	// Special case: Only do this if we are not a serial transaction
 	// because otherwise, we would interfere with the serial lock.
-	if (tx->shared_state != ~(typeof tx->shared_state)0)
+	if (!is_serial)
-	  tx->shared_state = v;
+	  tx->shared_state.store(v, memory_order_relaxed);
 
 	// We need a store-load barrier after this store to prevent it
 	// from becoming visible after later data loads because the
@@ -280,7 +277,7 @@ public:
 	// snapshot time (the lock bit had been set), which could break
 	// privatization safety. We do not need a barrier before this
 	// store (see pre_write() for an explanation).
-	__sync_synchronize();
+	atomic_thread_fence(memory_order_acq_rel);
       }
 
   }

libitm/stmlock.h

@@ -92,24 +92,23 @@ gtm_get_stmlock (const gtm_cacheline *addr)
 }
 
 /* The current global version number.  */
-extern gtm_version gtm_clock;
+extern atomic<gtm_version> gtm_clock;
 
 static inline gtm_version
 gtm_get_clock (void)
 {
-  gtm_version r;
+  atomic_thread_fence(memory_order_release);
-
+  return gtm_clock.load(memory_order_acquire);
-  __sync_synchronize ();
-  r = gtm_clock;
-  atomic_read_barrier ();
-
-  return r;
 }
 
 static inline gtm_version
 gtm_inc_clock (void)
 {
-  gtm_version r = __sync_add_and_fetch (&gtm_clock, 1);
+  /* ??? Here we have a choice, the pre-inc operator mapping to
+     __atomic_add_fetch with memory_order_seq_cst, or fetch_add
+     with memory_order_acq_rel plus another separate increment.
+     We really ought to recognize and optimize fetch_op(x) op x... */
+  gtm_version r = ++gtm_clock;
 
   /* ??? Ought to handle wraparound for 32-bit.  */
   if (sizeof(r) < 8 && r > GTM_VERSION_MAX)