path: root/src/atomic_ops/sysdeps/gcc/arm.h

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
 *
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 *
 */

#include "../test_and_set_t_is_ao_t.h" /* Probably suboptimal */

#if defined(__thumb__) && !defined(__thumb2__)
  /* Thumb One mode does not have ARM "mcr", "swp" and some load/store  */
  /* instructions, so we temporarily switch to ARM mode and go back     */
  /* afterwards (clobbering "r3" register).                             */
# define AO_THUMB_GO_ARM \
           "       adr     r3, 4f\n" \
           "       bx      r3\n" \
           "      .align\n" \
           "      .arm\n" \
           "4:\n"
# define AO_THUMB_RESTORE_MODE \
           "       adr     r3, 5f + 1\n" \
           "       bx      r3\n" \
           "       .thumb\n" \
           "5:\n"
# define AO_THUMB_SWITCH_CLOBBERS "r3",
#else
# define AO_THUMB_GO_ARM /* empty */
# define AO_THUMB_RESTORE_MODE /* empty */
# define AO_THUMB_SWITCH_CLOBBERS /* empty */
#endif /* !__thumb__ */

/* NEC LE-IT: gcc has no way to easily check the arm architecture       */
/* but it defines only one (or several) of __ARM_ARCH_x__ to be true.   */
#if !defined(__ARM_ARCH_2__) && !defined(__ARM_ARCH_3__) \
    && !defined(__ARM_ARCH_3M__) && !defined(__ARM_ARCH_4__) \
    && !defined(__ARM_ARCH_4T__) \
    && ((!defined(__ARM_ARCH_5__) && !defined(__ARM_ARCH_5E__) \
         && !defined(__ARM_ARCH_5T__) && !defined(__ARM_ARCH_5TE__) \
         && !defined(__ARM_ARCH_5TEJ__) && !defined(__ARM_ARCH_6M__)) \
        || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
# define AO_ARM_HAVE_LDREX
# if !defined(__ARM_ARCH_6__) && !defined(__ARM_ARCH_6J__) \
     && !defined(__ARM_ARCH_6T2__) && !defined(__ARM_ARCH_6Z__) \
     && !defined(__ARM_ARCH_6ZT2__)
#   if !defined(__ARM_ARCH_6K__) && !defined(__ARM_ARCH_6ZK__)
      /* DMB is present in ARMv6M and ARMv7+.   */
#     define AO_ARM_HAVE_DMB
#   endif
#   if !defined(__thumb__) \
       || (defined(__thumb2__) && !defined(__ARM_ARCH_7__) \
           && !defined(__ARM_ARCH_7M__) && !defined(__ARM_ARCH_7EM__))
      /* LDREXD/STREXD present in ARMv6K/M+ (see gas/config/tc-arm.c).  */
      /* In the Thumb mode, this works only starting from ARMv7 (except */
      /* for the base and 'M' models).                                  */
#     define AO_ARM_HAVE_LDREXD
#   endif /* !thumb || ARMv7A || ARMv7R+ */
# endif /* ARMv7+ */
#endif /* ARMv6+ */

#if !defined(__ARM_ARCH_2__) && !defined(__ARM_ARCH_6M__) \
    && !defined(__thumb2__)
# define AO_ARM_HAVE_SWP
                /* Note: ARMv6M is excluded due to no ARM mode support. */
#endif /* !__thumb2__ */

#ifdef AO_ARM_HAVE_LDREX

  /* AO_t/char/short/int load is simple reading.                */
  /* Unaligned accesses are not guaranteed to be atomic.        */
# define AO_ACCESS_CHECK_ALIGNED
# define AO_ACCESS_short_CHECK_ALIGNED
# define AO_ACCESS_int_CHECK_ALIGNED
# include "../all_atomic_only_load.h"

  /* ARMv6 is the first architecture providing support for simple       */
  /* LL/SC.  A data memory barrier must be raised via CP15 command (see */
  /* documentation).  ARMv7 is compatible to ARMv6 but has a simpler    */
  /* command for issuing a memory barrier (DMB).  Raising it via CP15   */
  /* should still work (but slightly less efficient because it requires */
  /* the use of a general-purpose register).  If only a single          */
  /* processor (core) is used, AO_UNIPROCESSOR could be defined by      */
  /* client to avoid unnecessary memory barrier.                        */

# if defined(AO_ARM_HAVE_DMB) && !defined(AO_UNIPROCESSOR)

    AO_INLINE void
    AO_nop_full(void)
    {
      /* AO_THUMB_GO_ARM is empty. */
      __asm__ __volatile__("dmb" : : : "memory");
    }
#   define AO_HAVE_nop_full

    AO_INLINE void
    AO_nop_write(void)
    {
      /* AO_THUMB_GO_ARM is empty. */
      __asm__ __volatile__("dmb st" : : : "memory");
    }
#   define AO_HAVE_nop_write

# else

    AO_INLINE void
    AO_nop_full(void)
    {
#     ifndef AO_UNIPROCESSOR
        unsigned dest = 0;

        /* Issue a data memory barrier (keeps ordering of memory        */
        /* transactions before and after this operation).               */
        __asm__ __volatile__("@AO_nop_full\n"
          AO_THUMB_GO_ARM
          "       mcr p15,0,%0,c7,c10,5\n"
          AO_THUMB_RESTORE_MODE
          : "=&r"(dest)
          : /* empty */
          : AO_THUMB_SWITCH_CLOBBERS "memory");
#     else
        AO_compiler_barrier();
#     endif
    }
#   define AO_HAVE_nop_full

# endif /* !AO_ARM_HAVE_DMB */

  /* "ARM Architecture Reference Manual" (chapter A3.5.3) says that the */
  /* single-copy atomic processor accesses are all byte accesses, all   */
  /* halfword accesses to halfword-aligned locations, all word accesses */
  /* to word-aligned locations.                                         */
# include "../loadstore/char_atomic_store.h"
# include "../loadstore/short_atomic_store.h"
  /* AO_int_store is defined in ao_t_is_int.h.  */

  /* There is only a single concern related to AO_store: a direct write */
  /* (by STR instruction) will not be recognized by the LL/SC construct */
  /* on the same CPU (i.e., according to ARM documentation, e.g., see   */
  /* documentation CortexA8 TRM, point 8.5, atomic "store" is the only  */
  /* safe way to set variables also used in LL/SC environment).         */
  /* This is only a problem if interrupt handlers do not clear the      */
  /* reservation (by CLREX instruction or a dummy STREX one), as they   */
  /* almost certainly should (e.g., see "switch_to" routine defined in  */
  /* arch/arm/kernel/entry-armv.S of Linux.  Nonetheless, there is some */
  /* doubt this was properly implemented in some ancient OS releases.   */
# ifdef AO_BROKEN_TASKSWITCH_CLREX
    AO_INLINE void AO_store(volatile AO_t *addr, AO_t value)
    {
      int flag;

      __asm__ __volatile__("@AO_store\n"
        AO_THUMB_GO_ARM
        "1:     ldrex %0, [%2]\n"
        "       strex %0, %3, [%2]\n"
        "       teq %0, #0\n"
        "       bne 1b\n"
        AO_THUMB_RESTORE_MODE
        : "=&r" (flag), "+m" (*addr)
        : "r" (addr), "r" (value)
        : AO_THUMB_SWITCH_CLOBBERS "cc");
    }
#   define AO_HAVE_store
# else
#   include "../loadstore/atomic_store.h"
# endif

/* NEC LE-IT: replace the SWAP as recommended by ARM:
   "Applies to: ARM11 Cores
      Though the SWP instruction will still work with ARM V6 cores, it is
      recommended to use the new V6 synchronization instructions.  The SWP
      instruction produces 'locked' read and write accesses which are atomic,
      i.e. another operation cannot be done between these locked accesses which
      ties up external bus (AHB, AXI) bandwidth and can increase worst case
      interrupt latencies. LDREX, STREX are more flexible, other instructions
      can be done between the LDREX and STREX accesses."
*/
#ifndef AO_PREFER_GENERALIZED
#if !defined(AO_FORCE_USE_SWP) || !defined(AO_ARM_HAVE_SWP)
  /* But, on the other hand, there could be a considerable performance  */
  /* degradation in case of a race.  Eg., test_atomic.c executing       */
  /* test_and_set test on a dual-core ARMv7 processor using LDREX/STREX */
  /* showed around 35 times lower performance than that using SWP.      */
  /* To force use of SWP instruction, use -D AO_FORCE_USE_SWP option    */
  /* (the latter is ignored if SWP instruction is unsupported).         */
  AO_INLINE AO_TS_VAL_t
  AO_test_and_set(volatile AO_TS_t *addr)
  {
    AO_TS_VAL_t oldval;
    int flag;

    __asm__ __volatile__("@AO_test_and_set\n"
      AO_THUMB_GO_ARM
      "1:     ldrex   %0, [%3]\n"
      "       strex   %1, %4, [%3]\n"
      "       teq     %1, #0\n"
      "       bne     1b\n"
      AO_THUMB_RESTORE_MODE
      : "=&r"(oldval), "=&r"(flag), "+m"(*addr)
      : "r"(addr), "r"(1)
      : AO_THUMB_SWITCH_CLOBBERS "cc");
    return oldval;
  }
# define AO_HAVE_test_and_set
#endif /* !AO_FORCE_USE_SWP */

AO_INLINE AO_t
AO_fetch_and_add(volatile AO_t *p, AO_t incr)
{
  AO_t result, tmp;
  int flag;

  __asm__ __volatile__("@AO_fetch_and_add\n"
    AO_THUMB_GO_ARM
    "1:     ldrex   %0, [%5]\n"         /* get original         */
    "       add     %2, %0, %4\n"       /* sum up in incr       */
    "       strex   %1, %2, [%5]\n"     /* store them           */
    "       teq     %1, #0\n"
    "       bne     1b\n"
    AO_THUMB_RESTORE_MODE
    : "=&r"(result), "=&r"(flag), "=&r"(tmp), "+m"(*p) /* 0..3 */
    : "r"(incr), "r"(p)                                /* 4..5 */
    : AO_THUMB_SWITCH_CLOBBERS "cc");
  return result;
}
#define AO_HAVE_fetch_and_add

AO_INLINE AO_t
AO_fetch_and_add1(volatile AO_t *p)
{
  AO_t result, tmp;
  int flag;

  __asm__ __volatile__("@AO_fetch_and_add1\n"
    AO_THUMB_GO_ARM
    "1:     ldrex   %0, [%4]\n"         /* get original */
    "       add     %1, %0, #1\n"       /* increment */
    "       strex   %2, %1, [%4]\n"     /* store them */
    "       teq     %2, #0\n"
    "       bne     1b\n"
    AO_THUMB_RESTORE_MODE
    : "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
    : "r"(p)
    : AO_THUMB_SWITCH_CLOBBERS "cc");
  return result;
}
#define AO_HAVE_fetch_and_add1

AO_INLINE AO_t
AO_fetch_and_sub1(volatile AO_t *p)
{
  AO_t result, tmp;
  int flag;

  __asm__ __volatile__("@AO_fetch_and_sub1\n"
    AO_THUMB_GO_ARM
    "1:     ldrex   %0, [%4]\n"         /* get original */
    "       sub     %1, %0, #1\n"       /* decrement */
    "       strex   %2, %1, [%4]\n"     /* store them */
    "       teq     %2, #0\n"
    "       bne     1b\n"
    AO_THUMB_RESTORE_MODE
    : "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
    : "r"(p)
    : AO_THUMB_SWITCH_CLOBBERS "cc");
  return result;
}
#define AO_HAVE_fetch_and_sub1
#endif /* !AO_PREFER_GENERALIZED */

#ifndef AO_GENERALIZE_ASM_BOOL_CAS
  /* Returns nonzero if the comparison succeeded.       */
  AO_INLINE int
  AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
  {
    AO_t result, tmp;

    __asm__ __volatile__("@AO_compare_and_swap\n"
      AO_THUMB_GO_ARM
      "1:     mov     %0, #2\n"         /* store a flag */
      "       ldrex   %1, [%3]\n"       /* get original */
      "       teq     %1, %4\n"         /* see if match */
#     ifdef __thumb2__
        "       it      eq\n"
#     endif
      "       strexeq %0, %5, [%3]\n"   /* store new one if matched */
      "       teq     %0, #1\n"
      "       beq     1b\n"             /* if update failed, repeat */
      AO_THUMB_RESTORE_MODE
      : "=&r"(result), "=&r"(tmp), "+m"(*addr)
      : "r"(addr), "r"(old_val), "r"(new_val)
      : AO_THUMB_SWITCH_CLOBBERS "cc");
    return !(result&2); /* if succeded, return 1, else 0 */
  }
# define AO_HAVE_compare_and_swap
#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */

AO_INLINE AO_t
AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
{
  AO_t fetched_val;
  int flag;

  __asm__ __volatile__("@AO_fetch_compare_and_swap\n"
    AO_THUMB_GO_ARM
    "1:     mov     %0, #2\n"           /* store a flag */
    "       ldrex   %1, [%3]\n"         /* get original */
    "       teq     %1, %4\n"           /* see if match */
#   ifdef __thumb2__
      "       it      eq\n"
#   endif
    "       strexeq %0, %5, [%3]\n"     /* store new one if matched */
    "       teq     %0, #1\n"
    "       beq     1b\n"               /* if update failed, repeat */
    AO_THUMB_RESTORE_MODE
    : "=&r"(flag), "=&r"(fetched_val), "+m"(*addr)
    : "r"(addr), "r"(old_val), "r"(new_val)
    : AO_THUMB_SWITCH_CLOBBERS "cc");
  return fetched_val;
}
#define AO_HAVE_fetch_compare_and_swap

#ifdef AO_ARM_HAVE_LDREXD
# include "../standard_ao_double_t.h"

  /* "ARM Architecture Reference Manual ARMv7-A/R edition" (chapter     */
  /* A3.5.3) says that memory accesses caused by LDREXD and STREXD      */
  /* instructions to doubleword-aligned locations are single-copy       */
  /* atomic; accesses to 64-bit elements by other instructions might    */
  /* not be single-copy atomic as they are executed as a sequence of    */
  /* 32-bit accesses.                                                   */
  AO_INLINE AO_double_t
  AO_double_load(const volatile AO_double_t *addr)
  {
    AO_double_t result;

    /* AO_THUMB_GO_ARM is empty. */
    __asm__ __volatile__("@AO_double_load\n"
      "       ldrexd  %0, [%1]"
      : "=&r" (result.AO_whole)
      : "r" (addr)
      /* : no clobber */);
    return result;
  }
# define AO_HAVE_double_load

  AO_INLINE void
  AO_double_store(volatile AO_double_t *addr, AO_double_t new_val)
  {
    AO_double_t old_val;
    int status;

    do {
      /* AO_THUMB_GO_ARM is empty. */
      __asm__ __volatile__("@AO_double_store\n"
        "       ldrexd  %0, [%3]\n"
        "       strexd  %1, %4, [%3]"
        : "=&r" (old_val.AO_whole), "=&r" (status), "+m" (*addr)
        : "r" (addr), "r" (new_val.AO_whole)
        : "cc");
    } while (AO_EXPECT_FALSE(status));
  }
# define AO_HAVE_double_store

  AO_INLINE int
  AO_double_compare_and_swap(volatile AO_double_t *addr,
                             AO_double_t old_val, AO_double_t new_val)
  {
    double_ptr_storage tmp;
    int result = 1;

    do {
      /* AO_THUMB_GO_ARM is empty. */
      __asm__ __volatile__("@AO_double_compare_and_swap\n"
        "       ldrexd  %0, [%1]\n"     /* get original to r1 & r2 */
        : "=&r"(tmp)
        : "r"(addr)
        /* : no clobber */);
      if (tmp != old_val.AO_whole)
        break;
      __asm__ __volatile__(
        "       strexd  %0, %2, [%3]\n" /* store new one if matched */
        : "=&r"(result), "+m"(*addr)
        : "r"(new_val.AO_whole), "r"(addr)
        : "cc");
    } while (AO_EXPECT_FALSE(result));
    return !result;   /* if succeded, return 1 else 0 */
  }
# define AO_HAVE_double_compare_and_swap
#endif /* AO_ARM_HAVE_LDREXD */

#else
/* pre ARMv6 architectures ... */

/* I found a slide set that, if I read it correctly, claims that        */
/* Loads followed by either a Load or Store are ordered, but nothing    */
/* else is.                                                             */
/* It appears that SWP is the only simple memory barrier.               */
#include "../all_aligned_atomic_load_store.h"

/* The code should run correctly on a multi-core ARMv6+ as well.        */

#endif /* !AO_ARM_HAVE_LDREX */

#if !defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_test_and_set) \
    && defined (AO_ARM_HAVE_SWP) && (!defined(AO_PREFER_GENERALIZED) \
                                || !defined(AO_HAVE_fetch_compare_and_swap))
  AO_INLINE AO_TS_VAL_t
  AO_test_and_set_full(volatile AO_TS_t *addr)
  {
    AO_TS_VAL_t oldval;
    /* SWP on ARM is very similar to XCHG on x86.                       */
    /* The first operand is the result, the second the value            */
    /* to be stored.  Both registers must be different from addr.       */
    /* Make the address operand an early clobber output so it           */
    /* doesn't overlap with the other operands.  The early clobber      */
    /* on oldval is necessary to prevent the compiler allocating        */
    /* them to the same register if they are both unused.               */

    __asm__ __volatile__("@AO_test_and_set_full\n"
      AO_THUMB_GO_ARM
      "       swp %0, %2, [%3]\n"
                /* Ignore GCC "SWP is deprecated for this architecture" */
                /* warning here (for ARMv6+).                           */
      AO_THUMB_RESTORE_MODE
      : "=&r"(oldval), "=&r"(addr)
      : "r"(1), "1"(addr)
      : AO_THUMB_SWITCH_CLOBBERS "memory");
    return oldval;
  }
# define AO_HAVE_test_and_set_full
#endif /* !AO_HAVE_test_and_set[_full] && AO_ARM_HAVE_SWP */

/* FIXME: 32-bit ABI is assumed.    */
#define AO_T_IS_INT

#include "../read_ordered.h"