Mnemosyne: usermode/library/atomic_ops/generalize.h Source File

00001 /*
00002  * Copyright (c) 2003-2004 Hewlett-Packard Development Company, L.P.
00003  *
00004  * Permission is hereby granted, free of charge, to any person obtaining a copy
00005  * of this software and associated documentation files (the "Software"), to deal
00006  * in the Software without restriction, including without limitation the rights
00007  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
00008  * copies of the Software, and to permit persons to whom the Software is
00009  * furnished to do so, subject to the following conditions:
00010  *
00011  * The above copyright notice and this permission notice shall be included in
00012  * all copies or substantial portions of the Software.
00013  *
00014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
00015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
00016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
00017  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
00018  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
00019  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
00020  * SOFTWARE.
00021  */
00022 
00023 /*
00024  * Generalize atomic operations for atomic_ops.h.
00025  * Should not be included directly.
00026  *
00027  * We make no attempt to define useless operations, such as
00028  * AO_nop_acquire
00029  * AO_nop_release
00030  *
00031  * We have also so far neglected to define some others, which
00032  * do not appear likely to be useful, e.g. stores with acquire
00033  * or read barriers.
00034  *
00035  * This file is sometimes included twice by atomic_ops.h.
00036  * All definitions include explicit checks that we are not replacing
00037  * an earlier definition.  In general, more desirable expansions
00038  * appear earlier so that we are more likely to use them.
00039  *
00040  * We only make safe generalizations, except that by default we define
00041  * the ...dd_acquire_read operations to be equivalent to those without
00042  * a barrier.  On platforms for which this is unsafe, the platform-specific
00043  * file must define AO_NO_DD_ORDERING.
00044  */
00045 
00046 #ifndef ATOMIC_OPS_H
00047 # error Atomic_ops_generalize.h should not be included directly.
00048 #endif
00049 
00050 #if AO_CHAR_TS_T
00051 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
00052          AO_char_compare_and_swap_full(a,o,n)
00053 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
00054          AO_char_compare_and_swap_acquire(a,o,n)
00055 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
00056          AO_char_compare_and_swap_release(a,o,n)
00057 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
00058          AO_char_compare_and_swap(a,o,n)
00059 #endif
00060 
00061 #if AO_AO_TS_T
00062 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
00063          AO_compare_and_swap_full(a,o,n)
00064 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
00065          AO_compare_and_swap_acquire(a,o,n)
00066 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
00067          AO_compare_and_swap_release(a,o,n)
00068 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
00069          AO_compare_and_swap(a,o,n)
00070 #endif
00071 
00072 /* Generate test_and_set_full, if necessary and possible.       */
00073 #if !defined(AO_HAVE_test_and_set) && \
00074     !defined(AO_HAVE_test_and_set_release) && \
00075     !defined(AO_HAVE_test_and_set_acquire) && \
00076     !defined(AO_HAVE_test_and_set_read) && \
00077     !defined(AO_HAVE_test_and_set_full)
00078 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_full) || \
00079       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_full)
00080      AO_INLINE AO_TS_VAL_t
00081      AO_test_and_set_full(volatile AO_TS_t *addr)
00082      {
00083        if (AO_TS_COMPARE_AND_SWAP_FULL(addr, AO_TS_CLEAR, AO_TS_SET))
00084          return AO_TS_CLEAR;
00085        else
00086          return AO_TS_SET;
00087      }
00088 #    define AO_HAVE_test_and_set_full
00089 #  endif /* AO_HAVE_compare_and_swap_full */
00090 
00091 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_acquire) || \
00092       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_acquire)
00093      AO_INLINE AO_TS_VAL_t
00094      AO_test_and_set_acquire(volatile AO_TS_t *addr)
00095      {
00096        if (AO_TS_COMPARE_AND_SWAP_ACQUIRE(addr, AO_TS_CLEAR, AO_TS_SET))
00097          return AO_TS_CLEAR;
00098        else
00099          return AO_TS_SET;
00100      }
00101 #    define AO_HAVE_test_and_set_acquire
00102 #  endif /* AO_HAVE_compare_and_swap_acquire */
00103 
00104 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_release) || \
00105       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_release)
00106      AO_INLINE AO_TS_VAL_t
00107      AO_test_and_set_release(volatile AO_TS_t *addr)
00108      {
00109        if (AO_TS_COMPARE_AND_SWAP_RELEASE(addr, AO_TS_CLEAR, AO_TS_SET))
00110          return AO_TS_CLEAR;
00111        else
00112          return AO_TS_SET;
00113      }
00114 #    define AO_HAVE_test_and_set_release
00115 #  endif /* AO_HAVE_compare_and_swap_release */
00116 
00117 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap) || \
00118       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap)
00119      AO_INLINE AO_TS_VAL_t
00120      AO_test_and_set(volatile AO_TS_t *addr)
00121      {
00122        if (AO_TS_COMPARE_AND_SWAP(addr, AO_TS_CLEAR, AO_TS_SET))
00123          return AO_TS_CLEAR;
00124        else
00125          return AO_TS_SET;
00126      }
00127 #    define AO_HAVE_test_and_set
00128 #  endif /* AO_HAVE_compare_and_swap */
00129 
00130 #  if defined(AO_HAVE_test_and_set) && defined(AO_HAVE_nop_full) \
00131       && !defined(AO_HAVE_test_and_set_acquire)
00132      AO_INLINE AO_TS_VAL_t
00133      AO_test_and_set_acquire(volatile AO_TS_t *addr)
00134      {
00135        AO_TS_VAL_t result = AO_test_and_set(addr);
00136        AO_nop_full();
00137        return result;
00138      }
00139 #    define AO_HAVE_test_and_set_acquire
00140 #  endif
00141 
00142 #endif /* No prior test and set */
00143 
00144 /* Nop */
00145 #if !defined(AO_HAVE_nop)
00146    AO_INLINE void AO_nop(void) {}
00147 #  define AO_HAVE_nop
00148 #endif
00149 
00150 #if defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_nop_full)
00151    AO_INLINE void
00152    AO_nop_full(void)
00153    {
00154      AO_TS_t dummy = AO_TS_INITIALIZER;
00155      AO_test_and_set_full(&dummy);
00156    }
00157 #  define AO_HAVE_nop_full
00158 #endif
00159 
00160 #if defined(AO_HAVE_nop_acquire)
00161 #  error AO_nop_acquire is useless: dont define.
00162 #endif
00163 #if defined(AO_HAVE_nop_release)
00164 #  error AO_nop_release is useless: dont define.
00165 #endif
00166 
00167 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_read)
00168 #  define AO_nop_read() AO_nop_full()
00169 #  define AO_HAVE_nop_read
00170 #endif
00171 
00172 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_write)
00173 #  define AO_nop_write() AO_nop_full()
00174 #  define AO_HAVE_nop_write
00175 #endif
00176 
00177 /* Load */
00178 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
00179 #  define AO_load_acquire(addr) AO_load_full(addr)
00180 #  define AO_HAVE_load_acquire
00181 #endif
00182 
00183 #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
00184 #  define AO_load(addr) AO_load_acquire(addr)
00185 #  define AO_HAVE_load
00186 #endif
00187 
00188 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
00189 #  define AO_load_read(addr) AO_load_full(addr)
00190 #  define AO_HAVE_load_read
00191 #endif
00192 
00193 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_acquire)
00194 #  define AO_load_acquire_read(addr) AO_load_acquire(addr)
00195 #  define AO_HAVE_load_acquire_read
00196 #endif
00197 
00198 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) && \
00199     !defined(AO_HAVE_load_acquire)
00200    AO_INLINE AO_t
00201    AO_load_acquire(const volatile AO_t *addr)
00202    {
00203      AO_t result = AO_load(addr);
00204      /* Acquire barrier would be useless, since the load could be delayed  */
00205      /* beyond it.                                                         */
00206      AO_nop_full();
00207      return result;
00208    }
00209 #  define AO_HAVE_load_acquire
00210 #endif
00211 
00212 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) && \
00213     !defined(AO_HAVE_load_read)
00214    AO_INLINE AO_t
00215    AO_load_read(const volatile AO_t *addr)
00216    {
00217      AO_t result = AO_load(addr);
00218      /* Acquire barrier would be useless, since the load could be delayed  */
00219      /* beyond it.                                                         */
00220      AO_nop_read();
00221      return result;
00222    }
00223 #  define AO_HAVE_load_read
00224 #endif
00225 
00226 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) && \
00227     !defined(AO_HAVE_load_full)
00228 #  define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
00229 #  define AO_HAVE_load_full
00230 #endif
00231 
00232 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_read)
00233 #  define AO_load_acquire_read(addr) AO_load_read(addr)
00234 #  define AO_HAVE_load_acquire_read
00235 #endif
00236 
00237 #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
00238 #  define AO_load(addr) AO_load_acquire_read(addr)
00239 #  define AO_HAVE_load
00240 #endif
00241 
00242 #ifdef AO_NO_DD_ORDERING
00243 #  if defined(AO_HAVE_load_acquire_read)
00244 #    define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
00245 #    define AO_HAVE_load_dd_acquire_read
00246 #  endif
00247 #else
00248 #  if defined(AO_HAVE_load)
00249 #    define AO_load_dd_acquire_read(addr) AO_load(addr)
00250 #    define AO_HAVE_load_dd_acquire_read
00251 #  endif
00252 #endif
00253 
00254 
00255 /* Store */
00256 
00257 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
00258 #  define AO_store_release(addr,val) AO_store_full(addr,val)
00259 #  define AO_HAVE_store_release
00260 #endif
00261 
00262 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
00263 #  define AO_store(addr, val) AO_store_release(addr,val)
00264 #  define AO_HAVE_store
00265 #endif
00266 
00267 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
00268 #  define AO_store_write(addr,val) AO_store_full(addr,val)
00269 #  define AO_HAVE_store_write
00270 #endif
00271 
00272 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store_release_write)
00273 #  define AO_store_release_write(addr, val) AO_store_release(addr,val)
00274 #  define AO_HAVE_store_release_write
00275 #endif
00276 
00277 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
00278 #  define AO_store(addr, val) AO_store_write(addr,val)
00279 #  define AO_HAVE_store
00280 #endif
00281 
00282 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) && \
00283     !defined(AO_HAVE_store_release)
00284 #  define AO_store_release(addr,val) (AO_nop_full(), AO_store(addr,val))
00285 #  define AO_HAVE_store_release
00286 #endif
00287 
00288 #if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) && \
00289      !defined(AO_HAVE_store_write)
00290 #  define AO_store_write(addr, val) (AO_nop_write(), AO_store(addr,val))
00291 #  define AO_HAVE_store_write
00292 #endif
00293 
00294 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store_release_write)
00295 #  define AO_store_release_write(addr, val) AO_store_write(addr,val)
00296 #  define AO_HAVE_store_release_write
00297 #endif
00298 
00299 #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) && \
00300     !defined(AO_HAVE_store_full)
00301 #  define AO_store_full(addr, val) (AO_store_release(addr, val), AO_nop_full())
00302 #  define AO_HAVE_store_full
00303 #endif
00304 
00305 /* NEC LE-IT: Test and set */
00306 #if defined(AO_HAVE_test_and_set) && \
00307         defined(AO_HAVE_nop_full) && \
00308     !defined(AO_HAVE_test_and_set_release)
00309 #       define AO_test_and_set_release(addr) \
00310         (AO_nop_full(), AO_test_and_set(addr))
00311 #  define AO_HAVE_test_and_set_release
00312 #endif
00313 
00314 #if defined(AO_HAVE_test_and_set) && \
00315         defined(AO_HAVE_nop_full) && \
00316     !defined(AO_HAVE_test_and_set_acquire)
00317 AO_INLINE AO_TS_t
00318 AO_test_and_set_acquire(volatile AO_TS_t *addr)
00319 {
00320         AO_TS_t res = AO_test_and_set(addr);
00321         AO_nop_full();
00322         return res;
00323 }
00324 #  define AO_HAVE_test_and_set_acquire
00325 #endif
00326 
00327 
00328 /* Fetch_and_add */
00329 /* We first try to implement fetch_and_add variants in terms    */
00330 /* of the corresponding compare_and_swap variants to minimize   */
00331 /* adding barriers.                                             */
00332 #if defined(AO_HAVE_compare_and_swap_full) && \
00333     !defined(AO_HAVE_fetch_and_add_full)
00334    AO_INLINE AO_t
00335    AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
00336    {
00337      AO_t old;
00338      do
00339        {
00340          old = *addr;
00341        }
00342      while (!AO_compare_and_swap_full(addr, old, old+incr));
00343      return old;
00344    }
00345 #  define AO_HAVE_fetch_and_add_full
00346 #endif
00347 
00348 #if defined(AO_HAVE_compare_and_swap_acquire) && \
00349     !defined(AO_HAVE_fetch_and_add_acquire)
00350    AO_INLINE AO_t
00351    AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
00352    {
00353      AO_t old;
00354      do
00355        {
00356          old = *addr;
00357        }
00358      while (!AO_compare_and_swap_acquire(addr, old, old+incr));
00359      return old;
00360    }
00361 #  define AO_HAVE_fetch_and_add_acquire
00362 #endif
00363 
00364 #if defined(AO_HAVE_compare_and_swap_release) && \
00365     !defined(AO_HAVE_fetch_and_add_release)
00366    AO_INLINE AO_t
00367    AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
00368    {
00369      AO_t old;
00370      do
00371        {
00372          old = *addr;
00373        }
00374      while (!AO_compare_and_swap_release(addr, old, old+incr));
00375      return old;
00376    }
00377 #  define AO_HAVE_fetch_and_add_release
00378 #endif
00379 
00380 #if defined(AO_HAVE_compare_and_swap) && \
00381     !defined(AO_HAVE_fetch_and_add)
00382    AO_INLINE AO_t
00383    AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
00384    {
00385      AO_t old;
00386      do
00387        {
00388          old = *addr;
00389        }
00390      while (!AO_compare_and_swap(addr, old, old+incr));
00391      return old;
00392    }
00393 #  define AO_HAVE_fetch_and_add
00394 #endif
00395 
00396 #if defined(AO_HAVE_fetch_and_add_full)
00397 #  if !defined(AO_HAVE_fetch_and_add_release)
00398 #    define AO_fetch_and_add_release(addr, val) \
00399          AO_fetch_and_add_full(addr, val)
00400 #    define AO_HAVE_fetch_and_add_release
00401 #  endif
00402 #  if !defined(AO_HAVE_fetch_and_add_acquire)
00403 #    define AO_fetch_and_add_acquire(addr, val) \
00404          AO_fetch_and_add_full(addr, val)
00405 #    define AO_HAVE_fetch_and_add_acquire
00406 #  endif
00407 #  if !defined(AO_HAVE_fetch_and_add_write)
00408 #    define AO_fetch_and_add_write(addr, val) \
00409          AO_fetch_and_add_full(addr, val)
00410 #    define AO_HAVE_fetch_and_add_write
00411 #  endif
00412 #  if !defined(AO_HAVE_fetch_and_add_read)
00413 #    define AO_fetch_and_add_read(addr, val) \
00414          AO_fetch_and_add_full(addr, val)
00415 #    define AO_HAVE_fetch_and_add_read
00416 #  endif
00417 #endif /* AO_HAVE_fetch_and_add_full */
00418 
00419 #if !defined(AO_HAVE_fetch_and_add) && \
00420     defined(AO_HAVE_fetch_and_add_release)
00421 #  define AO_fetch_and_add(addr, val) \
00422         AO_fetch_and_add_release(addr, val)
00423 #  define AO_HAVE_fetch_and_add
00424 #endif
00425 #if !defined(AO_HAVE_fetch_and_add) && \
00426     defined(AO_HAVE_fetch_and_add_acquire)
00427 #  define AO_fetch_and_add(addr, val) \
00428         AO_fetch_and_add_acquire(addr, val)
00429 #  define AO_HAVE_fetch_and_add
00430 #endif
00431 #if !defined(AO_HAVE_fetch_and_add) && \
00432     defined(AO_HAVE_fetch_and_add_write)
00433 #  define AO_fetch_and_add(addr, val) \
00434         AO_fetch_and_add_write(addr, val)
00435 #  define AO_HAVE_fetch_and_add
00436 #endif
00437 #if !defined(AO_HAVE_fetch_and_add) && \
00438     defined(AO_HAVE_fetch_and_add_read)
00439 #  define AO_fetch_and_add(addr, val) \
00440         AO_fetch_and_add_read(addr, val)
00441 #  define AO_HAVE_fetch_and_add
00442 #endif
00443 
00444 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
00445     defined(AO_HAVE_nop_full) && \
00446     !defined(AO_HAVE_fetch_and_add_full)
00447 #  define AO_fetch_and_add_full(addr, val) \
00448         (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
00449 #  define AO_HAVE_fetch_and_add_full
00450 #endif
00451 
00452 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
00453     defined(AO_HAVE_fetch_and_add_write)
00454 #  define AO_fetch_and_add_release_write(addr, val) \
00455         AO_fetch_and_add_write(addr, val)
00456 #  define AO_HAVE_fetch_and_add_release_write
00457 #endif
00458 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
00459     defined(AO_HAVE_fetch_and_add_release)
00460 #  define AO_fetch_and_add_release_write(addr, val) \
00461         AO_fetch_and_add_release(addr, val)
00462 #  define AO_HAVE_fetch_and_add_release_write
00463 #endif
00464 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
00465     defined(AO_HAVE_fetch_and_add_read)
00466 #  define AO_fetch_and_add_acquire_read(addr, val) \
00467         AO_fetch_and_add_read(addr, val)
00468 #  define AO_HAVE_fetch_and_add_acquire_read
00469 #endif
00470 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
00471     defined(AO_HAVE_fetch_and_add_acquire)
00472 #  define AO_fetch_and_add_acquire_read(addr, val) \
00473         AO_fetch_and_add_acquire(addr, val)
00474 #  define AO_HAVE_fetch_and_add_acquire_read
00475 #endif
00476 
00477 #ifdef AO_NO_DD_ORDERING
00478 #  if defined(AO_HAVE_fetch_and_add_acquire_read)
00479 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
00480         AO_fetch_and_add_acquire_read(addr, val)
00481 #    define AO_HAVE_fetch_and_add_dd_acquire_read
00482 #  endif
00483 #else
00484 #  if defined(AO_HAVE_fetch_and_add)
00485 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
00486         AO_fetch_and_add(addr, val)
00487 #    define AO_HAVE_fetch_and_add_dd_acquire_read
00488 #  endif
00489 #endif
00490 
00491 /* Fetch_and_add1 */
00492 
00493 #if defined(AO_HAVE_fetch_and_add_full) &&\
00494     !defined(AO_HAVE_fetch_and_add1_full)
00495 #  define AO_fetch_and_add1_full(addr) AO_fetch_and_add_full(addr,1)
00496 #  define AO_HAVE_fetch_and_add1_full
00497 #endif
00498 #if defined(AO_HAVE_fetch_and_add_release) &&\
00499     !defined(AO_HAVE_fetch_and_add1_release)
00500 #  define AO_fetch_and_add1_release(addr) AO_fetch_and_add_release(addr,1)
00501 #  define AO_HAVE_fetch_and_add1_release
00502 #endif
00503 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
00504     !defined(AO_HAVE_fetch_and_add1_acquire)
00505 #  define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add_acquire(addr,1)
00506 #  define AO_HAVE_fetch_and_add1_acquire
00507 #endif
00508 #if defined(AO_HAVE_fetch_and_add_write) &&\
00509     !defined(AO_HAVE_fetch_and_add1_write)
00510 #  define AO_fetch_and_add1_write(addr) AO_fetch_and_add_write(addr,1)
00511 #  define AO_HAVE_fetch_and_add1_write
00512 #endif
00513 #if defined(AO_HAVE_fetch_and_add_read) &&\
00514     !defined(AO_HAVE_fetch_and_add1_read)
00515 #  define AO_fetch_and_add1_read(addr) AO_fetch_and_add_read(addr,1)
00516 #  define AO_HAVE_fetch_and_add1_read
00517 #endif
00518 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
00519     !defined(AO_HAVE_fetch_and_add1_release_write)
00520 #  define AO_fetch_and_add1_release_write(addr) \
00521         AO_fetch_and_add_release_write(addr,1)
00522 #  define AO_HAVE_fetch_and_add1_release_write
00523 #endif
00524 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
00525     !defined(AO_HAVE_fetch_and_add1_acquire_read)
00526 #  define AO_fetch_and_add1_acquire_read(addr) \
00527         AO_fetch_and_add_acquire_read(addr,1)
00528 #  define AO_HAVE_fetch_and_add1_acquire_read
00529 #endif
00530 #if defined(AO_HAVE_fetch_and_add) &&\
00531     !defined(AO_HAVE_fetch_and_add1)
00532 #  define AO_fetch_and_add1(addr) \
00533         AO_fetch_and_add(addr,1)
00534 #  define AO_HAVE_fetch_and_add1
00535 #endif
00536 
00537 #if defined(AO_HAVE_fetch_and_add1_full)
00538 #  if !defined(AO_HAVE_fetch_and_add1_release)
00539 #    define AO_fetch_and_add1_release(addr) \
00540          AO_fetch_and_add1_full(addr)
00541 #    define AO_HAVE_fetch_and_add1_release
00542 #  endif
00543 #  if !defined(AO_HAVE_fetch_and_add1_acquire)
00544 #    define AO_fetch_and_add1_acquire(addr) \
00545          AO_fetch_and_add1_full(addr)
00546 #    define AO_HAVE_fetch_and_add1_acquire
00547 #  endif
00548 #  if !defined(AO_HAVE_fetch_and_add1_write)
00549 #    define AO_fetch_and_add1_write(addr) \
00550          AO_fetch_and_add1_full(addr)
00551 #    define AO_HAVE_fetch_and_add1_write
00552 #  endif
00553 #  if !defined(AO_HAVE_fetch_and_add1_read)
00554 #    define AO_fetch_and_add1_read(addr) \
00555          AO_fetch_and_add1_full(addr)
00556 #    define AO_HAVE_fetch_and_add1_read
00557 #  endif
00558 #endif /* AO_HAVE_fetch_and_add1_full */
00559 
00560 #if !defined(AO_HAVE_fetch_and_add1) && \
00561     defined(AO_HAVE_fetch_and_add1_release)
00562 #  define AO_fetch_and_add1(addr) \
00563         AO_fetch_and_add1_release(addr)
00564 #  define AO_HAVE_fetch_and_add1
00565 #endif
00566 #if !defined(AO_HAVE_fetch_and_add1) && \
00567     defined(AO_HAVE_fetch_and_add1_acquire)
00568 #  define AO_fetch_and_add1(addr) \
00569         AO_fetch_and_add1_acquire(addr)
00570 #  define AO_HAVE_fetch_and_add1
00571 #endif
00572 #if !defined(AO_HAVE_fetch_and_add1) && \
00573     defined(AO_HAVE_fetch_and_add1_write)
00574 #  define AO_fetch_and_add1(addr) \
00575         AO_fetch_and_add1_write(addr)
00576 #  define AO_HAVE_fetch_and_add1
00577 #endif
00578 #if !defined(AO_HAVE_fetch_and_add1) && \
00579     defined(AO_HAVE_fetch_and_add1_read)
00580 #  define AO_fetch_and_add1(addr) \
00581         AO_fetch_and_add1_read(addr)
00582 #  define AO_HAVE_fetch_and_add1
00583 #endif
00584 
00585 #if defined(AO_HAVE_fetch_and_add1_acquire) &&\
00586     defined(AO_HAVE_nop_full) && \
00587     !defined(AO_HAVE_fetch_and_add1_full)
00588 #  define AO_fetch_and_add1_full(addr) \
00589         (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
00590 #  define AO_HAVE_fetch_and_add1_full
00591 #endif
00592 
00593 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
00594     defined(AO_HAVE_fetch_and_add1_write)
00595 #  define AO_fetch_and_add1_release_write(addr) \
00596         AO_fetch_and_add1_write(addr)
00597 #  define AO_HAVE_fetch_and_add1_release_write
00598 #endif
00599 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
00600     defined(AO_HAVE_fetch_and_add1_release)
00601 #  define AO_fetch_and_add1_release_write(addr) \
00602         AO_fetch_and_add1_release(addr)
00603 #  define AO_HAVE_fetch_and_add1_release_write
00604 #endif
00605 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
00606     defined(AO_HAVE_fetch_and_add1_read)
00607 #  define AO_fetch_and_add1_acquire_read(addr) \
00608         AO_fetch_and_add1_read(addr)
00609 #  define AO_HAVE_fetch_and_add1_acquire_read
00610 #endif
00611 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
00612     defined(AO_HAVE_fetch_and_add1_acquire)
00613 #  define AO_fetch_and_add1_acquire_read(addr) \
00614         AO_fetch_and_add1_acquire(addr)
00615 #  define AO_HAVE_fetch_and_add1_acquire_read
00616 #endif
00617 
00618 #ifdef AO_NO_DD_ORDERING
00619 #  if defined(AO_HAVE_fetch_and_add1_acquire_read)
00620 #    define AO_fetch_and_add1_dd_acquire_read(addr) \
00621         AO_fetch_and_add1_acquire_read(addr)
00622 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
00623 #  endif
00624 #else
00625 #  if defined(AO_HAVE_fetch_and_add1)
00626 #    define AO_fetch_and_add1_dd_acquire_read(addr) AO_fetch_and_add1(addr)
00627 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
00628 #  endif
00629 #endif
00630 
00631 /* Fetch_and_sub1 */
00632 
00633 #if defined(AO_HAVE_fetch_and_add_full) &&\
00634     !defined(AO_HAVE_fetch_and_sub1_full)
00635 #  define AO_fetch_and_sub1_full(addr) AO_fetch_and_add_full(addr,(AO_t)(-1))
00636 #  define AO_HAVE_fetch_and_sub1_full
00637 #endif
00638 #if defined(AO_HAVE_fetch_and_add_release) &&\
00639     !defined(AO_HAVE_fetch_and_sub1_release)
00640 #  define AO_fetch_and_sub1_release(addr) \
00641         AO_fetch_and_add_release(addr,(AO_t)(-1))
00642 #  define AO_HAVE_fetch_and_sub1_release
00643 #endif
00644 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
00645     !defined(AO_HAVE_fetch_and_sub1_acquire)
00646 #  define AO_fetch_and_sub1_acquire(addr) \
00647         AO_fetch_and_add_acquire(addr,(AO_t)(-1))
00648 #  define AO_HAVE_fetch_and_sub1_acquire
00649 #endif
00650 #if defined(AO_HAVE_fetch_and_add_write) &&\
00651     !defined(AO_HAVE_fetch_and_sub1_write)
00652 #  define AO_fetch_and_sub1_write(addr) \
00653         AO_fetch_and_add_write(addr,(AO_t)(-1))
00654 #  define AO_HAVE_fetch_and_sub1_write
00655 #endif
00656 #if defined(AO_HAVE_fetch_and_add_read) &&\
00657     !defined(AO_HAVE_fetch_and_sub1_read)
00658 #  define AO_fetch_and_sub1_read(addr) \
00659         AO_fetch_and_add_read(addr,(AO_t)(-1))
00660 #  define AO_HAVE_fetch_and_sub1_read
00661 #endif
00662 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
00663     !defined(AO_HAVE_fetch_and_sub1_release_write)
00664 #  define AO_fetch_and_sub1_release_write(addr) \
00665         AO_fetch_and_add_release_write(addr,(AO_t)(-1))
00666 #  define AO_HAVE_fetch_and_sub1_release_write
00667 #endif
00668 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
00669     !defined(AO_HAVE_fetch_and_sub1_acquire_read)
00670 #  define AO_fetch_and_sub1_acquire_read(addr) \
00671         AO_fetch_and_add_acquire_read(addr,(AO_t)(-1))
00672 #  define AO_HAVE_fetch_and_sub1_acquire_read
00673 #endif
00674 #if defined(AO_HAVE_fetch_and_add) &&\
00675     !defined(AO_HAVE_fetch_and_sub1)
00676 #  define AO_fetch_and_sub1(addr) \
00677         AO_fetch_and_add(addr,(AO_t)(-1))
00678 #  define AO_HAVE_fetch_and_sub1
00679 #endif
00680 
00681 #if defined(AO_HAVE_fetch_and_sub1_full)
00682 #  if !defined(AO_HAVE_fetch_and_sub1_release)
00683 #    define AO_fetch_and_sub1_release(addr) \
00684          AO_fetch_and_sub1_full(addr)
00685 #    define AO_HAVE_fetch_and_sub1_release
00686 #  endif
00687 #  if !defined(AO_HAVE_fetch_and_sub1_acquire)
00688 #    define AO_fetch_and_sub1_acquire(addr) \
00689          AO_fetch_and_sub1_full(addr)
00690 #    define AO_HAVE_fetch_and_sub1_acquire
00691 #  endif
00692 #  if !defined(AO_HAVE_fetch_and_sub1_write)
00693 #    define AO_fetch_and_sub1_write(addr) \
00694          AO_fetch_and_sub1_full(addr)
00695 #    define AO_HAVE_fetch_and_sub1_write
00696 #  endif
00697 #  if !defined(AO_HAVE_fetch_and_sub1_read)
00698 #    define AO_fetch_and_sub1_read(addr) \
00699          AO_fetch_and_sub1_full(addr)
00700 #    define AO_HAVE_fetch_and_sub1_read
00701 #  endif
00702 #endif /* AO_HAVE_fetch_and_sub1_full */
00703 
00704 #if !defined(AO_HAVE_fetch_and_sub1) && \
00705     defined(AO_HAVE_fetch_and_sub1_release)
00706 #  define AO_fetch_and_sub1(addr) \
00707         AO_fetch_and_sub1_release(addr)
00708 #  define AO_HAVE_fetch_and_sub1
00709 #endif
00710 #if !defined(AO_HAVE_fetch_and_sub1) && \
00711     defined(AO_HAVE_fetch_and_sub1_acquire)
00712 #  define AO_fetch_and_sub1(addr) \
00713         AO_fetch_and_sub1_acquire(addr)
00714 #  define AO_HAVE_fetch_and_sub1
00715 #endif
00716 #if !defined(AO_HAVE_fetch_and_sub1) && \
00717     defined(AO_HAVE_fetch_and_sub1_write)
00718 #  define AO_fetch_and_sub1(addr) \
00719         AO_fetch_and_sub1_write(addr)
00720 #  define AO_HAVE_fetch_and_sub1
00721 #endif
00722 #if !defined(AO_HAVE_fetch_and_sub1) && \
00723     defined(AO_HAVE_fetch_and_sub1_read)
00724 #  define AO_fetch_and_sub1(addr) \
00725         AO_fetch_and_sub1_read(addr)
00726 #  define AO_HAVE_fetch_and_sub1
00727 #endif
00728 
00729 #if defined(AO_HAVE_fetch_and_sub1_acquire) &&\
00730     defined(AO_HAVE_nop_full) && \
00731     !defined(AO_HAVE_fetch_and_sub1_full)
00732 #  define AO_fetch_and_sub1_full(addr) \
00733         (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
00734 #  define AO_HAVE_fetch_and_sub1_full
00735 #endif
00736 
00737 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
00738     defined(AO_HAVE_fetch_and_sub1_write)
00739 #  define AO_fetch_and_sub1_release_write(addr) \
00740         AO_fetch_and_sub1_write(addr)
00741 #  define AO_HAVE_fetch_and_sub1_release_write
00742 #endif
00743 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
00744     defined(AO_HAVE_fetch_and_sub1_release)
00745 #  define AO_fetch_and_sub1_release_write(addr) \
00746         AO_fetch_and_sub1_release(addr)
00747 #  define AO_HAVE_fetch_and_sub1_release_write
00748 #endif
00749 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
00750     defined(AO_HAVE_fetch_and_sub1_read)
00751 #  define AO_fetch_and_sub1_acquire_read(addr) \
00752         AO_fetch_and_sub1_read(addr)
00753 #  define AO_HAVE_fetch_and_sub1_acquire_read
00754 #endif
00755 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
00756     defined(AO_HAVE_fetch_and_sub1_acquire)
00757 #  define AO_fetch_and_sub1_acquire_read(addr) \
00758         AO_fetch_and_sub1_acquire(addr)
00759 #  define AO_HAVE_fetch_and_sub1_acquire_read
00760 #endif
00761 
00762 #ifdef AO_NO_DD_ORDERING
00763 #  if defined(AO_HAVE_fetch_and_sub1_acquire_read)
00764 #    define AO_fetch_and_sub1_dd_acquire_read(addr) \
00765         AO_fetch_and_sub1_acquire_read(addr)
00766 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
00767 #  endif
00768 #else
00769 #  if defined(AO_HAVE_fetch_and_sub1)
00770 #    define AO_fetch_and_sub1_dd_acquire_read(addr) AO_fetch_and_sub1(addr)
00771 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
00772 #  endif
00773 #endif
00774 
00775 /* Atomic or */
00776 #if defined(AO_HAVE_compare_and_swap_full) && \
00777     !defined(AO_HAVE_or_full)
00778    AO_INLINE void
00779    AO_or_full(volatile AO_t *addr, AO_t incr)
00780    {
00781      AO_t old;
00782      do
00783        {
00784          old = *addr;
00785        }
00786      while (!AO_compare_and_swap_full(addr, old, (old | incr)));
00787    }
00788 #  define AO_HAVE_or_full
00789 #endif
00790 
00791 #if defined(AO_HAVE_or_full)
00792 #  if !defined(AO_HAVE_or_release)
00793 #    define AO_or_release(addr, val) \
00794          AO_or_full(addr, val)
00795 #    define AO_HAVE_or_release
00796 #  endif
00797 #  if !defined(AO_HAVE_or_acquire)
00798 #    define AO_or_acquire(addr, val) \
00799          AO_or_full(addr, val)
00800 #    define AO_HAVE_or_acquire
00801 #  endif
00802 #  if !defined(AO_HAVE_or_write)
00803 #    define AO_or_write(addr, val) \
00804          AO_or_full(addr, val)
00805 #    define AO_HAVE_or_write
00806 #  endif
00807 #  if !defined(AO_HAVE_or_read)
00808 #    define AO_or_read(addr, val) \
00809          AO_or_full(addr, val)
00810 #    define AO_HAVE_or_read
00811 #  endif
00812 #endif /* AO_HAVE_or_full */
00813 
00814 #if !defined(AO_HAVE_or) && \
00815     defined(AO_HAVE_or_release)
00816 #  define AO_or(addr, val) \
00817         AO_or_release(addr, val)
00818 #  define AO_HAVE_or
00819 #endif
00820 #if !defined(AO_HAVE_or) && \
00821     defined(AO_HAVE_or_acquire)
00822 #  define AO_or(addr, val) \
00823         AO_or_acquire(addr, val)
00824 #  define AO_HAVE_or
00825 #endif
00826 #if !defined(AO_HAVE_or) && \
00827     defined(AO_HAVE_or_write)
00828 #  define AO_or(addr, val) \
00829         AO_or_write(addr, val)
00830 #  define AO_HAVE_or
00831 #endif
00832 #if !defined(AO_HAVE_or) && \
00833     defined(AO_HAVE_or_read)
00834 #  define AO_or(addr, val) \
00835         AO_or_read(addr, val)
00836 #  define AO_HAVE_or
00837 #endif
00838 
00839 #if defined(AO_HAVE_or_acquire) &&\
00840     defined(AO_HAVE_nop_full) && \
00841     !defined(AO_HAVE_or_full)
00842 #  define AO_or_full(addr, val) \
00843         (AO_nop_full(), AO_or_acquire(addr, val))
00844 #endif
00845 
00846 #if !defined(AO_HAVE_or_release_write) && \
00847     defined(AO_HAVE_or_write)
00848 #  define AO_or_release_write(addr, val) \
00849         AO_or_write(addr, val)
00850 #  define AO_HAVE_or_release_write
00851 #endif
00852 #if !defined(AO_HAVE_or_release_write) && \
00853     defined(AO_HAVE_or_release)
00854 #  define AO_or_release_write(addr, val) \
00855         AO_or_release(addr, val)
00856 #  define AO_HAVE_or_release_write
00857 #endif
00858 #if !defined(AO_HAVE_or_acquire_read) && \
00859     defined(AO_HAVE_or_read)
00860 #  define AO_or_acquire_read(addr, val) \
00861         AO_or_read(addr, val)
00862 #  define AO_HAVE_or_acquire_read
00863 #endif
00864 #if !defined(AO_HAVE_or_acquire_read) && \
00865     defined(AO_HAVE_or_acquire)
00866 #  define AO_or_acquire_read(addr, val) \
00867         AO_or_acquire(addr, val)
00868 #  define AO_HAVE_or_acquire_read
00869 #endif
00870 
00871 /* dd_aquire_read is meaningless.       */
00872 
00873 /* Test_and_set */
00874 
00875 #if defined(AO_HAVE_test_and_set_full)
00876 #  if !defined(AO_HAVE_test_and_set_release)
00877 #    define AO_test_and_set_release(addr) \
00878          AO_test_and_set_full(addr)
00879 #    define AO_HAVE_test_and_set_release
00880 #  endif
00881 #  if !defined(AO_HAVE_test_and_set_acquire)
00882 #    define AO_test_and_set_acquire(addr) \
00883          AO_test_and_set_full(addr)
00884 #    define AO_HAVE_test_and_set_acquire
00885 #  endif
00886 #  if !defined(AO_HAVE_test_and_set_write)
00887 #    define AO_test_and_set_write(addr) \
00888          AO_test_and_set_full(addr)
00889 #    define AO_HAVE_test_and_set_write
00890 #  endif
00891 #  if !defined(AO_HAVE_test_and_set_read)
00892 #    define AO_test_and_set_read(addr) \
00893          AO_test_and_set_full(addr)
00894 #    define AO_HAVE_test_and_set_read
00895 #  endif
00896 #endif /* AO_HAVE_test_and_set_full */
00897 
00898 #if !defined(AO_HAVE_test_and_set) && \
00899     defined(AO_HAVE_test_and_set_release)
00900 #  define AO_test_and_set(addr) \
00901         AO_test_and_set_release(addr)
00902 #  define AO_HAVE_test_and_set
00903 #endif
00904 #if !defined(AO_HAVE_test_and_set) && \
00905     defined(AO_HAVE_test_and_set_acquire)
00906 #  define AO_test_and_set(addr) \
00907         AO_test_and_set_acquire(addr)
00908 #  define AO_HAVE_test_and_set
00909 #endif
00910 #if !defined(AO_HAVE_test_and_set) && \
00911     defined(AO_HAVE_test_and_set_write)
00912 #  define AO_test_and_set(addr) \
00913         AO_test_and_set_write(addr)
00914 #  define AO_HAVE_test_and_set
00915 #endif
00916 #if !defined(AO_HAVE_test_and_set) && \
00917     defined(AO_HAVE_test_and_set_read)
00918 #  define AO_test_and_set(addr) \
00919         AO_test_and_set_read(addr)
00920 #  define AO_HAVE_test_and_set
00921 #endif
00922 
00923 #if defined(AO_HAVE_test_and_set_acquire) &&\
00924     defined(AO_HAVE_nop_full) && \
00925     !defined(AO_HAVE_test_and_set_full)
00926 #  define AO_test_and_set_full(addr) \
00927         (AO_nop_full(), AO_test_and_set_acquire(addr))
00928 #  define AO_HAVE_test_and_set_full
00929 #endif
00930 
00931 #if !defined(AO_HAVE_test_and_set_release_write) && \
00932     defined(AO_HAVE_test_and_set_write)
00933 #  define AO_test_and_set_release_write(addr) \
00934         AO_test_and_set_write(addr)
00935 #  define AO_HAVE_test_and_set_release_write
00936 #endif
00937 #if !defined(AO_HAVE_test_and_set_release_write) && \
00938     defined(AO_HAVE_test_and_set_release)
00939 #  define AO_test_and_set_release_write(addr) \
00940         AO_test_and_set_release(addr)
00941 #  define AO_HAVE_test_and_set_release_write
00942 #endif
00943 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
00944     defined(AO_HAVE_test_and_set_read)
00945 #  define AO_test_and_set_acquire_read(addr) \
00946         AO_test_and_set_read(addr)
00947 #  define AO_HAVE_test_and_set_acquire_read
00948 #endif
00949 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
00950     defined(AO_HAVE_test_and_set_acquire)
00951 #  define AO_test_and_set_acquire_read(addr) \
00952         AO_test_and_set_acquire(addr)
00953 #  define AO_HAVE_test_and_set_acquire_read
00954 #endif
00955 
00956 #ifdef AO_NO_DD_ORDERING
00957 #  if defined(AO_HAVE_test_and_set_acquire_read)
00958 #    define AO_test_and_set_dd_acquire_read(addr) \
00959         AO_test_and_set_acquire_read(addr)
00960 #    define AO_HAVE_test_and_set_dd_acquire_read
00961 #  endif
00962 #else
00963 #  if defined(AO_HAVE_test_and_set)
00964 #    define AO_test_and_set_dd_acquire_read(addr) AO_test_and_set(addr)
00965 #    define AO_HAVE_test_and_set_dd_acquire_read
00966 #  endif
00967 #endif
00968 
00969 /* Compare_and_swap */
00970 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
00971     && !defined(AO_HAVE_compare_and_swap_acquire)
00972    AO_INLINE int
00973    AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
00974    {
00975      int result = AO_compare_and_swap(addr, old, new_val);
00976      AO_nop_full();
00977      return result;
00978    }
00979 #  define AO_HAVE_compare_and_swap_acquire
00980 #endif
00981 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
00982     && !defined(AO_HAVE_compare_and_swap_release)
00983 #  define AO_compare_and_swap_release(addr, old, new_val) \
00984         (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
00985 #  define AO_HAVE_compare_and_swap_release
00986 #endif
00987 #if defined(AO_HAVE_compare_and_swap_full)
00988 #  if !defined(AO_HAVE_compare_and_swap_release)
00989 #    define AO_compare_and_swap_release(addr, old, new_val) \
00990          AO_compare_and_swap_full(addr, old, new_val)
00991 #    define AO_HAVE_compare_and_swap_release
00992 #  endif
00993 #  if !defined(AO_HAVE_compare_and_swap_acquire)
00994 #    define AO_compare_and_swap_acquire(addr, old, new_val) \
00995          AO_compare_and_swap_full(addr, old, new_val)
00996 #    define AO_HAVE_compare_and_swap_acquire
00997 #  endif
00998 #  if !defined(AO_HAVE_compare_and_swap_write)
00999 #    define AO_compare_and_swap_write(addr, old, new_val) \
01000          AO_compare_and_swap_full(addr, old, new_val)
01001 #    define AO_HAVE_compare_and_swap_write
01002 #  endif
01003 #  if !defined(AO_HAVE_compare_and_swap_read)
01004 #    define AO_compare_and_swap_read(addr, old, new_val) \
01005          AO_compare_and_swap_full(addr, old, new_val)
01006 #    define AO_HAVE_compare_and_swap_read
01007 #  endif
01008 #endif /* AO_HAVE_compare_and_swap_full */
01009 
01010 #if !defined(AO_HAVE_compare_and_swap) && \
01011     defined(AO_HAVE_compare_and_swap_release)
01012 #  define AO_compare_and_swap(addr, old, new_val) \
01013         AO_compare_and_swap_release(addr, old, new_val)
01014 #  define AO_HAVE_compare_and_swap
01015 #endif
01016 #if !defined(AO_HAVE_compare_and_swap) && \
01017     defined(AO_HAVE_compare_and_swap_acquire)
01018 #  define AO_compare_and_swap(addr, old, new_val) \
01019         AO_compare_and_swap_acquire(addr, old, new_val)
01020 #  define AO_HAVE_compare_and_swap
01021 #endif
01022 #if !defined(AO_HAVE_compare_and_swap) && \
01023     defined(AO_HAVE_compare_and_swap_write)
01024 #  define AO_compare_and_swap(addr, old, new_val) \
01025         AO_compare_and_swap_write(addr, old, new_val)
01026 #  define AO_HAVE_compare_and_swap
01027 #endif
01028 #if !defined(AO_HAVE_compare_and_swap) && \
01029     defined(AO_HAVE_compare_and_swap_read)
01030 #  define AO_compare_and_swap(addr, old, new_val) \
01031         AO_compare_and_swap_read(addr, old, new_val)
01032 #  define AO_HAVE_compare_and_swap
01033 #endif
01034 
01035 #if defined(AO_HAVE_compare_and_swap_acquire) &&\
01036     defined(AO_HAVE_nop_full) && \
01037     !defined(AO_HAVE_compare_and_swap_full)
01038 #  define AO_compare_and_swap_full(addr, old, new_val) \
01039         (AO_nop_full(), AO_compare_and_swap_acquire(addr, old, new_val))
01040 #  define AO_HAVE_compare_and_swap_full
01041 #endif
01042 
01043 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
01044     defined(AO_HAVE_compare_and_swap_write)
01045 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
01046         AO_compare_and_swap_write(addr, old, new_val)
01047 #  define AO_HAVE_compare_and_swap_release_write
01048 #endif
01049 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
01050     defined(AO_HAVE_compare_and_swap_release)
01051 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
01052         AO_compare_and_swap_release(addr, old, new_val)
01053 #  define AO_HAVE_compare_and_swap_release_write
01054 #endif
01055 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
01056     defined(AO_HAVE_compare_and_swap_read)
01057 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
01058         AO_compare_and_swap_read(addr, old, new_val)
01059 #  define AO_HAVE_compare_and_swap_acquire_read
01060 #endif
01061 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
01062     defined(AO_HAVE_compare_and_swap_acquire)
01063 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
01064         AO_compare_and_swap_acquire(addr, old, new_val)
01065 #  define AO_HAVE_compare_and_swap_acquire_read
01066 #endif
01067 
01068 #ifdef AO_NO_DD_ORDERING
01069 #  if defined(AO_HAVE_compare_and_swap_acquire_read)
01070 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
01071         AO_compare_and_swap_acquire_read(addr, old, new_val)
01072 #    define AO_HAVE_compare_and_swap_dd_acquire_read
01073 #  endif
01074 #else
01075 #  if defined(AO_HAVE_compare_and_swap)
01076 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
01077         AO_compare_and_swap(addr, old, new_val)
01078 #    define AO_HAVE_compare_and_swap_dd_acquire_read
01079 #  endif
01080 #endif
01081 
01082 #include "generalize-small.h"
01083 
01084 /* Compare_double_and_swap_double */
01085 #if defined(AO_HAVE_compare_double_and_swap_double) && defined(AO_HAVE_nop_full)\
01086     && !defined(AO_HAVE_compare_double_and_swap_double_acquire)
01087    AO_INLINE int
01088    AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
01089                                              AO_t o1, AO_t o2,
01090                                              AO_t n1, AO_t n2)
01091    {
01092      int result = AO_compare_double_and_swap_double(addr, o1, o2, n1, n2);
01093      AO_nop_full();
01094      return result;
01095    }
01096 #  define AO_HAVE_compare_double_and_swap_double_acquire
01097 #endif
01098 #if defined(AO_HAVE_compare_double_and_swap_double) \
01099     && defined(AO_HAVE_nop_full)\
01100     && !defined(AO_HAVE_compare_double_and_swap_double_release)
01101 #  define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
01102         (AO_nop_full(), AO_compare_double_and_swap_double(addr, o1, o2, n1, n2))
01103 #  define AO_HAVE_compare_double_and_swap_double_release
01104 #endif
01105 #if defined(AO_HAVE_compare_double_and_swap_double_full)
01106 #  if !defined(AO_HAVE_compare_double_and_swap_double_release)
01107 #    define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
01108          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
01109 #    define AO_HAVE_compare_double_and_swap_double_release
01110 #  endif
01111 #  if !defined(AO_HAVE_compare_double_and_swap_double_acquire)
01112 #    define AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2) \
01113          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
01114 #    define AO_HAVE_compare_double_and_swap_double_acquire
01115 #  endif
01116 #  if !defined(AO_HAVE_compare_double_and_swap_double_write)
01117 #    define AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2) \
01118          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
01119 #    define AO_HAVE_compare_double_and_swap_double_write
01120 #  endif
01121 #  if !defined(AO_HAVE_compare_double_and_swap_double_read)
01122 #    define AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2) \
01123          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
01124 #    define AO_HAVE_compare_double_and_swap_double_read
01125 #  endif
01126 #endif /* AO_HAVE_compare_double_and_swap_double_full */
01127 
01128 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
01129     defined(AO_HAVE_compare_double_and_swap_double_release)
01130 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
01131         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
01132 #  define AO_HAVE_compare_double_and_swap_double
01133 #endif
01134 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
01135     defined(AO_HAVE_compare_double_and_swap_double_acquire)
01136 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
01137         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
01138 #  define AO_HAVE_compare_double_and_swap_double
01139 #endif
01140 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
01141     defined(AO_HAVE_compare_double_and_swap_double_write)
01142 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
01143         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
01144 #  define AO_HAVE_compare_double_and_swap_double
01145 #endif
01146 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
01147     defined(AO_HAVE_compare_double_and_swap_double_read)
01148 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
01149         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
01150 #  define AO_HAVE_compare_double_and_swap_double
01151 #endif
01152 
01153 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) &&\
01154     defined(AO_HAVE_nop_full) && \
01155     !defined(AO_HAVE_compare_double_and_swap_double_full)
01156 #  define AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2) \
01157         (AO_nop_full(), AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2))
01158 #  define AO_HAVE_compare_double_and_swap_double_full
01159 #endif
01160 
01161 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
01162     defined(AO_HAVE_compare_double_and_swap_double_write)
01163 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
01164         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
01165 #  define AO_HAVE_compare_double_and_swap_double_release_write
01166 #endif
01167 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
01168     defined(AO_HAVE_compare_double_and_swap_double_release)
01169 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
01170         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
01171 #  define AO_HAVE_compare_double_and_swap_double_release_write
01172 #endif
01173 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
01174     defined(AO_HAVE_compare_double_and_swap_double_read)
01175 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
01176         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
01177 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
01178 #endif
01179 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
01180     defined(AO_HAVE_compare_double_and_swap_double_acquire)
01181 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
01182         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
01183 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
01184 #endif
01185 
01186 #ifdef AO_NO_DD_ORDERING
01187 #  if defined(AO_HAVE_compare_double_and_swap_double_acquire_read)
01188 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
01189         AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2)
01190 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
01191 #  endif
01192 #else
01193 #  if defined(AO_HAVE_compare_double_and_swap_double)
01194 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
01195         AO_compare_double_and_swap_double(addr, o1, o2, n1, n2)
01196 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
01197 #  endif
01198 #endif
01199 
01200 /* Compare_and_swap_double */
01201 #if defined(AO_HAVE_compare_and_swap_double) && defined(AO_HAVE_nop_full)\
01202     && !defined(AO_HAVE_compare_and_swap_double_acquire)
01203    AO_INLINE int
01204    AO_compare_and_swap_double_acquire(volatile AO_double_t *addr,
01205                                              AO_t o1,
01206                                              AO_t n1, AO_t n2)
01207    {
01208      int result = AO_compare_and_swap_double(addr, o1, n1, n2);
01209      AO_nop_full();
01210      return result;
01211    }
01212 #  define AO_HAVE_compare_and_swap_double_acquire
01213 #endif
01214 #if defined(AO_HAVE_compare_and_swap_double) \
01215     && defined(AO_HAVE_nop_full)\
01216     && !defined(AO_HAVE_compare_and_swap_double_release)
01217 #  define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
01218         (AO_nop_full(), AO_compare_and_swap_double(addr, o1, n1, n2))
01219 #  define AO_HAVE_compare_and_swap_double_release
01220 #endif
01221 #if defined(AO_HAVE_compare_and_swap_double_full)
01222 #  if !defined(AO_HAVE_compare_and_swap_double_release)
01223 #    define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
01224          AO_compare_and_swap_double_full(addr, o1, n1, n2)
01225 #    define AO_HAVE_compare_and_swap_double_release
01226 #  endif
01227 #  if !defined(AO_HAVE_compare_and_swap_double_acquire)
01228 #    define AO_compare_and_swap_double_acquire(addr, o1, n1, n2) \
01229          AO_compare_and_swap_double_full(addr, o1, n1, n2)
01230 #    define AO_HAVE_compare_and_swap_double_acquire
01231 #  endif
01232 #  if !defined(AO_HAVE_compare_and_swap_double_write)
01233 #    define AO_compare_and_swap_double_write(addr, o1, n1, n2) \
01234          AO_compare_and_swap_double_full(addr, o1, n1, n2)
01235 #    define AO_HAVE_compare_and_swap_double_write
01236 #  endif
01237 #  if !defined(AO_HAVE_compare_and_swap_double_read)
01238 #    define AO_compare_and_swap_double_read(addr, o1, n1, n2) \
01239          AO_compare_and_swap_double_full(addr, o1, n1, n2)
01240 #    define AO_HAVE_compare_and_swap_double_read
01241 #  endif
01242 #endif /* AO_HAVE_compare_and_swap_double_full */
01243 
01244 #if !defined(AO_HAVE_compare_and_swap_double) && \
01245     defined(AO_HAVE_compare_and_swap_double_release)
01246 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
01247         AO_compare_and_swap_double_release(addr, o1, n1, n2)
01248 #  define AO_HAVE_compare_and_swap_double
01249 #endif
01250 #if !defined(AO_HAVE_compare_and_swap_double) && \
01251     defined(AO_HAVE_compare_and_swap_double_acquire)
01252 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
01253         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
01254 #  define AO_HAVE_compare_and_swap_double
01255 #endif
01256 #if !defined(AO_HAVE_compare_and_swap_double) && \
01257     defined(AO_HAVE_compare_and_swap_double_write)
01258 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
01259         AO_compare_and_swap_double_write(addr, o1, n1, n2)
01260 #  define AO_HAVE_compare_and_swap_double
01261 #endif
01262 #if !defined(AO_HAVE_compare_and_swap_double) && \
01263     defined(AO_HAVE_compare_and_swap_double_read)
01264 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
01265         AO_compare_and_swap_double_read(addr, o1, n1, n2)
01266 #  define AO_HAVE_compare_and_swap_double
01267 #endif
01268 
01269 #if defined(AO_HAVE_compare_and_swap_double_acquire) &&\
01270     defined(AO_HAVE_nop_full) && \
01271     !defined(AO_HAVE_compare_and_swap_double_full)
01272 #  define AO_compare_and_swap_double_full(addr, o1, n1, n2) \
01273         (AO_nop_full(), AO_compare_and_swap_double_acquire(addr, o1, n1, n2))
01274 #  define AO_HAVE_compare_and_swap_double_full
01275 #endif
01276 
01277 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
01278     defined(AO_HAVE_compare_and_swap_double_write)
01279 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
01280         AO_compare_and_swap_double_write(addr, o1, n1, n2)
01281 #  define AO_HAVE_compare_and_swap_double_release_write
01282 #endif
01283 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
01284     defined(AO_HAVE_compare_and_swap_double_release)
01285 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
01286         AO_compare_and_swap_double_release(addr, o1, n1, n2)
01287 #  define AO_HAVE_compare_and_swap_double_release_write
01288 #endif
01289 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
01290     defined(AO_HAVE_compare_and_swap_double_read)
01291 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
01292         AO_compare_and_swap_double_read(addr, o1, n1, n2)
01293 #  define AO_HAVE_compare_and_swap_double_acquire_read
01294 #endif
01295 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
01296     defined(AO_HAVE_compare_and_swap_double_acquire)
01297 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
01298         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
01299 #  define AO_HAVE_compare_and_swap_double_acquire_read
01300 #endif
01301 
01302 #ifdef AO_NO_DD_ORDERING
01303 #  if defined(AO_HAVE_compare_and_swap_double_acquire_read)
01304 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
01305         AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2)
01306 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
01307 #  endif
01308 #else
01309 #  if defined(AO_HAVE_compare_and_swap_double)
01310 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
01311         AO_compare_and_swap_double(addr, o1, n1, n2)
01312 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
01313 #  endif
01314 #endif
01315 
01316 /* NEC LE-IT: Convenience functions for AO_double compare and swap which */
01317 /* types and reads easier in code                                        */
01318 #if defined(AO_HAVE_compare_double_and_swap_double_release) && \
01319     !defined(AO_HAVE_double_compare_and_swap_release)
01320 AO_INLINE int
01321 AO_double_compare_and_swap_release(volatile AO_double_t *addr,
01322                                    AO_double_t old_val, AO_double_t new_val)
01323 {
01324         return AO_compare_double_and_swap_double_release(addr,
01325                                                          old_val.AO_val1, old_val.AO_val2,
01326                                                          new_val.AO_val1, new_val.AO_val2);
01327 }
01328 #define AO_HAVE_double_compare_and_swap_release
01329 #endif
01330 
01331 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) && \
01332     !defined(AO_HAVE_double_compare_and_swap_acquire)
01333 AO_INLINE int
01334 AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
01335                                    AO_double_t old_val, AO_double_t new_val)
01336 {
01337         return AO_compare_double_and_swap_double_acquire(addr,
01338                                                          old_val.AO_val1, old_val.AO_val2,
01339                                                          new_val.AO_val1, new_val.AO_val2);
01340 }
01341 #define AO_HAVE_double_compare_and_swap_acquire
01342 #endif
01343 
01344 #if defined(AO_HAVE_compare_double_and_swap_double_full) && \
01345     !defined(AO_HAVE_double_compare_and_swap_full)
01346 AO_INLINE int
01347 AO_double_compare_and_swap_full(volatile AO_double_t *addr,
01348                                          AO_double_t old_val, AO_double_t new_val)
01349 {
01350         return AO_compare_double_and_swap_double_full(addr,
01351                                                       old_val.AO_val1, old_val.AO_val2,
01352                                                       new_val.AO_val1, new_val.AO_val2);
01353 }
01354 #define AO_HAVE_double_compare_and_swap_full
01355 #endif