Implementation sources

libpthread/src/pthread_rwlock.c

1056 lines · original file on GitHub
1/* 2 * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. Please obtain a copy of the License at 10 * http://www.opensource.apple.com/apsl/ and read it before using this 11 * file. 12 * 13 * The Original Code and all software distributed under the License are 14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 18 * Please see the License for the specific language governing rights and 19 * limitations under the License. 20 * 21 * @APPLE_LICENSE_HEADER_END@ 22 */ 23/*- 24 * Copyright (c) 1998 Alex Nash 25 * All rights reserved. 26 * 27 * Redistribution and use in source and binary forms, with or without 28 * modification, are permitted provided that the following conditions 29 * are met: 30 * 1. Redistributions of source code must retain the above copyright 31 * notice, this list of conditions and the following disclaimer. 32 * 2. Redistributions in binary form must reproduce the above copyright 33 * notice, this list of conditions and the following disclaimer in the 34 * documentation and/or other materials provided with the distribution. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 37 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 38 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 39 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 40 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 41 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 42 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 44 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 45 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 46 * SUCH DAMAGE. 47 * 48 * $FreeBSD: src/lib/libc_r/uthread/uthread_rwlock.c,v 1.6 2001/04/10 04:19:20 deischen Exp $ 49 */ 50 51/* 52 * POSIX Pthread Library 53 * -- Read Write Lock support 54 * 4/24/02: A. Ramesh 55 * Ported from FreeBSD 56 */ 57 58#include "resolver.h" 59#include "internal.h" 60 61#ifdef PLOCKSTAT 62#include "plockstat.h" 63#else /* !PLOCKSTAT */ 64#define PLOCKSTAT_RW_ERROR(x, y, z) 65#define PLOCKSTAT_RW_BLOCK(x, y) 66#define PLOCKSTAT_RW_BLOCKED(x, y, z) 67#define PLOCKSTAT_RW_ACQUIRE(x, y) 68#define PLOCKSTAT_RW_RELEASE(x, y) 69#endif /* PLOCKSTAT */ 70 71#define READ_LOCK_PLOCKSTAT 0 72#define WRITE_LOCK_PLOCKSTAT 1 73 74#define BLOCK_FAIL_PLOCKSTAT 0 75#define BLOCK_SUCCESS_PLOCKSTAT 1 76 77#define PTHREAD_RWLOCK_INIT_UNUSED 1 78 79// maximum number of times a read lock may be obtained 80#define MAX_READ_LOCKS (INT_MAX - 1) 81 82 83#if defined(__LP64__) 84#define RWLOCK_USE_INT128 1 85#endif 86 87typedef union rwlock_seq { 88 uint32_t seq[4]; 89 struct { uint32_t lcntval; uint32_t rw_seq; uint32_t ucntval; }; 90 struct { uint32_t lgen; uint32_t rw_wc; uint32_t ugen; }; 91#if RWLOCK_USE_INT128 92 unsigned __int128 seq_LSU; 93 unsigned __int128 _Atomic atomic_seq_LSU; 94#endif 95 struct { 96 uint64_t seq_LS; 97 uint32_t seq_U; 98 uint32_t _pad; 99 }; 100 struct { 101 uint64_t _Atomic atomic_seq_LS; 102 uint32_t _Atomic atomic_seq_U; 103 uint32_t _Atomic _atomic_pad; 104 }; 105} rwlock_seq; 106 107_Static_assert(sizeof(rwlock_seq) == 4 * sizeof(uint32_t), 108 "Incorrect rwlock_seq size"); 109 110typedef enum rwlock_seqfields { 111 RWLOCK_SEQ_NONE = 0, 112 RWLOCK_SEQ_LS = 1, 113 RWLOCK_SEQ_U = 2, 114 RWLOCK_SEQ_LSU = RWLOCK_SEQ_LS | RWLOCK_SEQ_U, 115} rwlock_seqfields; 116 117#if PTHREAD_DEBUG_LOG 118#define RWLOCK_DEBUG_SEQ(op, rwlock, oldseq, newseq, updateval, f) \ 119 if (_pthread_debuglog >= 0) { \ 120 _simple_dprintf(_pthread_debuglog, "rw_" #op " %p tck %7llu thr %llx " \ 121 "L %x -> %x S %x -> %x U %x -> %x updt %x\n", rwlock, \ 122 mach_absolute_time() - _pthread_debugstart, _pthread_threadid_self_np_direct(), \ 123 (f) & RWLOCK_SEQ_LS ? (oldseq).lcntval : 0, \ 124 (f) & RWLOCK_SEQ_LS ? (newseq).lcntval : 0, \ 125 (f) & RWLOCK_SEQ_LS ? (oldseq).rw_seq : 0, \ 126 (f) & RWLOCK_SEQ_LS ? (newseq).rw_seq : 0, \ 127 (f) & RWLOCK_SEQ_U ? (oldseq).ucntval : 0, \ 128 (f) & RWLOCK_SEQ_U ? (newseq).ucntval : 0, updateval); } 129#else 130#define RWLOCK_DEBUG_SEQ(m, rwlock, oldseq, newseq, updateval, f) 131#endif 132 133#if !__LITTLE_ENDIAN__ 134#error RWLOCK_GETSEQ_ADDR assumes little endian layout of sequence words 135#endif 136 137OS_ALWAYS_INLINE 138static inline void 139RWLOCK_GETSEQ_ADDR(pthread_rwlock_t *rwlock, rwlock_seq **seqaddr) 140{ 141 // 128-bit aligned address inside rw_seq & rw_mis arrays 142 *seqaddr = (void*)(((uintptr_t)rwlock->rw_seq + 0xful) & ~0xful); 143} 144 145OS_ALWAYS_INLINE 146static inline void 147RWLOCK_GETTID_ADDR(pthread_rwlock_t *rwlock, uint64_t **tidaddr) 148{ 149 // 64-bit aligned address inside rw_tid array (&rw_tid[0] for aligned lock) 150 *tidaddr = (void*)(((uintptr_t)rwlock->rw_tid + 0x7ul) & ~0x7ul); 151} 152 153OS_ALWAYS_INLINE 154static inline void 155rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval, 156 const rwlock_seqfields seqfields) 157{ 158 switch (seqfields) { 159 case RWLOCK_SEQ_LSU: 160#if RWLOCK_USE_INT128 161 oldseqval->seq_LSU = seqaddr->seq_LSU; 162#else 163 oldseqval->seq_LS = seqaddr->seq_LS; 164 oldseqval->seq_U = seqaddr->seq_U; 165#endif 166 break; 167 case RWLOCK_SEQ_LS: 168 oldseqval->seq_LS = seqaddr->seq_LS; 169 break; 170#if DEBUG // unused 171 case RWLOCK_SEQ_U: 172 oldseqval->seq_U = seqaddr->seq_U; 173 break; 174#endif // unused 175 default: 176 __builtin_trap(); 177 } 178} 179 180OS_ALWAYS_INLINE 181static inline void 182rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, 183 const rwlock_seqfields seqfields) 184{ 185 switch (seqfields) { 186 case RWLOCK_SEQ_LSU: 187#if RWLOCK_USE_INT128 188#if defined(__arm64__) && defined(__ARM_ARCH_8_2__) 189 // Workaround clang armv81 codegen bug for 128bit os_atomic_load 190 // rdar://problem/31213932 191 oldseqval->seq_LSU = seqaddr->seq_LSU; 192 while (!os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, 193 oldseqval->seq_LSU, oldseqval->seq_LSU, &oldseqval->seq_LSU, 194 relaxed)); 195#else 196 oldseqval->seq_LSU = os_atomic_load_wide(&seqaddr->atomic_seq_LSU, relaxed); 197#endif 198#else 199 oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed); 200 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed); 201#endif 202 break; 203 case RWLOCK_SEQ_LS: 204 oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed); 205 break; 206#if DEBUG // unused 207 case RWLOCK_SEQ_U: 208 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed); 209 break; 210#endif // unused 211 default: 212 __builtin_trap(); 213 } 214} 215 216#define rwlock_seq_atomic_load(seqaddr, oldseqval, seqfields, m) \ 217 rwlock_seq_atomic_load_##m(seqaddr, oldseqval, seqfields) 218 219OS_ALWAYS_INLINE 220static inline rwlock_seqfields 221rwlock_seq_atomic_cmpxchgv_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, 222 rwlock_seq *newseqval, const rwlock_seqfields seqfields) 223{ 224 bool r; 225 rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE; 226 switch (seqfields) { 227#if DEBUG // unused 228 case RWLOCK_SEQ_LSU: 229#if RWLOCK_USE_INT128 230 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU, 231 newseqval->seq_LSU, &oldseqval->seq_LSU, relaxed); 232 if (r) updated_seqfields = RWLOCK_SEQ_LSU; 233#else 234 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 235 newseqval->seq_LS, &oldseqval->seq_LS, relaxed); 236 if (r) { 237 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 238 newseqval->seq_U, &oldseqval->seq_U, relaxed); 239 if (!r) oldseqval->seq_LS = newseqval->seq_LS; 240 updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS; 241 } else { 242 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed); 243 } 244#endif 245 break; 246 case RWLOCK_SEQ_U: 247 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 248 newseqval->seq_U, &oldseqval->seq_U, relaxed); 249 if (r) updated_seqfields = RWLOCK_SEQ_U; 250 break; 251#endif // unused 252 case RWLOCK_SEQ_LS: 253 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 254 newseqval->seq_LS, &oldseqval->seq_LS, relaxed); 255 if (r) updated_seqfields = RWLOCK_SEQ_LS; 256 break; 257 default: 258 __builtin_trap(); 259 } 260 return updated_seqfields; 261} 262 263OS_ALWAYS_INLINE 264static inline rwlock_seqfields 265rwlock_seq_atomic_cmpxchgv_acquire(rwlock_seq *seqaddr, rwlock_seq *oldseqval, 266 rwlock_seq *newseqval, const rwlock_seqfields seqfields) 267{ 268 bool r; 269 rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE; 270 switch (seqfields) { 271#if DEBUG // unused 272 case RWLOCK_SEQ_LSU: 273#if RWLOCK_USE_INT128 274 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU, 275 newseqval->seq_LSU, &oldseqval->seq_LSU, acquire); 276 if (r) updated_seqfields = RWLOCK_SEQ_LSU; 277#else 278 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 279 newseqval->seq_LS, &oldseqval->seq_LS, acquire); 280 if (r) { 281 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 282 newseqval->seq_U, &oldseqval->seq_U, relaxed); 283 if (!r) oldseqval->seq_LS = newseqval->seq_LS; 284 updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_LS; 285 } else { 286 oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed); 287 } 288#endif 289 break; 290 case RWLOCK_SEQ_U: 291 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 292 newseqval->seq_U, &oldseqval->seq_U, acquire); 293 if (r) updated_seqfields = RWLOCK_SEQ_U; 294 break; 295#endif // unused 296 case RWLOCK_SEQ_LS: 297 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 298 newseqval->seq_LS, &oldseqval->seq_LS, acquire); 299 if (r) updated_seqfields = RWLOCK_SEQ_LS; 300 break; 301 default: 302 __builtin_trap(); 303 } 304 return updated_seqfields; 305} 306 307OS_ALWAYS_INLINE 308static inline rwlock_seqfields 309rwlock_seq_atomic_cmpxchgv_release(rwlock_seq *seqaddr, rwlock_seq *oldseqval, 310 rwlock_seq *newseqval, const rwlock_seqfields seqfields) 311{ 312 bool r; 313 rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE; 314 switch (seqfields) { 315 case RWLOCK_SEQ_LSU: 316#if RWLOCK_USE_INT128 317 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU, 318 newseqval->seq_LSU, &oldseqval->seq_LSU, release); 319 if (r) updated_seqfields = RWLOCK_SEQ_LSU; 320#else 321 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 322 newseqval->seq_U, &oldseqval->seq_U, release); 323 if (r) { 324 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 325 newseqval->seq_LS, &oldseqval->seq_LS, relaxed); 326 if (!r) oldseqval->seq_U = newseqval->seq_U; 327 updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_U; 328 } else { 329 oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, 330 relaxed); 331 } 332#endif 333 break; 334 case RWLOCK_SEQ_LS: 335 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_LS, oldseqval->seq_LS, 336 newseqval->seq_LS, &oldseqval->seq_LS, release); 337 if (r) updated_seqfields = RWLOCK_SEQ_LS; 338 break; 339#if DEBUG // unused 340 case RWLOCK_SEQ_U: 341 r = os_atomic_cmpxchgv(&seqaddr->atomic_seq_U, oldseqval->seq_U, 342 newseqval->seq_U, &oldseqval->seq_U, release); 343 if (r) updated_seqfields = RWLOCK_SEQ_U; 344 break; 345#endif // unused 346 default: 347 __builtin_trap(); 348 } 349 return updated_seqfields; 350} 351 352#define rwlock_seq_atomic_cmpxchgv(seqaddr, oldseqval, newseqval, seqfields, m)\ 353 rwlock_seq_atomic_cmpxchgv_##m(seqaddr, oldseqval, newseqval, seqfields) 354 355#ifndef BUILDING_VARIANT /* [ */ 356 357int 358pthread_rwlockattr_init(pthread_rwlockattr_t *attr) 359{ 360 attr->sig = _PTHREAD_RWLOCK_ATTR_SIG; 361 attr->pshared = _PTHREAD_DEFAULT_PSHARED; 362 return 0; 363} 364 365int 366pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr) 367{ 368 attr->sig = _PTHREAD_NO_SIG; 369 attr->pshared = 0; 370 return 0; 371} 372 373int 374pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, int *pshared) 375{ 376 int res = EINVAL; 377 if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) { 378 *pshared = (int)attr->pshared; 379 res = 0; 380 } 381 return res; 382} 383 384int 385pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared) 386{ 387 int res = EINVAL; 388 if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) { 389 if (( pshared == PTHREAD_PROCESS_PRIVATE) || 390 (pshared == PTHREAD_PROCESS_SHARED)) 391 { 392 attr->pshared = pshared ; 393 res = 0; 394 } 395 } 396 return res; 397} 398 399#endif /* !BUILDING_VARIANT ] */ 400 401OS_ALWAYS_INLINE 402static inline int 403_pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) 404{ 405 uint64_t *tidaddr; 406 RWLOCK_GETTID_ADDR(rwlock, &tidaddr); 407 408 rwlock_seq *seqaddr; 409 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 410 411#if PTHREAD_RWLOCK_INIT_UNUSED 412 if ((uint32_t*)tidaddr != rwlock->rw_tid) { 413 rwlock->misalign = 1; 414 __builtin_memset(rwlock->rw_tid, 0xff, sizeof(rwlock->rw_tid)); 415 } 416 if ((uint32_t*)seqaddr != rwlock->rw_seq) { 417 __builtin_memset(rwlock->rw_seq, 0xff, sizeof(rwlock->rw_seq)); 418 } 419 __builtin_memset(rwlock->rw_mis, 0xff, sizeof(rwlock->rw_mis)); 420#endif // PTHREAD_MUTEX_INIT_UNUSED 421 *tidaddr = 0; 422 *seqaddr = (rwlock_seq){ 423 .lcntval = PTHRW_RWLOCK_INIT, 424 .rw_seq = PTHRW_RWS_INIT, 425 .ucntval = 0, 426 }; 427 428 if (attr != NULL && attr->pshared == PTHREAD_PROCESS_SHARED) { 429 rwlock->pshared = PTHREAD_PROCESS_SHARED; 430 rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED; 431 } else { 432 rwlock->pshared = _PTHREAD_DEFAULT_PSHARED; 433 rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE; 434 } 435 436 long sig = _PTHREAD_RWLOCK_SIG; 437 438#if DEBUG 439 bzero(rwlock->_reserved, sizeof(rwlock->_reserved)); 440#endif 441#if PTHREAD_RWLOCK_INIT_UNUSED 442 // For detecting copied rwlocks and smashes during debugging 443 uint32_t sig32 = (uint32_t)sig; 444 uintptr_t guard = ~(uintptr_t)rwlock; // use ~ to hide from leaks 445 __builtin_memcpy(rwlock->_reserved, &guard, sizeof(guard)); 446#define countof(x) (sizeof(x) / sizeof(x[0])) 447 rwlock->_reserved[countof(rwlock->_reserved) - 1] = sig32; 448#if defined(__LP64__) 449 rwlock->_pad = sig32; 450#endif 451#endif // PTHREAD_RWLOCK_INIT_UNUSED 452 453 // Ensure all contents are properly set before setting signature. 454#if defined(__LP64__) 455 // For binary compatibility reasons we cannot require natural alignment of 456 // the 64bit 'sig' long value in the struct. rdar://problem/21610439 457 uint32_t *sig32_ptr = (uint32_t*)&rwlock->sig; 458 uint32_t *sig32_val = (uint32_t*)&sig; 459 *(sig32_ptr + 1) = *(sig32_val + 1); 460 os_atomic_store(sig32_ptr, *sig32_val, release); 461#else 462 os_atomic_store(&rwlock->sig, sig, release); 463#endif 464 465 return 0; 466} 467 468static uint32_t 469_pthread_rwlock_modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits) 470{ 471 uint32_t lval = lgenval & PTHRW_BIT_MASK; 472 uint32_t uval = updateval & PTHRW_BIT_MASK; 473 uint32_t rval, nlval; 474 475 nlval = (lval | uval) & ~(PTH_RWL_MBIT); 476 477 // reconcile bits on the lock with what kernel needs to set 478 if ((uval & PTH_RWL_KBIT) == 0 && (lval & PTH_RWL_WBIT) == 0) { 479 nlval &= ~PTH_RWL_KBIT; 480 } 481 482 if (savebits != 0) { 483 if ((savebits & PTH_RWS_WSVBIT) != 0 && (nlval & PTH_RWL_WBIT) == 0 && 484 (nlval & PTH_RWL_EBIT) == 0) { 485 nlval |= (PTH_RWL_WBIT | PTH_RWL_KBIT); 486 } 487 } 488 rval = (lgenval & PTHRW_COUNT_MASK) | nlval; 489 return(rval); 490} 491 492OS_ALWAYS_INLINE 493static inline void 494_pthread_rwlock_updateval(pthread_rwlock_t *rwlock, uint32_t updateval) 495{ 496 bool isoverlap = (updateval & PTH_RWL_MBIT) != 0; 497 498 // TBD: restore U bit 499 rwlock_seq *seqaddr; 500 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 501 502 rwlock_seq oldseq, newseq; 503 rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LS); 504 do { 505 newseq = oldseq; 506 if (isoverlap || is_rws_unlockinit_set(oldseq.rw_seq)) { 507 // Set S word to the specified value 508 uint32_t savebits = (oldseq.rw_seq & PTHRW_RWS_SAVEMASK); 509 newseq.lcntval = _pthread_rwlock_modbits(oldseq.lcntval, updateval, 510 savebits); 511 newseq.rw_seq += (updateval & PTHRW_COUNT_MASK); 512 if (!isoverlap) { 513 newseq.rw_seq &= PTHRW_COUNT_MASK; 514 } 515 newseq.rw_seq &= ~PTHRW_RWS_SAVEMASK; 516 } 517 } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, 518 RWLOCK_SEQ_LS, relaxed)); 519 RWLOCK_DEBUG_SEQ(update, rwlock, oldseq, newseq, updateval, RWLOCK_SEQ_LS); 520} 521 522OS_ALWAYS_INLINE 523static inline int 524_pthread_rwlock_check_busy(pthread_rwlock_t *rwlock) 525{ 526 int res = 0; 527 528 rwlock_seq *seqaddr; 529 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 530 531 rwlock_seq seq; 532 rwlock_seq_atomic_load(seqaddr, &seq, RWLOCK_SEQ_LSU, relaxed); 533 if ((seq.lcntval & PTHRW_COUNT_MASK) != seq.ucntval) { 534 res = EBUSY; 535 } 536 537 return res; 538} 539 540PTHREAD_NOEXPORT_VARIANT 541int 542pthread_rwlock_destroy(pthread_rwlock_t *rwlock) 543{ 544 int res = 0; 545 546 _pthread_lock_lock(&rwlock->lock); 547 if (_pthread_rwlock_check_signature(rwlock)) { 548 res = _pthread_rwlock_check_busy(rwlock); 549 } else if (!_pthread_rwlock_check_signature_init(rwlock)) { 550 res = EINVAL; 551 } 552 if (res == 0) { 553 rwlock->sig = _PTHREAD_NO_SIG; 554 } 555 _pthread_lock_unlock(&rwlock->lock); 556 return res; 557} 558 559PTHREAD_NOEXPORT_VARIANT 560int 561pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) 562{ 563 int res = 0; 564 565 if (attr && attr->sig != _PTHREAD_RWLOCK_ATTR_SIG) { 566 res = EINVAL; 567 } 568 569 if (res == 0 && _pthread_rwlock_check_signature(rwlock)) { 570 res = _pthread_rwlock_check_busy(rwlock); 571 } 572 if (res == 0) { 573 _pthread_lock_init(&rwlock->lock); 574 res = _pthread_rwlock_init(rwlock, attr); 575 } 576 return res; 577} 578 579OS_NOINLINE 580static int 581_pthread_rwlock_check_init_slow(pthread_rwlock_t *rwlock) 582{ 583 int res = EINVAL; 584 585 if (_pthread_rwlock_check_signature_init(rwlock)) { 586 _pthread_lock_lock(&rwlock->lock); 587 if (_pthread_rwlock_check_signature_init(rwlock)) { 588 res = _pthread_rwlock_init(rwlock, NULL); 589 } else if (_pthread_rwlock_check_signature(rwlock)){ 590 res = 0; 591 } 592 _pthread_lock_unlock(&rwlock->lock); 593 } else if (_pthread_rwlock_check_signature(rwlock)){ 594 res = 0; 595 } 596 if (res != 0) { 597 PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, res); 598 } 599 return res; 600} 601 602OS_ALWAYS_INLINE 603static inline int 604_pthread_rwlock_check_init(pthread_rwlock_t *rwlock) 605{ 606 int res = 0; 607 608 if (!_pthread_rwlock_check_signature(rwlock)) { 609 return _pthread_rwlock_check_init_slow(rwlock); 610 } 611 return res; 612} 613 614OS_NOINLINE 615static int 616_pthread_rwlock_lock_wait(pthread_rwlock_t *rwlock, bool readlock, 617 rwlock_seq newseq) 618{ 619 int res; 620 621#ifdef PLOCKSTAT 622 int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT; 623#endif 624 625 if (readlock) { 626 RWLOCK_DEBUG_SEQ(rdlock, rwlock, oldseq, newseq, gotlock, 627 RWLOCK_SEQ_LSU); 628 } else { 629 RWLOCK_DEBUG_SEQ(wrlock, rwlock, oldseq, newseq, gotlock, 630 RWLOCK_SEQ_LSU); 631 } 632 633 uint32_t updateval; 634 635 PLOCKSTAT_RW_BLOCK(rwlock, plockstat); 636 637 do { 638 if (readlock) { 639 updateval = __psynch_rw_rdlock(rwlock, newseq.lcntval, 640 newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); 641 } else { 642 updateval = __psynch_rw_wrlock(rwlock, newseq.lcntval, 643 newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); 644 } 645 if (updateval == (uint32_t)-1) { 646 res = errno; 647 } else { 648 res = 0; 649 } 650 } while (res == EINTR); 651 652 if (res == 0) { 653 _pthread_rwlock_updateval(rwlock, updateval); 654 PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_SUCCESS_PLOCKSTAT); 655 } else { 656 PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_FAIL_PLOCKSTAT); 657 PTHREAD_INTERNAL_CRASH(res, "kernel rwlock returned unknown error"); 658 } 659 660 return res; 661} 662 663OS_NOINLINE 664int 665_pthread_rwlock_lock_slow(pthread_rwlock_t *rwlock, bool readlock, 666 bool trylock) 667{ 668 int res; 669 670#ifdef PLOCKSTAT 671 int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT; 672#endif 673 674 res = _pthread_rwlock_check_init(rwlock); 675 if (res != 0) return res; 676 677 rwlock_seq *seqaddr; 678 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 679 680 rwlock_seq oldseq, newseq; 681 rwlock_seq_atomic_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU, relaxed); 682 683 uint64_t *tidaddr; 684 RWLOCK_GETTID_ADDR(rwlock, &tidaddr); 685 uint64_t selfid = _pthread_threadid_self_np_direct(); 686 if (is_rwl_ebit_set(oldseq.lcntval)) { 687 if (os_atomic_load_wide(tidaddr, relaxed) == selfid) return EDEADLK; 688 } 689 690 int retry_count; 691 bool gotlock; 692 do { 693 retry_count = 0; 694retry: 695 newseq = oldseq; 696 697 // if W and K bit are clear or U bit is on, acquire lock in userland 698 if (readlock) { 699 gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0; 700 } else { 701 gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0; 702 } 703 704 if (trylock && !gotlock) { 705 // A trylock on a held lock will fail immediately. But since 706 // we did not load the sequence words atomically, perform a 707 // no-op CAS to ensure that nobody has unlocked concurrently. 708 } else if (gotlock) { 709 if (readlock) { 710 if (diff_genseq(oldseq.lcntval, oldseq.ucntval) >= 711 PTHRW_MAX_READERS) { 712 // since ucntval may be newer, just redo 713 retry_count++; 714 if (retry_count > 1024) { 715 gotlock = false; 716 res = EAGAIN; 717 goto out; 718 } else { 719 sched_yield(); 720 rwlock_seq_atomic_load(seqaddr, &oldseq, 721 RWLOCK_SEQ_LSU, relaxed); 722 goto retry; 723 } 724 } 725 // Need to update L (remove U bit) and S word 726 newseq.lcntval &= ~PTH_RWL_UBIT; 727 } else { 728 newseq.lcntval &= PTHRW_COUNT_MASK; 729 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT; 730 } 731 newseq.lcntval += PTHRW_INC; 732 newseq.rw_seq += PTHRW_INC; 733 } else { 734 if (readlock) { 735 // Need to block in kernel. Remove U bit. 736 newseq.lcntval &= ~PTH_RWL_UBIT; 737 } else { 738 newseq.lcntval |= PTH_RWL_KBIT | PTH_RWL_WBIT; 739 } 740 newseq.lcntval += PTHRW_INC; 741 if (is_rws_sbit_set(oldseq.rw_seq)) { 742 // Clear the S bit and set S to L 743 newseq.rw_seq &= (PTHRW_BIT_MASK & ~PTH_RWS_SBIT); 744 newseq.rw_seq |= (oldseq.lcntval & PTHRW_COUNT_MASK); 745 } 746 } 747 } while (!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, 748 RWLOCK_SEQ_LS, acquire)); 749 750 if (gotlock) { 751 if (!readlock) os_atomic_store_wide(tidaddr, selfid, relaxed); 752 res = 0; 753 } else if (trylock) { 754 res = EBUSY; 755 } else { 756 res = _pthread_rwlock_lock_wait(rwlock, readlock, newseq); 757 } 758 759out: 760#ifdef PLOCKSTAT 761 if (res == 0) { 762 PLOCKSTAT_RW_ACQUIRE(rwlock, plockstat); 763 } else { 764 PLOCKSTAT_RW_ERROR(rwlock, plockstat, res); 765 } 766#endif 767 768 return res; 769} 770 771OS_ALWAYS_INLINE 772static inline int 773_pthread_rwlock_lock(pthread_rwlock_t *rwlock, bool readlock, bool trylock) 774{ 775#if PLOCKSTAT 776 if (PLOCKSTAT_RW_ACQUIRE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) { 777 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); 778 } 779#endif 780 781 if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) { 782 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); 783 } 784 785 rwlock_seq *seqaddr; 786 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 787 788 rwlock_seq oldseq, newseq; 789 // no need to perform a single-copy-atomic 128-bit load in the fastpath, 790 // if stores to L and U are seen out of order, we will fallback to the 791 // slowpath below (which has rwlock_seq_atomic_load) 792 rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU); 793 794 if (os_unlikely(is_rwl_ebit_set(oldseq.lcntval))) { 795 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); 796 } 797 798 bool gotlock; 799 do { 800 newseq = oldseq; 801 802 // if W and K bit are clear or U bit is on, acquire lock in userland 803 if (readlock) { 804 gotlock = (oldseq.lcntval & (PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0; 805 } else { 806 gotlock = (oldseq.lcntval & PTH_RWL_UBIT) != 0; 807 } 808 809 if (trylock && !gotlock) { 810 // A trylock on a held lock will fail immediately. But since 811 // we did not load the sequence words atomically, perform a 812 // no-op CAS to ensure that nobody has unlocked concurrently. 813 } else if (os_likely(gotlock)) { 814 if (readlock) { 815 if (os_unlikely(diff_genseq(oldseq.lcntval, oldseq.ucntval) >= 816 PTHRW_MAX_READERS)) { 817 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); 818 } 819 // Need to update L (remove U bit) and S word 820 newseq.lcntval &= ~PTH_RWL_UBIT; 821 } else { 822 newseq.lcntval &= PTHRW_COUNT_MASK; 823 newseq.lcntval |= PTH_RWL_IBIT | PTH_RWL_KBIT | PTH_RWL_EBIT; 824 } 825 newseq.lcntval += PTHRW_INC; 826 newseq.rw_seq += PTHRW_INC; 827 } else { 828 return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); 829 } 830 } while (os_unlikely(!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, 831 RWLOCK_SEQ_LS, acquire))); 832 833 if (os_likely(gotlock)) { 834 if (!readlock) { 835 uint64_t *tidaddr; 836 RWLOCK_GETTID_ADDR(rwlock, &tidaddr); 837 uint64_t selfid = _pthread_threadid_self_np_direct(); 838 os_atomic_store_wide(tidaddr, selfid, relaxed); 839 } 840 return 0; 841 } else if (trylock) { 842 return EBUSY; 843 } else { 844 __builtin_trap(); 845 } 846} 847 848PTHREAD_NOEXPORT_VARIANT 849int 850pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) 851{ 852 // read lock, no try 853 return _pthread_rwlock_lock(rwlock, true, false); 854} 855 856PTHREAD_NOEXPORT_VARIANT 857int 858pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) 859{ 860 // read lock, try lock 861 return _pthread_rwlock_lock(rwlock, true, true); 862} 863 864PTHREAD_NOEXPORT_VARIANT 865int 866pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) 867{ 868 // write lock, no try 869 return _pthread_rwlock_lock(rwlock, false, false); 870} 871 872PTHREAD_NOEXPORT_VARIANT 873int 874pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) 875{ 876 // write lock, try lock 877 return _pthread_rwlock_lock(rwlock, false, true); 878} 879 880OS_NOINLINE 881static int 882_pthread_rwlock_unlock_drop(pthread_rwlock_t *rwlock, rwlock_seq oldseq, 883 rwlock_seq newseq) 884{ 885 int res; 886 887 RWLOCK_DEBUG_SEQ(unlock, rwlock, oldseq, newseq, !droplock, RWLOCK_SEQ_LSU); 888 uint32_t updateval; 889 do { 890 updateval = __psynch_rw_unlock(rwlock, oldseq.lcntval, 891 newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); 892 if (updateval == (uint32_t)-1) { 893 res = errno; 894 } else { 895 res = 0; 896 RWLOCK_DEBUG_SEQ(wakeup, rwlock, oldseq, newseq, updateval, 897 RWLOCK_SEQ_LSU); 898 } 899 } while (res == EINTR); 900 901 if (res != 0) { 902 PTHREAD_INTERNAL_CRASH(res, "kernel rwunlock returned unknown error"); 903 } 904 905 return res; 906} 907 908OS_NOINLINE 909int 910_pthread_rwlock_unlock_slow(pthread_rwlock_t *rwlock, 911 rwlock_seqfields updated_seqfields) 912{ 913 int res; 914 rwlock_seqfields seqfields = RWLOCK_SEQ_LSU; 915#ifdef PLOCKSTAT 916 int wrlock = 0; 917#endif 918 919 res = _pthread_rwlock_check_init(rwlock); 920 if (res != 0) return res; 921 922 rwlock_seq *seqaddr; 923 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 924 925 rwlock_seq oldseq, newseq; 926 rwlock_seq_load(seqaddr, &oldseq, seqfields); 927 928 if ((oldseq.lcntval & PTH_RWL_UBIT) != 0) { 929 // spurious unlock (unlock of unlocked lock) 930 return 0; 931 } 932 933 if (is_rwl_ebit_set(oldseq.lcntval)) { 934#ifdef PLOCKSTAT 935 wrlock = 1; 936#endif 937 uint64_t *tidaddr; 938 RWLOCK_GETTID_ADDR(rwlock, &tidaddr); 939 os_atomic_store_wide(tidaddr, 0, relaxed); 940 } 941 942 bool droplock; 943 do { 944 // stop loading & updating fields that have successfully been stored 945 seqfields &= ~updated_seqfields; 946 947 newseq = oldseq; 948 if (seqfields & RWLOCK_SEQ_U) { 949 newseq.ucntval += PTHRW_INC; 950 } 951 952 droplock = false; 953 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK); 954 if (newseq.ucntval == oldlcnt) { 955 // last unlock, set L with U and init bits and set S to L with S bit 956 newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT; 957 newseq.rw_seq = oldlcnt | PTHRW_RWS_INIT; 958 } else { 959 // no L/S update if lock is not exclusive or no writer pending 960 if ((oldseq.lcntval & 961 (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0) { 962 continue; 963 } 964 965 // kernel transition only needed if U == S 966 if (newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK)) { 967 continue; 968 } 969 970 droplock = true; 971 // reset all bits and set K 972 newseq.lcntval = oldlcnt | PTH_RWL_KBIT; 973 // set I bit on S word 974 newseq.rw_seq |= PTH_RWS_IBIT; 975 if ((oldseq.lcntval & PTH_RWL_WBIT) != 0) { 976 newseq.rw_seq |= PTH_RWS_WSVBIT; 977 } 978 } 979 } while (seqfields != (updated_seqfields = rwlock_seq_atomic_cmpxchgv( 980 seqaddr, &oldseq, &newseq, seqfields, release))); 981 982 if (droplock) { 983 res = _pthread_rwlock_unlock_drop(rwlock, oldseq, newseq); 984 } 985 986 PLOCKSTAT_RW_RELEASE(rwlock, wrlock); 987 988 return res; 989} 990 991PTHREAD_NOEXPORT_VARIANT 992int 993pthread_rwlock_unlock(pthread_rwlock_t *rwlock) 994{ 995 rwlock_seqfields seqfields = RWLOCK_SEQ_LSU; 996 rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE; 997 998#if PLOCKSTAT 999 if (PLOCKSTAT_RW_RELEASE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) { 1000 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); 1001 } 1002#endif 1003 1004 if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) { 1005 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); 1006 } 1007 1008 rwlock_seq *seqaddr; 1009 RWLOCK_GETSEQ_ADDR(rwlock, &seqaddr); 1010 1011 rwlock_seq oldseq, newseq; 1012 rwlock_seq_load(seqaddr, &oldseq, seqfields); 1013 1014 if (os_unlikely(oldseq.lcntval & PTH_RWL_UBIT)) { 1015 // spurious unlock (unlock of unlocked lock) 1016 return 0; 1017 } 1018 1019 if (is_rwl_ebit_set(oldseq.lcntval)) { 1020 uint64_t *tidaddr; 1021 RWLOCK_GETTID_ADDR(rwlock, &tidaddr); 1022 os_atomic_store_wide(tidaddr, 0, relaxed); 1023 } 1024 1025 do { 1026 if (updated_seqfields) { 1027 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); 1028 } 1029 1030 newseq = oldseq; 1031 if (seqfields & RWLOCK_SEQ_U) { 1032 newseq.ucntval += PTHRW_INC; 1033 } 1034 1035 uint32_t oldlcnt = (oldseq.lcntval & PTHRW_COUNT_MASK); 1036 if (os_likely(newseq.ucntval == oldlcnt)) { 1037 // last unlock, set L with U and init bits and set S to L with S bit 1038 newseq.lcntval = oldlcnt | PTHRW_RWLOCK_INIT; 1039 newseq.rw_seq = oldlcnt | PTHRW_RWS_INIT; 1040 } else { 1041 if (os_likely((oldseq.lcntval & 1042 (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_KBIT)) == 0 || 1043 newseq.ucntval != (oldseq.rw_seq & PTHRW_COUNT_MASK))) { 1044 // no L/S update if lock is not exclusive or no writer pending 1045 // kernel transition only needed if U == S 1046 } else { 1047 return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); 1048 } 1049 } 1050 } while (os_unlikely(seqfields != (updated_seqfields = 1051 rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, seqfields, 1052 release)))); 1053 1054 return 0; 1055} 1056