libpthread/src/pthread_cond.c
1/*
2 * Copyright (c) 2000-2003, 2007, 2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23/*
24 * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
25 * All Rights Reserved
26 *
27 * Permission to use, copy, modify, and distribute this software and
28 * its documentation for any purpose and without fee is hereby granted,
29 * provided that the above copyright notice appears in all copies and
30 * that both the copyright notice and this permission notice appear in
31 * supporting documentation.
32 *
33 * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
34 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE.
36 *
37 * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
38 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
39 * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
40 * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
41 * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
42 */
43/*
44 * MkLinux
45 */
46
47/*
48 * POSIX Pthread Library
49 */
50
51#include "resolver.h"
52#include "internal.h"
53#include <sys/time.h> /* For struct timespec and getclock(). */
54
55#ifdef PLOCKSTAT
56#include "plockstat.h"
57#else /* !PLOCKSTAT */
58#define PLOCKSTAT_MUTEX_RELEASE(x, y)
59#endif /* PLOCKSTAT */
60
61typedef union {
62 uint64_t val;
63 struct {
64 uint32_t seq;
65 uint16_t waiters;
66 uint16_t signal;
67 };
68} pthread_ulock_cond_state_u;
69
70#define _PTHREAD_COND_WAITERS_INC \
71 (1ull << (offsetof(pthread_ulock_cond_state_u, waiters) * CHAR_BIT))
72
73OS_ALWAYS_INLINE
74static inline void
75COND_GETSEQ_ADDR(pthread_cond_t *cond,
76 volatile uint64_t **c_lsseqaddr,
77 volatile uint32_t **c_lseqcnt,
78 volatile uint32_t **c_useqcnt,
79 volatile uint32_t **c_sseqcnt)
80{
81 if (cond->misalign) {
82 *c_lseqcnt = &cond->c_seq[1];
83 *c_sseqcnt = &cond->c_seq[2];
84 *c_useqcnt = &cond->c_seq[0];
85 } else {
86 *c_lseqcnt = &cond->c_seq[0];
87 *c_sseqcnt = &cond->c_seq[1];
88 *c_useqcnt = &cond->c_seq[2];
89 }
90 *c_lsseqaddr = (volatile uint64_t *)*c_lseqcnt;
91}
92
93OS_ALWAYS_INLINE
94static inline pthread_ulock_cond_state_u *
95_pthread_ulock_cond_state(pthread_cond_t *cond)
96{
97 return (pthread_ulock_cond_state_u *)&cond->c_seq[cond->misalign];
98}
99
100#ifndef BUILDING_VARIANT /* [ */
101
102static void _pthread_psynch_cond_cleanup(void *arg);
103static void _pthread_cond_updateval(pthread_cond_t *cond, pthread_mutex_t *mutex,
104 int error, uint32_t updateval);
105
106static int
107_pthread_ulock_cond_wait_complete(pthread_ulock_cond_state_u *state,
108 pthread_mutex_t *mutex, int rc);
109static void
110_pthread_ulock_cond_cleanup(void *arg);
111
112
113int
114pthread_condattr_init(pthread_condattr_t *attr)
115{
116 attr->sig = _PTHREAD_COND_ATTR_SIG;
117 attr->pshared = _PTHREAD_DEFAULT_PSHARED;
118 return 0;
119}
120
121int
122pthread_condattr_destroy(pthread_condattr_t *attr)
123{
124 attr->sig = _PTHREAD_NO_SIG;
125 return 0;
126}
127
128int
129pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared)
130{
131 int res = EINVAL;
132 if (attr->sig == _PTHREAD_COND_ATTR_SIG) {
133 *pshared = (int)attr->pshared;
134 res = 0;
135 }
136 return res;
137}
138
139int
140pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared)
141{
142 int res = EINVAL;
143 if (attr->sig == _PTHREAD_COND_ATTR_SIG) {
144 if (pshared == PTHREAD_PROCESS_PRIVATE || pshared == PTHREAD_PROCESS_SHARED) {
145 attr->pshared = pshared;
146 res = 0;
147 }
148 }
149 return res;
150}
151
152int
153pthread_cond_timedwait_relative_np(pthread_cond_t *cond, pthread_mutex_t *mutex,
154 const struct timespec *abstime)
155{
156 return _pthread_cond_wait(cond, mutex, abstime, 1,
157 PTHREAD_CONFORM_UNIX03_NOCANCEL);
158}
159
160#endif /* !BUILDING_VARIANT ] */
161
162OS_ALWAYS_INLINE
163static inline int
164_pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr,
165 uint32_t sig)
166{
167 volatile uint64_t *c_lsseqaddr;
168 volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt;
169
170 cond->busy = NULL;
171 cond->c_seq[0] = 0;
172 cond->c_seq[1] = 0;
173 cond->c_seq[2] = 0;
174 cond->unused = 0;
175
176 // TODO: PTHREAD_STRICT candidate
177 cond->misalign = (((uintptr_t)&cond->c_seq[0]) & 0x7) != 0;
178 COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt);
179 *c_sseqcnt = PTH_RWS_CV_CBIT; // set Sword to 0c
180
181 if (attr) {
182 cond->pshared = attr->pshared;
183 } else {
184 cond->pshared = _PTHREAD_DEFAULT_PSHARED;
185 }
186
187 // Ensure all contents are properly set before setting signature.
188#if defined(__LP64__)
189 // For binary compatibility reasons we cannot require natural alignment of
190 // the 64bit 'sig' long value in the struct. rdar://problem/21610439
191 cond->sig._pad = 0;
192#endif
193 os_atomic_store(&cond->sig.val, sig, release);
194
195 return 0;
196}
197
198#ifndef BUILDING_VARIANT /* [ */
199
200OS_ALWAYS_INLINE
201static int
202_pthread_cond_check_signature(pthread_cond_t *cond, uint32_t sig_current,
203 uint32_t *sig_inout)
204{
205 int res = 0;
206 switch (sig_current) {
207 case _PTHREAD_COND_SIG_init:
208 __builtin_unreachable();
209 break;
210 case _PTHREAD_COND_SIG_pristine:
211 if (*sig_inout != _PTHREAD_COND_SIG_pristine) {
212 os_atomic_store(&cond->sig.val, *sig_inout, relaxed);
213 }
214 break;
215 case _PTHREAD_COND_SIG_psynch:
216 case _PTHREAD_COND_SIG_ulock:
217 if (*sig_inout == _PTHREAD_COND_SIG_pristine) {
218 *sig_inout = sig_current;
219 } else if (*sig_inout != sig_current) {
220 PTHREAD_INTERNAL_CRASH(0, "Mixed ulock and psych condvar use");
221 }
222 break;
223 default:
224 // TODO: PTHREAD_STRICT candidate
225 res = EINVAL;
226 break;
227 }
228
229 return res;
230}
231
232OS_NOINLINE
233static int
234_pthread_cond_check_init_slow(pthread_cond_t *cond, uint32_t *sig_inout)
235{
236 int res;
237 _pthread_lock_lock(&cond->lock);
238
239 uint32_t sig_current = os_atomic_load(&cond->sig.val, relaxed);
240 if (sig_current == _PTHREAD_COND_SIG_init) {
241 res = _pthread_cond_init(cond, NULL, *sig_inout);
242 } else {
243 res = _pthread_cond_check_signature(cond, sig_current, sig_inout);
244 }
245
246 _pthread_lock_unlock(&cond->lock);
247 return res;
248}
249
250/*
251 * These routines maintain the signature of the condition variable, which
252 * encodes a small state machine:
253 * - a statically initialized condvar begins with SIG_init
254 * - explicit initialization via _cond_init() and implicit initialization
255 * transition to SIG_pristine, as there have been no waiters so we don't know
256 * what kind of mutex we'll be used with
257 * - the first _cond_wait() transitions to one of SIG_psynch or SIG_ulock
258 * according to the mutex being waited on
259 *
260 * On entry, *sig_inout is the furthest state we can transition to given the
261 * calling context. On exit, it is the actual state we observed, after any
262 * possible advancement.
263 */
264OS_ALWAYS_INLINE
265static inline int
266_pthread_cond_check_init(pthread_cond_t *cond, uint32_t *sig_inout)
267{
268 uint32_t sig_current = os_atomic_load(&cond->sig.val, relaxed);
269 if (sig_current == _PTHREAD_COND_SIG_init) {
270 return _pthread_cond_check_init_slow(cond, sig_inout);
271 } else {
272 return _pthread_cond_check_signature(cond, sig_current, sig_inout);
273 }
274}
275
276PTHREAD_NOEXPORT_VARIANT
277int
278pthread_cond_destroy(pthread_cond_t *cond)
279{
280 int res = EINVAL;
281 uint32_t sig = os_atomic_load(&cond->sig.val, relaxed);
282 switch (sig) {
283 case _PTHREAD_COND_SIG_psynch:
284 _pthread_lock_lock(&cond->lock);
285
286 uint64_t oldval64, newval64;
287 uint32_t lcntval, ucntval, scntval;
288 volatile uint64_t *c_lsseqaddr;
289 volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt;
290
291 COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt);
292
293 do {
294 lcntval = *c_lseqcnt;
295 ucntval = *c_useqcnt;
296 scntval = *c_sseqcnt;
297
298 // validate it is not busy
299 if ((lcntval & PTHRW_COUNT_MASK) != (scntval & PTHRW_COUNT_MASK)) {
300 //res = EBUSY;
301 break;
302 }
303 oldval64 = (((uint64_t)scntval) << 32);
304 oldval64 |= lcntval;
305 newval64 = oldval64;
306 } while (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst));
307
308 // <rdar://problem/13782056> Need to clear preposts.
309 uint32_t flags = 0;
310 bool needclearpre = ((scntval & PTH_RWS_CV_PBIT) != 0);
311 if (needclearpre && cond->pshared == PTHREAD_PROCESS_SHARED) {
312 flags |= _PTHREAD_MTX_OPT_PSHARED;
313 }
314
315 os_atomic_store(&cond->sig.val, _PTHREAD_NO_SIG, relaxed);
316 res = 0;
317
318 _pthread_lock_unlock(&cond->lock);
319
320 if (needclearpre) {
321 (void)__psynch_cvclrprepost(cond, lcntval, ucntval, scntval, 0, lcntval, flags);
322 }
323 break;
324 case _PTHREAD_COND_SIG_init:
325 // Compatibility for misbehaving applications that attempt to
326 // destroy a statically initialized condition variable.
327 //
328 // fall through
329 case _PTHREAD_COND_SIG_pristine:
330 case _PTHREAD_COND_SIG_ulock:
331 os_atomic_store(&cond->sig.val, _PTHREAD_NO_SIG, relaxed);
332 res = 0;
333 break;
334 default:
335 // TODO: PTHREAD_STRICT candidate
336 break;
337 }
338 return res;
339}
340
341OS_ALWAYS_INLINE
342static inline int
343_pthread_psynch_cond_signal(pthread_cond_t *cond, bool broadcast,
344 mach_port_t thread)
345{
346 uint32_t updateval;
347 uint32_t diffgen;
348 uint32_t ulval;
349
350 uint64_t oldval64, newval64;
351 uint32_t lcntval, ucntval, scntval;
352 volatile uint64_t *c_lsseqaddr;
353 volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt;
354
355 int retry_count = 0, uretry_count = 0;
356 int ucountreset = 0;
357
358 COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt);
359
360 bool retry;
361 do {
362 retry = false;
363
364 lcntval = *c_lseqcnt;
365 ucntval = *c_useqcnt;
366 scntval = *c_sseqcnt;
367 diffgen = 0;
368 ulval = 0;
369
370 if (((lcntval & PTHRW_COUNT_MASK) == (scntval & PTHRW_COUNT_MASK)) ||
371 (thread == MACH_PORT_NULL && ((lcntval & PTHRW_COUNT_MASK) == (ucntval & PTHRW_COUNT_MASK)))) {
372 /* validate it is spurious and return */
373 oldval64 = (((uint64_t)scntval) << 32);
374 oldval64 |= lcntval;
375 newval64 = oldval64;
376
377 if (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst)) {
378 retry = true;
379 continue;
380 } else {
381 return 0;
382 }
383 }
384
385 if (thread) {
386 break;
387 }
388
389 /* validate to eliminate spurious values, race snapshots */
390 if (is_seqhigher((scntval & PTHRW_COUNT_MASK), (lcntval & PTHRW_COUNT_MASK))) {
391 /* since ucntval may be newer, just redo */
392 retry_count++;
393 if (retry_count > 8192) {
394 return EAGAIN;
395 } else {
396 sched_yield();
397 retry = true;
398 continue;
399 }
400 } else if (is_seqhigher((ucntval & PTHRW_COUNT_MASK), (lcntval & PTHRW_COUNT_MASK))) {
401 /* since ucntval may be newer, just redo */
402 uretry_count++;
403 if (uretry_count > 8192) {
404 /*
405 * U value if not used for a while can go out of sync
406 * set this to S value and try one more time.
407 */
408 if (ucountreset != 0) {
409 return EAGAIN;
410 } else if (os_atomic_cmpxchg(c_useqcnt, ucntval, (scntval & PTHRW_COUNT_MASK), seq_cst)) {
411 /* now the U is reset to S value */
412 ucountreset = 1;
413 uretry_count = 0;
414 }
415 }
416 sched_yield();
417 retry = true;
418 continue;
419 }
420
421 if (is_seqlower(ucntval & PTHRW_COUNT_MASK, scntval & PTHRW_COUNT_MASK) != 0) {
422 /* If U < S, set U = S+diff due to intr's TO, etc */
423 ulval = (scntval & PTHRW_COUNT_MASK);
424 } else {
425 /* If U >= S, set U = U+diff due to intr's TO, etc */
426 ulval = (ucntval & PTHRW_COUNT_MASK);
427 }
428
429 if (broadcast) {
430 diffgen = diff_genseq(lcntval, ulval);
431 // Set U = L
432 ulval = (lcntval & PTHRW_COUNT_MASK);
433 } else {
434 ulval += PTHRW_INC;
435 }
436
437 } while (retry || !os_atomic_cmpxchg(c_useqcnt, ucntval, ulval, seq_cst));
438
439 uint32_t flags = 0;
440 if (cond->pshared == PTHREAD_PROCESS_SHARED) {
441 flags |= _PTHREAD_MTX_OPT_PSHARED;
442 }
443
444 uint64_t cvlsgen = ((uint64_t)scntval << 32) | lcntval;
445
446 if (broadcast) {
447 // pass old U val so kernel will know the diffgen
448 uint64_t cvudgen = ((uint64_t)ucntval << 32) | diffgen;
449 updateval = __psynch_cvbroad(cond, cvlsgen, cvudgen, flags, NULL, 0, 0);
450 } else {
451 updateval = __psynch_cvsignal(cond, cvlsgen, ucntval, thread, NULL, 0, 0, flags);
452 }
453
454 if (updateval != (uint32_t)-1 && updateval != 0) {
455 _pthread_cond_updateval(cond, NULL, 0, updateval);
456 }
457
458 return 0;
459}
460
461OS_ALWAYS_INLINE
462static inline int
463_pthread_ulock_cond_signal(pthread_cond_t *cond, bool broadcast,
464 mach_port_t thread)
465{
466 pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(cond);
467
468 pthread_ulock_cond_state_u oldstate, newstate;
469 // release to pair with acquire after wait
470 os_atomic_rmw_loop(&state->val, oldstate.val, newstate.val, release, {
471 if (!oldstate.waiters || oldstate.waiters == oldstate.signal) {
472 os_atomic_rmw_loop_give_up(return 0);
473 }
474
475 newstate = (pthread_ulock_cond_state_u){
476 .seq = oldstate.seq + 1,
477 .waiters = oldstate.waiters,
478 .signal = broadcast ? oldstate.waiters :
479 MIN(oldstate.signal + 1, oldstate.waiters),
480 };
481 });
482
483 PTHREAD_TRACE(ulcond_signal, cond, oldstate.val, newstate.val, broadcast);
484
485 // Priority hole: if we're pre-empted here, nobody else can signal the
486 // waiter we took responsibility for signaling by incrementing the signal
487 // count.
488
489 if (oldstate.signal < oldstate.waiters) {
490 uint32_t wake_op = UL_COMPARE_AND_WAIT | ULF_NO_ERRNO;
491 if (broadcast) {
492 wake_op |= ULF_WAKE_ALL;
493 } else if (thread) {
494 wake_op |= ULF_WAKE_THREAD;
495 }
496
497 for (;;) {
498 int rc = __ulock_wake(wake_op, &state->seq, thread);
499 if (rc < 0) {
500 switch (-rc) {
501 case EINTR:
502 continue;
503 case ENOENT:
504 break;
505 case EALREADY:
506 if (!thread) {
507 PTHREAD_INTERNAL_CRASH(0, "EALREADY from ulock_wake");
508 }
509 // Compatibility with psynch: promote to broadcast
510 return pthread_cond_broadcast(cond);
511 default:
512 PTHREAD_INTERNAL_CRASH(-rc, "ulock_wake failure");
513 }
514 }
515 break;
516 }
517 }
518
519 return 0;
520}
521
522OS_ALWAYS_INLINE
523static inline int
524_pthread_cond_signal(pthread_cond_t *cond, bool broadcast, mach_port_t thread)
525{
526 uint32_t sig = _PTHREAD_COND_SIG_pristine;
527 int res = _pthread_cond_check_init(cond, &sig);
528 if (res != 0 || sig == _PTHREAD_COND_SIG_pristine) {
529 return res;
530 }
531
532 switch (sig) {
533 case _PTHREAD_COND_SIG_psynch:
534 return _pthread_psynch_cond_signal(cond, broadcast, thread);
535 case _PTHREAD_COND_SIG_ulock:
536 return _pthread_ulock_cond_signal(cond, broadcast, thread);
537 default:
538 PTHREAD_INTERNAL_CRASH(sig, "impossible cond signature");
539 }
540}
541
542/*
543 * Signal a condition variable, waking up all threads waiting for it.
544 */
545PTHREAD_NOEXPORT_VARIANT
546int
547pthread_cond_broadcast(pthread_cond_t *cond)
548{
549 return _pthread_cond_signal(cond, true, MACH_PORT_NULL);
550}
551
552/*
553 * Signal a condition variable, waking a specified thread.
554 */
555PTHREAD_NOEXPORT_VARIANT
556int
557pthread_cond_signal_thread_np(pthread_cond_t *cond, pthread_t thread)
558{
559 mach_port_t mp = MACH_PORT_NULL;
560 if (thread) {
561 mp = pthread_mach_thread_np((_Nonnull pthread_t)thread);
562 }
563 return _pthread_cond_signal(cond, false, mp);
564}
565
566/*
567 * Signal a condition variable, waking only one thread.
568 */
569PTHREAD_NOEXPORT_VARIANT
570int
571pthread_cond_signal(pthread_cond_t *cond)
572{
573 return _pthread_cond_signal(cond, false, MACH_PORT_NULL);
574}
575
576static int
577_pthread_psynch_cond_wait(pthread_cond_t *cond,
578 pthread_mutex_t *mutex,
579 const struct timespec *then,
580 pthread_conformance_t conforming)
581{
582 uint32_t mtxgen, mtxugen, flags=0, updateval;
583 uint32_t lcntval, ucntval, scntval;
584 uint32_t nlval, ulval, savebits;
585 volatile uint64_t *c_lsseqaddr;
586 volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt;
587 uint64_t oldval64, newval64, mugen, cvlsgen;
588 uint32_t *npmtx = NULL;
589
590 COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt);
591
592 do {
593 lcntval = *c_lseqcnt;
594 ucntval = *c_useqcnt;
595 scntval = *c_sseqcnt;
596
597 oldval64 = (((uint64_t)scntval) << 32);
598 oldval64 |= lcntval;
599
600 /* remove c and p bits on S word */
601 savebits = scntval & PTH_RWS_CV_BITSALL;
602 ulval = (scntval & PTHRW_COUNT_MASK);
603 nlval = lcntval + PTHRW_INC;
604 newval64 = (((uint64_t)ulval) << 32);
605 newval64 |= nlval;
606 } while (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst));
607
608 cond->busy = mutex;
609
610 int res = _pthread_mutex_droplock(mutex, &flags, &npmtx, &mtxgen, &mtxugen);
611
612 /* TBD: cases are for normal (non owner for recursive mutex; error checking)*/
613 if (res != 0) {
614 return EINVAL;
615 }
616 if ((flags & _PTHREAD_MTX_OPT_NOTIFY) == 0) {
617 npmtx = NULL;
618 mugen = 0;
619 } else {
620 mugen = ((uint64_t)mtxugen << 32) | mtxgen;
621 }
622 flags &= ~_PTHREAD_MTX_OPT_MUTEX; /* reset the mutex bit as this is cvar */
623
624 cvlsgen = ((uint64_t)(ulval | savebits)<< 32) | nlval;
625
626 // SUSv3 requires pthread_cond_wait to be a cancellation point
627 if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) {
628 pthread_cleanup_push(_pthread_psynch_cond_cleanup, (void *)cond);
629 updateval = __psynch_cvwait(cond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)(then->tv_sec), (int32_t)(then->tv_nsec));
630 pthread_testcancel();
631 pthread_cleanup_pop(0);
632 } else {
633 updateval = __psynch_cvwait(cond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)(then->tv_sec), (int32_t)(then->tv_nsec));
634 }
635
636 if (updateval == (uint32_t)-1) {
637 int err = errno;
638 switch (err & 0xff) {
639 case ETIMEDOUT:
640 res = ETIMEDOUT;
641 break;
642 case EINTR:
643 // spurious wakeup (unless canceled)
644 res = 0;
645 break;
646 default:
647 res = EINVAL;
648 break;
649 }
650
651 // add unlock ref to show one less waiter
652 _pthread_cond_updateval(cond, mutex, err, 0);
653 } else if (updateval != 0) {
654 // Successful wait
655 // The return due to prepost and might have bit states
656 // update S and return for prepo if needed
657 _pthread_cond_updateval(cond, mutex, 0, updateval);
658 }
659
660 pthread_mutex_lock(mutex);
661
662 return res;
663}
664
665struct pthread_ulock_cond_cancel_ctx_s {
666 pthread_cond_t *cond;
667 pthread_mutex_t *mutex;
668};
669
670static int
671_pthread_ulock_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex,
672 const struct timespec *then, pthread_conformance_t conforming)
673{
674 bool cancelable = (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE);
675
676 uint64_t timeout_ns = 0;
677 if (then->tv_sec || then->tv_nsec) {
678 // psynch compatibility: cast and bitwise-truncate tv_nsec
679 uint64_t fraction_ns = ((uint32_t)then->tv_nsec) & 0x3fffffff;
680 if (os_mul_and_add_overflow(then->tv_sec, NSEC_PER_SEC, fraction_ns,
681 &timeout_ns)) {
682 // saturate (can't wait longer than 584 years...)
683 timeout_ns = UINT64_MAX;
684 }
685 }
686
687 pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(cond);
688
689 pthread_ulock_cond_state_u origstate = {
690 .val = os_atomic_add(&state->val, _PTHREAD_COND_WAITERS_INC, relaxed)
691 };
692
693 int rc = _pthread_mutex_ulock_unlock(mutex);
694 if (rc) {
695 return _pthread_ulock_cond_wait_complete(state, NULL, rc);
696 }
697
698 PTHREAD_TRACE(ulcond_wait, cond, origstate.val, timeout_ns, 0);
699
700 do {
701 const uint32_t wait_op = UL_COMPARE_AND_WAIT | ULF_NO_ERRNO;
702 if (cancelable) {
703 struct pthread_ulock_cond_cancel_ctx_s ctx = {
704 .cond = cond,
705 .mutex = mutex,
706 };
707 pthread_cleanup_push(_pthread_ulock_cond_cleanup, &ctx);
708 rc = __ulock_wait2(wait_op | ULF_WAIT_CANCEL_POINT, &state->seq,
709 origstate.seq, timeout_ns, 0);
710 pthread_testcancel();
711 pthread_cleanup_pop(0);
712 } else {
713 rc = __ulock_wait2(wait_op, &state->seq, origstate.seq, timeout_ns, 0);
714 }
715 if (rc < 0) {
716 switch (-rc) {
717 case EFAULT:
718 continue;
719 case EINTR:
720 // "These functions shall not return an error code of [EINTR]."
721 // => promote to spurious wake-up
722 rc = 0;
723 goto out;
724 case ETIMEDOUT:
725 rc = ETIMEDOUT;
726 goto out;
727 default:
728 PTHREAD_INTERNAL_CRASH(-rc, "ulock_wait failure");
729 }
730 } else {
731 // XXX for now don't care about other waiters
732 rc = 0;
733 }
734 } while (os_atomic_load(&state->seq, relaxed) == origstate.seq);
735
736out:
737 return _pthread_ulock_cond_wait_complete(state, mutex, rc);
738}
739
740static int
741_pthread_ulock_cond_wait_complete(pthread_ulock_cond_state_u *state,
742 pthread_mutex_t *mutex, int rc)
743{
744 if (mutex) {
745 // XXX Check this return value? Historically we haven't, but if rc == 0
746 // we could promote the return value to this one.
747 _pthread_mutex_ulock_lock(mutex, false);
748 }
749
750 pthread_ulock_cond_state_u oldstate, newstate;
751 // acquire to pair with release upon signal
752 os_atomic_rmw_loop(&state->val, oldstate.val, newstate.val, acquire, {
753 newstate = (pthread_ulock_cond_state_u){
754 .seq = oldstate.seq,
755 .waiters = oldstate.waiters - 1,
756 .signal = oldstate.signal ? oldstate.signal - 1 : 0,
757 };
758 });
759
760 return rc;
761}
762
763/*
764 * Suspend waiting for a condition variable.
765 * If conformance is not cancelable, we skip the pthread_testcancel(),
766 * but keep the remaining conforming behavior.
767 */
768PTHREAD_NOEXPORT OS_NOINLINE
769int
770_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex,
771 const struct timespec *abstime, int isRelative,
772 pthread_conformance_t conforming)
773{
774 int res;
775 struct timespec then = { 0, 0 };
776 bool timeout_elapsed = false;
777
778 if (!_pthread_mutex_check_signature(mutex) &&
779 !_pthread_mutex_check_signature_init(mutex)) {
780 return EINVAL;
781 }
782
783 bool ulock = _pthread_mutex_uses_ulock(mutex);
784 uint32_t sig = ulock ? _PTHREAD_COND_SIG_ulock : _PTHREAD_COND_SIG_psynch;
785 res = _pthread_cond_check_init(cond, &sig);
786 if (res != 0) {
787 return res;
788 }
789
790 if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) {
791 pthread_testcancel();
792 }
793
794 /* send relative time to kernel */
795 if (abstime) {
796 if (abstime->tv_nsec < 0 || abstime->tv_nsec >= NSEC_PER_SEC) {
797 // TODO: PTHREAD_STRICT candidate
798 return EINVAL;
799 }
800
801 if (isRelative == 0) {
802 struct timespec now;
803 struct timeval tv;
804 __gettimeofday(&tv, NULL);
805 TIMEVAL_TO_TIMESPEC(&tv, &now);
806
807 if ((abstime->tv_sec == now.tv_sec) ?
808 (abstime->tv_nsec <= now.tv_nsec) :
809 (abstime->tv_sec < now.tv_sec)) {
810 timeout_elapsed = true;
811 } else {
812 /* Compute relative time to sleep */
813 then.tv_nsec = abstime->tv_nsec - now.tv_nsec;
814 then.tv_sec = abstime->tv_sec - now.tv_sec;
815 if (then.tv_nsec < 0) {
816 then.tv_nsec += NSEC_PER_SEC;
817 then.tv_sec--;
818 }
819 }
820 } else {
821 then.tv_sec = abstime->tv_sec;
822 then.tv_nsec = abstime->tv_nsec;
823 if ((then.tv_sec == 0) && (then.tv_nsec == 0)) {
824 timeout_elapsed = true;
825 }
826 }
827 }
828
829 if (!ulock && cond->busy != NULL && cond->busy != mutex) {
830 // TODO: PTHREAD_STRICT candidate
831 return EINVAL;
832 }
833
834 /*
835 * If timeout is known to have elapsed, we still need to unlock and
836 * relock the mutex to allow other waiters to get in line and
837 * modify the condition state.
838 */
839 if (timeout_elapsed) {
840 res = pthread_mutex_unlock(mutex);
841 if (res != 0) {
842 return res;
843 }
844 res = pthread_mutex_lock(mutex);
845 if (res != 0) {
846 return res;
847 }
848
849 return ETIMEDOUT;
850 }
851
852 if (ulock) {
853 return _pthread_ulock_cond_wait(cond, mutex, &then, conforming);
854 } else {
855 return _pthread_psynch_cond_wait(cond, mutex, &then, conforming);
856 }
857}
858
859static void
860_pthread_ulock_cond_cleanup(void *arg)
861{
862 struct pthread_ulock_cond_cancel_ctx_s *ctx = arg;
863 pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(ctx->cond);
864
865 (void)_pthread_ulock_cond_wait_complete(state, ctx->mutex, 0);
866
867 // "A thread that has been unblocked because it has been canceled while
868 // blocked in a call to pthread_cond_timedwait() or pthread_cond_wait()
869 // shall not consume any condition signal that may be directed concurrently
870 // at the condition variable if there are other threads blocked on the
871 // condition variable."
872 //
873 // Since we have no way to know if we've eaten somebody else's signal, just
874 // signal again pessimistically.
875 pthread_cond_signal(ctx->cond);
876}
877
878static void
879_pthread_psynch_cond_cleanup(void *arg)
880{
881 pthread_cond_t *cond = (pthread_cond_t *)arg;
882 pthread_t thread = pthread_self();
883 pthread_mutex_t *mutex;
884
885// 4597450: begin
886 if (!thread->canceled) {
887 return;
888 }
889// 4597450: end
890
891 mutex = cond->busy;
892
893 // add unlock ref to show one less waiter
894 _pthread_cond_updateval(cond, mutex, thread->cancel_error, 0);
895
896 /*
897 ** Can't do anything if this fails -- we're on the way out
898 */
899 if (mutex != NULL) {
900 (void)pthread_mutex_lock(mutex);
901 }
902}
903
904static void
905_pthread_cond_updateval(pthread_cond_t *cond, pthread_mutex_t *mutex,
906 int error, uint32_t updateval)
907{
908 int needclearpre;
909
910 uint32_t diffgen, nsval;
911 uint64_t oldval64, newval64;
912 uint32_t lcntval, ucntval, scntval;
913 volatile uint64_t *c_lsseqaddr;
914 volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt;
915
916 if (error != 0) {
917 updateval = PTHRW_INC;
918 if (error & ECVCLEARED) {
919 updateval |= PTH_RWS_CV_CBIT;
920 }
921 if (error & ECVPREPOST) {
922 updateval |= PTH_RWS_CV_PBIT;
923 }
924 }
925
926 COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt);
927
928 do {
929 lcntval = *c_lseqcnt;
930 ucntval = *c_useqcnt;
931 scntval = *c_sseqcnt;
932 nsval = 0;
933 needclearpre = 0;
934
935 diffgen = diff_genseq(lcntval, scntval); // pending waiters
936
937 oldval64 = (((uint64_t)scntval) << 32);
938 oldval64 |= lcntval;
939
940 PTHREAD_TRACE(psynch_cvar_updateval | DBG_FUNC_START, cond, oldval64,
941 updateval, 0);
942
943 if (diffgen <= 0 && !is_rws_pbit_set(updateval)) {
944 /* TBD: Assert, should not be the case */
945 /* validate it is spurious and return */
946 newval64 = oldval64;
947 } else {
948 // update S by one
949
950 // update scntval with number of expected returns and bits
951 nsval = (scntval & PTHRW_COUNT_MASK) + (updateval & PTHRW_COUNT_MASK);
952 // set bits
953 nsval |= ((scntval & PTH_RWS_CV_BITSALL) | (updateval & PTH_RWS_CV_BITSALL));
954
955 // if L==S and c&p bits are set, needs clearpre
956 if (((nsval & PTHRW_COUNT_MASK) == (lcntval & PTHRW_COUNT_MASK)) &&
957 ((nsval & PTH_RWS_CV_BITSALL) == PTH_RWS_CV_BITSALL)) {
958 // reset p bit but retain c bit on the sword
959 nsval &= PTH_RWS_CV_RESET_PBIT;
960 needclearpre = 1;
961 }
962
963 newval64 = (((uint64_t)nsval) << 32);
964 newval64 |= lcntval;
965 }
966 } while (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst));
967
968 PTHREAD_TRACE(psynch_cvar_updateval | DBG_FUNC_END, cond, newval64,
969 (uint64_t)diffgen << 32 | needclearpre, 0);
970
971 if (diffgen > 0) {
972 // if L == S, then reset associated mutex
973 if ((nsval & PTHRW_COUNT_MASK) == (lcntval & PTHRW_COUNT_MASK)) {
974 cond->busy = NULL;
975 }
976 }
977
978 if (needclearpre) {
979 uint32_t flags = 0;
980 if (cond->pshared == PTHREAD_PROCESS_SHARED) {
981 flags |= _PTHREAD_MTX_OPT_PSHARED;
982 }
983 (void)__psynch_cvclrprepost(cond, lcntval, ucntval, nsval, 0, lcntval, flags);
984 }
985}
986
987#endif /* !BUILDING_VARIANT ] */
988
989PTHREAD_NOEXPORT_VARIANT
990int
991pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
992{
993 _pthread_lock_init(&cond->lock);
994 return _pthread_cond_init(cond, attr, _PTHREAD_COND_SIG_pristine);
995}
996