pthread mutexes

static void
_plockstat_never_fired(void)
{
	PLOCKSTAT_MUTEX_SPIN(NULL);
	PLOCKSTAT_MUTEX_SPUN(NULL, 0, 0);
}

static inline bool
_pthread_mutex_policy_validate(int policy)
{
	return (policy >= 0 && policy < _PTHREAD_MUTEX_POLICY_LAST);
}

static inline int
_pthread_mutex_policy_to_opt(int policy)
{
	switch (policy) {
	case PTHREAD_MUTEX_POLICY_FAIRSHARE_NP:
		return _PTHREAD_MTX_OPT_POLICY_FAIRSHARE;
	case PTHREAD_MUTEX_POLICY_FIRSTFIT_NP:
		return _PTHREAD_MTX_OPT_POLICY_FIRSTFIT;
	default:
		__builtin_unreachable();
	}
}

void
_pthread_mutex_global_init(const char *envp[],
		struct _pthread_registration_data *registration_data)
{
	int opt = _PTHREAD_MTX_OPT_POLICY_DEFAULT;
	if (registration_data->mutex_default_policy) {
		int policy = registration_data->mutex_default_policy &
				_PTHREAD_REG_DEFAULT_POLICY_MASK;
		if (_pthread_mutex_policy_validate(policy)) {
			opt = _pthread_mutex_policy_to_opt(policy);
		}
	}

	const char *envvar = _simple_getenv(envp, "PTHREAD_MUTEX_DEFAULT_POLICY");
	if (envvar) {
		int policy = envvar[0] - '0';
		if (_pthread_mutex_policy_validate(policy)) {
			opt = _pthread_mutex_policy_to_opt(policy);
		}
	}

	if (opt != __pthread_mutex_default_opt_policy) {
		__pthread_mutex_default_opt_policy = opt;
	}

	bool use_ulock = _PTHREAD_MTX_OPT_ULOCK_DEFAULT;
	if (_os_xbs_chrooted) {
		use_ulock = false;
	} else {
		envvar = _simple_getenv(envp, "PTHREAD_MUTEX_USE_ULOCK");
… more in source

OS_ALWAYS_INLINE
static inline void
mutex_seq_load(mutex_seq *seqaddr, mutex_seq *oldseqval)
{
	oldseqval->seq_LU = seqaddr->seq_LU;
}

OS_ALWAYS_INLINE OS_USED
static inline bool
mutex_seq_atomic_cmpxchgv_relaxed(mutex_seq *seqaddr, mutex_seq *oldseqval,
		mutex_seq *newseqval)
{
	return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
			newseqval->seq_LU, &oldseqval->seq_LU, relaxed);
}

OS_ALWAYS_INLINE OS_USED
static inline bool
mutex_seq_atomic_cmpxchgv_acquire(mutex_seq *seqaddr, mutex_seq *oldseqval,
		mutex_seq *newseqval)
{
	return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
			newseqval->seq_LU, &oldseqval->seq_LU, acquire);
}

OS_ALWAYS_INLINE OS_USED
static inline bool
mutex_seq_atomic_cmpxchgv_release(mutex_seq *seqaddr, mutex_seq *oldseqval,
		mutex_seq *newseqval)
{
	return os_atomic_cmpxchgv(&seqaddr->atomic_seq_LU, oldseqval->seq_LU,
			newseqval->seq_LU, &oldseqval->seq_LU, release);
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
{
#if 0
	/* conformance tests depend on not having this behavior */
	/* The test for this behavior is optional */
	if (_pthread_mutex_check_signature(mutex))
		return EBUSY;
#endif
	_pthread_lock_init(&mutex->lock);
	return (_pthread_mutex_init(mutex, attr, 0x7));
}

int
pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling)
{
	int res = EINVAL;
	pthread_mutex_t *mutex = (pthread_mutex_t *)omutex;
	if (_pthread_mutex_check_signature(mutex)) {
		_pthread_lock_lock(&mutex->lock);
		*prioceiling = mutex->prioceiling;
		res = 0;
		_pthread_lock_unlock(&mutex->lock);
	}
	return res;
}

int
pthread_mutex_setprioceiling(pthread_mutex_t *mutex, int prioceiling,
		int *old_prioceiling)
{
	int res = EINVAL;
	if (_pthread_mutex_check_signature(mutex)) {
		_pthread_lock_lock(&mutex->lock);
		if (prioceiling >= -999 && prioceiling <= 999) {
			*old_prioceiling = mutex->prioceiling;
			mutex->prioceiling = (int16_t)prioceiling;
			res = 0;
		}
		_pthread_lock_unlock(&mutex->lock);
	}
	return res;
}

int
pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr,
		int *prioceiling)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		*prioceiling = attr->prioceiling;
		res = 0;
	}
	return res;
}

int
pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		*protocol = attr->protocol;
		res = 0;
	}
	return res;
}

int
pthread_mutexattr_getpolicy_np(const pthread_mutexattr_t *attr, int *policy)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		switch (attr->opt) {
		case _PTHREAD_MTX_OPT_POLICY_FAIRSHARE:
			*policy = PTHREAD_MUTEX_POLICY_FAIRSHARE_NP;
			res = 0;
			break;
		case _PTHREAD_MTX_OPT_POLICY_FIRSTFIT:
			*policy = PTHREAD_MUTEX_POLICY_FIRSTFIT_NP;
			res = 0;
			break;
		}
	}
	return res;
}

int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		*type = attr->type;
		res = 0;
	}
	return res;
}

int
pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		*pshared = (int)attr->pshared;
		res = 0;
	}
	return res;
}

int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
	attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
	attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;
	attr->opt = __pthread_mutex_default_opt_policy;
	attr->type = PTHREAD_MUTEX_DEFAULT;
	attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
	attr->pshared = _PTHREAD_DEFAULT_PSHARED;
	return 0;
}

int
pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		if (prioceiling >= -999 && prioceiling <= 999) {
			attr->prioceiling = prioceiling;
			res = 0;
		}
	}
	return res;
}

int
pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		switch (protocol) {
		case PTHREAD_PRIO_NONE:
		case PTHREAD_PRIO_INHERIT:
		case PTHREAD_PRIO_PROTECT:
			attr->protocol = protocol;
			res = 0;
			break;
		}
	}
	return res;
}

int
pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *attr, int policy)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		// <rdar://problem/35844519> the first-fit implementation was broken
		// pre-Liberty so this mapping exists to ensure that the old first-fit
		// define (2) is no longer valid when used on older systems.
		switch (policy) {
		case PTHREAD_MUTEX_POLICY_FAIRSHARE_NP:
			attr->opt = _PTHREAD_MTX_OPT_POLICY_FAIRSHARE;
			res = 0;
			break;
		case PTHREAD_MUTEX_POLICY_FIRSTFIT_NP:
			attr->opt = _PTHREAD_MTX_OPT_POLICY_FIRSTFIT;
			res = 0;
			break;
		}
	}
	return res;
}

int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		switch (type) {
		case PTHREAD_MUTEX_NORMAL:
		case PTHREAD_MUTEX_ERRORCHECK:
		case PTHREAD_MUTEX_RECURSIVE:
		//case PTHREAD_MUTEX_DEFAULT:
			attr->type = type;
			res = 0;
			break;
		}
	}
	return res;
}

int
pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
{
	int res = EINVAL;
	if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
		if (( pshared == PTHREAD_PROCESS_PRIVATE) ||
				(pshared == PTHREAD_PROCESS_SHARED))
		{
			attr->pshared = pshared;
			res = 0;
		}
	}
	return res;
}

OS_NOINLINE
int
_pthread_mutex_corruption_abort(pthread_mutex_t *mutex)
{
	PTHREAD_CLIENT_CRASH(0, "pthread_mutex corruption: mutex owner changed "
			"in the middle of lock/unlock");
}

OS_NOINLINE
static int
_pthread_mutex_check_init_slow(pthread_mutex_t *mutex)
{
	int res = EINVAL;

	if (_pthread_mutex_check_signature_init(mutex)) {
		_pthread_lock_lock(&mutex->lock);
		if (_pthread_mutex_check_signature_init(mutex)) {
			// initialize a statically initialized mutex to provide
			// compatibility for misbehaving applications.
			// (unlock should not be the first operation on a mutex)
			res = _pthread_mutex_init(mutex, NULL, (mutex->sig & 0xf));
		} else if (_pthread_mutex_check_signature(mutex)) {
			res = 0;
		}
		_pthread_lock_unlock(&mutex->lock);
	} else if (_pthread_mutex_check_signature(mutex)) {
		res = 0;
	}
	if (res != 0) {
		PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, res);
	}
	return res;
}

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_check_init(pthread_mutex_t *mutex)
{
	int res = 0;
	if (!_pthread_mutex_check_signature(mutex)) {
		return _pthread_mutex_check_init_slow(mutex);
	}
	return res;
}

OS_ALWAYS_INLINE
static inline bool
_pthread_mutex_is_fairshare(pthread_mutex_t *mutex)
{
	return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FAIRSHARE);
}

OS_ALWAYS_INLINE
static inline bool
_pthread_mutex_is_firstfit(pthread_mutex_t *mutex)
{
	return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FIRSTFIT);
}

OS_ALWAYS_INLINE
static inline bool
_pthread_mutex_is_recursive(pthread_mutex_t *mutex)
{
	return (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE);
}

OS_ALWAYS_INLINE
static int
_pthread_mutex_lock_handle_options(pthread_mutex_t *mutex, bool trylock,
		uint64_t *tidaddr)
{
	if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) {
		// NORMAL does not do EDEADLK checking
		return 0;
	}

	uint64_t selfid = _pthread_threadid_self_np_direct();
	if (os_atomic_load_wide(tidaddr, relaxed) == selfid) {
		if (_pthread_mutex_is_recursive(mutex)) {
			if (mutex->mtxopts.options.lock_count < USHRT_MAX) {
				mutex->mtxopts.options.lock_count += 1;
				return mutex->mtxopts.options.lock_count;
			} else {
				return -EAGAIN;
			}
		} else if (trylock) { /* PTHREAD_MUTEX_ERRORCHECK */
			// <rdar://problem/16261552> as per OpenGroup, trylock cannot
			// return EDEADLK on a deadlock, it should return EBUSY.
			return -EBUSY;
		} else { /* PTHREAD_MUTEX_ERRORCHECK */
			return -EDEADLK;
		}
	}

	// Not recursive, or recursive but first lock.
	return 0;
… more in source

OS_ALWAYS_INLINE
static int
_pthread_mutex_unlock_handle_options(pthread_mutex_t *mutex, uint64_t *tidaddr)
{
	if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) {
		// NORMAL does not do EDEADLK checking
		return 0;
	}

	uint64_t selfid = _pthread_threadid_self_np_direct();
	if (os_atomic_load_wide(tidaddr, relaxed) != selfid) {
		return -EPERM;
	} else if (_pthread_mutex_is_recursive(mutex) &&
			--mutex->mtxopts.options.lock_count) {
		return 1;
	}
	return 0;
}

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_fairshare_unlock_updatebits(pthread_mutex_t *mutex,
		uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
	uint32_t flags = mutex->mtxopts.value;
	flags &= ~_PTHREAD_MTX_OPT_NOTIFY; // no notification by default

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t oldtid, newtid;

	int res = _pthread_mutex_unlock_handle_options(mutex, tidaddr);
	if (res > 0) {
		// Valid recursive unlock
		if (flagsp) {
			*flagsp = flags;
		}
		PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
		return 0;
	} else if (res < 0) {
		PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, -res);
		return -res;
	}
… more in source

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_fairshare_lock_updatebits(pthread_mutex_t *mutex, uint64_t selfid)
{
	bool firstfit = _pthread_mutex_is_firstfit(mutex);
	bool gotlock = true;

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);

	do {
		newseq = oldseq;

		if (firstfit) {
			// firstfit locks can have the lock stolen out from under a locker
			// between the unlock from the kernel and this lock path. When this
			// happens, we still want to set the K bit before leaving the loop
			// (or notice if the lock unlocks while we try to update).
			gotlock = !is_rwl_ebit_set(oldseq.lgenval);
		} else if ((oldseq.lgenval & (PTH_RWL_KBIT | PTH_RWL_EBIT)) == 
				(PTH_RWL_KBIT | PTH_RWL_EBIT)) {
			// bit are already set, just update the owner tidaddr
			break;
		}
… more in source

OS_NOINLINE
static int
_pthread_mutex_fairshare_lock_wait(pthread_mutex_t *mutex, mutex_seq newseq,
		uint64_t oldtid)
{
	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t selfid = _pthread_threadid_self_np_direct();

	PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex);
	do {
		uint32_t updateval;
		do {
			updateval = __psynch_mutexwait(mutex, newseq.lgenval,
					newseq.ugenval, oldtid, mutex->mtxopts.value);
			oldtid = os_atomic_load_wide(tidaddr, relaxed);
		} while (updateval == (uint32_t)-1);

		// returns 0 on succesful update; in firstfit it may fail with 1
	} while (_pthread_mutex_fairshare_lock_updatebits(mutex, selfid) == 1);
	PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT);

	return 0;
}

OS_NOINLINE
int
_pthread_mutex_fairshare_lock_slow(pthread_mutex_t *mutex, bool trylock)
{
	int res, recursive = 0;

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t oldtid, selfid = _pthread_threadid_self_np_direct();

	res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr);
	if (res > 0) {
		recursive = 1;
		res = 0;
		goto out;
	} else if (res < 0) {
		res = -res;
		goto out;
	}

	bool gotlock;
	do {
		newseq = oldseq;
		oldtid = os_atomic_load_wide(tidaddr, relaxed);
… more in source

OS_NOINLINE
static inline int
_pthread_mutex_fairshare_lock(pthread_mutex_t *mutex, bool trylock)
{
#if ENABLE_USERSPACE_TRACE
	return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
#elif PLOCKSTAT
	if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
		return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
	}
#endif

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t selfid = _pthread_threadid_self_np_direct();

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	if (os_unlikely(oldseq.lgenval & PTH_RWL_EBIT)) {
		return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
	}

	bool gotlock;
	do {
		newseq = oldseq;
… more in source

OS_NOINLINE
static int
_pthread_mutex_fairshare_unlock_drop(pthread_mutex_t *mutex, mutex_seq newseq,
		uint32_t flags)
{
	int res;
	uint32_t updateval;

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);

	PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_START, mutex, newseq.lgenval,
			newseq.ugenval, os_atomic_load_wide(tidaddr, relaxed));

	updateval = __psynch_mutexdrop(mutex, newseq.lgenval, newseq.ugenval,
			os_atomic_load_wide(tidaddr, relaxed), flags);

	PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_END, mutex, updateval, 0, 0);

	if (updateval == (uint32_t)-1) {
		res = errno;

		if (res == EINTR) {
			res = 0;
		}
		if (res != 0) {
			PTHREAD_INTERNAL_CRASH(res, "__psynch_mutexdrop failed");
		}
		return res;
	}
… more in source

OS_NOINLINE
int
_pthread_mutex_fairshare_unlock_slow(pthread_mutex_t *mutex)
{
	int res;
	mutex_seq newseq;
	uint32_t flags;

	res = _pthread_mutex_fairshare_unlock_updatebits(mutex, &flags, NULL,
			&newseq.lgenval, &newseq.ugenval);
	if (res != 0) return res;

	if ((flags & _PTHREAD_MTX_OPT_NOTIFY) != 0) {
		return _pthread_mutex_fairshare_unlock_drop(mutex, newseq, flags);
	} else {
		uint64_t *tidaddr;
		MUTEX_GETTID_ADDR(mutex, &tidaddr);
		PTHREAD_TRACE(psynch_mutex_uunlock, mutex, newseq.lgenval,
				newseq.ugenval, os_atomic_load_wide(tidaddr, relaxed));
	}

	return 0;
}

OS_NOINLINE
static int
_pthread_mutex_fairshare_unlock(pthread_mutex_t *mutex)
{
#if ENABLE_USERSPACE_TRACE
	return _pthread_mutex_fairshare_unlock_slow(mutex);
#elif PLOCKSTAT
	if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
		return _pthread_mutex_fairshare_unlock_slow(mutex);
	}
#endif

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	int numwaiters = diff_genseq(oldseq.lgenval, oldseq.ugenval);
	if (os_unlikely(numwaiters == 0)) {
		// spurious unlock (unlock of unlocked lock)
		return 0;
	}

	// We're giving up the mutex one way or the other, so go ahead and
	// update the owner to 0 so that once the CAS below succeeds, there
	// is no stale ownership information. If the CAS of the seqaddr
… more in source

OS_ALWAYS_INLINE
static inline uint32_t
_pthread_mutex_ulock_self_owner_value(void)
{
	mach_port_t self_port = _pthread_mach_thread_self_direct();
	return self_port & _PTHREAD_MUTEX_ULOCK_OWNER_MASK;
}

OS_NOINLINE
static int
_pthread_mutex_ulock_lock_slow(pthread_mutex_t *mutex, uint32_t self_ownerval,
		uint32_t state)
{
	bool success = false, kernel_waiters = false;

	uint32_t wait_op = UL_UNFAIR_LOCK | ULF_NO_ERRNO;
	if (__pthread_mutex_ulock_adaptive_spin) {
		wait_op |= ULF_WAIT_ADAPTIVE_SPIN;
	}

	PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex);
	do {
		bool owner_dead = false;

		do {
			uint32_t current_ownerval = state & _PTHREAD_MUTEX_ULOCK_OWNER_MASK;
			if (os_unlikely(owner_dead)) {
				// TODO: PTHREAD_STRICT candidate
				//
				// For a non-recursive mutex, this indicates that it's really
				// being used as a semaphore: even though we're the current
				// owner, in reality we're expecting another thread to 'unlock'
				// this mutex on our behalf later.
				//
				// __ulock_wait(2) doesn't permit you to wait for yourself, so
				// we need to first swap our ownership for the anonymous owner
				current_ownerval =
						MACH_PORT_DEAD & _PTHREAD_MUTEX_ULOCK_OWNER_MASK;
… more in source

PTHREAD_NOEXPORT_VARIANT
int
_pthread_mutex_ulock_lock(pthread_mutex_t *mutex, bool trylock)
{
	uint32_t unlocked = _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE;
	uint32_t locked = _pthread_mutex_ulock_self_owner_value();
	uint32_t state;

	bool success = os_atomic_cmpxchgv(&mutex->ulock.uval, unlocked, locked,
			&state, acquire);

	if (trylock) {
		PTHREAD_TRACE(ulmutex_trylock, mutex, locked, state, success);
	} else {
		PTHREAD_TRACE(ulmutex_lock, mutex, locked, state, success);
	}

	int rc = 0;
	if (!success) {
		if (trylock) {
			rc = EBUSY;
		} else {
			rc = _pthread_mutex_ulock_lock_slow(mutex, locked, state);
		}
	}

	if (rc) {
		PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, rc);
	} else {
		PLOCKSTAT_MUTEX_ACQUIRE((pthread_mutex_t *)mutex, /* recursive */ 0, 0);
… more in source

OS_NOINLINE
static int
_pthread_mutex_ulock_unlock_slow(pthread_mutex_t *mutex, uint32_t self_ownerval,
		uint32_t orig_state)
{
	if (os_unlikely(orig_state == _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE)) {
		// XXX This is illegal, but psynch permitted it...
		// TODO: PTHREAD_STRICT candidate
		return 0;
	}

	uint32_t wake_flags = 0;

	uint32_t orig_ownerval = orig_state & _PTHREAD_MUTEX_ULOCK_OWNER_MASK;
	bool orig_waiters = orig_state & _PTHREAD_MUTEX_ULOCK_WAITERS_BIT;
	if (os_unlikely(orig_ownerval != self_ownerval)) {
		// XXX This is illegal, but psynch permitted it...
		// TODO: PTHREAD_STRICT candidate
		if (!orig_waiters) {
			return 0;
		}

		wake_flags |= ULF_WAKE_ALLOW_NON_OWNER;
	} else if (os_unlikely(!orig_waiters)) {
		PTHREAD_INTERNAL_CRASH(0, "unlock_slow without orig_waiters");
	}

	for (;;) {
		int rc = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO | wake_flags,
				&mutex->ulock, 0);
… more in source

PTHREAD_NOEXPORT_VARIANT
int
_pthread_mutex_ulock_unlock(pthread_mutex_t *mutex)
{
	uint32_t locked_uncontended = _pthread_mutex_ulock_self_owner_value();
	uint32_t unlocked = _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE;
	uint32_t state = os_atomic_xchg(&mutex->ulock.uval, unlocked, release);

	PTHREAD_TRACE(ulmutex_unlock, mutex, locked_uncontended, state, 0);

	int rc = 0;
	if (state != locked_uncontended) {
		rc = _pthread_mutex_ulock_unlock_slow(mutex, locked_uncontended,
				state);
	}

	if (rc) {
		PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, rc);
	} else {
		PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, /* recursive */ 0);
	}

	return rc;
}

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_firstfit_unlock_updatebits(pthread_mutex_t *mutex,
		uint32_t *flagsp, uint32_t **mutexp, uint32_t *lvalp, uint32_t *uvalp)
{
	uint32_t flags = mutex->mtxopts.value & ~_PTHREAD_MTX_OPT_NOTIFY;
	bool kernel_wake;

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t oldtid;

	int res = _pthread_mutex_unlock_handle_options(mutex, tidaddr);
	if (res > 0) {
		// Valid recursive unlock
		if (flagsp) {
			*flagsp = flags;
		}
		PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
		return 0;
	} else if (res < 0) {
		PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, -res);
		return -res;
	}
… more in source

OS_NOINLINE
static int
_pthread_mutex_firstfit_wake(pthread_mutex_t *mutex, mutex_seq newseq,
		uint32_t flags)
{
	PTHREAD_TRACE(psynch_ffmutex_wake, mutex, newseq.lgenval, newseq.ugenval,
			0);
	int res = __psynch_mutexdrop(mutex, newseq.lgenval, newseq.ugenval, 0,
			flags);

	if (res == -1) {
		res = errno;
		if (res == EINTR) {
			res = 0;
		}
		if (res != 0) {
			PTHREAD_INTERNAL_CRASH(res, "__psynch_mutexdrop failed");
		}
		return res;
	}
	return 0;
}

OS_NOINLINE
int
_pthread_mutex_firstfit_unlock_slow(pthread_mutex_t *mutex)
{
	mutex_seq newseq;
	uint32_t flags;
	int res;

	res = _pthread_mutex_firstfit_unlock_updatebits(mutex, &flags, NULL,
			&newseq.lgenval, &newseq.ugenval);
	if (res != 0) return res;

	if (flags & _PTHREAD_MTX_OPT_NOTIFY) {
		return _pthread_mutex_firstfit_wake(mutex, newseq, flags);
	}
	return 0;
}

OS_ALWAYS_INLINE
static bool
_pthread_mutex_firstfit_lock_updatebits(pthread_mutex_t *mutex, uint64_t selfid,
		mutex_seq *newseqp)
{
	bool gotlock;

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);

	PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_START, mutex,
			oldseq.lgenval, oldseq.ugenval, 0);

	do {
		newseq = oldseq;
		gotlock = is_rwl_ebit_clear(oldseq.lgenval);

		if (gotlock) {
			// If we see the E-bit cleared, we should just attempt to take it.
			newseq.lgenval |= PTH_RWL_EBIT;
		} else {
			// If we failed to get the lock then we need to put ourselves back
			// in the queue of waiters. The previous unlocker that woke us out
			// of the kernel consumed the S-count for our previous wake. So
… more in source

OS_NOINLINE
static int
_pthread_mutex_firstfit_lock_wait(pthread_mutex_t *mutex, mutex_seq newseq,
		uint64_t oldtid)
{
	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t selfid = _pthread_threadid_self_np_direct();

	PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex);
	do {
		uint32_t uval;
		do {
			PTHREAD_TRACE(psynch_ffmutex_wait | DBG_FUNC_START, mutex,
					newseq.lgenval, newseq.ugenval, mutex->mtxopts.value);
			uval = __psynch_mutexwait(mutex, newseq.lgenval, newseq.ugenval,
					oldtid, mutex->mtxopts.value);
			PTHREAD_TRACE(psynch_ffmutex_wait | DBG_FUNC_END, mutex,
					uval, 0, 0);
			oldtid = os_atomic_load_wide(tidaddr, relaxed);
		} while (uval == (uint32_t)-1);
	} while (!_pthread_mutex_firstfit_lock_updatebits(mutex, selfid, &newseq));
	PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT);

	return 0;
}

OS_NOINLINE
int
_pthread_mutex_firstfit_lock_slow(pthread_mutex_t *mutex, bool trylock)
{
	int res, recursive = 0;

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t oldtid, selfid = _pthread_threadid_self_np_direct();

	res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr);
	if (res > 0) {
		recursive = 1;
		res = 0;
		goto out;
	} else if (res < 0) {
		res = -res;
		goto out;
	}

	PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_START, mutex,
			oldseq.lgenval, oldseq.ugenval, 0);

	bool gotlock;
… more in source

OS_NOINLINE
int
_pthread_mutex_droplock(pthread_mutex_t *mutex, uint32_t *flagsp,
		uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
	if (_pthread_mutex_is_fairshare(mutex)) {
		return _pthread_mutex_fairshare_unlock_updatebits(mutex, flagsp,
				pmtxp, mgenp, ugenp);
	}
	return _pthread_mutex_firstfit_unlock_updatebits(mutex, flagsp, pmtxp,
			mgenp, ugenp);
}

OS_NOINLINE
int
_pthread_mutex_lock_init_slow(pthread_mutex_t *mutex, bool trylock)
{
	int res;

	res = _pthread_mutex_check_init(mutex);
	if (res != 0) return res;

	if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
		return _pthread_mutex_fairshare_lock_slow(mutex, trylock);
	} else if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) {
		return _pthread_mutex_ulock_lock(mutex, trylock);
	}
	return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
}

OS_NOINLINE
static int
_pthread_mutex_unlock_init_slow(pthread_mutex_t *mutex)
{
	int res;

	// Initialize static mutexes for compatibility with misbehaving
	// applications (unlock should not be the first operation on a mutex).
	res = _pthread_mutex_check_init(mutex);
	if (res != 0) return res;

	if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
		return _pthread_mutex_fairshare_unlock_slow(mutex);
	} else if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) {
		return _pthread_mutex_ulock_unlock(mutex);
	}
	return _pthread_mutex_firstfit_unlock_slow(mutex);
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_unlock(pthread_mutex_t *mutex)
{
	if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
		return _pthread_mutex_unlock_init_slow(mutex);
	}

	if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
		return _pthread_mutex_fairshare_unlock(mutex);
	}

	if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) {
		return _pthread_mutex_ulock_unlock(mutex);
	}

#if ENABLE_USERSPACE_TRACE
	return _pthread_mutex_firstfit_unlock_slow(mutex);
#elif PLOCKSTAT
	if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
		return _pthread_mutex_firstfit_unlock_slow(mutex);
	}
#endif

	/*
	 * This is the first-fit fast path. The fairshare fast-ish path is in
	 * _pthread_mutex_firstfit_unlock()
	 */
	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
… more in source

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_firstfit_lock(pthread_mutex_t *mutex, bool trylock)
{
	/*
	 * This is the first-fit fast path. The fairshare fast-ish path is in
	 * _pthread_mutex_fairshare_lock()
	 */
	uint64_t *tidaddr;
	MUTEX_GETTID_ADDR(mutex, &tidaddr);
	uint64_t selfid = _pthread_threadid_self_np_direct();

	mutex_seq *seqaddr;
	MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

	mutex_seq oldseq, newseq;
	mutex_seq_load(seqaddr, &oldseq);

	if (os_unlikely(!trylock && (oldseq.lgenval & PTH_RWL_EBIT))) {
		return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
	}

	bool gotlock;
	do {
		newseq = oldseq;
		gotlock = is_rwl_ebit_clear(oldseq.lgenval);

		if (trylock && !gotlock) {
#if __LP64__
			// The sequence load is atomic, so we can bail here without writing
… more in source

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_lock(pthread_mutex_t *mutex, bool trylock)
{
	if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
		return _pthread_mutex_lock_init_slow(mutex, trylock);
	}

	if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) {
		return _pthread_mutex_fairshare_lock(mutex, trylock);
	}

	if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) {
		return _pthread_mutex_ulock_lock(mutex, trylock);
	}

#if ENABLE_USERSPACE_TRACE
	return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
#elif PLOCKSTAT
	if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {
		return _pthread_mutex_firstfit_lock_slow(mutex, trylock);
	}
#endif

	return _pthread_mutex_firstfit_lock(mutex, trylock);
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_lock(pthread_mutex_t *mutex)
{
	return _pthread_mutex_lock(mutex, false);
}

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_trylock(pthread_mutex_t *mutex)
{
	return _pthread_mutex_lock(mutex, true);
}

OS_ALWAYS_INLINE
static inline int
_pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr,
		uint32_t static_type)
{
	mutex->mtxopts.value = 0;
	mutex->mtxopts.options.mutex = 1;
	if (attr) {
		if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
			return EINVAL;
		}
		mutex->prioceiling = (int16_t)attr->prioceiling;
		mutex->mtxopts.options.protocol = attr->protocol;
		mutex->mtxopts.options.policy = attr->opt;
		mutex->mtxopts.options.type = attr->type;
		mutex->mtxopts.options.pshared = attr->pshared;
	} else {
		switch (static_type) {
		case 1:
			mutex->mtxopts.options.type = PTHREAD_MUTEX_ERRORCHECK;
			break;
		case 2:
			mutex->mtxopts.options.type = PTHREAD_MUTEX_RECURSIVE;
			break;
		case 3:
			/* firstfit fall thru */
		case 7:
			mutex->mtxopts.options.type = PTHREAD_MUTEX_DEFAULT;
			break;
		default:
… more in source

PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_destroy(pthread_mutex_t *mutex)
{
	int res = EINVAL;

	_pthread_lock_lock(&mutex->lock);
	if (_pthread_mutex_check_signature(mutex)) {
		// TODO: PTHREAD_STRICT candidate
		res = EBUSY;

		if (_pthread_mutex_uses_ulock(mutex) &&
				mutex->ulock.uval == _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE) {
			res = 0;
		} else {
			mutex_seq *seqaddr;
			MUTEX_GETSEQ_ADDR(mutex, &seqaddr);

			mutex_seq seq;
			mutex_seq_load(seqaddr, &seq);

			uint64_t *tidaddr;
			MUTEX_GETTID_ADDR(mutex, &tidaddr);

			if ((os_atomic_load_wide(tidaddr, relaxed) == 0) &&
					(seq.lgenval & PTHRW_COUNT_MASK) ==
					(seq.ugenval & PTHRW_COUNT_MASK)) {
				res = 0;
			}
		}
… more in source

int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
	if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
		return EINVAL;
	}

	attr->sig = _PTHREAD_NO_SIG;
	return 0;
}

typedef union mutex_seq { uint32_t seq[2]; struct { uint32_t lgenval; uint32_t ugenval; }; struct { uint32_t mgen; uint32_t ugen; }; uint64_t seq_LU; uint64_t _Atomic atomic_seq_LU; } mutex_seq;