Index: kern_switch.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_switch.c,v
retrieving revision 1.136
diff -p -u -r1.136 kern_switch.c
--- kern_switch.c	20 Sep 2007 20:38:43 -0000	1.136
+++ kern_switch.c	2 Oct 2007 21:41:10 -0000
@@ -133,16 +133,6 @@ choosethread(void)
 {
 	struct thread *td;
 
-#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
-	if (smp_active == 0 && PCPU_GET(cpuid) != 0) {
-		/* Shutting down, run idlethread on AP's */
-		td = PCPU_GET(idlethread);
-		CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td);
-		TD_SET_RUNNING(td);
-		return (td);
-	}
-#endif
-
 retry:
 	td = sched_choose();
 
@@ -184,7 +174,7 @@ critical_exit(void)
 	td = curthread;
 	KASSERT(td->td_critnest != 0,
 	    ("critical_exit: td_critnest == 0"));
-#ifdef PREEMPTION
+
 	if (td->td_critnest == 1) {
 		td->td_critnest = 0;
 		if (td->td_owepreempt) {
@@ -196,7 +186,6 @@ critical_exit(void)
 			thread_unlock(td);
 		}
 	} else
-#endif
 		td->td_critnest--;
 
 	CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td,
Index: kern_synch.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_synch.c,v
retrieving revision 1.301
diff -p -u -r1.301 kern_synch.c
--- kern_synch.c	17 Sep 2007 05:27:20 -0000	1.301
+++ kern_synch.c	2 Oct 2007 08:18:19 -0000
@@ -553,8 +553,11 @@ synch_setup(dummy)
 int
 yield(struct thread *td, struct yield_args *uap)
 {
-	mtx_assert(&Giant, MA_NOTOWNED);
-	(void)uap;
-	sched_relinquish(td);
+
+	thread_lock(td);
+	sched_prio(td, PRI_MAX_TIMESHARE);
+	mi_switch(SW_VOL, NULL);
+	thread_unlock(td);
+	td->td_retval[0] = 0;
 	return (0);
 }
Index: p1003_1b.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/p1003_1b.c,v
retrieving revision 1.35
diff -p -u -r1.35 p1003_1b.c
--- p1003_1b.c	5 Mar 2007 13:10:57 -0000	1.35
+++ p1003_1b.c	2 Oct 2007 21:55:48 -0000
@@ -241,7 +241,8 @@ int
 sched_yield(struct thread *td, struct sched_yield_args *uap)
 {
 
-	return (ksched_yield(ksched));
+	sched_relinquish(curthread);
+	return 0;
 }
 
 int
Index: sched_4bsd.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/sched_4bsd.c,v
retrieving revision 1.105
diff -p -u -r1.105 sched_4bsd.c
--- sched_4bsd.c	21 Sep 2007 04:10:23 -0000	1.105
+++ sched_4bsd.c	2 Oct 2007 08:08:36 -0000
@@ -1324,8 +1324,6 @@ void
 sched_relinquish(struct thread *td)
 {
 	thread_lock(td);
-	if (td->td_pri_class == PRI_TIMESHARE)
-		sched_prio(td, PRI_MAX_TIMESHARE);
 	SCHED_STAT_INC(switch_relinquish);
 	mi_switch(SW_VOL, NULL);
 	thread_unlock(td);
Index: sched_ule.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/sched_ule.c,v
retrieving revision 1.209
diff -p -u -r1.209 sched_ule.c
--- sched_ule.c	24 Sep 2007 00:28:54 -0000	1.209
+++ sched_ule.c	2 Oct 2007 22:26:14 -0000
@@ -376,6 +376,7 @@ tdq_runq_add(struct tdq *tdq, struct td_
 {
 	TDQ_LOCK_ASSERT(tdq, MA_OWNED);
 	THREAD_LOCK_ASSERT(ts->ts_thread, MA_OWNED);
+	TD_SET_RUNQ(ts->ts_thread);
 #ifdef SMP
 	if (THREAD_CAN_MIGRATE(ts->ts_thread)) {
 		tdq->tdq_transferable++;
@@ -411,6 +412,23 @@ tdq_runq_add(struct tdq *tdq, struct td_
 		runq_add(ts->ts_runq, ts, flags);
 }
 
+/*
+ * Pick the run queue based on priority.
+ */
+static __inline void
+tdq_runq_pick(struct tdq *tdq, struct td_sched *ts)
+{
+	int pri;
+
+	pri = ts->ts_thread->td_priority;
+	if (pri <= PRI_MAX_REALTIME)
+		ts->ts_runq = &tdq->tdq_realtime;
+	else if (pri <= PRI_MAX_TIMESHARE)
+		ts->ts_runq = &tdq->tdq_timeshare;
+	else
+		ts->ts_runq = &tdq->tdq_idle;
+}
+
 /* 
  * Remove a thread from a run-queue.  This typically happens when a thread
  * is selected to run.  Running threads are not on the queue and the
@@ -434,13 +452,6 @@ tdq_runq_rem(struct tdq *tdq, struct td_
 			runq_remove_idx(ts->ts_runq, ts, &tdq->tdq_ridx);
 		else
 			runq_remove_idx(ts->ts_runq, ts, NULL);
-		/*
-		 * For timeshare threads we update the priority here so
-		 * the priority reflects the time we've been sleeping.
-		 */
-		ts->ts_ltick = ticks;
-		sched_pctcpu_update(ts);
-		sched_priority(ts->ts_thread);
 	} else
 		runq_remove(ts->ts_runq, ts);
 }
@@ -1099,6 +1110,15 @@ sched_pickcpu(struct td_sched *ts, int f
 
 #endif	/* SMP */
 
+#if RQB_LEN == 1
+#define	runq_empty(rq)	((rq)->rq_status.rqb_bits[0] == 0)
+#elif RQB_LEN == 2
+#define	runq_empty(rq)							\
+    (((rq)->rq_status.rqb_bits[0] | (rq)->rq_status.rqb_bits[1]) == 0)
+#else
+#error "Unsupported RQB_LEN"
+#endif
+
 /*
  * Pick the highest priority task we have and return it.
  */
@@ -1108,25 +1128,29 @@ tdq_choose(struct tdq *tdq)
 	struct td_sched *ts;
 
 	TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-	ts = runq_choose(&tdq->tdq_realtime);
-	if (ts != NULL)
-		return (ts);
-	ts = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
-	if (ts != NULL) {
-		KASSERT(ts->ts_thread->td_priority >= PRI_MIN_TIMESHARE,
-		    ("tdq_choose: Invalid priority on timeshare queue %d",
-		    ts->ts_thread->td_priority));
-		return (ts);
+	if (!runq_empty(&tdq->tdq_realtime)) {
+		ts = runq_choose(&tdq->tdq_realtime);
+		if (ts != NULL)
+			return (ts);
+	}
+	if (!runq_empty(&tdq->tdq_timeshare)) {
+		ts = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
+		if (ts != NULL) {
+			KASSERT(ts->ts_thread->td_priority >= PRI_MIN_TIMESHARE,
+			    ("tdq_choose: Invalid timeshare priority %d",
+			    ts->ts_thread->td_priority));
+			return (ts);
+		}
 	}
-
-	ts = runq_choose(&tdq->tdq_idle);
-	if (ts != NULL) {
-		KASSERT(ts->ts_thread->td_priority >= PRI_MIN_IDLE,
-		    ("tdq_choose: Invalid priority on idle queue %d",
-		    ts->ts_thread->td_priority));
-		return (ts);
+	if (!runq_empty(&tdq->tdq_idle)) {
+		ts = runq_choose(&tdq->tdq_idle);
+		if (ts != NULL) {
+			KASSERT(ts->ts_thread->td_priority >= PRI_MIN_IDLE,
+			    ("tdq_choose: Invalid priority on idle queue %d",
+			    ts->ts_thread->td_priority));
+			return (ts);
+		}
 	}
-
 	return (NULL);
 }
 
@@ -1302,6 +1326,7 @@ sched_setup(void *dummy)
 	/* Add thread0's load since it's running. */
 	TDQ_LOCK(tdq);
 	thread0.td_lock = TDQ_LOCKPTR(TDQ_SELF());
+	tdq_runq_pick(tdq, &td_sched0);
 	tdq_load_add(tdq, &td_sched0);
 	TDQ_UNLOCK(tdq);
 }
@@ -1565,15 +1590,20 @@ static void
 sched_thread_priority(struct thread *td, u_char prio)
 {
 	struct td_sched *ts;
+	struct tdq *tdq;
 
 	CTR6(KTR_SCHED, "sched_prio: %p(%s) prio %d newprio %d by %p(%s)",
 	    td, td->td_proc->p_comm, td->td_priority, prio, curthread,
 	    curthread->td_proc->p_comm);
 	ts = td->td_sched;
+	tdq = TDQ_CPU(ts->ts_cpu);
 	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	if (td->td_priority == prio)
 		return;
-
+#ifdef SMP
+	if (prio < tdq->tdq_lowpri)
+		tdq->tdq_lowpri = prio;
+#endif
 	if (TD_ON_RUNQ(td) && prio < td->td_priority) {
 		/*
 		 * If the priority has been elevated due to priority
@@ -1585,14 +1615,8 @@ sched_thread_priority(struct thread *td,
 		td->td_priority = prio;
 		sched_add(td, SRQ_BORROWING);
 	} else {
-#ifdef SMP
-		struct tdq *tdq;
-
-		tdq = TDQ_CPU(ts->ts_cpu);
-		if (prio < tdq->tdq_lowpri)
-			tdq->tdq_lowpri = prio;
-#endif
 		td->td_priority = prio;
+		tdq_runq_pick(tdq, ts);
 	}
 }
 
@@ -1739,6 +1763,8 @@ sched_switch_migrate(struct tdq *tdq, st
 
 	tdn = TDQ_CPU(td->td_sched->ts_cpu);
 #ifdef SMP
+	/* The load is being removed from the current cpu. */
+	tdq_load_rem(tdq, td->td_sched);
 	/*
 	 * Do the lock dance required to avoid LOR.  We grab an extra
 	 * spinlock nesting to prevent preemption while we're
@@ -1830,12 +1856,11 @@ sched_switch(struct thread *td, struct t
 		TD_SET_CAN_RUN(td);
 	} else if (TD_IS_RUNNING(td)) {
 		MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-		tdq_load_rem(tdq, ts);
 		srqflag = (flags & SW_PREEMPT) ?
 		    SRQ_OURSELF|SRQ_YIELDING|SRQ_PREEMPTED :
 		    SRQ_OURSELF|SRQ_YIELDING;
 		if (ts->ts_cpu == cpuid)
-			tdq_add(tdq, td, srqflag);
+			tdq_runq_add(tdq, ts, srqflag);
 		else
 			mtx = sched_switch_migrate(tdq, td, srqflag);
 	} else {
@@ -1949,7 +1974,6 @@ sched_wakeup(struct thread *td)
 		ts->ts_slptime += hzticks;
 		sched_interact_update(td);
 		sched_pctcpu_update(ts);
-		sched_priority(td);
 	}
 	/* Reset the slice value after we sleep. */
 	ts->ts_slice = sched_slice;
@@ -2154,16 +2178,17 @@ sched_clock(struct thread *td)
 	 */
 	td->td_sched->ts_runtime += tickincr;
 	sched_interact_update(td);
+	sched_priority(td);
 	/*
 	 * We used up one time slice.
 	 */
 	if (--ts->ts_slice > 0)
 		return;
 	/*
-	 * We're out of time, recompute priorities and requeue.
+	 * We're out of time, force a requeue later.
 	 */
-	sched_priority(td);
 	td->td_flags |= TDF_NEEDRESCHED;
+	ts->ts_slice = sched_slice;
 }
 
 /*
@@ -2284,11 +2309,10 @@ void
 tdq_add(struct tdq *tdq, struct thread *td, int flags)
 {
 	struct td_sched *ts;
-	int class;
 #ifdef SMP
+	int class;
 	int cpumask;
 #endif
-
 	TDQ_LOCK_ASSERT(tdq, MA_OWNED);
 	KASSERT((td->td_inhibitors == 0),
 	    ("sched_add: trying to run inhibited thread"));
@@ -2298,20 +2322,11 @@ tdq_add(struct tdq *tdq, struct thread *
 	    ("sched_add: thread swapped out"));
 
 	ts = td->td_sched;
-	class = PRI_BASE(td->td_pri_class);
-        TD_SET_RUNQ(td);
-	if (ts->ts_slice == 0)
-		ts->ts_slice = sched_slice;
-	/*
-	 * Pick the run queue based on priority.
-	 */
-	if (td->td_priority <= PRI_MAX_REALTIME)
-		ts->ts_runq = &tdq->tdq_realtime;
-	else if (td->td_priority <= PRI_MAX_TIMESHARE)
-		ts->ts_runq = &tdq->tdq_timeshare;
-	else
-		ts->ts_runq = &tdq->tdq_idle;
+	tdq_runq_pick(tdq, ts);
+	tdq_runq_add(tdq, ts, flags);
+	tdq_load_add(tdq, ts);
 #ifdef SMP
+	class = PRI_BASE(td->td_pri_class);
 	cpumask = 1 << ts->ts_cpu;
 	/*
 	 * If we had been idle, clear our bit in the group and potentially
@@ -2334,8 +2349,6 @@ tdq_add(struct tdq *tdq, struct thread *
 	if (td->td_priority < tdq->tdq_lowpri)
 		tdq->tdq_lowpri = td->td_priority;
 #endif
-	tdq_runq_add(tdq, ts, flags);
-	tdq_load_add(tdq, ts);
 }
 
 /*
@@ -2502,8 +2515,6 @@ void
 sched_relinquish(struct thread *td)
 {
 	thread_lock(td);
-	if (td->td_pri_class == PRI_TIMESHARE)
-		sched_prio(td, PRI_MAX_TIMESHARE);
 	SCHED_STAT_INC(switch_relinquish);
 	mi_switch(SW_VOL, NULL);
 	thread_unlock(td);