Re: [patch and review please] 64 CPU Support

From: John Baldwin <jhb_at_freebsd.org>
Date: Tue, 3 Aug 2010 16:39:38 -0400
On Monday, July 26, 2010 10:43:59 am John Baldwin wrote:
> On Sunday, July 25, 2010 4:48:53 pm Sean Bruno wrote:
> > Reposting from -stable.  
> > 
> > Kind of a large patch, but in order to make an omlette, you need to
> > break a few servers.
> > 
> > This is a diff against -CURRENT, not stable-8 as I didn't get a chance
> > to test it.  It is directly based off of changes that peter_at_ made to the
> > Yahoo FreeBSD 7 tree.
> > 
> > I have compile and boot tested this on my local machines, but I don't
> > have 64 CPU machines to test upon.
> 
> I think IPI_AST in the first hunk should be using ipi_cpu().  I would perhaps 
> tackle ipi_cpu() as a first step: introduce ipi_cpu() on both i386 and amd64 
> (it should be ok to add a real version for i386 rather than the current macro, 
> it should be a copy of the amd64 code).  Other folks can help with other 
> architectures.  ipi_selected() should generally be a good clue as to how to 
> implement ipi_cpu().

So I have a patch to add ipi_cpu() for HEAD.  I've cross-compiled it for each
SMP architecture and have booted it on amd64.  It uses ipi_cpu() instead of
ipi_selected() whenever possible.

--- //depot/vendor/freebsd/src/sys/amd64/amd64/mp_machdep.c	2010-06-22 16:25:14.000000000 0000
+++ //depot/projects/smpng/sys/amd64/amd64/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -1239,15 +1239,51 _at__at_
 			do {
 				old_pending = cpu_ipi_pending[cpu];
 				new_pending = old_pending | bitmap;
-			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));	
-
+			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+			    old_pending, new_pending));	
 			if (old_pending)
 				continue;
 		}
+		lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+	}
+}
 
-		lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+	u_int bitmap = 0;
+	u_int old_pending;
+	u_int new_pending;
+
+	if (IPI_IS_BITMAPED(ipi)) { 
+		bitmap = 1 << ipi;
+		ipi = IPI_BITMAP_VECTOR;
 	}
 
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+	CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+	KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
+
+	if (bitmap) {
+		do {
+			old_pending = cpu_ipi_pending[cpu];
+			new_pending = old_pending | bitmap;
+		} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+		    old_pending, new_pending));	
+		if (old_pending)
+			return;
+	}
+	lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
 }
 
 /*
--- //depot/vendor/freebsd/src/sys/amd64/include/smp.h	2010-06-17 12:00:13.000000000 0000
+++ //depot/projects/smpng/sys/amd64/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -60,10 +60,11 _at__at_
 void	cpustop_handler(void);
 void	cpususpend_handler(void);
 void	init_secondary(void);
+void	ipi_all_but_self(u_int ipi);
+void 	ipi_bitmap_handler(struct trapframe frame);
+void	ipi_cpu(int cpu, u_int ipi);
 int	ipi_nmi_handler(void);
 void	ipi_selected(cpumask_t cpus, u_int ipi);
-void	ipi_all_but_self(u_int ipi);
-void 	ipi_bitmap_handler(struct trapframe frame);
 u_int	mp_bootaddress(u_int);
 int	mp_grab_cpu_hlt(void);
 void	smp_cache_flush(void);
--- //depot/vendor/freebsd/src/sys/i386/i386/mp_machdep.c	2010-06-22 16:25:14.000000000 0000
+++ //depot/projects/smpng/sys/i386/i386/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -1327,15 +1327,51 _at__at_
 			do {
 				old_pending = cpu_ipi_pending[cpu];
 				new_pending = old_pending | bitmap;
-			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));	
-
+			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+			    old_pending, new_pending));	
 			if (old_pending)
 				continue;
 		}
+		lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+	}
+}
 
-		lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+	u_int bitmap = 0;
+	u_int old_pending;
+	u_int new_pending;
+
+	if (IPI_IS_BITMAPED(ipi)) { 
+		bitmap = 1 << ipi;
+		ipi = IPI_BITMAP_VECTOR;
 	}
 
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+	CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+	KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
+
+	if (bitmap) {
+		do {
+			old_pending = cpu_ipi_pending[cpu];
+			new_pending = old_pending | bitmap;
+		} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+		    old_pending, new_pending));	
+		if (old_pending)
+			return;
+	}
+	lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
 }
 
 /*
--- //depot/vendor/freebsd/src/sys/i386/include/smp.h	2009-08-13 17:30:15.000000000 0000
+++ //depot/projects/smpng/sys/i386/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -60,12 +60,13 _at__at_
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
 void	init_secondary(void);
-int	ipi_nmi_handler(void);
-void	ipi_selected(cpumask_t cpus, u_int ipi);
 void	ipi_all_but_self(u_int ipi);
 #ifndef XEN
 void 	ipi_bitmap_handler(struct trapframe frame);
 #endif
+void	ipi_cpu(int cpu, u_int ipi);
+int	ipi_nmi_handler(void);
+void	ipi_selected(cpumask_t cpus, u_int ipi);
 u_int	mp_bootaddress(u_int);
 int	mp_grab_cpu_hlt(void);
 void	smp_cache_flush(void);
--- //depot/vendor/freebsd/src/sys/i386/xen/mp_machdep.c	2010-03-10 19:55:15.000000000 0000
+++ //depot/projects/smpng/sys/i386/xen/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -1121,19 +1121,14 _at__at_
 		cpu--;
 		cpus &= ~(1 << cpu);
 
-		KASSERT(cpu_apic_ids[cpu] != -1,
-		    ("IPI to non-existent CPU %d", cpu));
-
 		if (bitmap) {
 			do {
 				old_pending = cpu_ipi_pending[cpu];
 				new_pending = old_pending | bitmap;
-			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));	
-
+			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+			    old_pending, new_pending));	
 			if (!old_pending)
 				ipi_pcpu(cpu, RESCHEDULE_VECTOR);
-			continue;
-			
 		} else {
 			KASSERT(call_data != NULL, ("call_data not set"));
 			ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
_at__at_ -1142,6 +1137,45 _at__at_
 }
 
 /*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+	u_int bitmap = 0;
+	u_int old_pending;
+	u_int new_pending;
+	
+	if (IPI_IS_BITMAPED(ipi)) { 
+		bitmap = 1 << ipi;
+		ipi = IPI_BITMAP_VECTOR;
+	} 
+
+	/*
+	 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+	 * of help in order to understand what is the source.
+	 * Set the mask of receiving CPUs for this purpose.
+	 */
+	if (ipi == IPI_STOP_HARD)
+		atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+	CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+
+	if (bitmap) {
+		do {
+			old_pending = cpu_ipi_pending[cpu];
+			new_pending = old_pending | bitmap;
+		} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+		    old_pending, new_pending));	
+		if (!old_pending)
+			ipi_pcpu(cpu, RESCHEDULE_VECTOR);
+	} else {
+		KASSERT(call_data != NULL, ("call_data not set"));
+		ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
+	}
+}
+
+/*
  * send an IPI to all CPUs EXCEPT myself
  */
 void
--- //depot/vendor/freebsd/src/sys/ia64/ia64/mp_machdep.c	2010-07-03 20:20:15.000000000 0000
+++ //depot/projects/smpng/sys/ia64/ia64/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -408,6 +408,16 _at__at_
 }
 
 /*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+	ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+/*
  * send an IPI to all CPUs EXCEPT myself.
  */
 void
--- //depot/vendor/freebsd/src/sys/ia64/include/smp.h	2010-03-17 00:40:15.000000000 0000
+++ //depot/projects/smpng/sys/ia64/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -25,6 +25,7 _at__at_
 extern int ia64_ipi_wakeup;
 
 void	ipi_all_but_self(int ipi);
+void	ipi_cpu(int cpu, u_int ipi);
 void	ipi_selected(cpumask_t cpus, int ipi);
 void	ipi_send(struct pcpu *, int ipi);
 
--- //depot/vendor/freebsd/src/sys/kern/sched_4bsd.c	2010-06-11 18:50:16.000000000 0000
+++ //depot/projects/smpng/sys/kern/sched_4bsd.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -1154,7 +1154,7 _at__at_
 	pcpu = pcpu_find(cpuid);
 	if (idle_cpus_mask & pcpu->pc_cpumask) {
 		forward_wakeups_delivered++;
-		ipi_selected(pcpu->pc_cpumask, IPI_AST);
+		ipi_cpu(cpuid, IPI_AST);
 		return;
 	}
 
_at__at_ -1167,13 +1167,13 _at__at_
 	if (pri <= PRI_MAX_ITHD)
 #endif /* ! FULL_PREEMPTION */
 	{
-		ipi_selected(pcpu->pc_cpumask, IPI_PREEMPT);
+		ipi_cpu(cpuid, IPI_PREEMPT);
 		return;
 	}
 #endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */
 
 	pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED;
-	ipi_selected(pcpu->pc_cpumask, IPI_AST);
+	ipi_cpu(cpuid, IPI_AST);
 	return;
 }
 #endif /* SMP */
_at__at_ -1666,7 +1666,7 _at__at_
 
 		td->td_flags |= TDF_NEEDRESCHED;
 		if (td != curthread)
-			ipi_selected(1 << cpu, IPI_AST);
+			ipi_cpu(cpu, IPI_AST);
 		break;
 	default:
 		break;
--- //depot/vendor/freebsd/src/sys/kern/sched_ule.c	2010-07-15 13:50:13.000000000 0000
+++ //depot/projects/smpng/sys/kern/sched_ule.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -851,7 +851,7 _at__at_
 		 * IPI the target cpu to force it to reschedule with the new
 		 * workload.
 		 */
-		ipi_selected(1 << TDQ_ID(low), IPI_PREEMPT);
+		ipi_cpu(TDQ_ID(low), IPI_PREEMPT);
 	}
 	tdq_unlock_pair(high, low);
 	return (moved);
_at__at_ -974,7 +974,7 _at__at_
 			return;
 	}
 	tdq->tdq_ipipending = 1;
-	ipi_selected(1 << cpu, IPI_PREEMPT);
+	ipi_cpu(cpu, IPI_PREEMPT);
 }
 
 /*
_at__at_ -2411,7 +2411,7 _at__at_
 	cpu = ts->ts_cpu;
 	ts->ts_cpu = sched_pickcpu(td, 0);
 	if (cpu != PCPU_GET(cpuid))
-		ipi_selected(1 << cpu, IPI_PREEMPT);
+		ipi_cpu(cpu, IPI_PREEMPT);
 #endif
 }
 
--- //depot/vendor/freebsd/src/sys/kern/subr_smp.c	2010-06-11 18:50:16.000000000 0000
+++ //depot/projects/smpng/sys/kern/subr_smp.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -181,7 +181,7 _at__at_
 	id = td->td_oncpu;
 	if (id == NOCPU)
 		return;
-	ipi_selected(1 << id, IPI_AST);
+	ipi_cpu(id, IPI_AST);
 }
 
 /*
--- //depot/vendor/freebsd/src/sys/mips/include/smp.h	2010-05-16 19:50:44.000000000 0000
+++ //depot/projects/smpng/sys/mips/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -30,6 +30,7 _at__at_
 
 #ifndef LOCORE
 
+void	ipi_cpu(int cpu, u_int ipi);
 void	ipi_selected(cpumask_t cpus, int ipi);
 void	smp_init_secondary(u_int32_t cpuid);
 void	mpentry(void);
--- //depot/vendor/freebsd/src/sys/mips/mips/mp_machdep.c	2010-06-17 05:05:14.000000000 0000
+++ //depot/projects/smpng/sys/mips/mips/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -85,6 +85,15 _at__at_
 	}
 }
 
+/* Send an IPI to a specific CPU. */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+	CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
+	ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
 /*
  * Handle an IPI sent to this processor.
  */
--- //depot/vendor/freebsd/src/sys/powerpc/include/smp.h	2010-07-13 05:39:35.000000000 0000
+++ //depot/projects/smpng/sys/powerpc/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -40,6 +40,7 _at__at_
 #ifndef LOCORE
 
 void	ipi_all_but_self(int ipi);
+void	ipi_cpu(int cpu, u_int ipi);
 void	ipi_selected(cpumask_t cpus, int ipi);
 
 struct cpuref {
--- //depot/vendor/freebsd/src/sys/powerpc/powerpc/mp_machdep.c	2010-07-13 05:39:35.000000000 0000
+++ //depot/projects/smpng/sys/powerpc/powerpc/mp_machdep.c	2010-08-01 18:48:37.000000000 0000
_at__at_ -336,6 +336,14 _at__at_
 	}
 }
 
+/* Send an IPI to a specific CPU. */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+	ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
 /* Send an IPI to all CPUs EXCEPT myself. */
 void
 ipi_all_but_self(int ipi)
--- //depot/vendor/freebsd/src/sys/sparc64/include/smp.h	2010-07-04 12:45:48.000000000 0000
+++ //depot/projects/smpng/sys/sparc64/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -133,6 +133,17 _at__at_
 	cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
 }
 
+static __inline void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+	/*
+	 * XXX: Not ideal, but would require more work to add a cpu_ipi_cpu
+	 * function pointer.
+	 */
+	cpu_ipi_selected(1 << cpu, 0, (u_long)tl_ipi_level, ipi);
+}
+
 #if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
 
 static __inline void *
--- //depot/vendor/freebsd/src/sys/sun4v/include/smp.h	2009-08-13 17:30:15.000000000 0000
+++ //depot/projects/smpng/sys/sun4v/include/smp.h	2010-08-01 18:48:37.000000000 0000
_at__at_ -82,8 +82,9 _at__at_
 void cpu_ipi_stop(struct trapframe *tf);
 void cpu_ipi_preempt(struct trapframe *tf);
 
+void	ipi_all_but_self(u_int ipi);
+void	ipi_cpu(int cpu, u_int ipi);
 void	ipi_selected(u_int cpus, u_int ipi);
-void	ipi_all_but_self(u_int ipi);
 
 vm_offset_t mp_tramp_alloc(void);
 void        mp_set_tsb_desc_ra(vm_paddr_t);
--- //depot/vendor/freebsd/src/sys/sun4v/sun4v/mp_machdep.c	2009-06-23 22:54:36.000000000 0000
+++ //depot/projects/smpng/sys/sun4v/sun4v/mp_machdep.c	2010-08-03 17:34:33.000000000 0000
_at__at_ -518,7 +518,6 _at__at_
 	}
 }
 
-
 void
 ipi_selected(u_int icpus, u_int ipi)
 {
_at__at_ -533,7 +532,6 _at__at_
 	 * 4) handling 4-way threading vs 2-way threading should happen here
 	 *    and not in forward wakeup
 	 */
-	
 	cpulist = PCPU_GET(cpulist);
 	cpus = (icpus & ~PCPU_GET(cpumask));
 	
_at__at_ -545,8 +543,32 _at__at_
 		cpu_count++;
 	}
 
-	cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0, &ackmask);
-	
+	cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
+	    &ackmask);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+	int cpu_count;
+	uint16_t *cpulist;
+	uint64_t ackmask;
+
+	/* 
+	 * 
+	 * 3) forward_wakeup appears to abuse ASTs
+	 * 4) handling 4-way threading vs 2-way threading should happen here
+	 *    and not in forward wakeup
+	 */
+	cpulist = PCPU_GET(cpulist);
+	if (PCPU_GET(cpumask) & (1 << cpu))
+		cpu_count = 0;
+	else {
+		cpulist[0] = (uint16_t)cpu;
+		cpu_count = 1;
+	}
+	cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
+	    &ackmask);
 }
 
 void

-- 
John Baldwin
Received on Tue Aug 03 2010 - 18:50:31 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:40:05 UTC