On Thu, Dec 19, 2013 at 07:54:49PM +0100, Roger Pau Monne wrote: > AP startup on PVH follows the PV method, so we need to add a hook in > order to diverge from bare metal. > --- > sys/amd64/amd64/mp_machdep.c | 16 ++++--- > sys/amd64/include/cpu.h | 1 + > sys/x86/xen/hvm.c | 17 +++++++- > sys/x86/xen/pv.c | 90 ++++++++++++++++++++++++++++++++++++++++++ > sys/xen/pv.h | 1 + > 5 files changed, 117 insertions(+), 8 deletions(-) > > diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c > index 4ef4b3d..e302886 100644 > --- a/sys/amd64/amd64/mp_machdep.c > +++ b/sys/amd64/amd64/mp_machdep.c > _at__at_ -90,7 +90,7 _at__at_ extern struct pcpu __pcpu[]; > > /* AP uses this during bootstrap. Do not staticize. */ > char *bootSTK; > -static int bootAP; > +int bootAP; > > /* Free these after use */ > void *bootstacks[MAXCPU]; > _at__at_ -122,9 +122,12 _at__at_ u_long *ipi_rendezvous_counts[MAXCPU]; > static u_long *ipi_hardclock_counts[MAXCPU]; > #endif > > +int native_start_all_aps(void); > + > /* Default cpu_ops implementation. */ > struct cpu_ops cpu_ops = { > - .ipi_vectored = lapic_ipi_vectored > + .ipi_vectored = lapic_ipi_vectored, > + .start_all_aps = native_start_all_aps, > }; > > extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); > _at__at_ -138,7 +141,7 _at__at_ extern int pmap_pcid_enabled; > static volatile cpuset_t ipi_nmi_pending; > > /* used to hold the AP's until we are ready to release them */ > -static struct mtx ap_boot_mtx; > +struct mtx ap_boot_mtx; > > /* Set to 1 once we're ready to let the APs out of the pen. */ > static volatile int aps_ready = 0; > _at__at_ -165,7 +168,6 _at__at_ static int cpu_cores; /* cores per package */ > > static void assign_cpu_ids(void); > static void set_interrupt_apic_ids(void); > -static int start_all_aps(void); > static int start_ap(int apic_id); > static void release_aps(void *dummy); > > _at__at_ -569,7 +571,7 _at__at_ cpu_mp_start(void) > assign_cpu_ids(); > > /* Start each Application Processor */ > - start_all_aps(); > + cpu_ops.start_all_aps(); > > set_interrupt_apic_ids(); > } > _at__at_ -908,8 +910,8 _at__at_ assign_cpu_ids(void) > /* > * start each AP in our list > */ > -static int > -start_all_aps(void) > +int > +native_start_all_aps(void) > { > vm_offset_t va = boot_address + KERNBASE; > u_int64_t *pt4, *pt3, *pt2; > diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h > index 3d9ff531..ed9f1db 100644 > --- a/sys/amd64/include/cpu.h > +++ b/sys/amd64/include/cpu.h > _at__at_ -64,6 +64,7 _at__at_ struct cpu_ops { > void (*cpu_init)(void); > void (*cpu_resume)(void); > void (*ipi_vectored)(u_int, int); > + int (*start_all_aps)(void); > }; > > extern struct cpu_ops cpu_ops; > diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c > index fb1ed79..5ec9f3a 100644 > --- a/sys/x86/xen/hvm.c > +++ b/sys/x86/xen/hvm.c > _at__at_ -53,6 +53,9 _at__at_ __FBSDID("$FreeBSD$"); > #include <xen/hypervisor.h> > #include <xen/hvm.h> > #include <xen/xen_intr.h> > +#ifdef __amd64__ > +#include <xen/pv.h> > +#endif > > #include <xen/interface/hvm/params.h> > #include <xen/interface/vcpu.h> > _at__at_ -97,6 +100,11 _at__at_ extern void pmap_lazyfix_action(void); > /* Variables used by mp_machdep to perform the bitmap IPI */ > extern volatile u_int cpu_ipi_pending[MAXCPU]; > > +#ifdef __amd64__ > +/* Native AP start used on PVHVM */ > +extern int native_start_all_aps(void); > +#endif > + > /*---------------------------------- Macros ----------------------------------*/ > #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) > > _at__at_ -119,7 +127,10 _at__at_ enum xen_domain_type xen_domain_type = XEN_NATIVE; > struct cpu_ops xen_hvm_cpu_ops = { > .ipi_vectored = lapic_ipi_vectored, > .cpu_init = xen_hvm_cpu_init, > - .cpu_resume = xen_hvm_cpu_resume > + .cpu_resume = xen_hvm_cpu_resume, > +#ifdef __amd64__ > + .start_all_aps = native_start_all_aps, > +#endif > }; > > static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support"); > _at__at_ -698,6 +709,10 _at__at_ xen_hvm_init(enum xen_hvm_init_type init_type) > setup_xen_features(); > cpu_ops = xen_hvm_cpu_ops; > vm_guest = VM_GUEST_XEN; > +#ifdef __amd64__ > + if (xen_pv_domain()) > + cpu_ops.start_all_aps = xen_pv_start_all_aps; > +#endif > break; > case XEN_HVM_INIT_RESUME: > if (error != 0) > diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c > index db576e0..7e45a83 100644 > --- a/sys/x86/xen/pv.c > +++ b/sys/x86/xen/pv.c > _at__at_ -34,21 +34,43 _at__at_ __FBSDID("$FreeBSD$"); > #include <sys/kernel.h> > #include <sys/reboot.h> > #include <sys/systm.h> > +#include <sys/malloc.h> > +#include <sys/lock.h> > +#include <sys/mutex.h> > +#include <sys/smp.h> > + > +#include <vm/vm.h> > +#include <vm/pmap.h> > +#include <vm/vm_extern.h> > +#include <vm/vm_kern.h> > > #include <machine/sysarch.h> > #include <machine/clock.h> > #include <machine/pc/bios.h> > +#include <machine/smp.h> > > #include <xen/xen-os.h> > #include <xen/pv.h> > #include <xen/hypervisor.h> > > +#include <xen/interface/vcpu.h> > + > #define MAX_E820_ENTRIES 128 > > /*--------------------------- Forward Declarations ---------------------------*/ > static caddr_t xen_pv_parse_preload_data(u_int64_t); > static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *); > > +/*---------------------------- Extern Declarations ---------------------------*/ > +/* Variables used by amd64 mp_machdep to start APs */ > +extern struct mtx ap_boot_mtx; > +extern void *bootstacks[]; > +extern char *doublefault_stack; > +extern char *nmi_stack; > +extern void *dpcpu; > +extern int bootAP; > +extern char *bootSTK; > + > /*-------------------------------- Global Data -------------------------------*/ > /* Xen init_ops implementation. */ > struct init_ops xen_init_ops = { > _at__at_ -78,6 +100,74 _at__at_ static struct > > static struct bios_smap xen_smap[MAX_E820_ENTRIES]; > > +static int > +start_xen_ap(int cpu) > +{ > + struct vcpu_guest_context *ctxt; > + int ms, cpus = mp_naps; > + > + ctxt = malloc(sizeof(*ctxt), M_TEMP, M_NOWAIT | M_ZERO); > + if (ctxt == NULL) > + panic("unable to allocate memory"); > + > + ctxt->flags = VGCF_IN_KERNEL; > + ctxt->user_regs.rip = (unsigned long) init_secondary; > + ctxt->user_regs.rsp = (unsigned long) bootSTK; > + > + /* Set the AP to use the same page tables */ > + ctxt->ctrlreg[3] = KPML4phys; > + > + if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt)) > + panic("unable to initialize AP#%d\n", cpu); > + > + free(ctxt, M_TEMP); > + > + /* Launch the vCPU */ > + if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL)) > + panic("unable to start AP#%d\n", cpu); Why the panic? Why not just return 0 (which I presume would mean we failed to init the CPU?) > + > + /* Wait up to 5 seconds for it to start. */ > + for (ms = 0; ms < 5000; ms++) { > + if (mp_naps > cpus) > + return (1); /* return SUCCESS */ > + DELAY(1000); > + } > + Wow. So much simpler than the PV path. > + return (0); There isn't a doorbell mechanism for the booted processor to tell the BSP it is up? > +} > + > +int > +xen_pv_start_all_aps(void) > +{ > + int cpu; > + > + mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); > + > + for (cpu = 1; cpu < mp_ncpus; cpu++) { > + > + /* allocate and set up an idle stack data page */ > + bootstacks[cpu] = (void *)kmem_malloc(kernel_arena, > + KSTACK_PAGES * PAGE_SIZE, M_WAITOK | M_ZERO); > + doublefault_stack = (char *)kmem_malloc(kernel_arena, > + PAGE_SIZE, M_WAITOK | M_ZERO); > + nmi_stack = (char *)kmem_malloc(kernel_arena, PAGE_SIZE, > + M_WAITOK | M_ZERO); > + dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, > + M_WAITOK | M_ZERO); > + > + bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8; > + bootAP = cpu; > + > + /* attempt to start the Application Processor */ > + if (!start_xen_ap(cpu)) > + panic("AP #%d failed to start!", cpu); > + > + CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ > + } > + > + return (mp_naps); > +} > + > /* > * Functions to convert the "extra" parameters passed by Xen > * into FreeBSD boot options (from the i386 Xen port). > diff --git a/sys/xen/pv.h b/sys/xen/pv.h > index 71b8776..60d9def 100644 > --- a/sys/xen/pv.h > +++ b/sys/xen/pv.h > _at__at_ -24,5 +24,6 _at__at_ > #define __XEN_PV_H__ > > void xen_pv_set_init_ops(void); > +int xen_pv_start_all_aps(void); > > #endif /* __XEN_PV_H__ */ > -- > 1.7.7.5 (Apple Git-26) > > > _______________________________________________ > Xen-devel mailing list > Xen-devel_at_lists.xen.org > http://lists.xen.org/xen-develReceived on Fri Jan 03 2014 - 19:54:04 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:40:46 UTC