Re: number of CPUs and IPI panic

I'm still getting this panic with the patch:

panic: APIC: Previous IPI is stuck
cpuid = 3
boot() called on cpu#0
Uptime: 4m16s

:(

--- Stephan Uphoff <ups_at_tree.com> wrote:

> On Wed, 2004-11-10 at 11:05, Nik Azim Azam wrote:
> > hmm, you've forgotten to include the patch with
> the
> > mail?
> > 
> > --- Stephan Uphoff <ups_at_tree.com> wrote:
> 
> > > OK - lets try this again.
> > > The attached patch should prevent blocking on
> > > redundant IPIs caused by
> > > the apic not being able to queue the request.
> > > If this does not work we will have to instrument
> the
> > > code.
> > > 
> > > 	Stephan
> 
> Sight - one more time ...
> 
> 
> > Index: sys/i386/i386/apic_vector.s
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/apic_vector.s,v
> retrieving revision 1.101
> diff -u -r1.101 apic_vector.s
> --- sys/i386/i386/apic_vector.s	26 May 2004 07:43:41
> -0000	1.101
> +++ sys/i386/i386/apic_vector.s	3 Nov 2004 15:43:07
> -0000
> _at__at_ -286,7 +286,11 _at__at_
>  	movl	%eax, %es
>  	movl	$KPSEL, %eax
>  	movl	%eax, %fs
> -
> +		
> +	movl	PCPU(CPUID), %edx
> +	lock
> +	btrl	%edx,CNAME(ipi_ast_pending)  
> +	
>  	movl	lapic, %edx
>  	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC
> */
>  
> Index: sys/i386/i386/local_apic.c
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/local_apic.c,v
> retrieving revision 1.9
> diff -u -r1.9 local_apic.c
> --- sys/i386/i386/local_apic.c	14 Jul 2004 18:12:15
> -0000	1.9
> +++ sys/i386/i386/local_apic.c	4 Oct 2004 02:24:17
> -0000
> _at__at_ -693,6 +693,62 _at__at_
>  	intr_restore(eflags);
>  }
>  
> +
> +static void
> +lapic_ipi_wait_and_raw(int delay,register_t icrlo,
> u_int dest)
> +{
> +    int x, incr,idle;
> +    register_t value, eflags;
> +
> +    
> +/* XXX: Need more sanity checking of icrlo? */
> +    KASSERT(lapic != NULL, ("%s called too early",
> __func__));
> +	KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT))
> == 0,
> +	    ("%s: invalid dest field", __func__));
> +	KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
> +	    ("%s: reserved bits set in ICR LO register",
> __func__));
> +
> +	if (delay == -1) {
> +		incr = 0;
> +		delay = 1;
> +	} else
> +		incr = 1;
> +
> +	/* Set destination in ICR HI register if it is
> being used. */
> +	eflags = intr_disable();
> +
> +	idle = 0;
> +
> +	for (x = 0; x < delay; x += incr) {
> +		if ((lapic->icr_lo & APIC_DELSTAT_MASK) ==
> APIC_DELSTAT_IDLE)
> +		{
> +		    idle = 1;
> +		    break;
> +		}
> +		intr_restore(eflags);
> +		ia32_pause();
> +		eflags = intr_disable();
> +
> +	}
> +	
> +	if (!idle)  panic("APIC: Previous IPI is stuck");
> +
> +	if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD)
> {
> +		value = lapic->icr_hi;
> +		value &= ~APIC_ID_MASK;
> +		value |= dest << APIC_ID_SHIFT;
> +		lapic->icr_hi = value;
> +	}
> +
> +	/* Program the contents of the IPI and dispatch
> it. */
> +	value = lapic->icr_lo;
> +	value &= APIC_ICRLO_RESV_MASK;
> +	value |= icrlo;
> +	lapic->icr_lo = value;
> +	intr_restore(eflags);
> +}
> +
> +
>  #define	BEFORE_SPIN	1000000
>  #ifdef DETECT_DEADLOCK
>  #define	AFTER_SPIN	1000
> _at__at_ -725,11 +781,8 _at__at_
>  		destfield = dest;
>  	}
>  
> -	/* Wait for an earlier IPI to finish. */
> -	if (!lapic_ipi_wait(BEFORE_SPIN))
> -		panic("APIC: Previous IPI is stuck");
> -
> -	lapic_ipi_raw(icrlo, destfield);
> +	
> +	lapic_ipi_wait_and_raw(BEFORE_SPIN,icrlo,
> destfield);
>  
>  #ifdef DETECT_DEADLOCK
>  	/* Wait for IPI to be delivered. */
> Index: sys/i386/i386/mp_machdep.c
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/mp_machdep.c,v
> retrieving revision 1.240
> diff -u -r1.240 mp_machdep.c
> --- sys/i386/i386/mp_machdep.c	1 Nov 2004 22:11:27
> -0000	1.240
> +++ sys/i386/i386/mp_machdep.c	10 Nov 2004 01:48:57
> -0000
> _at__at_ -212,6 +212,14 _at__at_
>  static int	hlt_logical_cpus;
>  static struct	sysctl_ctx_list logical_cpu_clist;
>  
> +
> +static int ipi_statclock_pending;
> +static int ipi_hardclock_pending;
> +int ipi_ast_pending;
> +
> +
> +
> +
>  static void
>  mem_range_AP_init(void)
>  {
> _at__at_ -1017,12 +1025,24 _at__at_
>  	smp_tlb_addr1 = addr1;
>  	smp_tlb_addr2 = addr2;
>  	atomic_store_rel_int(&smp_tlb_wait, 0);
> +
> +
> +	/* Enable interrupts */
> +	/* Thread switching still disabled */
> +
> +	enable_intr();
> +
>  	if (mask == (u_int)-1)
>  		ipi_all_but_self(vector);
>  	else
>  		ipi_selected(mask, vector);
> +
>  	while (smp_tlb_wait < ncpu)
>  		ia32_pause();
> +
> +	/* disable interrupts */
> +	disable_intr();
> +
>  }
>  
>  void
> _at__at_ -1104,6 +1124,9 _at__at_
>  	struct thread *td;
>  
>  	CTR0(KTR_SMP, "forwarded_statclock");
> +
> +
>
atomic_clear_int(&ipi_statclock_pending,PCPU_GET(cpumask));
> 
> +
>  	td = curthread;
>  	td->td_intr_nesting_level++;
>  	if (profprocs != 0)
> _at__at_ -1123,9 +1146,11 _at__at_
>  	if (!smp_started || cold || panicstr)
>  		return;
>  
> -	map = PCPU_GET(other_cpus) &
> ~(stopped_cpus|hlt_cpus_mask);
> -	if (map != 0)
> +	map = PCPU_GET(other_cpus) &
> ~(stopped_cpus|hlt_cpus_mask|ipi_statclock_pending);
> +	if (map != 0) {
> +		atomic_set_int(&ipi_statclock_pending,map);
>  		ipi_selected(map, IPI_STATCLOCK);
> +	}
>  }
>  
>  /*
> _at__at_ -1141,6 +1166,9 _at__at_
>  	struct thread *td;
>  
>  	CTR0(KTR_SMP, "forwarded_hardclock");
> +
> +
>
atomic_clear_int(&ipi_hardclock_pending,PCPU_GET(cpumask));

=== message truncated ===



		
__________________________________ 
Do you Yahoo!? 
Check out the new Yahoo! Front Page. 
www.yahoo.com