Subject: separate statclock(9) with mips3 CP0 timer interrupt
To: None <port-mips@NetBSD.org>
From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
List: port-cobalt
Date: 04/22/2006 14:41:22
Hi,

Current several mips ports with MIP3 CPUs (evbmips, sgimips etc.)
use CPU internal CP0 clock timer interrupts for hardclock(9),
but some other ports (arc, cobalt etc) use their own external
timer interrupts for hardclock(9) and CPU CP0 clock interrupt
(i.e. CPU INT5) is not used.

Is it worth to prepare a separate interrupt handler for
statclock(9) with CP0 clock interrupt on such mips port?
I guess it may provide more precise statistics on the system,
but I'm afraid number of interrupts (and its overhead) is also
increased ~two times.

I've written some code for NetBSD/cobalt (derived from hp300 and
arm/footbridge), but is there any good benchmark for this?
---
Izumi Tsutsui


Index: cobalt/autoconf.c
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/autoconf.c,v
retrieving revision 1.19
diff -u -r1.19 autoconf.c
--- cobalt/autoconf.c	21 Apr 2006 18:17:45 -0000	1.19
+++ cobalt/autoconf.c	22 Apr 2006 01:21:48 -0000
@@ -52,11 +52,13 @@
 	(void)splhigh();
 
 	evcnt_attach_static(&hardclock_ev);
+	evcnt_attach_static(&statclock_ev);
 
 	if (config_rootfound("mainbus", NULL) == NULL)
 		panic("no mainbus found");
 
-	_splnone();
+	/* turn on interrupts except cpu clock */
+	_spllower(MIPS_INT_MASK_5);
 }
 
 void
Index: cobalt/clock.c
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/clock.c,v
retrieving revision 1.12
diff -u -r1.12 clock.c
--- cobalt/clock.c	21 Apr 2006 16:52:15 -0000	1.12
+++ cobalt/clock.c	22 Apr 2006 01:21:48 -0000
@@ -47,6 +47,20 @@
 void *timer_cookie;
 
 /*
+ * Statistics clock variance, in usec.  Variance must be a
+ * power of two.  Since this gives us an even number, not an odd number,
+ * we discard one case and compensate.  That is, a variance of 1024 would
+ * give us offsets in [0..1023].  Instead, we take offsets in [1..1023].
+ * This is symmetric about the point 512, or statvar/2, and thus averages
+ * to that value (assuming uniform random numbers).
+ */
+static const uint32_t statvar = 1024;
+static uint32_t statint;	/* number of clock ticks for stathz */
+static uint32_t statmin;	/* minimum stat clock count in ticks */
+static uint32_t statprev;/* last value of we set statclock to */
+static u_int statcountperusec;	/* number of ticks per usec at current stathz */
+
+/*
  * Common parts of todclock autoconfiguration.
  */
 void
@@ -63,12 +77,21 @@
 cpu_initclocks(void)
 {
 
-	/* start timer */
+	if (stathz == 0)
+		stathz = hz;
+
+	if (profhz == 0)
+		profhz = hz * 5;
+
+	setstatclockrate(stathz);
+
+	/* start timer interrups for hardclock */
 	if (timer_start == NULL)
 		panic("cpu_initclocks(): no timer configured");
 	(*timer_start)(timer_cookie);
 
-	return;
+	/* enable statclock intr (CPU INT5) */
+	_splnone();
 }
 
 /*
@@ -139,11 +162,59 @@
 }
 
 void
-setstatclockrate(int arg)
+setstatclockrate(int newhz)
 {
-	/* XXX */
+	uint32_t countpersecond, statvarticks;
+
+	statprev = mips3_cp0_count_read();
+
+	statint = ((curcpu()->ci_cpu_freq + newhz / 2) / newhz) / 2;
+
+	/* Get the total ticks a second */
+	countpersecond = statint * newhz;
+
+	/* now work out how many ticks per usec */
+	statcountperusec = countpersecond / 1000000;
+
+	/* calculate a variance range of statvar */
+	statvarticks = statcountperusec * statvar;
+
+	/* minimum is statint - 50% of variant */
+	statmin = statint - (statvarticks / 2);
+
+	mips3_cp0_compare_write(statprev + statint);
+}
+
+void
+statclockintr(struct clockframe *cfp)
+{
+	uint32_t curcount, statnext, delta, r;
+	int lost;
+
+	lost = 0;
+
+	do {
+		r = (uint32_t)random() & (statvar - 1);
+	} while (r == 0);
+	statnext = statprev + statmin + (r * statcountperusec);
+
+	mips3_cp0_compare_write(statnext);
+	curcount = mips3_cp0_count_read();
+	delta = statnext - curcount;
+
+	while ((int32_t)delta < 0) {
+		lost++;
+		delta += statint;
+	}
+	if (lost > 0) {
+		statnext = curcount + delta;
+		mips3_cp0_compare_write(statnext);
+		for (; lost > 0; lost--)
+			statclock(cfp);
+	}
+	statclock(cfp);
 
-	return;
+	statprev = statnext;
 }
 
 void
Index: cobalt/clockvar.h
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/clockvar.h,v
retrieving revision 1.3
diff -u -r1.3 clockvar.h
--- cobalt/clockvar.h	21 Apr 2006 18:17:45 -0000	1.3
+++ cobalt/clockvar.h	22 Apr 2006 01:21:48 -0000
@@ -25,7 +25,10 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+void statclockintr(struct clockframe *);
+
 extern struct evcnt hardclock_ev;
+extern struct evcnt statclock_ev;
 
 extern void (*timer_start)(void *);
 extern long (*timer_read)(void *);
Index: cobalt/machdep.c
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/cobalt/machdep.c,v
retrieving revision 1.66
diff -u -r1.66 machdep.c
--- cobalt/machdep.c	21 Apr 2006 18:21:30 -0000	1.66
+++ cobalt/machdep.c	22 Apr 2006 01:21:48 -0000
@@ -110,6 +110,9 @@
 
 struct evcnt hardclock_ev =
     EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL, "cpu", "hardclock");
+struct evcnt statclock_ev =
+    EVCNT_INITIALIZER(EVCNT_TYPE_INTR, NULL, "cpu", "statclock");
+
 
 u_int cobalt_id;
 static const char * const cobalt_model[] =
@@ -451,11 +454,22 @@
 cpu_intr(uint32_t status, uint32_t cause, uint32_t pc, uint32_t ipending)
 {
 	struct clockframe cf;
-	static uint32_t cycles;
 	struct cobalt_intrhand *ih;
 
 	uvmexp.intrs++;
 
+	if (ipending & MIPS_INT_MASK_5) {
+		/* statclock */
+		cf.pc = pc;
+		cf.sr = status;
+
+		statclockintr(&cf);
+		statclock_ev.ev_count++;
+
+		cause &= ~MIPS_INT_MASK_5;
+	}
+	_splset((status & MIPS_INT_MASK_5) | MIPS_SR_INT_IE);
+
 	if (ipending & MIPS_INT_MASK_0) {
 		/* GT64x11 timer0 for hardclock */
 		volatile uint32_t *irq_src =
@@ -467,6 +481,29 @@
 			cf.pc = pc;
 			cf.sr = status;
 
+			if ((status & MIPS_INT_MASK) == MIPS_INT_MASK) {
+				if ((ipending & MIPS_INT_MASK &
+				     ~MIPS_INT_MASK_0) == 0) {
+					/*
+					 * If all interrupts were enabled and
+					 * there is no pending interrupts,
+					 * set MIPS_SR_INT_IE so that
+					 * spllowerclock() in hardclock()
+					 * works properly.
+					 */
+#if 0					/* MIPS_SR_INT_IE is enabled above */
+					_splset(MIPS_SR_INT_IE);
+#endif
+ 				} else {
+					/*
+					 * If there are any pending interrputs,
+					 * clear MIPS_SR_INT_IE in cf.sr so that
+					 * spllowerclock() in hardclock() will
+					 * not happen.
+					 */
+					cf.sr &= ~MIPS_SR_INT_IE;
+				}
+			}
 			hardclock(&cf);
 			hardclock_ev.ev_count++;
 		}
@@ -474,20 +511,6 @@
 	}
 	_splset((status & ~cause & MIPS_HARD_INT_MASK) | MIPS_SR_INT_IE);
 
-	if (ipending & MIPS_INT_MASK_5) {
-		cycles = mips3_cp0_count_read();
-		mips3_cp0_compare_write(cycles + 1250000);	/* XXX */
-
-#if 0
-		cf.pc = pc;
-		cf.sr = status;
-
-		statclock(&cf);
-#endif
-		cause &= ~MIPS_INT_MASK_5;
-	}
-	_splset((status & ~cause & MIPS_HARD_INT_MASK) | MIPS_SR_INT_IE);
-
 	if (ipending & MIPS_INT_MASK_3) {
 		/* 16650 serial */
 		ih = &intrtab[3];
Index: conf/std.cobalt
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/conf/std.cobalt,v
retrieving revision 1.12
diff -u -r1.12 std.cobalt
--- conf/std.cobalt	11 Dec 2005 12:17:06 -0000	1.12
+++ conf/std.cobalt	22 Apr 2006 01:21:48 -0000
@@ -5,6 +5,7 @@
 makeoptions	MACHINE_ARCH="mipsel"
 
 options 	MIPS3
+options 	MIPS3_ENABLE_CLOCK_INTR
 
 options 	EXEC_ELF32	# exec ELF32 binaries
 options 	EXEC_SCRIPT	# exec #! scripts
Index: include/intr.h
===================================================================
RCS file: /cvsroot/src/sys/arch/cobalt/include/intr.h,v
retrieving revision 1.20
diff -u -r1.20 intr.h
--- include/intr.h	21 Apr 2006 18:17:45 -0000	1.20
+++ include/intr.h	22 Apr 2006 01:21:48 -0000
@@ -80,7 +80,8 @@
 #define SPLBIO		(SPLSOFT | MIPS_INT_MASK_4)
 #define SPLNET		(SPLBIO | MIPS_INT_MASK_1 | MIPS_INT_MASK_2)
 #define SPLTTY		(SPLNET | MIPS_INT_MASK_3)
-#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0 | MIPS_INT_MASK_5)
+#define SPLCLOCK	(SPLTTY | MIPS_INT_MASK_0)
+#define SPLSTATCLOCK	(SPLCLOCK | MIPS_INT_MASK_5)
 #define splbio()	_splraise(SPLBIO)
 #define splnet()	_splraise(SPLNET)
 #define spltty()	_splraise(SPLTTY)
@@ -88,7 +89,7 @@
 #define splserial()	_splraise(SPLTTY)
 #define splclock()	_splraise(SPLCLOCK)
 #define splvm()		splclock()
-#define splstatclock()	splclock()
+#define splstatclock()	_splraise(SPLSTATCLOCK)
 #define spllowersoftclock() _spllower(MIPS_SOFT_INT_MASK_0)
 
 #define	splsched()	splhigh()