From e2d4b1f051d1a67568d73038a83333b75577d34d Mon Sep 17 00:00:00 2001
From: Garrett Wollman <wollman@FreeBSD.org>
Date: Thu, 12 Oct 1995 20:39:49 +0000
Subject: [PATCH] Reduce jitter of Pentium microtime() implementation by
 letting the counter free-run and doing a subtract in microtime() rather than
 resetting the counter to zero at every clock tick.  In combination with the
 changes to kern_clock.c, this should eliminate all the immediately obvious
 sources of systematic jitter in timekeeping on Pentium machines.

---
 sys/amd64/amd64/tsc.c     | 21 ++++++++++----
 sys/amd64/include/clock.h | 61 ++++++++++++++++++++++++---------------
 sys/amd64/isa/clock.c     | 21 ++++++++++----
 sys/i386/i386/microtime.s |  8 +++--
 sys/i386/i386/tsc.c       | 21 ++++++++++----
 sys/i386/include/clock.h  | 61 ++++++++++++++++++++++++---------------
 sys/i386/isa/clock.c      | 21 ++++++++++----
 sys/isa/atrtc.c           | 21 ++++++++++----
 8 files changed, 156 insertions(+), 79 deletions(-)

diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c
index 188e25bd2899..310972fc8864 100644
--- a/sys/amd64/amd64/tsc.c
+++ b/sys/amd64/amd64/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.35 1995/06/11 19:31:18 rgrimes Exp $
+ *	$Id: clock.c,v 1.36 1995/08/25 19:24:56 bde Exp $
  */
 
 /*
@@ -95,6 +95,8 @@ int	disable_rtc_set	= 0;	/* disable resettodr() if != 0 */
 u_int	idelayed;
 #ifdef I586_CPU
 int	pentium_mhz;
+long long	i586_ctr_bias;
+long long	i586_last_tick;
 #endif
 u_int	stat_imask = SWI_CLOCK_MASK;
 int 	timer0_max_count;
@@ -279,8 +281,6 @@ getit(void)
 }
 
 #ifdef I586_CPU
-static long long cycles_per_sec = 0;
-
 /*
  * Figure out how fast the cyclecounter runs.  This must be run with
  * clock interrupts disabled, but with the timer/counter programmed
@@ -293,15 +293,15 @@ calibrate_cyclecounter(void)
 	 * Don't need volatile; should always use unsigned if 2's
 	 * complement arithmetic is desired.
 	 */
-	unsigned long long count, last_count;
+	unsigned long long count;
 
-	__asm __volatile(".byte 0xf,0x31" : "=A" (last_count));
+	__asm __volatile(".byte 0x0f, 0x30" : : "A"(0LL), "c" (0x10));
 	DELAY(1000000);
 	__asm __volatile(".byte 0xf,0x31" : "=A" (count));
 	/*
 	 * XX lose if the clock rate is not nearly a multiple of 1000000.
 	 */
-	pentium_mhz = ((count - last_count) + 500000) / 1000000;
+	pentium_mhz = (count + 500000) / 1000000;
 }
 #endif
 
@@ -569,6 +569,15 @@ cpu_initclocks()
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
+#ifdef I586_CPU
+	/*
+	 * Finish setting up anti-jitter measures.
+	 */
+	if (pentium_mhz) {
+		I586_CYCLECTR(i586_last_tick);
+		i586_ctr_bias = i586_last_tick;
+	}
+#endif
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h
index 3d87a07cee46..96476a7ecff4 100644
--- a/sys/amd64/include/clock.h
+++ b/sys/amd64/include/clock.h
@@ -8,40 +8,35 @@
 #define	_MACHINE_CLOCK_H_
 
 #ifdef I586_CPU
-	/*
-	 * This resets the CPU cycle counter to zero, to make our
-	 * job easier in microtime().  Some fancy ifdefs could speed
-	 * this up for Pentium-only kernels.
-	 * We want this to be done as close as possible to the actual
-	 * timer incrementing in hardclock(), because there is a window
-	 * between the two where the value is no longer valid.  Experimentation
-	 * may reveal a good precompensation to apply in microtime().
-	 */
+
+#define I586_CYCLECTR(x) \
+	__asm __volatile(".byte 0x0f, 0x31" : "=A" (x))
+
+/*
+ * When we update the clock, we also update this bias value which is
+ * automatically subtracted in microtime().  We assume that CPU_THISTICKLEN()
+ * has been called at some point in the past, so that an appropriate value is
+ * set up in i586_last_tick.  (This works even if we are not being called
+ * from hardclock because hardclock will have run before and will made the
+ * call.)
+ */
 #define CPU_CLOCKUPDATE(otime, ntime) \
 	do { \
 	if(pentium_mhz) { \
-		__asm __volatile("cli\n" \
-				 "movl (%2),%%eax\n" \
-				 "movl %%eax,(%1)\n" \
-				 "movl 4(%2),%%eax\n" \
-				 "movl %%eax,4(%1)\n" \
-				 "movl $0x10,%%ecx\n" \
-				 "xorl %%eax,%%eax\n" \
-				 "movl %%eax,%%edx\n" \
-				 ".byte 0x0f, 0x30\n" \
-				 "sti\n" \
-				 "#%0%1%2" \
-				 : "=m"(*otime)	/* no outputs */ \
-				 : "c"(otime), "b"(ntime) /* fake input */ \
-				 : "ax", "cx", "dx"); \
+		disable_intr(); \
+		i586_ctr_bias = i586_last_tick; \
+		*(otime) = *(ntime); \
+		enable_intr(); \
 	} else { \
 		*(otime) = *(ntime); \
 	} \
 	} while(0)
 
+#define	CPU_THISTICKLEN(dflt) cpu_thisticklen(dflt)
 #else
 #define CPU_CLOCKUPDATE(otime, ntime) \
 		(*(otime) = *(ntime))
+#define CPU_THISTICKLEN(dflt) dflt
 #endif
 
 #if defined(KERNEL) && !defined(LOCORE)
@@ -57,6 +52,8 @@ extern int	adjkerntz;
 extern int	disable_rtc_set;
 #ifdef I586_CPU
 extern int	pentium_mhz;
+extern long long i586_last_tick;
+extern long long i586_ctr_bias;
 #endif
 extern int 	timer0_max_count;
 extern u_int 	timer0_overflow_threshold;
@@ -68,6 +65,24 @@ void	calibrate_cyclecounter __P((void));
 void	clkintr __P((struct clockframe frame));
 void	rtcintr __P((struct clockframe frame));
 
+#ifdef I586_CPU
+static __inline u_long 
+cpu_thisticklen(u_long dflt)
+{
+	long long old;
+	long rv;
+	
+	if (pentium_mhz) {
+		old = i586_last_tick;
+		I586_CYCLECTR(i586_last_tick);
+		rv = (i586_last_tick - old) / pentium_mhz;
+	} else {
+		rv = dflt;
+	}
+	return rv;
+}
+#endif
+
 /*
  * Driver to clock driver interface.
  */
diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c
index 188e25bd2899..310972fc8864 100644
--- a/sys/amd64/isa/clock.c
+++ b/sys/amd64/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.35 1995/06/11 19:31:18 rgrimes Exp $
+ *	$Id: clock.c,v 1.36 1995/08/25 19:24:56 bde Exp $
  */
 
 /*
@@ -95,6 +95,8 @@ int	disable_rtc_set	= 0;	/* disable resettodr() if != 0 */
 u_int	idelayed;
 #ifdef I586_CPU
 int	pentium_mhz;
+long long	i586_ctr_bias;
+long long	i586_last_tick;
 #endif
 u_int	stat_imask = SWI_CLOCK_MASK;
 int 	timer0_max_count;
@@ -279,8 +281,6 @@ getit(void)
 }
 
 #ifdef I586_CPU
-static long long cycles_per_sec = 0;
-
 /*
  * Figure out how fast the cyclecounter runs.  This must be run with
  * clock interrupts disabled, but with the timer/counter programmed
@@ -293,15 +293,15 @@ calibrate_cyclecounter(void)
 	 * Don't need volatile; should always use unsigned if 2's
 	 * complement arithmetic is desired.
 	 */
-	unsigned long long count, last_count;
+	unsigned long long count;
 
-	__asm __volatile(".byte 0xf,0x31" : "=A" (last_count));
+	__asm __volatile(".byte 0x0f, 0x30" : : "A"(0LL), "c" (0x10));
 	DELAY(1000000);
 	__asm __volatile(".byte 0xf,0x31" : "=A" (count));
 	/*
 	 * XX lose if the clock rate is not nearly a multiple of 1000000.
 	 */
-	pentium_mhz = ((count - last_count) + 500000) / 1000000;
+	pentium_mhz = (count + 500000) / 1000000;
 }
 #endif
 
@@ -569,6 +569,15 @@ cpu_initclocks()
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
+#ifdef I586_CPU
+	/*
+	 * Finish setting up anti-jitter measures.
+	 */
+	if (pentium_mhz) {
+		I586_CYCLECTR(i586_last_tick);
+		i586_ctr_bias = i586_last_tick;
+	}
+#endif
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
diff --git a/sys/i386/i386/microtime.s b/sys/i386/i386/microtime.s
index 19031c727c0b..23268560682d 100644
--- a/sys/i386/i386/microtime.s
+++ b/sys/i386/i386/microtime.s
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  *	from: Steve McCanne's microtime code
- *	$Id: microtime.s,v 1.6 1994/08/13 17:45:09 wollman Exp $
+ *	$Id: microtime.s,v 1.7 1994/11/05 23:53:46 bde Exp $
  */
 
 #include <machine/asmacros.h>
@@ -46,8 +46,6 @@ ENTRY(microtime)
 	movl	_pentium_mhz, %ecx
 	testl	%ecx, %ecx
 	jne	pentium_microtime
-#else
-	xorl	%ecx, %ecx	# clear ecx
 #endif
 
 	movb	$TIMER_SEL0|TIMER_LATCH, %al	# prepare to latch
@@ -173,9 +171,13 @@ common_microtime:
 
 	ret
 
+	.extern _i586_ctr_bias
+
 	ALIGN_TEXT
 pentium_microtime:
 	cli
 	.byte	0x0f, 0x31	# RDTSC
+	subl	_i586_ctr_bias, %eax
+	sbbl	_i586_ctr_bias+4, %edx
 	divl	%ecx		# get value in usec
 	jmp	common_microtime
diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c
index 188e25bd2899..310972fc8864 100644
--- a/sys/i386/i386/tsc.c
+++ b/sys/i386/i386/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.35 1995/06/11 19:31:18 rgrimes Exp $
+ *	$Id: clock.c,v 1.36 1995/08/25 19:24:56 bde Exp $
  */
 
 /*
@@ -95,6 +95,8 @@ int	disable_rtc_set	= 0;	/* disable resettodr() if != 0 */
 u_int	idelayed;
 #ifdef I586_CPU
 int	pentium_mhz;
+long long	i586_ctr_bias;
+long long	i586_last_tick;
 #endif
 u_int	stat_imask = SWI_CLOCK_MASK;
 int 	timer0_max_count;
@@ -279,8 +281,6 @@ getit(void)
 }
 
 #ifdef I586_CPU
-static long long cycles_per_sec = 0;
-
 /*
  * Figure out how fast the cyclecounter runs.  This must be run with
  * clock interrupts disabled, but with the timer/counter programmed
@@ -293,15 +293,15 @@ calibrate_cyclecounter(void)
 	 * Don't need volatile; should always use unsigned if 2's
 	 * complement arithmetic is desired.
 	 */
-	unsigned long long count, last_count;
+	unsigned long long count;
 
-	__asm __volatile(".byte 0xf,0x31" : "=A" (last_count));
+	__asm __volatile(".byte 0x0f, 0x30" : : "A"(0LL), "c" (0x10));
 	DELAY(1000000);
 	__asm __volatile(".byte 0xf,0x31" : "=A" (count));
 	/*
 	 * XX lose if the clock rate is not nearly a multiple of 1000000.
 	 */
-	pentium_mhz = ((count - last_count) + 500000) / 1000000;
+	pentium_mhz = (count + 500000) / 1000000;
 }
 #endif
 
@@ -569,6 +569,15 @@ cpu_initclocks()
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
+#ifdef I586_CPU
+	/*
+	 * Finish setting up anti-jitter measures.
+	 */
+	if (pentium_mhz) {
+		I586_CYCLECTR(i586_last_tick);
+		i586_ctr_bias = i586_last_tick;
+	}
+#endif
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
diff --git a/sys/i386/include/clock.h b/sys/i386/include/clock.h
index 3d87a07cee46..96476a7ecff4 100644
--- a/sys/i386/include/clock.h
+++ b/sys/i386/include/clock.h
@@ -8,40 +8,35 @@
 #define	_MACHINE_CLOCK_H_
 
 #ifdef I586_CPU
-	/*
-	 * This resets the CPU cycle counter to zero, to make our
-	 * job easier in microtime().  Some fancy ifdefs could speed
-	 * this up for Pentium-only kernels.
-	 * We want this to be done as close as possible to the actual
-	 * timer incrementing in hardclock(), because there is a window
-	 * between the two where the value is no longer valid.  Experimentation
-	 * may reveal a good precompensation to apply in microtime().
-	 */
+
+#define I586_CYCLECTR(x) \
+	__asm __volatile(".byte 0x0f, 0x31" : "=A" (x))
+
+/*
+ * When we update the clock, we also update this bias value which is
+ * automatically subtracted in microtime().  We assume that CPU_THISTICKLEN()
+ * has been called at some point in the past, so that an appropriate value is
+ * set up in i586_last_tick.  (This works even if we are not being called
+ * from hardclock because hardclock will have run before and will made the
+ * call.)
+ */
 #define CPU_CLOCKUPDATE(otime, ntime) \
 	do { \
 	if(pentium_mhz) { \
-		__asm __volatile("cli\n" \
-				 "movl (%2),%%eax\n" \
-				 "movl %%eax,(%1)\n" \
-				 "movl 4(%2),%%eax\n" \
-				 "movl %%eax,4(%1)\n" \
-				 "movl $0x10,%%ecx\n" \
-				 "xorl %%eax,%%eax\n" \
-				 "movl %%eax,%%edx\n" \
-				 ".byte 0x0f, 0x30\n" \
-				 "sti\n" \
-				 "#%0%1%2" \
-				 : "=m"(*otime)	/* no outputs */ \
-				 : "c"(otime), "b"(ntime) /* fake input */ \
-				 : "ax", "cx", "dx"); \
+		disable_intr(); \
+		i586_ctr_bias = i586_last_tick; \
+		*(otime) = *(ntime); \
+		enable_intr(); \
 	} else { \
 		*(otime) = *(ntime); \
 	} \
 	} while(0)
 
+#define	CPU_THISTICKLEN(dflt) cpu_thisticklen(dflt)
 #else
 #define CPU_CLOCKUPDATE(otime, ntime) \
 		(*(otime) = *(ntime))
+#define CPU_THISTICKLEN(dflt) dflt
 #endif
 
 #if defined(KERNEL) && !defined(LOCORE)
@@ -57,6 +52,8 @@ extern int	adjkerntz;
 extern int	disable_rtc_set;
 #ifdef I586_CPU
 extern int	pentium_mhz;
+extern long long i586_last_tick;
+extern long long i586_ctr_bias;
 #endif
 extern int 	timer0_max_count;
 extern u_int 	timer0_overflow_threshold;
@@ -68,6 +65,24 @@ void	calibrate_cyclecounter __P((void));
 void	clkintr __P((struct clockframe frame));
 void	rtcintr __P((struct clockframe frame));
 
+#ifdef I586_CPU
+static __inline u_long 
+cpu_thisticklen(u_long dflt)
+{
+	long long old;
+	long rv;
+	
+	if (pentium_mhz) {
+		old = i586_last_tick;
+		I586_CYCLECTR(i586_last_tick);
+		rv = (i586_last_tick - old) / pentium_mhz;
+	} else {
+		rv = dflt;
+	}
+	return rv;
+}
+#endif
+
 /*
  * Driver to clock driver interface.
  */
diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c
index 188e25bd2899..310972fc8864 100644
--- a/sys/i386/isa/clock.c
+++ b/sys/i386/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.35 1995/06/11 19:31:18 rgrimes Exp $
+ *	$Id: clock.c,v 1.36 1995/08/25 19:24:56 bde Exp $
  */
 
 /*
@@ -95,6 +95,8 @@ int	disable_rtc_set	= 0;	/* disable resettodr() if != 0 */
 u_int	idelayed;
 #ifdef I586_CPU
 int	pentium_mhz;
+long long	i586_ctr_bias;
+long long	i586_last_tick;
 #endif
 u_int	stat_imask = SWI_CLOCK_MASK;
 int 	timer0_max_count;
@@ -279,8 +281,6 @@ getit(void)
 }
 
 #ifdef I586_CPU
-static long long cycles_per_sec = 0;
-
 /*
  * Figure out how fast the cyclecounter runs.  This must be run with
  * clock interrupts disabled, but with the timer/counter programmed
@@ -293,15 +293,15 @@ calibrate_cyclecounter(void)
 	 * Don't need volatile; should always use unsigned if 2's
 	 * complement arithmetic is desired.
 	 */
-	unsigned long long count, last_count;
+	unsigned long long count;
 
-	__asm __volatile(".byte 0xf,0x31" : "=A" (last_count));
+	__asm __volatile(".byte 0x0f, 0x30" : : "A"(0LL), "c" (0x10));
 	DELAY(1000000);
 	__asm __volatile(".byte 0xf,0x31" : "=A" (count));
 	/*
 	 * XX lose if the clock rate is not nearly a multiple of 1000000.
 	 */
-	pentium_mhz = ((count - last_count) + 500000) / 1000000;
+	pentium_mhz = (count + 500000) / 1000000;
 }
 #endif
 
@@ -569,6 +569,15 @@ cpu_initclocks()
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
+#ifdef I586_CPU
+	/*
+	 * Finish setting up anti-jitter measures.
+	 */
+	if (pentium_mhz) {
+		I586_CYCLECTR(i586_last_tick);
+		i586_ctr_bias = i586_last_tick;
+	}
+#endif
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
diff --git a/sys/isa/atrtc.c b/sys/isa/atrtc.c
index 188e25bd2899..310972fc8864 100644
--- a/sys/isa/atrtc.c
+++ b/sys/isa/atrtc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.35 1995/06/11 19:31:18 rgrimes Exp $
+ *	$Id: clock.c,v 1.36 1995/08/25 19:24:56 bde Exp $
  */
 
 /*
@@ -95,6 +95,8 @@ int	disable_rtc_set	= 0;	/* disable resettodr() if != 0 */
 u_int	idelayed;
 #ifdef I586_CPU
 int	pentium_mhz;
+long long	i586_ctr_bias;
+long long	i586_last_tick;
 #endif
 u_int	stat_imask = SWI_CLOCK_MASK;
 int 	timer0_max_count;
@@ -279,8 +281,6 @@ getit(void)
 }
 
 #ifdef I586_CPU
-static long long cycles_per_sec = 0;
-
 /*
  * Figure out how fast the cyclecounter runs.  This must be run with
  * clock interrupts disabled, but with the timer/counter programmed
@@ -293,15 +293,15 @@ calibrate_cyclecounter(void)
 	 * Don't need volatile; should always use unsigned if 2's
 	 * complement arithmetic is desired.
 	 */
-	unsigned long long count, last_count;
+	unsigned long long count;
 
-	__asm __volatile(".byte 0xf,0x31" : "=A" (last_count));
+	__asm __volatile(".byte 0x0f, 0x30" : : "A"(0LL), "c" (0x10));
 	DELAY(1000000);
 	__asm __volatile(".byte 0xf,0x31" : "=A" (count));
 	/*
 	 * XX lose if the clock rate is not nearly a multiple of 1000000.
 	 */
-	pentium_mhz = ((count - last_count) + 500000) / 1000000;
+	pentium_mhz = (count + 500000) / 1000000;
 }
 #endif
 
@@ -569,6 +569,15 @@ cpu_initclocks()
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
+#ifdef I586_CPU
+	/*
+	 * Finish setting up anti-jitter measures.
+	 */
+	if (pentium_mhz) {
+		I586_CYCLECTR(i586_last_tick);
+		i586_ctr_bias = i586_last_tick;
+	}
+#endif
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);