Index: sys/amd64/amd64/cpu_switch.S
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/cpu_switch.S,v
retrieving revision 1.160
diff -u -r1.160 cpu_switch.S
--- sys/amd64/amd64/cpu_switch.S	22 Aug 2007 05:06:14 -0000	1.160
+++ sys/amd64/amd64/cpu_switch.S	12 Mar 2008 11:13:35 -0000
@@ -80,6 +80,7 @@
 	movq	TD_PCB(%rsi),%rdx		/* newtd->td_proc */
 	movq	PCB_CR3(%rdx),%rdx
 	movq	%rdx,%cr3			/* new address space */
+	movq    TD_PCB(%rsi),%r8
 	jmp	swact
 END(cpu_throw)
 
@@ -97,43 +98,22 @@
 	movq	TD_PCB(%rdi),%r8
 
 	movq	(%rsp),%rax			/* Hardware registers */
-	movq	%rax,PCB_RIP(%r8)
-	movq	%rbx,PCB_RBX(%r8)
-	movq	%rsp,PCB_RSP(%r8)
-	movq	%rbp,PCB_RBP(%r8)
-	movq	%r12,PCB_R12(%r8)
-	movq	%r13,PCB_R13(%r8)
-	movq	%r14,PCB_R14(%r8)
 	movq	%r15,PCB_R15(%r8)
+	movq	%r14,PCB_R14(%r8)
+	movq	%r13,PCB_R13(%r8)
+	movq	%r12,PCB_R12(%r8)
+	movq	%rbp,PCB_RBP(%r8)
+	movq	%rsp,PCB_RSP(%r8)
+	movq	%rbx,PCB_RBX(%r8)
+	movq	%rax,PCB_RIP(%r8)
 
 	testl	$PCB_32BIT,PCB_FLAGS(%r8)
-	jz	1f				/* no, skip over */
-
-	/* Save userland %gs */
-	movl	%gs,PCB_GS(%r8)
-	movq	PCB_GS32P(%r8),%rax
-	movq	(%rax),%rax
-	movq	%rax,PCB_GS32SD(%r8)
+	jnz	store_gs			/* static predict not taken */
+done_store_gs:
 
-1:
-	/* Test if debug registers should be saved. */
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
-	jz	1f				/* no, skip over */
-	movq	%dr7,%rax			/* yes, do the save */
-	movq	%rax,PCB_DR7(%r8)
-	andq	$0x0000fc00, %rax		/* disable all watchpoints */
-	movq	%rax,%dr7
-	movq	%dr6,%rax
-	movq	%rax,PCB_DR6(%r8)
-	movq	%dr3,%rax
-	movq	%rax,PCB_DR3(%r8)
-	movq	%dr2,%rax
-	movq	%rax,PCB_DR2(%r8)
-	movq	%dr1,%rax
-	movq	%rax,PCB_DR1(%r8)
-	movq	%dr0,%rax
-	movq	%rax,PCB_DR0(%r8)
-1:
+	jnz	store_dr			/* static predict not taken */
+done_store_dr:
 
 	/* have we used fp, and need a save? */
 	cmpq	%rdi,PCPU(FPCURTHREAD)
@@ -181,82 +161,135 @@
 	cmpq	%rcx, %rdx
 	pause
 	je	1b
-	lfence
 #endif
 	/*
 	 * At this point, we've switched address spaces and are ready
 	 * to load up the rest of the next context.
 	 */
-	movq	TD_PCB(%rsi),%r8
 
+	/* Skip loading user fsbase/gsbase for kthreads */
+	testl	$TDP_KTHREAD,TD_PFLAGS(%rsi)
+	jnz	do_tss
+
+	movq	PCPU(FSBASE),%r10
+	movq	PCPU(GSBASE),%r11
+
+	cmpq	PCB_FSBASE(%r8),%r10
+	jz	1f
 	/* Restore userland %fs */
 	movl	$MSR_FSBASE,%ecx
 	movl	PCB_FSBASE(%r8),%eax
 	movl	PCB_FSBASE+4(%r8),%edx
 	wrmsr
+	movq	PCB_FSBASE(%r8),%rax
+	movq	%rax,PCPU(FSBASE)
+1:
 
+	cmpq	PCB_GSBASE(%r8),%r11
+	jz	2f
 	/* Restore userland %gs */
 	movl	$MSR_KGSBASE,%ecx
 	movl	PCB_GSBASE(%r8),%eax
 	movl	PCB_GSBASE+4(%r8),%edx
 	wrmsr
+	movq	PCB_GSBASE(%r8),%rax
+	movq	%rax,PCPU(GSBASE)
+2:
+do_tss:
 
 	/* Update the TSS_RSP0 pointer for the next interrupt */
 	movq	PCPU(TSSP), %rax
-	addq	$COMMON_TSS_RSP0, %rax
-	leaq	-16(%r8), %rbx
-	movq	%rbx, (%rax)
-	movq	%rbx, PCPU(RSP0)
-
+	movq	%r8, PCPU(RSP0)
 	movq	%r8, PCPU(CURPCB)
+	addq	$COMMON_TSS_RSP0, %rax
 	movq	%rsi, PCPU(CURTHREAD)		/* into next thread */
+	movq	%r8, (%rax)
+
+	/* Test if debug registers should be restored. */
+	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
+	jnz	load_dr				/* static predict not taken */
+done_load_dr:
 
 	testl	$PCB_32BIT,PCB_FLAGS(%r8)
-	jz	1f				/* no, skip over */
+	jnz	load_gs				/* static predict not taken */
+done_load_gs:
+
+	/* Restore context. */
+	movq	PCB_R15(%r8),%r15
+	movq	PCB_R14(%r8),%r14
+	movq	PCB_R13(%r8),%r13
+	movq	PCB_R12(%r8),%r12
+	movq	PCB_RBP(%r8),%rbp
+	movq	PCB_RSP(%r8),%rsp
+	movq	PCB_RBX(%r8),%rbx
+	movq	PCB_RIP(%r8),%rax
+	movq	%rax,(%rsp)
+	ret
+
+	/*
+	 * We order these strangely for several reasons.
+	 * 1: I wanted to use static branch prediction hints
+	 * 2: Most athlon64/opteron cpus don't have them.  They define
+	 *    a forward branch as 'predict not taken'.  Intel cores have
+	 *    the 'rep' prefix to invert this.
+	 * So, to make it work on both forms of cpu we do the detour.
+	 * We use jumps rather than call in order to avoid the stack.
+	 */
+store_gs:
+	movl	%gs,PCB_GS(%r8)
+	movq	PCB_GS32P(%r8),%rax
+	movq	(%rax),%rax
+	movq	%rax,PCB_GS32SD(%r8)
+	jmp	done_store_gs
 
+load_gs:
 	/* Restore userland %gs while preserving kernel gsbase */
 	movq	PCB_GS32P(%r8),%rax
-	movq	PCB_GS32SD(%r8),%rbx
-	movq	%rbx,(%rax)
+	movq	PCB_GS32SD(%r8),%rcx
+	movq	%rcx,(%rax)
 	movl	$MSR_GSBASE,%ecx
 	rdmsr
 	movl	PCB_GS(%r8),%gs
 	wrmsr
+	jmp	done_load_gs
 
-1:
-	/* Restore context. */
-	movq	PCB_RBX(%r8),%rbx
-	movq	PCB_RSP(%r8),%rsp
-	movq	PCB_RBP(%r8),%rbp
-	movq	PCB_R12(%r8),%r12
-	movq	PCB_R13(%r8),%r13
-	movq	PCB_R14(%r8),%r14
-	movq	PCB_R15(%r8),%r15
-	movq	PCB_RIP(%r8),%rax
-	movq	%rax,(%rsp)
+store_dr:
+	movq	%dr7,%rax			/* yes, do the save */
+	movq	%dr0,%r15
+	movq	%dr1,%r14
+	movq	%dr2,%r13
+	movq	%dr3,%r12
+	movq	%dr6,%r11
+	andq	$0x0000fc00, %rax		/* disable all watchpoints */
+	movq	%r15,PCB_DR0(%r8)
+	movq	%r14,PCB_DR1(%r8)
+	movq	%r13,PCB_DR2(%r8)
+	movq	%r12,PCB_DR3(%r8)
+	movq	%r11,PCB_DR6(%r8)
+	movq	%rax,PCB_DR7(%r8)
+	movq	%rax,%dr7
+	jmp	done_store_dr
 
-	/* Test if debug registers should be restored. */
-	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
-	jz	1f
-	movq	PCB_DR6(%r8),%rax
-	movq	%rax,%dr6
-	movq	PCB_DR3(%r8),%rax
-	movq	%rax,%dr3
-	movq	PCB_DR2(%r8),%rax
-	movq	%rax,%dr2
-	movq	PCB_DR1(%r8),%rax
-	movq	%rax,%dr1
-	movq	PCB_DR0(%r8),%rax
-	movq	%rax,%dr0
-	/* But preserve reserved bits in %dr7 */
+load_dr:
 	movq	%dr7,%rax
-	andq	$0x0000fc00,%rax
+	movq	PCB_DR0(%r8),%r15
+	movq	PCB_DR1(%r8),%r14
+	movq	PCB_DR2(%r8),%r13
+	movq	PCB_DR3(%r8),%r12
+	movq	PCB_DR6(%r8),%r11
 	movq	PCB_DR7(%r8),%rcx
+	movq	%r15,%dr0
+	movq	%r14,%dr1
+	/* Preserve reserved bits in %dr7 */
+	andq	$0x0000fc00,%rax
 	andq	$~0x0000fc00,%rcx
+	movq	%r13,%dr2
+	movq	%r12,%dr3
 	orq	%rcx,%rax
+	movq	%r11,%dr6
 	movq	%rax,%dr7
-1:
-	ret
+	jmp	done_load_dr
+
 END(cpu_switch)
 	
 /*
Index: sys/amd64/amd64/genassym.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/genassym.c,v
retrieving revision 1.166
diff -u -r1.166 genassym.c
--- sys/amd64/amd64/genassym.c	23 Nov 2007 03:03:29 -0000	1.166
+++ sys/amd64/amd64/genassym.c	12 Mar 2008 11:13:35 -0000
@@ -86,6 +86,7 @@
 ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
 
 ASSYM(TDP_CALLCHAIN, TDP_CALLCHAIN);
+ASSYM(TDP_KTHREAD, TDP_KTHREAD);
 
 ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap));
 ASSYM(V_SYSCALL, offsetof(struct vmmeter, v_syscall));
@@ -197,6 +198,8 @@
 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
 ASSYM(PC_TSSP, offsetof(struct pcpu, pc_tssp));
 ASSYM(PC_RSP0, offsetof(struct pcpu, pc_rsp0));
+ASSYM(PC_FSBASE, offsetof(struct pcpu, pc_fsbase));
+ASSYM(PC_GSBASE, offsetof(struct pcpu, pc_gsbase));
  
 ASSYM(LA_VER, offsetof(struct LAPIC, version));
 ASSYM(LA_TPR, offsetof(struct LAPIC, tpr));
Index: sys/amd64/amd64/machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/machdep.c,v
retrieving revision 1.680
diff -u -r1.680 machdep.c
--- sys/amd64/amd64/machdep.c	25 Dec 2007 17:51:55 -0000	1.680
+++ sys/amd64/amd64/machdep.c	12 Mar 2008 11:13:35 -0000
@@ -601,6 +601,8 @@
 	critical_enter();
 	wrmsr(MSR_FSBASE, 0);
 	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
+	PCPU_SET(fsbase, 0);
+	PCPU_SET(gsbase, 0);
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
 	critical_exit();
Index: sys/amd64/amd64/sys_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/sys_machdep.c,v
retrieving revision 1.90
diff -u -r1.90 sys_machdep.c
--- sys/amd64/amd64/sys_machdep.c	10 Jul 2005 23:31:10 -0000	1.90
+++ sys/amd64/amd64/sys_machdep.c	12 Mar 2008 11:13:35 -0000
@@ -73,6 +73,7 @@
 		if (!error) {
 			critical_enter();
 			wrmsr(MSR_FSBASE, i386base);
+			PCPU_SET(fsbase, i386base);
 			pcb->pcb_fsbase = i386base;
 			critical_exit();
 		}
@@ -86,6 +87,7 @@
 		if (!error) {
 			critical_enter();
 			wrmsr(MSR_KGSBASE, i386base);
+			PCPU_SET(gsbase, i386base);
 			pcb->pcb_gsbase = i386base;
 			critical_exit();
 		}
@@ -100,6 +102,7 @@
 			if (a64base < VM_MAXUSER_ADDRESS) {
 				critical_enter();
 				wrmsr(MSR_FSBASE, a64base);
+				PCPU_SET(fsbase, a64base);
 				pcb->pcb_fsbase = a64base;
 				critical_exit();
 			} else {
@@ -118,6 +121,7 @@
 			if (a64base < VM_MAXUSER_ADDRESS) {
 				critical_enter();
 				wrmsr(MSR_KGSBASE, a64base);
+				PCPU_SET(gsbase, a64base);
 				pcb->pcb_gsbase = a64base;
 				critical_exit();
 			} else {
Index: sys/amd64/amd64/vm_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/amd64/vm_machdep.c,v
retrieving revision 1.256
diff -u -r1.256 vm_machdep.c
--- sys/amd64/amd64/vm_machdep.c	14 Nov 2007 20:21:52 -0000	1.256
+++ sys/amd64/amd64/vm_machdep.c	12 Mar 2008 11:13:35 -0000
@@ -385,6 +385,7 @@
 			critical_enter();
 			td->td_pcb->pcb_gsbase = (register_t)tls_base;
 			wrmsr(MSR_KGSBASE, td->td_pcb->pcb_gsbase);
+			PCPU_SET(gsbase, td->td_pcb->pcb_gsbase);
 			critical_exit();
 		} else {
 			td->td_pcb->pcb_gsbase = (register_t)tls_base;
@@ -396,6 +397,7 @@
 		critical_enter();
 		td->td_pcb->pcb_fsbase = (register_t)tls_base;
 		wrmsr(MSR_FSBASE, td->td_pcb->pcb_fsbase);
+		PCPU_SET(fsbase, td->td_pcb->pcb_fsbase);
 		critical_exit();
 	} else {
 		td->td_pcb->pcb_fsbase = (register_t)tls_base;
Index: sys/amd64/ia32/ia32_signal.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/ia32/ia32_signal.c,v
retrieving revision 1.15
diff -u -r1.15 ia32_signal.c
--- sys/amd64/ia32/ia32_signal.c	5 Oct 2006 01:56:10 -0000	1.15
+++ sys/amd64/ia32/ia32_signal.c	12 Mar 2008 11:13:35 -0000
@@ -715,10 +715,14 @@
 	struct trapframe *regs = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
 	
+	critical_enter();
 	wrmsr(MSR_FSBASE, 0);
 	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
+	PCPU_SET(fsbase, 0);
+	PCPU_SET(gsbase, 0);
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
+	critical_exit();
 	load_ds(_udatasel);
 	load_es(_udatasel);
 	load_fs(_udatasel);
Index: sys/amd64/include/pcpu.h
===================================================================
RCS file: /home/ncvs/src/sys/amd64/include/pcpu.h,v
retrieving revision 1.48
diff -u -r1.48 pcpu.h
--- sys/amd64/include/pcpu.h	4 Jun 2007 21:38:45 -0000	1.48
+++ sys/amd64/include/pcpu.h	12 Mar 2008 11:13:35 -0000
@@ -49,7 +49,9 @@
 	register_t pc_rsp0;						\
 	register_t pc_scratch_rsp;	/* User %rsp in syscall */	\
 	u_int	pc_apic_id;						\
-	u_int   pc_acpi_id		/* ACPI CPU id */
+	u_int   pc_acpi_id;		/* ACPI CPU id */		\
+	register_t pc_fsbase;		/* User values of fsbase */	\
+	register_t pc_gsbase		/* User values of gsbase */
 
 #ifdef lint
 
Index: sys/amd64/linux32/linux32_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/linux32/linux32_machdep.c,v
retrieving revision 1.46
diff -u -r1.46 linux32_machdep.c
--- sys/amd64/linux32/linux32_machdep.c	11 Feb 2008 19:35:03 -0000	1.46
+++ sys/amd64/linux32/linux32_machdep.c	12 Mar 2008 11:13:35 -0000
@@ -1356,6 +1356,7 @@
 	td->td_pcb->pcb_gs32p = &gdt[GUGS32_SEL];
 	td->td_pcb->pcb_flags |= PCB_32BIT;
 	wrmsr(MSR_KGSBASE, td->td_pcb->pcb_gsbase);
+	PCPU_SET(gsbase, td->td_pcb->pcb_gsbase);
 	critical_exit();
 
 	return (0);
Index: sys/amd64/linux32/linux32_sysvec.c
===================================================================
RCS file: /home/ncvs/src/sys/amd64/linux32/linux32_sysvec.c,v
retrieving revision 1.31
diff -u -r1.31 linux32_sysvec.c
--- sys/amd64/linux32/linux32_sysvec.c	20 Sep 2007 13:46:26 -0000	1.31
+++ sys/amd64/linux32/linux32_sysvec.c	12 Mar 2008 11:13:35 -0000
@@ -820,6 +820,8 @@
 	critical_enter();
 	wrmsr(MSR_FSBASE, 0);
 	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
+	PCPU_SET(fsbase, 0);
+	PCPU_SET(gsbase, 0);
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
 	critical_exit();