Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys:r177533,177535,180992 Property changes on: powerpc/include/atomic.h ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/powerpc/include/atomic.h:r180992 Property changes on: conf ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/conf:r177533,177535,180992 Property changes on: nlm ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/nlm:r177533,177535,180992 Property changes on: kern ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/kern:r177533,177535,180992 Property changes on: cam/scsi ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/cam/scsi:r177533,177535,180992 Property changes on: boot/forth/support.4th ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/boot/forth/support.4th:r180992 Property changes on: modules/geom/geom_part ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/modules/geom/geom_part:r180992 Property changes on: modules/cxgb ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/modules/cxgb:r180992 Property changes on: pci ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/pci:r177533,177535,180992 Property changes on: dev/mfi ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev/mfi:r180992 Property changes on: dev/gem ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev/gem:r177533,177535,180992 Property changes on: dev/cxgb ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev/cxgb:r177533,177535,180992 Property changes on: dev/usb ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev/usb:r177533,177535,180992 Property changes on: geom/part ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/geom/part:r180992 Property changes on: contrib/pf ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/contrib/pf:r177533,177535,180992 Property changes on: security/audit ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/security/audit:r177533,177535,180992 Index: amd64/include/pcb.h =================================================================== --- amd64/include/pcb.h (revision 181682) +++ amd64/include/pcb.h (working copy) @@ -70,6 +70,7 @@ u_long pcb_flags; #define PCB_DBREGS 0x02 /* process using debug registers */ #define PCB_FPUINITDONE 0x08 /* fpu state is initialized */ +#define PCB_GS32BIT 0x20 /* linux gs switch */ #define PCB_32BIT 0x40 /* process has 32 bit context (segs etc) */ #define PCB_FULLCTX 0x80 /* full context restore on sysret */ Index: amd64/linux32/linux32_machdep.c =================================================================== --- amd64/linux32/linux32_machdep.c (revision 181682) +++ amd64/linux32/linux32_machdep.c (working copy) @@ -696,7 +696,7 @@ td2->td_pcb->pcb_gs32sd = sd; td2->td_pcb->pcb_gs32p = &gdt[GUGS32_SEL]; td2->td_pcb->pcb_gs = GSEL(GUGS32_SEL, SEL_UPL); - td2->td_pcb->pcb_flags |= PCB_32BIT; + td2->td_pcb->pcb_flags |= PCB_GS32BIT | PCB_32BIT; } } Index: amd64/amd64/genassym.c =================================================================== --- amd64/amd64/genassym.c (revision 181682) +++ amd64/amd64/genassym.c (working copy) @@ -136,6 +136,7 @@ ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7)); ASSYM(PCB_DBREGS, PCB_DBREGS); ASSYM(PCB_32BIT, PCB_32BIT); +ASSYM(PCB_GS32BIT, PCB_GS32BIT); ASSYM(PCB_FULLCTX, PCB_FULLCTX); ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); Index: amd64/amd64/cpu_switch.S =================================================================== --- amd64/amd64/cpu_switch.S (revision 181682) +++ amd64/amd64/cpu_switch.S (working copy) @@ -69,16 +69,20 @@ * %rsi = newtd */ ENTRY(cpu_throw) + testq %rdi,%rdi + jnz 1f + movq PCPU(IDLETHREAD),%rdi +1: + movq TD_PCB(%rdi),%r8 /* Old pcb */ movl PCPU(CPUID), %eax - testq %rdi,%rdi /* no thread? */ - jz 1f + movq PCB_FSBASE(%r8),%r9 + movq PCB_GSBASE(%r8),%r10 /* release bit from old pm_active */ movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ -1: - movq TD_PCB(%rsi),%rdx /* newtd->td_proc */ - movq PCB_CR3(%rdx),%rdx + movq TD_PCB(%rsi),%r8 /* newtd->td_proc */ + movq PCB_CR3(%r8),%rdx movq %rdx,%cr3 /* new address space */ jmp swact END(cpu_throw) @@ -97,43 +101,24 @@ movq TD_PCB(%rdi),%r8 movq (%rsp),%rax /* Hardware registers */ + movq %r15,PCB_R15(%r8) + movq %r14,PCB_R14(%r8) + movq %r13,PCB_R13(%r8) + movq %r12,PCB_R12(%r8) + movq %rbp,PCB_RBP(%r8) + movq %rsp,PCB_RSP(%r8) + movq %rbx,PCB_RBX(%r8) movq %rax,PCB_RIP(%r8) - movq %rbx,PCB_RBX(%r8) - movq %rsp,PCB_RSP(%r8) - movq %rbp,PCB_RBP(%r8) - movq %r12,PCB_R12(%r8) - movq %r13,PCB_R13(%r8) - movq %r14,PCB_R14(%r8) - movq %r15,PCB_R15(%r8) + movq PCB_FSBASE(%r8),%r9 + movq PCB_GSBASE(%r8),%r10 testl $PCB_32BIT,PCB_FLAGS(%r8) - jz 1f /* no, skip over */ + jnz store_seg +done_store_seg: - /* Save userland %gs */ - movl %gs,PCB_GS(%r8) - movq PCB_GS32P(%r8),%rax - movq (%rax),%rax - movq %rax,PCB_GS32SD(%r8) - -1: - /* Test if debug registers should be saved. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) - jz 1f /* no, skip over */ - movq %dr7,%rax /* yes, do the save */ - movq %rax,PCB_DR7(%r8) - andq $0x0000fc00, %rax /* disable all watchpoints */ - movq %rax,%dr7 - movq %dr6,%rax - movq %rax,PCB_DR6(%r8) - movq %dr3,%rax - movq %rax,PCB_DR3(%r8) - movq %dr2,%rax - movq %rax,PCB_DR2(%r8) - movq %dr1,%rax - movq %rax,PCB_DR1(%r8) - movq %dr0,%rax - movq %rax,PCB_DR0(%r8) -1: + jnz store_dr /* static predict not taken */ +done_store_dr: /* have we used fp, and need a save? */ cmpq %rdi,PCPU(FPCURTHREAD) @@ -181,82 +166,152 @@ cmpq %rcx, %rdx pause je 1b - lfence #endif /* * At this point, we've switched address spaces and are ready * to load up the rest of the next context. */ - movq TD_PCB(%rsi),%r8 + /* Skip loading user fsbase/gsbase for kthreads */ +/* + * XXX Commented out since TDP_KTHREAD code was not merged + * to the RELENG_7. + testl $TDP_KTHREAD,TD_PFLAGS(%rsi) + jnz do_kthread +*/ + testl $PCB_32BIT,PCB_FLAGS(%r8) + jnz load_seg +done_load_seg: + + cmpq PCB_FSBASE(%r8),%r9 + jz 1f /* Restore userland %fs */ movl $MSR_FSBASE,%ecx movl PCB_FSBASE(%r8),%eax movl PCB_FSBASE+4(%r8),%edx wrmsr - +1: + cmpq PCB_GSBASE(%r8),%r10 + jz 2f /* Restore userland %gs */ movl $MSR_KGSBASE,%ecx movl PCB_GSBASE(%r8),%eax movl PCB_GSBASE+4(%r8),%edx wrmsr +2: +do_tss: /* Update the TSS_RSP0 pointer for the next interrupt */ movq PCPU(TSSP), %rax + movq %r8, PCPU(RSP0) + movq %r8, PCPU(CURPCB) addq $COMMON_TSS_RSP0, %rax - leaq -16(%r8), %rbx - movq %rbx, (%rax) - movq %rbx, PCPU(RSP0) - - movq %r8, PCPU(CURPCB) movq %rsi, PCPU(CURTHREAD) /* into next thread */ + movq %r8, (%rax) - testl $PCB_32BIT,PCB_FLAGS(%r8) - jz 1f /* no, skip over */ + /* Test if debug registers should be restored. */ + testl $PCB_DBREGS,PCB_FLAGS(%r8) + jnz load_dr /* static predict not taken */ +done_load_dr: - /* Restore userland %gs while preserving kernel gsbase */ - movq PCB_GS32P(%r8),%rax - movq PCB_GS32SD(%r8),%rbx - movq %rbx,(%rax) - movl $MSR_GSBASE,%ecx - rdmsr - movl PCB_GS(%r8),%gs - wrmsr - -1: /* Restore context. */ + movq PCB_R15(%r8),%r15 + movq PCB_R14(%r8),%r14 + movq PCB_R13(%r8),%r13 + movq PCB_R12(%r8),%r12 + movq PCB_RBP(%r8),%rbp + movq PCB_RSP(%r8),%rsp movq PCB_RBX(%r8),%rbx - movq PCB_RSP(%r8),%rsp - movq PCB_RBP(%r8),%rbp - movq PCB_R12(%r8),%r12 - movq PCB_R13(%r8),%r13 - movq PCB_R14(%r8),%r14 - movq PCB_R15(%r8),%r15 movq PCB_RIP(%r8),%rax movq %rax,(%rsp) + ret - /* Test if debug registers should be restored. */ - testl $PCB_DBREGS,PCB_FLAGS(%r8) - jz 1f - movq PCB_DR6(%r8),%rax - movq %rax,%dr6 - movq PCB_DR3(%r8),%rax - movq %rax,%dr3 - movq PCB_DR2(%r8),%rax - movq %rax,%dr2 - movq PCB_DR1(%r8),%rax - movq %rax,%dr1 - movq PCB_DR0(%r8),%rax - movq %rax,%dr0 - /* But preserve reserved bits in %dr7 */ + /* + * We order these strangely for several reasons. + * 1: I wanted to use static branch prediction hints + * 2: Most athlon64/opteron cpus don't have them. They define + * a forward branch as 'predict not taken'. Intel cores have + * the 'rep' prefix to invert this. + * So, to make it work on both forms of cpu we do the detour. + * We use jumps rather than call in order to avoid the stack. + */ + +do_kthread: + /* + * Copy old fs/gsbase to new kthread pcb for future switches + * This maintains curpcb->pcb_[fg]sbase as caches of the MSR + */ + movq %r9,PCB_FSBASE(%r8) + movq %r10,PCB_GSBASE(%r8) + jmp do_tss + +store_seg: + movl %gs,PCB_GS(%r8) + testl $PCB_GS32BIT,PCB_FLAGS(%r8) + jnz 2f +1: movl %ds,PCB_DS(%r8) + movl %es,PCB_ES(%r8) + movl %fs,PCB_FS(%r8) + jmp done_store_seg +2: movq PCB_GS32P(%r8),%rax + movq (%rax),%rax + movq %rax,PCB_GS32SD(%r8) + jmp 1b + +load_seg: + testl $PCB_GS32BIT,PCB_FLAGS(%r8) + jnz 2f +1: movl $MSR_GSBASE,%ecx + rdmsr + movl PCB_GS(%r8),%gs + wrmsr + movl PCB_DS(%r8),%ds + movl PCB_ES(%r8),%es + movl PCB_FS(%r8),%fs + jmp done_load_seg + /* Restore userland %gs while preserving kernel gsbase */ +2: movq PCB_GS32P(%r8),%rax + movq PCB_GS32SD(%r8),%rcx + movq %rcx,(%rax) + jmp 1b + +store_dr: + movq %dr7,%rax /* yes, do the save */ + movq %dr0,%r15 + movq %dr1,%r14 + movq %dr2,%r13 + movq %dr3,%r12 + movq %dr6,%r11 + andq $0x0000fc00, %rax /* disable all watchpoints */ + movq %r15,PCB_DR0(%r8) + movq %r14,PCB_DR1(%r8) + movq %r13,PCB_DR2(%r8) + movq %r12,PCB_DR3(%r8) + movq %r11,PCB_DR6(%r8) + movq %rax,PCB_DR7(%r8) + movq %rax,%dr7 + jmp done_store_dr + +load_dr: movq %dr7,%rax + movq PCB_DR0(%r8),%r15 + movq PCB_DR1(%r8),%r14 + movq PCB_DR2(%r8),%r13 + movq PCB_DR3(%r8),%r12 + movq PCB_DR6(%r8),%r11 + movq PCB_DR7(%r8),%rcx + movq %r15,%dr0 + movq %r14,%dr1 + /* Preserve reserved bits in %dr7 */ andq $0x0000fc00,%rax - movq PCB_DR7(%r8),%rcx andq $~0x0000fc00,%rcx + movq %r13,%dr2 + movq %r12,%dr3 orq %rcx,%rax + movq %r11,%dr6 movq %rax,%dr7 -1: - ret + jmp done_load_dr + END(cpu_switch) /* Index: amd64/ia32/ia32_signal.c =================================================================== --- amd64/ia32/ia32_signal.c (revision 181682) +++ amd64/ia32/ia32_signal.c (working copy) @@ -739,6 +739,6 @@ fpstate_drop(td); /* Return via doreti so that we can change to a different %cs */ - pcb->pcb_flags |= PCB_FULLCTX; + pcb->pcb_flags |= PCB_FULLCTX | PCB_32BIT; td->td_retval[1] = 0; }