diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S index 505adcbf134..3fb0c225567 100644 --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -161,6 +161,50 @@ ENTRY(bcopy) ret END(bcopy) +ENTRY(bcopy_noxchg) + PUSH_FRAME_POINTER + movq %rsi,%r8 + movq %rdi,%rsi + movq %r8,%rdi + movq %rdx,%rcx + + movq %rdi,%rax + subq %rsi,%rax + cmpq %rcx,%rax /* overlapping && src < dst? */ + jb 1f + + shrq $3,%rcx /* copy by 64-bit words */ + rep + movsq + movq %rdx,%rcx + andq $7,%rcx /* any bytes left? */ + rep + movsb + POP_FRAME_POINTER + ret + + /* ALIGN_TEXT */ +1: + addq %rcx,%rdi /* copy backwards */ + addq %rcx,%rsi + decq %rdi + decq %rsi + andq $7,%rcx /* any fractional bytes? */ + std + rep + movsb + movq %rdx,%rcx /* copy remainder by 32-bit words */ + shrq $3,%rcx + subq $7,%rsi + subq $7,%rdi + rep + movsq + cld + POP_FRAME_POINTER + ret +END(bcopy_noxchg) + + /* * Note: memcpy does not support overlapping copies */ @@ -179,6 +223,46 @@ ENTRY(memcpy) ret END(memcpy) +ENTRY(oolmemcpy) + PUSH_FRAME_POINTER + movq %rdi,%rax + movq %rdx,%rcx + shrq $3,%rcx /* copy by 64-bit words */ + rep + movsq + movq %rdx,%rcx + andq $7,%rcx /* any bytes left? */ + rep + movsb + POP_FRAME_POINTER + ret +END(oolmemcpy) + +ENTRY(oolmemcpy2) + PUSH_FRAME_POINTER + movq %rdi,%rax + movq %rdx,%rcx + shrq $3,%rcx /* copy by 64-bit words */ + andq $7,%rdx /* any bytes left? */ + rep + movsq + movq %rdx,%rcx + rep + movsb + POP_FRAME_POINTER + ret +END(oolmemcpy2) + +ENTRY(oolmemcpy_es) + PUSH_FRAME_POINTER + movq %rdi,%rax + movq %rdx,%rcx + rep + movsb + POP_FRAME_POINTER + ret +END(oolmemcpy_es) + /* * pagecopy(%rdi=from, %rsi=to) */ diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index c90853b1b79..6f10b5b8ba2 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -923,6 +923,271 @@ cpu_fetch_syscall_args(struct thread *td) return (error); } +int +cpu_fetch_syscall_args_bcopy(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + register_t *argp; + struct syscall_args *sa; + caddr_t params; + int reg, regcnt, error; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + reg = 0; + regcnt = 6; + + params = (caddr_t)frame->tf_rsp + sizeof(register_t); + sa->code = frame->tf_rax; + + if (sa->code == SYS_syscall || sa->code == SYS___syscall) { + sa->code = frame->tf_rdi; + reg++; + regcnt--; + } + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (sa->code >= p->p_sysent->sv_size) + sa->callp = &p->p_sysent->sv_table[0]; + else + sa->callp = &p->p_sysent->sv_table[sa->code]; + + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), + ("Too many syscall arguments!")); + error = 0; + argp = &frame->tf_rdi; + argp += reg; + bcopy(argp, sa->args, sizeof(sa->args[0]) * regcnt); + if (sa->narg > regcnt) { + KASSERT(params != NULL, ("copyin args with no params!")); + error = copyin(params, &sa->args[regcnt], + (sa->narg - regcnt) * sizeof(sa->args[0])); + } + + if (error == 0) { + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + } + + return (error); +} + +int +cpu_fetch_syscall_args_bcopy_noxchg(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + register_t *argp; + struct syscall_args *sa; + caddr_t params; + int reg, regcnt, error; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + reg = 0; + regcnt = 6; + + params = (caddr_t)frame->tf_rsp + sizeof(register_t); + sa->code = frame->tf_rax; + + if (sa->code == SYS_syscall || sa->code == SYS___syscall) { + sa->code = frame->tf_rdi; + reg++; + regcnt--; + } + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (sa->code >= p->p_sysent->sv_size) + sa->callp = &p->p_sysent->sv_table[0]; + else + sa->callp = &p->p_sysent->sv_table[sa->code]; + + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), + ("Too many syscall arguments!")); + error = 0; + argp = &frame->tf_rdi; + argp += reg; + bcopy_noxchg(argp, sa->args, sizeof(sa->args[0]) * regcnt); + if (sa->narg > regcnt) { + KASSERT(params != NULL, ("copyin args with no params!")); + error = copyin(params, &sa->args[regcnt], + (sa->narg - regcnt) * sizeof(sa->args[0])); + } + + if (error == 0) { + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + } + + return (error); +} + +int +cpu_fetch_syscall_args_oolmemcpy(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + register_t *argp; + struct syscall_args *sa; + caddr_t params; + int reg, regcnt, error; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + reg = 0; + regcnt = 6; + + params = (caddr_t)frame->tf_rsp + sizeof(register_t); + sa->code = frame->tf_rax; + + if (sa->code == SYS_syscall || sa->code == SYS___syscall) { + sa->code = frame->tf_rdi; + reg++; + regcnt--; + } + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (sa->code >= p->p_sysent->sv_size) + sa->callp = &p->p_sysent->sv_table[0]; + else + sa->callp = &p->p_sysent->sv_table[sa->code]; + + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), + ("Too many syscall arguments!")); + error = 0; + argp = &frame->tf_rdi; + argp += reg; + oolmemcpy(sa->args, argp, sizeof(sa->args[0]) * 6); + if (sa->narg > regcnt) { + KASSERT(params != NULL, ("copyin args with no params!")); + error = copyin(params, &sa->args[regcnt], + (sa->narg - regcnt) * sizeof(sa->args[0])); + } + + if (error == 0) { + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + } + + return (error); +} + + +int +cpu_fetch_syscall_args_oolmemcpy2(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + register_t *argp; + struct syscall_args *sa; + caddr_t params; + int reg, regcnt, error; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + reg = 0; + regcnt = 6; + + params = (caddr_t)frame->tf_rsp + sizeof(register_t); + sa->code = frame->tf_rax; + + if (sa->code == SYS_syscall || sa->code == SYS___syscall) { + sa->code = frame->tf_rdi; + reg++; + regcnt--; + } + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (sa->code >= p->p_sysent->sv_size) + sa->callp = &p->p_sysent->sv_table[0]; + else + sa->callp = &p->p_sysent->sv_table[sa->code]; + + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), + ("Too many syscall arguments!")); + error = 0; + argp = &frame->tf_rdi; + argp += reg; + oolmemcpy2(sa->args, argp, sizeof(sa->args[0]) * 6); + if (sa->narg > regcnt) { + KASSERT(params != NULL, ("copyin args with no params!")); + error = copyin(params, &sa->args[regcnt], + (sa->narg - regcnt) * sizeof(sa->args[0])); + } + + if (error == 0) { + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + } + + return (error); +} + +int +cpu_fetch_syscall_args_oolmemcpy_es(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + register_t *argp; + struct syscall_args *sa; + caddr_t params; + int reg, regcnt, error; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + reg = 0; + regcnt = 6; + + params = (caddr_t)frame->tf_rsp + sizeof(register_t); + sa->code = frame->tf_rax; + + if (sa->code == SYS_syscall || sa->code == SYS___syscall) { + sa->code = frame->tf_rdi; + reg++; + regcnt--; + } + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (sa->code >= p->p_sysent->sv_size) + sa->callp = &p->p_sysent->sv_table[0]; + else + sa->callp = &p->p_sysent->sv_table[sa->code]; + + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), + ("Too many syscall arguments!")); + error = 0; + argp = &frame->tf_rdi; + argp += reg; + oolmemcpy_es(sa->args, argp, sizeof(sa->args[0]) * 6); + if (sa->narg > regcnt) { + KASSERT(params != NULL, ("copyin args with no params!")); + error = copyin(params, &sa->args[regcnt], + (sa->narg - regcnt) * sizeof(sa->args[0])); + } + + if (error == 0) { + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + } + + return (error); +} #include "../../kern/subr_syscall.c" /*