diff --git a/sys/mips/cavium/cryptocteon/cavium_crypto.c b/sys/mips/cavium/cryptocteon/cavium_crypto.c index f983cfa..ea01b8d 100644 --- a/sys/mips/cavium/cryptocteon/cavium_crypto.c +++ b/sys/mips/cavium/cryptocteon/cavium_crypto.c @@ -87,22 +87,6 @@ __FBSDID("$FreeBSD$"); } \ } while (0) -static inline unsigned long octeon_crypto_enable(void) -{ - register_t s; - - s = intr_disable(); - mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT); - - return (s); -} - -static inline void octeon_crypto_disable(register_t s) -{ - mips_wr_status(mips_rd_status() & ~MIPS_SR_COP_2_BIT); - intr_restore(s); -} - #define ESP_HEADER_LENGTH 8 #define DES_CBC_IV_LENGTH 8 #define AES_CBC_IV_LENGTH 16 @@ -252,14 +236,12 @@ octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *oute uint64_t *key1; register uint64_t xor1 = 0x3636363636363636ULL; register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; - register_t s; dprintf("%s()\n", __func__); memset(hash_key, 0, sizeof(hash_key)); memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); key1 = (uint64_t *) hash_key; - s = octeon_crypto_enable(); if (auth) { CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); @@ -332,7 +314,6 @@ octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *oute outer[2] = 0; CVMX_MF_HSH_IV(outer[2], 2); } - octeon_crypto_disable(s); return; } @@ -349,7 +330,6 @@ octo_des_cbc_encrypt( { uint64_t *data; int data_i, data_l; - register_t s; dprintf("%s()\n", __func__); @@ -367,7 +347,6 @@ octo_des_cbc_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); @@ -378,7 +357,6 @@ octo_des_cbc_encrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -397,7 +375,6 @@ octo_des_cbc_encrypt( crypt_len -= 8; } - octeon_crypto_disable(s); return 0; } @@ -412,7 +389,6 @@ octo_des_cbc_decrypt( { uint64_t *data; int data_i, data_l; - register_t s; dprintf("%s()\n", __func__); @@ -430,8 +406,6 @@ octo_des_cbc_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { @@ -441,7 +415,6 @@ octo_des_cbc_decrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -460,7 +433,6 @@ octo_des_cbc_decrypt( crypt_len -= 8; } - octeon_crypto_disable(s); return 0; } @@ -477,7 +449,6 @@ octo_aes_cbc_encrypt( { uint64_t *data, *pdata; int data_i, data_l; - register_t s; dprintf("%s()\n", __func__); @@ -495,8 +466,6 @@ octo_aes_cbc_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -511,7 +480,6 @@ octo_aes_cbc_encrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -536,7 +504,6 @@ octo_aes_cbc_encrypt( crypt_len -= 16; } - octeon_crypto_disable(s); return 0; } @@ -551,7 +518,6 @@ octo_aes_cbc_decrypt( { uint64_t *data, *pdata; int data_i, data_l; - register_t s; dprintf("%s()\n", __func__); @@ -569,8 +535,6 @@ octo_aes_cbc_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -585,7 +549,6 @@ octo_aes_cbc_decrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -610,7 +573,6 @@ octo_aes_cbc_decrypt( crypt_len -= 16; } - octeon_crypto_disable(s); return 0; } @@ -629,7 +591,6 @@ octo_null_md5_encrypt( uint64_t *data; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -644,8 +605,6 @@ octo_null_md5_encrypt( IOV_INIT(iov, data, data_i, data_l); - s = octeon_crypto_enable(); - /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); @@ -716,7 +675,6 @@ octo_null_md5_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *(uint32_t *)data = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -735,7 +693,6 @@ octo_null_sha1_encrypt( uint64_t *data; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -750,8 +707,6 @@ octo_null_sha1_encrypt( IOV_INIT(iov, data, data_i, data_l); - s = octeon_crypto_enable(); - /* Load SHA1 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); @@ -825,7 +780,6 @@ octo_null_sha1_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *(uint32_t *)data = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -849,7 +803,6 @@ octo_des_cbc_md5_encrypt( uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -870,8 +823,6 @@ octo_des_cbc_md5_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { @@ -881,7 +832,6 @@ octo_des_cbc_md5_encrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -981,7 +931,6 @@ octo_des_cbc_md5_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1002,7 +951,6 @@ octo_des_cbc_md5_decrypt( uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1023,8 +971,6 @@ octo_des_cbc_md5_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { @@ -1034,7 +980,6 @@ octo_des_cbc_md5_decrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -1134,7 +1079,6 @@ octo_des_cbc_md5_decrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1158,7 +1102,6 @@ octo_des_cbc_sha1_encrypt( uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1179,8 +1122,6 @@ octo_des_cbc_sha1_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { @@ -1190,7 +1131,6 @@ octo_des_cbc_sha1_encrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -1293,7 +1233,6 @@ octo_des_cbc_sha1_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1314,7 +1253,6 @@ octo_des_cbc_sha1_decrypt( uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1335,8 +1273,6 @@ octo_des_cbc_sha1_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { @@ -1346,7 +1282,6 @@ octo_des_cbc_sha1_decrypt( CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -1448,7 +1383,6 @@ octo_des_cbc_sha1_decrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1473,7 +1407,6 @@ octo_aes_cbc_md5_encrypt( uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1494,8 +1427,6 @@ octo_aes_cbc_md5_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -1510,7 +1441,6 @@ octo_aes_cbc_md5_encrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -1633,7 +1563,6 @@ octo_aes_cbc_md5_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1655,7 +1584,6 @@ octo_aes_cbc_md5_decrypt( uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1676,8 +1604,6 @@ octo_aes_cbc_md5_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -1692,7 +1618,6 @@ octo_aes_cbc_md5_decrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -1811,7 +1736,6 @@ octo_aes_cbc_md5_decrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -1836,7 +1760,6 @@ octo_aes_cbc_sha1_encrypt( uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -1857,8 +1780,6 @@ octo_aes_cbc_sha1_encrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -1873,7 +1794,6 @@ octo_aes_cbc_sha1_encrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -2015,7 +1935,6 @@ octo_aes_cbc_sha1_encrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } @@ -2037,7 +1956,6 @@ octo_aes_cbc_sha1_decrypt( uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; - register_t s; dprintf("%s()\n", __func__); @@ -2058,8 +1976,6 @@ octo_aes_cbc_sha1_decrypt( CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); - s = octeon_crypto_enable(); - /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); @@ -2074,7 +1990,6 @@ octo_aes_cbc_sha1_decrypt( CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { - octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } @@ -2215,7 +2130,6 @@ octo_aes_cbc_sha1_decrypt( CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); - octeon_crypto_disable(s); return 0; } diff --git a/sys/mips/cavium/files.octeon1 b/sys/mips/cavium/files.octeon1 index b6c7e79..bf3e2b9 100644 --- a/sys/mips/cavium/files.octeon1 +++ b/sys/mips/cavium/files.octeon1 @@ -21,6 +21,8 @@ mips/cavium/octeon_nmi.S optional octeon_wdog mips/cavium/cryptocteon/cavium_crypto.c optional cryptocteon mips/cavium/cryptocteon/cryptocteon.c optional cryptocteon +mips/mips/octeon_cop2_swtch.S standard +mips/mips/octeon_cop2.c standard mips/cavium/octe/ethernet.c optional octe mips/cavium/octe/ethernet-mv88e61xx.c optional octe octeon_vendor_lanner diff --git a/sys/mips/include/cpuregs.h b/sys/mips/include/cpuregs.h index b47b264..eec9621 100644 --- a/sys/mips/include/cpuregs.h +++ b/sys/mips/include/cpuregs.h @@ -233,6 +233,7 @@ #define MIPS3_CR_EXC_CODE 0x0000007C /* five bits */ #define MIPS_CR_IP 0x0000FF00 #define MIPS_CR_EXC_CODE_SHIFT 2 +#define MIPS_CR_COP_ERR_SHIFT 28 /* * The bits in the status register. All bits are active when set to 1. diff --git a/sys/mips/include/frame.h b/sys/mips/include/frame.h index b08d88c..de7f94c 100644 --- a/sys/mips/include/frame.h +++ b/sys/mips/include/frame.h @@ -122,11 +122,6 @@ struct trapframe { f_register_t f31; register_t fsr; register_t fdummy; - /* - * COP2 registers may need to be saved here based on the CPU, and those - * might need to be per process, or even for the kernel, so we need - * some thought here. - */ }; /* REVISIT */ diff --git a/sys/mips/include/proc.h b/sys/mips/include/proc.h index 8b575dd..24f0838 100644 --- a/sys/mips/include/proc.h +++ b/sys/mips/include/proc.h @@ -39,6 +39,10 @@ #ifndef _MACHINE_PROC_H_ #define _MACHINE_PROC_H_ +#ifdef CPU_CNMIPS +#include +#endif + /* * Machine-dependent part of the proc structure. */ @@ -58,10 +62,15 @@ struct mdthread { int md_pc_count; /* performance counter */ int md_pc_spill; /* performance counter spill */ void *md_tls; +#ifdef CPU_CNMIPS + struct octeon_cop2_state *md_cop2; /* kernel context */ + struct octeon_cop2_state *md_ucop2; /* userland context */ +#endif }; /* md_flags */ #define MDTD_FPUSED 0x0001 /* Process used the FPU */ +#define MDTD_COP2USED 0x0002 /* Process used the COP2 */ struct mdproc { /* empty */ diff --git a/sys/mips/mips/exception.S b/sys/mips/mips/exception.S index 8b7307c..1882def 100644 --- a/sys/mips/mips/exception.S +++ b/sys/mips/mips/exception.S @@ -375,15 +375,15 @@ NNON_LEAF(MipsKernGenException, KERN_EXC_FRAME_SIZE, ra) REG_S a3, CALLFRAME_RA + KERN_REG_SIZE(sp) # for debugging /* - * Update interrupt mask in saved status register + * Update interrupt and CPU mask in saved status register * Some of interrupts could be disabled by * intr filters if interrupts are enabled later * in trap handler */ mfc0 a0, MIPS_COP_0_STATUS - and a0, a0, MIPS_SR_INT_MASK + and a0, a0, (MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) RESTORE_REG(a1, SR, sp) - and a1, a1, ~MIPS_SR_INT_MASK + and a1, a1, ~(MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) or a1, a1, a0 SAVE_REG(a1, SR, sp) RESTORE_CPU # v0 contains the return address. @@ -631,15 +631,15 @@ NNON_LEAF(MipsKernIntr, KERN_EXC_FRAME_SIZE, ra) REG_S a3, CALLFRAME_RA + KERN_REG_SIZE(sp) # for debugging /* - * Update interrupt mask in saved status register + * Update interrupt and CPU mask in saved status register * Some of interrupts could be disabled by * intr filters if interrupts are enabled later * in trap handler */ mfc0 a0, MIPS_COP_0_STATUS - and a0, a0, MIPS_SR_INT_MASK + and a0, a0, (MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) RESTORE_REG(a1, SR, sp) - and a1, a1, ~MIPS_SR_INT_MASK + and a1, a1, ~(MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) or a1, a1, a0 SAVE_REG(a1, SR, sp) REG_L v0, CALLFRAME_RA + KERN_REG_SIZE(sp) diff --git a/sys/mips/mips/genassym.c b/sys/mips/mips/genassym.c index 2cb046d..54e7523 100644 --- a/sys/mips/mips/genassym.c +++ b/sys/mips/mips/genassym.c @@ -58,11 +58,14 @@ __FBSDID("$FreeBSD$"); #include #include +#ifdef CPU_CNMIPS +#include +#endif + #ifndef offsetof #define offsetof(t,m) (int)((&((t *)0L)->m)) #endif - ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); ASSYM(TD_UPTE, offsetof(struct thread, td_md.md_upte)); ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); @@ -70,6 +73,7 @@ ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TD_TLS, offsetof(struct thread, td_md.md_tls)); +ASSYM(TD_MDFLAGS, offsetof(struct thread, td_md.md_flags)); ASSYM(TF_REG_SR, offsetof(struct trapframe, sr)); @@ -99,8 +103,72 @@ ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); ASSYM(PCPU_SIZE, sizeof(struct pcpu)); ASSYM(MAXCOMLEN, MAXCOMLEN); +ASSYM(MDTD_COP2USED, MDTD_COP2USED); ASSYM(MIPS_KSEG0_START, MIPS_KSEG0_START); ASSYM(MIPS_KSEG1_START, MIPS_KSEG1_START); ASSYM(MIPS_KSEG2_START, MIPS_KSEG2_START); ASSYM(MIPS_XKSEG_START, MIPS_XKSEG_START); + +#ifdef CPU_CNMIPS +ASSYM(TD_COP2, offsetof(struct thread, td_md.md_cop2)); +ASSYM(TD_UCOP2, offsetof(struct thread, td_md.md_ucop2)); +ASSYM(COP2_CRC_IV_OFFSET, offsetof(struct octeon_cop2_state, crc_iv)); +ASSYM(COP2_CRC_LENGTH_OFFSET, offsetof(struct octeon_cop2_state, crc_length)); +ASSYM(COP2_CRC_POLY_OFFSET, offsetof(struct octeon_cop2_state, crc_poly)); +ASSYM(COP2_LLM_DAT0_OFFSET, offsetof(struct octeon_cop2_state, llm_dat)); +ASSYM(COP2_LLM_DAT1_OFFSET, offsetof(struct octeon_cop2_state, llm_dat) + 8); +ASSYM(COP2_3DES_IV_OFFSET, offsetof(struct octeon_cop2_state, _3des_iv)); +ASSYM(COP2_3DES_KEY0_OFFSET, offsetof(struct octeon_cop2_state, _3des_key)); +ASSYM(COP2_3DES_KEY1_OFFSET, offsetof(struct octeon_cop2_state, _3des_key) + 8); +ASSYM(COP2_3DES_KEY2_OFFSET, offsetof(struct octeon_cop2_state, _3des_key) + 16); +ASSYM(COP2_3DES_RESULT_OFFSET, offsetof(struct octeon_cop2_state, _3des_result)); +ASSYM(COP2_AES_INP0_OFFSET, offsetof(struct octeon_cop2_state, aes_inp0)); +ASSYM(COP2_AES_IV0_OFFSET, offsetof(struct octeon_cop2_state, aes_iv)); +ASSYM(COP2_AES_IV1_OFFSET, offsetof(struct octeon_cop2_state, aes_iv) + 8); +ASSYM(COP2_AES_KEY0_OFFSET, offsetof(struct octeon_cop2_state, aes_key)); +ASSYM(COP2_AES_KEY1_OFFSET, offsetof(struct octeon_cop2_state, aes_key) + 8); +ASSYM(COP2_AES_KEY2_OFFSET, offsetof(struct octeon_cop2_state, aes_key) + 16); +ASSYM(COP2_AES_KEY3_OFFSET, offsetof(struct octeon_cop2_state, aes_key) + 24); +ASSYM(COP2_AES_KEYLEN_OFFSET, offsetof(struct octeon_cop2_state, aes_keylen)); +ASSYM(COP2_AES_RESULT0_OFFSET, offsetof(struct octeon_cop2_state, aes_result)); +ASSYM(COP2_AES_RESULT1_OFFSET, offsetof(struct octeon_cop2_state, aes_result) + 8); +ASSYM(COP2_HSH_DATW0_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw)); +ASSYM(COP2_HSH_DATW1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 8); +ASSYM(COP2_HSH_DATW2_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 16); +ASSYM(COP2_HSH_DATW3_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 24); +ASSYM(COP2_HSH_DATW4_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 32); +ASSYM(COP2_HSH_DATW5_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 40); +ASSYM(COP2_HSH_DATW6_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 48); +ASSYM(COP2_HSH_DATW7_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 56); +ASSYM(COP2_HSH_DATW8_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 64); +ASSYM(COP2_HSH_DATW9_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 72); +ASSYM(COP2_HSH_DATW10_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 80); +ASSYM(COP2_HSH_DATW11_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 88); +ASSYM(COP2_HSH_DATW12_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 96); +ASSYM(COP2_HSH_DATW13_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 104); +ASSYM(COP2_HSH_DATW14_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 112); +ASSYM(COP2_HSH_IVW0_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw)); +ASSYM(COP2_HSH_IVW1_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 8); +ASSYM(COP2_HSH_IVW2_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 16); +ASSYM(COP2_HSH_IVW3_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 24); +ASSYM(COP2_HSH_IVW4_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 32); +ASSYM(COP2_HSH_IVW5_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 40); +ASSYM(COP2_HSH_IVW6_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 48); +ASSYM(COP2_HSH_IVW7_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 56); +ASSYM(COP2_GFM_MULT0_OFFSET, offsetof(struct octeon_cop2_state, gfm_mult)); +ASSYM(COP2_GFM_MULT1_OFFSET, offsetof(struct octeon_cop2_state, gfm_mult) + 8); +ASSYM(COP2_GFM_POLY_OFFSET, offsetof(struct octeon_cop2_state, gfm_poly)); +ASSYM(COP2_GFM_RESULT0_OFFSET, offsetof(struct octeon_cop2_state, gfm_result)); +ASSYM(COP2_GFM_RESULT1_OFFSET, offsetof(struct octeon_cop2_state, gfm_result) + 8); +ASSYM(COP2_HSH_DATW0_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw)); +ASSYM(COP2_HSH_DATW1_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 8); +ASSYM(COP2_HSH_DATW2_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 16); +ASSYM(COP2_HSH_DATW3_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 24); +ASSYM(COP2_HSH_DATW4_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 32); +ASSYM(COP2_HSH_DATW5_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 40); +ASSYM(COP2_HSH_DATW6_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_datw) + 48); +ASSYM(COP2_HSH_IVW0_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw)); +ASSYM(COP2_HSH_IVW1_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 8); +ASSYM(COP2_HSH_IVW2_PASS1_OFFSET, offsetof(struct octeon_cop2_state, hsh_ivw) + 16); +#endif diff --git a/sys/mips/mips/locore.S b/sys/mips/mips/locore.S index 4b28f19..c6e78a2 100644 --- a/sys/mips/mips/locore.S +++ b/sys/mips/mips/locore.S @@ -95,10 +95,10 @@ VECTOR(_locore, unknown) */ /* Set these bits */ - li t1, (MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT | MIPS_SR_PX | MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_SX | MIPS_SR_BEV) + li t1, (MIPS_SR_COP_0_BIT | MIPS_SR_PX | MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_SX | MIPS_SR_BEV) /* Reset these bits */ - li t0, ~(MIPS_SR_DE | MIPS_SR_SOFT_RESET | MIPS_SR_ERL | MIPS_SR_EXL | MIPS_SR_INT_IE) + li t0, ~(MIPS_SR_DE | MIPS_SR_SOFT_RESET | MIPS_SR_ERL | MIPS_SR_EXL | MIPS_SR_INT_IE | MIPS_SR_COP_2_BIT) #elif defined (CPU_RMI) || defined (CPU_NLM) /* Set these bits */ li t1, (MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT | MIPS_SR_KX | MIPS_SR_UX) diff --git a/sys/mips/mips/pm_machdep.c b/sys/mips/mips/pm_machdep.c index d730ccb..d136f5f 100644 --- a/sys/mips/mips/pm_machdep.c +++ b/sys/mips/mips/pm_machdep.c @@ -520,7 +520,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) td->td_frame->sr |= MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX; #endif #ifdef CPU_CNMIPS - td->td_frame->sr |= MIPS_SR_COP_2_BIT | MIPS_SR_PX | MIPS_SR_UX | + td->td_frame->sr |= MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX; #endif /* diff --git a/sys/mips/mips/swtch.S b/sys/mips/mips/swtch.S index 699c206..cadf883 100644 --- a/sys/mips/mips/swtch.S +++ b/sys/mips/mips/swtch.S @@ -250,6 +250,58 @@ NON_LEAF(cpu_switch, CALLFRAME_SIZ, ra) getpc: SAVE_U_PCB_CONTEXT(ra, PREG_PC, a0) # save return address +#ifdef CPU_CNMIPS + + lw t2, TD_MDFLAGS(a3) # get md_flags + and t1, t2, MDTD_COP2USED + beqz t1, cop2_untouched + nop + + /* Clear cop2used flag */ + and t2, t2, ~MDTD_COP2USED + sw t2, TD_MDFLAGS(a3) + + li t3, ~MIPS_SR_COP_2_BIT + and t2, t0, t3 # clear COP_2 enable bit + SAVE_U_PCB_CONTEXT(t2, PREG_SR, a0) # save status register + + /* preserve a0..a3 */ + move s0, a0 + move s1, a1 + move s2, a2 + move s3, a3 + + /* did kernel part touch CP2? */ + and t1, t0, MIPS_SR_COP_2_BIT + beqz t1, userland_cop2 + nop + + PTR_L a0, TD_COP2(a3) + beqz a0, no_cop2_context + nop + + j do_cop2_save + nop + +userland_cop2: + + PTR_L a0, TD_UCOP2(a3) + beqz a0, no_cop2_context + nop + +do_cop2_save: + jal octeon_cop2_save + nop + +no_cop2_context: + move a3, s3 + move a2, s2 + move a1, s1 + move a0, s0 + +cop2_untouched: +#endif + PTR_S a2, TD_LOCK(a3) # Switchout td_lock mips_sw1: diff --git a/sys/mips/mips/trap.c b/sys/mips/mips/trap.c index 97374a7..6e3ef2c 100644 --- a/sys/mips/mips/trap.c +++ b/sys/mips/mips/trap.c @@ -251,6 +251,9 @@ char *access_name[] = { "Store Doubleword" }; +#ifdef CPU_CNMIPS +#include +#endif static int allow_unaligned_acc = 1; @@ -410,6 +413,7 @@ trap(struct trapframe *trapframe) char *msg = NULL; intptr_t addr = 0; register_t pc; + int cop; trapdebug_enter(trapframe, 0); @@ -767,28 +771,87 @@ dofault: goto err; break; case T_COP_UNUSABLE: +#ifdef CPU_CNMIPS + cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT; + /* Handle only COP2 exception */ + if (cop != 2) + goto err; + + addr = trapframe->pc; + /* save userland cop2 context if it has been touched */ + if (td->td_frame->sr & MIPS_SR_COP_2_BIT) { + if (td->td_md.md_ucop2) + octeon_cop2_save(td->td_md.md_ucop2); + else + panic("COP2 was used in user mode but md_ucop2 is NULL"); + } + + if (td->td_md.md_cop2 == NULL) { + td->td_md.md_cop2 = octeon_cop2_alloc_ctx(); + if (td->td_md.md_cop2 == NULL) + panic("Failed to allocate COP2 context"); + memset(td->td_md.md_cop2, 0, sizeof(*td->td_md.md_cop2)); + } + + octeon_cop2_restore(td->td_md.md_cop2); + + /* Make userland re-request its context */ + td->td_frame->sr &= ~MIPS_SR_COP_2_BIT; + td->td_md.md_flags |= MDTD_COP2USED; + /* Enable COP2, it will be disabled in cpu_switch */ + mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT); + return (trapframe->pc); +#else goto err; break; +#endif + case T_COP_UNUSABLE + T_USER: + cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT; + if (cop == 1) { #if !defined(CPU_HAVEFPU) /* FP (COP1) instruction */ - if ((trapframe->cause & MIPS_CR_COP_ERR) == 0x10000000) { log_illegal_instruction("COP1_UNUSABLE", trapframe); i = SIGILL; break; +#else + addr = trapframe->pc; + MipsSwitchFPState(PCPU_GET(fpcurthread), td->td_frame); + PCPU_SET(fpcurthread, td); + td->td_frame->sr |= MIPS_SR_COP_1_BIT; + td->td_md.md_flags |= MDTD_FPUSED; + goto out; +#endif + } +#ifdef CPU_CNMIPS + else if (cop == 2) { + addr = trapframe->pc; + if (td->td_md.md_flags & MDTD_COP2USED) { + if (td->td_md.md_cop2) + octeon_cop2_save(td->td_md.md_cop2); + else + panic("COP2 was used in kernel mode but md_cop2 is NULL"); + } + + if (td->td_md.md_ucop2 == NULL) { + td->td_md.md_ucop2 = octeon_cop2_alloc_ctx(); + if (td->td_md.md_ucop2 == NULL) + panic("Failed to allocate userland COP2 context"); + memset(td->td_md.md_ucop2, 0, sizeof(*td->td_md.md_ucop2)); + } + + octeon_cop2_restore(td->td_md.md_ucop2); + + td->td_frame->sr |= MIPS_SR_COP_2_BIT; + td->td_md.md_flags |= MDTD_COP2USED; + goto out; } #endif - if ((trapframe->cause & MIPS_CR_COP_ERR) != 0x10000000) { + else { log_illegal_instruction("COPn_UNUSABLE", trapframe); i = SIGILL; /* only FPU instructions allowed */ break; } - addr = trapframe->pc; - MipsSwitchFPState(PCPU_GET(fpcurthread), td->td_frame); - PCPU_SET(fpcurthread, td); - td->td_frame->sr |= MIPS_SR_COP_1_BIT; - td->td_md.md_flags |= MDTD_FPUSED; - goto out; case T_FPE: #if !defined(SMP) && (defined(DDB) || defined(DEBUG)) diff --git a/sys/mips/mips/vm_machdep.c b/sys/mips/mips/vm_machdep.c index 47058ee..e5f21d5 100644 --- a/sys/mips/mips/vm_machdep.c +++ b/sys/mips/mips/vm_machdep.c @@ -163,7 +163,31 @@ cpu_fork(register struct thread *td1,register struct proc *p2, td2->td_md.md_saved_intr = MIPS_SR_INT_IE; td2->td_md.md_spinlock_count = 1; #ifdef CPU_CNMIPS - pcb2->pcb_context[PCB_REG_SR] |= MIPS_SR_COP_2_BIT | MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX; + if (td1->td_frame->sr & MIPS_SR_COP_2_BIT) { + if (td1->td_md.md_ucop2) + octeon_cop2_save(td1->td_md.md_ucop2); + else + panic("cpu_fork: ucop2 is NULL but COP2 is enabled"); + } + else if (mips_rd_status() & MIPS_SR_COP_2_BIT) { + if (td1->td_md.md_cop2) + octeon_cop2_save(td1->td_md.md_cop2); + else + panic("cpu_fork: cop2 is NULL but COP2 is enabled"); + } + + if (td1->td_md.md_cop2) { + td2->td_md.md_cop2 = octeon_cop2_alloc_ctx(); + memcpy(td2->td_md.md_cop2, td1->td_md.md_cop2, + sizeof(*td1->td_md.md_cop2)); + } + if (td1->td_md.md_ucop2) { + td2->td_md.md_ucop2 = octeon_cop2_alloc_ctx(); + memcpy(td2->td_md.md_ucop2, td1->td_md.md_ucop2, + sizeof(*td1->td_md.md_ucop2)); + } + pcb2->pcb_context[PCB_REG_SR] |= MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX; + td2->td_frame->sr &= ~MIPS_SR_COP_2_BIT; #endif } @@ -195,11 +219,27 @@ cpu_thread_exit(struct thread *td) if (PCPU_GET(fpcurthread) == td) PCPU_GET(fpcurthread) = (struct thread *)0; +#ifdef CPU_CNMIPS + if (td->td_md.md_cop2) + memset(td->td_md.md_cop2, 0, + sizeof(*td->td_md.md_cop2)); + if (td->td_md.md_ucop2) + memset(td->td_md.md_ucop2, 0, + sizeof(*td->td_md.md_ucop2)); +#endif } void cpu_thread_free(struct thread *td) { +#ifdef CPU_CNMIPS + if (td->td_md.md_cop2) + octeon_cop2_free_ctx(td->td_md.md_cop2); + if (td->td_md.md_ucop2) + octeon_cop2_free_ctx(td->td_md.md_ucop2); + td->td_md.md_cop2 = NULL; + td->td_md.md_ucop2 = NULL; +#endif } void @@ -357,7 +397,7 @@ cpu_set_upcall(struct thread *td, struct thread *td0) (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_INT_MASK); #ifdef CPU_CNMIPS - pcb2->pcb_context[PCB_REG_SR] |= MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT | + pcb2->pcb_context[PCB_REG_SR] |= MIPS_SR_COP_0_BIT | MIPS_SR_PX | MIPS_SR_UX | MIPS_SR_KX | MIPS_SR_SX; #endif diff --git a/tools/install.sh b/tools/install.sh old mode 100644 new mode 100755