commit a85acdb705bb9e215d16d56389f29969ab15c492 Author: Drew Gallatin Date: Thu Oct 9 15:49:58 2025 -0400 rss: Enable portions of RSS globally to enable symmetric hashing We use the fact that all NICs that support hashing are using the same hash algorithm and hash key to enable symmetic hashing in TCP, where a software version of the same hash is used to establish hashes on outgoing connections. diff --git a/sys/conf/files b/sys/conf/files index 87c8830b192e..fa8aa341edf2 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -4234,10 +4234,10 @@ net/route/route_rtentry.c standard net/route/route_subscription.c standard net/route/route_tables.c standard net/route/route_temporal.c standard -net/rss_config.c optional inet rss | inet6 rss +net/rss_config.c standard net/rtsock.c standard net/slcompress.c optional netgraph_vjc -net/toeplitz.c optional inet rss | inet6 rss | route_mpath +net/toeplitz.c optional inet | inet6 | route_mpath net/vnet.c optional vimage net80211/ieee80211.c optional wlan net80211/ieee80211_acl.c optional wlan wlan_acl @@ -4380,7 +4380,7 @@ netinet/in_pcb.c optional inet | inet6 netinet/in_prot.c optional inet | inet6 netinet/in_proto.c optional inet | inet6 netinet/in_rmx.c optional inet -netinet/in_rss.c optional inet rss +netinet/in_rss.c optional inet netinet/ip_divert.c optional ipdivert inet | ipdivert inet6 netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 @@ -4482,7 +4482,7 @@ netinet6/in6_mcast.c optional inet6 netinet6/in6_pcb.c optional inet6 netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 -netinet6/in6_rss.c optional inet6 rss +netinet6/in6_rss.c optional inet6 netinet6/in6_src.c optional inet6 netinet6/ip6_fastfwd.c optional inet6 netinet6/ip6_forward.c optional inet6 diff --git a/sys/net/rss_config.c b/sys/net/rss_config.c index 266ea57b2dc9..9e4120a4e9dd 100644 --- a/sys/net/rss_config.c +++ b/sys/net/rss_config.c @@ -29,6 +29,8 @@ #include "opt_inet6.h" +#include "opt_inet.h" +#include "opt_rss.h" #include #include @@ -72,6 +74,10 @@ * placement and pcbgroup expectations. */ +#if !defined(INET) && !defined(INET6) +#define _net_inet _net +#define _net_inet_rss _net_rss +#endif SYSCTL_DECL(_net_inet); SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Receive-side steering"); @@ -84,6 +90,7 @@ static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, "RSS hash algorithm"); +#ifdef RSS /* * Size of the indirection table; at most 128 entries per the RSS spec. We * size it to at least 2 times the number of CPUs by default to allow useful @@ -132,6 +139,7 @@ static const u_int rss_basecpu; SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); +#endif /* * Print verbose debugging messages. * 0 - disable @@ -159,6 +167,7 @@ static uint8_t rss_key[RSS_KEYSIZE] = { 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, }; +#ifdef RSS /* * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. * Drivers may supplement this table with a separate CPU<->queue table when @@ -168,13 +177,15 @@ struct rss_table_entry { uint8_t rte_cpu; /* CPU affinity of bucket. */ }; static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; +#endif static void rss_init(__unused void *arg) { +#ifdef RSS u_int i; u_int cpuid; - +#endif /* * Validate tunables, coerce to sensible values. */ @@ -189,6 +200,7 @@ rss_init(__unused void *arg) rss_hashalgo = RSS_HASH_TOEPLITZ; } +#ifdef RSS /* * Count available CPUs. * @@ -248,7 +260,7 @@ rss_init(__unused void *arg) rss_table[i].rte_cpu = cpuid; cpuid = CPU_NEXT(cpuid); } - +#endif /* RSS */ /* * Randomize rrs_key. * @@ -292,6 +304,30 @@ rss_hash(u_int datalen, const uint8_t *data) } } +/* + * Query the current RSS key; likely to be used by device drivers when + * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. + * + * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? + */ +void +rss_getkey(uint8_t *key) +{ + + bcopy(rss_key, key, sizeof(rss_key)); +} + +/* + * Query the RSS hash algorithm. + */ +u_int +rss_gethashalgo(void) +{ + + return (rss_hashalgo); +} + +#ifdef RSS /* * Query the number of RSS bits in use. */ @@ -406,29 +442,6 @@ rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) bucket_id)); } -/* - * Query the RSS hash algorithm. - */ -u_int -rss_gethashalgo(void) -{ - - return (rss_hashalgo); -} - -/* - * Query the current RSS key; likely to be used by device drivers when - * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. - * - * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? - */ -void -rss_getkey(uint8_t *key) -{ - - bcopy(rss_key, key, sizeof(rss_key)); -} - /* * Query the number of buckets; this may be used by both network device * drivers, which will need to populate hardware shadows of the software @@ -454,6 +467,7 @@ rss_getnumcpus(void) return (rss_ncpus); } +#endif /* * Return the supported RSS hash configuration. * @@ -517,6 +531,7 @@ SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, "", "RSS keying material"); +#ifdef RSS static int sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) { @@ -544,3 +559,4 @@ sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); +#endif diff --git a/sys/net/rss_config.h b/sys/net/rss_config.h index 07c2d09b44c5..aaa282b12e72 100644 --- a/sys/net/rss_config.h +++ b/sys/net/rss_config.h @@ -104,6 +104,7 @@ extern int rss_debug; +#ifdef RSS /* * Device driver interfaces to query RSS properties that must be programmed * into hardware. @@ -112,16 +113,8 @@ u_int rss_getbits(void); u_int rss_getbucket(u_int hash); u_int rss_get_indirection_to_bucket(u_int index); u_int rss_getcpu(u_int bucket); -void rss_getkey(uint8_t *key); -u_int rss_gethashalgo(void); u_int rss_getnumbuckets(void); u_int rss_getnumcpus(void); -u_int rss_gethashconfig(void); - -/* - * Hash calculation functions. - */ -uint32_t rss_hash(u_int datalen, const uint8_t *data); /* * Network stack interface to query desired CPU affinity of a packet. @@ -132,4 +125,15 @@ int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id); int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id); +#endif /* RSS */ + +void rss_getkey(uint8_t *key); +u_int rss_gethashalgo(void); +u_int rss_gethashconfig(void); +/* + * Hash calculation functions. + */ +uint32_t rss_hash(u_int datalen, const uint8_t *data); + + #endif /* !_NET_RSS_CONFIG_H_ */ diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c index f93a1d2bfd7b..4854265bd9f4 100644 --- a/sys/netinet/in_rss.c +++ b/sys/netinet/in_rss.c @@ -29,6 +29,7 @@ #include "opt_inet6.h" +#include "opt_rss.h" #include #include @@ -350,6 +351,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval, } } +#ifdef RSS /* * Similar to rss_m2cpuid, but designed to be used by the IP NETISR * on incoming frames. @@ -387,3 +389,4 @@ rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source, u_int *cpuid) } return (m); } +#endif diff --git a/sys/netinet6/in6_rss.c b/sys/netinet6/in6_rss.c index 79c7bfa6e68c..3d98d0065d1e 100644 --- a/sys/netinet6/in6_rss.c +++ b/sys/netinet6/in6_rss.c @@ -29,6 +29,7 @@ #include "opt_inet6.h" +#include "opt_rss.h" #include #include @@ -375,6 +376,7 @@ rss_mbuf_software_hash_v6(const struct mbuf *m, int dir, uint32_t *hashval, } } +#ifdef RSS /* * Similar to rss_m2cpuid, but designed to be used by the IPv6 NETISR * on incoming frames. @@ -412,3 +414,4 @@ rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source, u_int *cpuid) } return (m); } +#endif