diff --git a/FreeBSD/sys/netinet/in_pcb.c b/FreeBSD/sys/netinet/in_pcb.c index 9610ce2fe363..2dae62683b70 100644 --- a/FreeBSD/sys/netinet/in_pcb.c +++ b/FreeBSD/sys/netinet/in_pcb.c @@ -223,6 +223,18 @@ SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD, #endif /* INET */ +int inplb_use_cpu; +counter_u64_t inplb_used_cpu; +counter_u64_t inplb_used_hash; +SYSCTL_INT(_net_inet_ip, OID_AUTO, inplb_use_cpu, + CTLFLAG_RWTUN, &inplb_use_cpu, 0, + "Prefer CPU to hash for inpcb load balancing"); +SYSCTL_COUNTER_U64(_net_inet_ip, OID_AUTO, used_cpu, CTLFLAG_RD, + &inplb_used_cpu, "Selected listener via CPU"); +SYSCTL_COUNTER_U64(_net_inet_ip, OID_AUTO, used_hash, CTLFLAG_RD, + &inplb_used_hash, "Selected listener via hash"); + + VNET_DEFINE(uint32_t, in_pcbhashseed); static void in_pcbhashseed_init(void) @@ -2172,6 +2184,15 @@ in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, count = atomic_load_acq_int(&grp->il_inpcnt); if (count == 0) return (NULL); + if (inplb_use_cpu) { + int cpu = PCPU_GET(cpuid); + if (cpu < count && + (inp = grp->il_inp[cpu]) != NULL) { + counter_u64_add(inplb_used_cpu, 1); + return (inp); + } + } + counter_u64_add(inplb_used_hash, 1); inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count]; KASSERT(inp != NULL, ("%s: inp == NULL", __func__)); return (inp); @@ -3421,4 +3442,14 @@ rl_init(void *st) SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL); #endif + +static void +inplb_init(void *st) +{ + inplb_used_hash = counter_u64_alloc(M_WAITOK); + inplb_used_cpu = counter_u64_alloc(M_WAITOK); +} + +SYSINIT(inplb, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, inplb_init, NULL); + #endif /* RATELIMIT */ diff --git a/FreeBSD/sys/netinet6/in6_pcb.c b/FreeBSD/sys/netinet6/in6_pcb.c index f2eb00161690..ca5750d95c6c 100644 --- a/FreeBSD/sys/netinet6/in6_pcb.c +++ b/FreeBSD/sys/netinet6/in6_pcb.c @@ -893,6 +893,10 @@ in6_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib) (fib == RT_ALL_FIBS || fib == grp->il_fibnum)); } +extern int inplb_use_cpu; +extern counter_u64_t inplb_used_cpu; +extern counter_u64_t inplb_used_hash; + static struct inpcb * in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, const struct in6_addr *faddr, uint16_t fport, const struct in6_addr *laddr, @@ -971,6 +975,15 @@ in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, count = atomic_load_acq_int(&grp->il_inpcnt); if (count == 0) return (NULL); + if (inplb_use_cpu) { + int cpu = PCPU_GET(cpuid); + if (cpu < count && + (inp = grp->il_inp[cpu]) != NULL) { + counter_u64_add(inplb_used_cpu, 1); + return (inp); + } + } + counter_u64_add(inplb_used_hash, 1); inp = grp->il_inp[INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count]; KASSERT(inp != NULL, ("%s: inp == NULL", __func__)); return (inp);