commit 48e5dccc89235f3bdf9d1a11a21228f7b242450f Author: Mark Johnston Date: Fri Sep 13 15:05:27 2019 -0400 Add global FT and IL policies. Add thread_set_domainset(). diff --git a/sys/kern/kern_cpuset.c b/sys/kern/kern_cpuset.c index d42fcda80b51..eb5ce238be55 100644 --- a/sys/kern/kern_cpuset.c +++ b/sys/kern/kern_cpuset.c @@ -119,7 +119,9 @@ __FBSDID("$FreeBSD$"); */ LIST_HEAD(domainlist, domainset); +struct domainset __read_mostly domainset_firsttouch; struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; +struct domainset __read_mostly domainset_interleave; struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; struct domainset __read_mostly domainset_roundrobin; @@ -130,7 +132,7 @@ static struct setlist cpuset_ids; static struct domainlist cpuset_domains; static struct unrhdr *cpuset_unr; static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; -static struct domainset domainset0, domainset2; +static struct domainset *domainset0, *domainset2; /* Return the size of cpuset_t at the kernel level */ SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, @@ -545,7 +547,7 @@ domainset_create(const struct domainset *domain) if (domain->ds_policy == DOMAINSET_POLICY_PREFER && !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) return (NULL); - if (!DOMAINSET_SUBSET(&domainset0.ds_mask, &domain->ds_mask)) + if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) return (NULL); ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); domainset_copy(domain, ndomain); @@ -1393,54 +1395,51 @@ cpuset_setithread(lwpid_t id, int cpu) /* * Initialize static domainsets after NUMA information is available. This is - * called before memory allocators are initialized. + * called before memory allocators are initialized. Also create the domainsets + * for cpusets 0, 1 and 2. */ void domainset_init(void) { - struct domainset *dset; + struct domainset *dset, *tmp; int i; + mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); + + dset = &domainset_firsttouch; + DOMAINSET_COPY(&all_domains, &dset->ds_mask); + dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; + dset->ds_prefer = -1; + (void)_domainset_create(dset, NULL); + + dset = &domainset_interleave; + DOMAINSET_COPY(&all_domains, &dset->ds_mask); + dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; + dset->ds_prefer = -1; + (void)_domainset_create(dset, NULL); + dset = &domainset_roundrobin; DOMAINSET_COPY(&all_domains, &dset->ds_mask); dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; dset->ds_prefer = -1; - _domainset_create(dset, NULL); + (void)_domainset_create(dset, NULL); for (i = 0; i < vm_ndomains; i++) { dset = &domainset_fixed[i]; DOMAINSET_ZERO(&dset->ds_mask); DOMAINSET_SET(i, &dset->ds_mask); dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; - _domainset_create(dset, NULL); + (void)_domainset_create(dset, NULL); dset = &domainset_prefer[i]; DOMAINSET_COPY(&all_domains, &dset->ds_mask); dset->ds_policy = DOMAINSET_POLICY_PREFER; dset->ds_prefer = i; - _domainset_create(dset, NULL); + (void)_domainset_create(dset, NULL); } -} -/* - * Create the domainset for cpuset 0, 1 and cpuset 2. - */ -void -domainset_zero(void) -{ - struct domainset *dset, *tmp; - - mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); - - dset = &domainset0; - DOMAINSET_COPY(&all_domains, &dset->ds_mask); - dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; - dset->ds_prefer = -1; - curthread->td_domain.dr_policy = _domainset_create(dset, NULL); - - domainset_copy(dset, &domainset2); - domainset2.ds_policy = DOMAINSET_POLICY_INTERLEAVE; - kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL); + domainset0 = &domainset_firsttouch; + domainset2 = &domainset_interleave; /* Remove empty domains from the global policies. */ LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) @@ -1484,7 +1483,7 @@ cpuset_thread0(void) LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); set->cs_ref = 1; set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; - set->cs_domain = &domainset0; + set->cs_domain = domainset0; cpuset_zero = set; cpuset_root = &set->cs_mask; @@ -1495,17 +1494,18 @@ cpuset_thread0(void) error = _cpuset_create(set, cpuset_zero, NULL, NULL, 1); KASSERT(error == 0, ("Error creating default set: %d\n", error)); cpuset_default = set; + /* * Create the kernel set (2). */ set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); error = _cpuset_create(set, cpuset_zero, NULL, NULL, 2); KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); - set->cs_domain = &domainset2; + set->cs_domain = domainset2; cpuset_kernel = set; /* - * Initialize the unit allocator. 0 and 1 are allocated above. + * Initialize the unit allocator. 0, 1 and 2 are allocated above. */ cpuset_unr = new_unrhdr(3, INT_MAX, NULL); @@ -2181,7 +2181,7 @@ kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, * across all domains. */ if (domainset_empty_vm(&domain)) - domainset_copy(&domainset2, &domain); + domainset_copy(domainset2, &domain); switch (level) { case CPU_LEVEL_ROOT: diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 378bdb88d3a6..39d45d78e46c 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -1199,6 +1199,13 @@ thread_single_end(struct proc *p, int mode) kick_proc0(); } +void +thread_set_domainset(struct thread *td, struct domainset *ds) +{ + + td->td_domain.dr_policy = ds; +} + struct thread * thread_find(struct proc *p, lwpid_t tid) { diff --git a/sys/sys/domainset.h b/sys/sys/domainset.h index 5a00347f29d3..fb8dcb34921b 100644 --- a/sys/sys/domainset.h +++ b/sys/sys/domainset.h @@ -96,6 +96,10 @@ struct domainset { domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */ }; +extern struct domainset domainset_firsttouch; +#define DOMAINSET_FT() (&domainset_firsttouch) +extern struct domainset domainset_interleave; +#define DOMAINSET_IL() (&domainset_interleave) extern struct domainset domainset_fixed[MAXMEMDOM], domainset_prefer[MAXMEMDOM]; #define DOMAINSET_FIXED(domain) (&domainset_fixed[(domain)]) #define DOMAINSET_PREF(domain) (&domainset_prefer[(domain)]) @@ -103,7 +107,6 @@ extern struct domainset domainset_roundrobin; #define DOMAINSET_RR() (&domainset_roundrobin) void domainset_init(void); -void domainset_zero(void); /* * Add a domainset to the system based on a key initializing policy, prefer, diff --git a/sys/sys/proc.h b/sys/sys/proc.h index ae605a1266dd..9b038ec2cb25 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -174,6 +174,7 @@ struct pargs { * for write access. */ struct cpuset; +struct domainset; struct filecaps; struct filemon; struct kaioinfo; @@ -1128,6 +1129,7 @@ void thread_exit(void) __dead2; void thread_free(struct thread *td); void thread_link(struct thread *td, struct proc *p); void thread_reap(void); +void thread_set_domainset(struct thread *td, struct domainset *ds); int thread_single(struct proc *p, int how); void thread_single_end(struct proc *p, int how); void thread_stash(struct thread *td); diff --git a/sys/vm/vm_init.c b/sys/vm/vm_init.c index 98f7b2b76131..67e41db3b3ec 100644 --- a/sys/vm/vm_init.c +++ b/sys/vm/vm_init.c @@ -118,7 +118,8 @@ vm_mem_init(void *dummy) { /* - * Initialize static domainsets, used by various allocators. + * Initialize static domainsets, used by various allocators and + * as default policies. */ domainset_init(); @@ -129,12 +130,6 @@ vm_mem_init(void *dummy) vm_set_page_size(); virtual_avail = vm_page_startup(virtual_avail); - /* - * Set an initial domain policy for thread0 so that allocations - * can work. - */ - domainset_zero(); - #ifdef UMA_MD_SMALL_ALLOC /* Announce page availability to UMA. */ uma_startup1();