commit 0a40794813d296944a0ee4b9a10db0aa7e2e3ddd Author: Mikolaj Golub Date: Sun Sep 1 14:55:36 2013 +0300 Make hastctl list command output current queue sizes. diff --git a/sbin/hastctl/hastctl.c b/sbin/hastctl/hastctl.c index c2057dd..9348c57 100644 --- a/sbin/hastctl/hastctl.c +++ b/sbin/hastctl/hastctl.c @@ -355,6 +355,13 @@ control_list(struct nv *nv) (uintmax_t)nv_get_uint64(nv, "stat_write_error%u", ii), (uintmax_t)nv_get_uint64(nv, "stat_delete_error%u", ii), (uintmax_t)nv_get_uint64(nv, "stat_flush_error%u", ii)); + printf(" queues: " + "local: %ju, send: %ju, recv: %ju, done: %ju, idle: %ju\n", + (uintmax_t)nv_get_uint64(nv, "local_queue_size%u", ii), + (uintmax_t)nv_get_uint64(nv, "send_queue_size%u", ii), + (uintmax_t)nv_get_uint64(nv, "recv_queue_size%u", ii), + (uintmax_t)nv_get_uint64(nv, "done_queue_size%u", ii), + (uintmax_t)nv_get_uint64(nv, "idle_queue_size%u", ii)); } return (ret); } diff --git a/sbin/hastd/control.c b/sbin/hastd/control.c index 486d06f..865ec96 100644 --- a/sbin/hastd/control.c +++ b/sbin/hastd/control.c @@ -215,6 +215,16 @@ control_status_worker(struct hast_resource *res, struct nv *nvout, "stat_delete_error%u", no); nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"), "stat_flush_error%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"), + "idle_queue_size%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"), + "local_queue_size%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"), + "send_queue_size%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"), + "recv_queue_size%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"), + "done_queue_size%u", no); end: if (cnvin != NULL) nv_free(cnvin); @@ -478,6 +488,7 @@ ctrl_thread(void *arg) nv_add_uint64(nvout, res->hr_stat_flush_error + res->hr_stat_activemap_flush_error, "stat_flush_error"); + res->output_status_aux(nvout); nv_add_int16(nvout, 0, "error"); break; case CONTROL_RELOAD: diff --git a/sbin/hastd/hast.h b/sbin/hastd/hast.h index 6e6c999..86501c9 100644 --- a/sbin/hastd/hast.h +++ b/sbin/hastd/hast.h @@ -137,6 +137,8 @@ struct hastd_config { #define HAST_CHECKSUM_CRC32 1 #define HAST_CHECKSUM_SHA256 2 +struct nv; + /* * Structure that describes single resource. */ @@ -254,6 +256,9 @@ struct hast_resource { /* Number of activemap flush errors. */ uint64_t hr_stat_activemap_flush_error; + /* Function to output worker specific info on control status request. */ + void (*output_status_aux)(struct nv*); + /* Next resource. */ TAILQ_ENTRY(hast_resource) hr_next; }; diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c index 83770fa..a552731 100644 --- a/sbin/hastd/primary.c +++ b/sbin/hastd/primary.c @@ -112,6 +112,7 @@ struct hio { * until some in-progress requests are freed. */ static TAILQ_HEAD(, hio) hio_free_list; +static size_t hio_free_list_size; static pthread_mutex_t hio_free_list_lock; static pthread_cond_t hio_free_list_cond; /* @@ -120,6 +121,7 @@ static pthread_cond_t hio_free_list_cond; * responsible for managing his own send list. */ static TAILQ_HEAD(, hio) *hio_send_list; +static size_t *hio_send_list_size; static pthread_mutex_t *hio_send_list_lock; static pthread_cond_t *hio_send_list_cond; /* @@ -127,6 +129,7 @@ static pthread_cond_t *hio_send_list_cond; * use recv lists as local requests are done synchronously. */ static TAILQ_HEAD(, hio) *hio_recv_list; +static size_t *hio_recv_list_size; static pthread_mutex_t *hio_recv_list_lock; static pthread_cond_t *hio_recv_list_cond; /* @@ -134,6 +137,7 @@ static pthread_cond_t *hio_recv_list_cond; * decreased hio_countdown from 1 to 0). */ static TAILQ_HEAD(, hio) hio_done_list; +static size_t hio_done_list_size; static pthread_mutex_t hio_done_list_lock; static pthread_cond_t hio_done_list_cond; /* @@ -174,6 +178,7 @@ static pthread_mutex_t metadata_lock; _wakeup = TAILQ_EMPTY(&hio_##name##_list[(ncomp)]); \ TAILQ_INSERT_TAIL(&hio_##name##_list[(ncomp)], (hio), \ hio_next[(ncomp)]); \ + hio_##name##_list_size[(ncomp)]++; \ mtx_unlock(&hio_##name##_list_lock[ncomp]); \ if (_wakeup) \ cv_signal(&hio_##name##_list_cond[(ncomp)]); \ @@ -184,6 +189,7 @@ static pthread_mutex_t metadata_lock; mtx_lock(&hio_##name##_list_lock); \ _wakeup = TAILQ_EMPTY(&hio_##name##_list); \ TAILQ_INSERT_TAIL(&hio_##name##_list, (hio), hio_##name##_next);\ + hio_##name##_list_size++; \ mtx_unlock(&hio_##name##_list_lock); \ if (_wakeup) \ cv_broadcast(&hio_##name##_list_cond); \ @@ -200,6 +206,8 @@ static pthread_mutex_t metadata_lock; _last = true; \ } \ if (hio != NULL) { \ + PJDLOG_ASSERT(hio_##name##_list_size[(ncomp)] != 0); \ + hio_##name##_list_size[(ncomp)]--; \ TAILQ_REMOVE(&hio_##name##_list[(ncomp)], (hio), \ hio_next[(ncomp)]); \ } \ @@ -211,6 +219,8 @@ static pthread_mutex_t metadata_lock; cv_wait(&hio_##name##_list_cond, \ &hio_##name##_list_lock); \ } \ + PJDLOG_ASSERT(hio_##name##_list_size != 0); \ + hio_##name##_list_size--; \ TAILQ_REMOVE(&hio_##name##_list, (hio), hio_##name##_next); \ mtx_unlock(&hio_##name##_list_lock); \ } while (0) @@ -243,6 +253,17 @@ static void *sync_thread(void *arg); static void *guard_thread(void *arg); static void +output_status_aux(struct nv *nvout) +{ + + nv_add_uint64(nvout, (uint64_t)hio_free_list_size, "idle_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_send_list_size[0], "local_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_send_list_size[1], "send_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_recv_list_size[1], "recv_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_done_list_size, "done_queue_size"); +} + +static void cleanup(struct hast_resource *res) { int rerrno; @@ -358,6 +379,12 @@ init_environment(struct hast_resource *res __unused) "Unable to allocate %zu bytes of memory for send lists.", sizeof(hio_send_list[0]) * ncomps); } + hio_send_list_size = malloc(sizeof(hio_send_list_size[0]) * ncomps); + if (hio_send_list_size == NULL) { + primary_exitx(EX_TEMPFAIL, + "Unable to allocate %zu bytes of memory for send list counters.", + sizeof(hio_send_list_size[0]) * ncomps); + } hio_send_list_lock = malloc(sizeof(hio_send_list_lock[0]) * ncomps); if (hio_send_list_lock == NULL) { primary_exitx(EX_TEMPFAIL, @@ -376,6 +403,12 @@ init_environment(struct hast_resource *res __unused) "Unable to allocate %zu bytes of memory for recv lists.", sizeof(hio_recv_list[0]) * ncomps); } + hio_recv_list_size = malloc(sizeof(hio_recv_list_size[0]) * ncomps); + if (hio_recv_list_size == NULL) { + primary_exitx(EX_TEMPFAIL, + "Unable to allocate %zu bytes of memory for recv list counters.", + sizeof(hio_recv_list_size[0]) * ncomps); + } hio_recv_list_lock = malloc(sizeof(hio_recv_list_lock[0]) * ncomps); if (hio_recv_list_lock == NULL) { primary_exitx(EX_TEMPFAIL, @@ -396,16 +429,18 @@ init_environment(struct hast_resource *res __unused) } /* - * Initialize lists, their locks and theirs condition variables. + * Initialize lists, their counters, locks and condition variables. */ TAILQ_INIT(&hio_free_list); mtx_init(&hio_free_list_lock); cv_init(&hio_free_list_cond); for (ii = 0; ii < HAST_NCOMPONENTS; ii++) { TAILQ_INIT(&hio_send_list[ii]); + hio_send_list_size[ii] = 0; mtx_init(&hio_send_list_lock[ii]); cv_init(&hio_send_list_cond[ii]); TAILQ_INIT(&hio_recv_list[ii]); + hio_recv_list_size[ii] = 0; mtx_init(&hio_recv_list_lock[ii]); cv_init(&hio_recv_list_cond[ii]); rw_init(&hio_remote_lock[ii]); @@ -448,6 +483,7 @@ init_environment(struct hast_resource *res __unused) hio->hio_ggio.gctl_length = MAXPHYS; hio->hio_ggio.gctl_error = 0; TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_free_next); + hio_free_list_size++; } } @@ -966,6 +1002,7 @@ hastd_primary(struct hast_resource *res) } gres = res; + res->output_status_aux = output_status_aux; mode = pjdlog_mode_get(); debuglevel = pjdlog_debug_get(); @@ -1621,6 +1658,7 @@ remote_send_thread(void *arg) mtx_lock(&hio_recv_list_lock[ncomp]); wakeup = TAILQ_EMPTY(&hio_recv_list[ncomp]); TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]++; mtx_unlock(&hio_recv_list_lock[ncomp]); if (hast_proto_send(res, res->hr_remoteout, nv, data, data != NULL ? length : 0) == -1) { @@ -1639,6 +1677,7 @@ remote_send_thread(void *arg) mtx_lock(&hio_recv_list_lock[ncomp]); TAILQ_REMOVE(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]--; mtx_unlock(&hio_recv_list_lock[ncomp]); goto done_queue; } @@ -1723,6 +1762,7 @@ remote_recv_thread(void *arg) PJDLOG_ASSERT(hio != NULL); TAILQ_REMOVE(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]--; mtx_unlock(&hio_recv_list_lock[ncomp]); hio->hio_errors[ncomp] = ENOTCONN; goto done_queue; @@ -1747,6 +1787,7 @@ remote_recv_thread(void *arg) if (hio->hio_ggio.gctl_seq == seq) { TAILQ_REMOVE(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]--; break; } } @@ -1850,6 +1891,7 @@ done_queue: mtx_lock(&hio_recv_list_lock[ncomp]); TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]++; mtx_unlock(&hio_recv_list_lock[ncomp]); } else { /* @@ -1878,6 +1920,7 @@ done_queue: mtx_lock(&hio_recv_list_lock[ncomp]); TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio, hio_next[ncomp]); + hio_recv_list_size[ncomp]++; mtx_unlock(&hio_recv_list_lock[ncomp]); continue; default: diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c index d659367..4d6f965 100644 --- a/sbin/hastd/secondary.c +++ b/sbin/hastd/secondary.c @@ -82,18 +82,21 @@ static struct hast_resource *gres; * until some in-progress requests are freed. */ static TAILQ_HEAD(, hio) hio_free_list; +static size_t hio_free_list_size; static pthread_mutex_t hio_free_list_lock; static pthread_cond_t hio_free_list_cond; /* * Disk thread (the one that does I/O requests) takes requests from this list. */ static TAILQ_HEAD(, hio) hio_disk_list; +static size_t hio_disk_list_size; static pthread_mutex_t hio_disk_list_lock; static pthread_cond_t hio_disk_list_cond; /* * Thread that sends requests back to primary takes requests from this list. */ static TAILQ_HEAD(, hio) hio_send_list; +static size_t hio_send_list_size; static pthread_mutex_t hio_send_list_lock; static pthread_cond_t hio_send_list_cond; @@ -112,6 +115,7 @@ static void *send_thread(void *arg); mtx_lock(&hio_##name##_list_lock); \ _wakeup = TAILQ_EMPTY(&hio_##name##_list); \ TAILQ_INSERT_TAIL(&hio_##name##_list, (hio), hio_next); \ + hio_##name##_list_size++; \ mtx_unlock(&hio_##name##_list_lock); \ if (_wakeup) \ cv_signal(&hio_##name##_list_cond); \ @@ -122,11 +126,22 @@ static void *send_thread(void *arg); cv_wait(&hio_##name##_list_cond, \ &hio_##name##_list_lock); \ } \ + PJDLOG_ASSERT(hio_##name##_list_size != 0); \ + hio_##name##_list_size--; \ TAILQ_REMOVE(&hio_##name##_list, (hio), hio_next); \ mtx_unlock(&hio_##name##_list_lock); \ } while (0) static void +output_status_aux(struct nv *nvout) +{ + + nv_add_uint64(nvout, (uint64_t)hio_free_list_size, "idle_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_disk_list_size, "local_queue_size"); + nv_add_uint64(nvout, (uint64_t)hio_send_list_size, "send_queue_size"); +} + +static void hio_clear(struct hio *hio) { @@ -190,6 +205,7 @@ init_environment(void) } hio_clear(hio); TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next); + hio_free_list_size++; } } @@ -441,6 +457,7 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin) } gres = res; + res->output_status_aux = output_status_aux; mode = pjdlog_mode_get(); debuglevel = pjdlog_debug_get();