Index: include/netdb.h =================================================================== --- include/netdb.h (revision 253898) +++ include/netdb.h (working copy) @@ -83,6 +83,7 @@ #define _PATH_PROTOCOLS "/etc/protocols" #define _PATH_SERVICES "/etc/services" #define _PATH_SERVICES_DB "/var/db/services.db" +#define _PATH_SERVICES_CDB "/var/db/services.cdb" #define h_errno (*__h_errno()) Index: lib/libc/gen/Makefile.inc =================================================================== --- lib/libc/gen/Makefile.inc (revision 253898) +++ lib/libc/gen/Makefile.inc (working copy) @@ -1,6 +1,10 @@ # @(#)Makefile.inc 8.6 (Berkeley) 5/4/95 # $FreeBSD$ +.PATH: ${.CURDIR}/../../contrib/libcdbrw +SRCS+= cdbr.c mi_vector_hash.c +CFLAGS+= -I${.CURDIR}/../../contrib/libcdbrw + # machine-independent gen sources .PATH: ${.CURDIR}/${LIBC_ARCH}/gen ${.CURDIR}/gen Index: lib/libc/net/getservent.c =================================================================== --- lib/libc/net/getservent.c (revision 253898) +++ lib/libc/net/getservent.c (working copy) @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include #include @@ -100,6 +102,7 @@ struct db_state { DB *db; + struct cdbr *cdb; int stayopen; int keynum; }; @@ -223,6 +226,78 @@ } static int +cdb_parse_result(struct servent *s, char *buffer, size_t bufsize, + const uint8_t *data, size_t len, int *errnop, struct db_state *st) +{ + size_t i; + char **aliases; + size_t maxaliases; + + if (len > bufsize) { + *errnop = ERANGE; + return (NS_RETURN); + } + + memcpy(buffer, data, len); + data = buffer; + + if (len < 2) + return (NS_NOTFOUND); + + s->s_port = htobe16(be16dec(data)); + data += 2; + len -= 2; + + if (len == 0 || len < (size_t)data[0] + 2) + return (NS_NOTFOUND); + + s->s_proto = __DECONST(uint8_t *, data + 1); + + if (s->s_proto[data[0]] != '\0') + return (NS_NOTFOUND); + + len -= 2 + data[0]; + data += 2 + data[0]; + + if (len == 0) + return (NS_NOTFOUND); + if (len < (size_t)data[0] + 2) + return (NS_NOTFOUND); + + s->s_name = __DECONST(uint8_t *, data + 1); + len -= 2 + data[0]; + data += 2 + data[0]; + + maxaliases = 10; + aliases = malloc(maxaliases * sizeof(char *)); + if (aliases == NULL) + return (NS_NOTFOUND); + + s->s_aliases = aliases; + i = 0; + while (len) { + if (len < (size_t)data[0] + 2) { + free(aliases); + return (NS_NOTFOUND); + } + if (i == maxaliases - 2) { + maxaliases *= 2; + aliases = realloc(aliases, maxaliases * sizeof(char *)); + if (aliases == NULL) + return (NS_NOTFOUND); + s->s_aliases = aliases; + } + + s->s_aliases[i++] = __DECONST(uint8_t *, data + 1); + len -= 2 + data[0]; + data += 2 + data[0]; + } + s->s_aliases[i] = NULL; + + return (NS_SUCCESS); +} + +static int parse_result(struct servent *serv, char *buffer, size_t bufsize, char *resultbuf, size_t resultbuflen, int *errnop) { @@ -489,9 +564,12 @@ db_servent(void *retval, void *mdata, va_list ap) { char buf[BUFSIZ]; - DBT key, data, *result; + DBT key, data; DB *db; + const void *rawdata; + size_t datalen; + struct db_state *st; int rv; int stayopen; @@ -506,6 +584,10 @@ size_t bufsize; int *errnop; + uint8_t cdbkey[BUFSIZ]; + size_t namelen, protolen; + const uint8_t *cdbdata, *cdbdata_end; + name = NULL; proto = NULL; how = (enum nss_lookup_type)mdata; @@ -536,12 +618,12 @@ if (how == nss_lt_all && st->keynum < 0) return (NS_NOTFOUND); - if (st->db == NULL) { - st->db = dbopen(_PATH_SERVICES_DB, O_RDONLY, 0, DB_HASH, NULL); - if (st->db == NULL) { - *errnop = errno; - return (NS_UNAVAIL); - } + if (st->cdb == NULL && + (st->cdb = cdbr_open(_PATH_SERVICES_CDB, CDBR_DEFAULT)) == NULL && + st->db == NULL && + (st->db = dbopen(_PATH_SERVICES_DB, O_RDONLY, 0, DB_HASH, NULL)) == NULL) { + *errnop = errno; + return (NS_UNAVAIL); } stayopen = (how == nss_lt_all) ? 1 : st->stayopen; @@ -550,54 +632,195 @@ do { switch (how) { case nss_lt_name: - key.data = buf; - if (proto == NULL) - key.size = snprintf(buf, sizeof(buf), - "\376%s", name); - else - key.size = snprintf(buf, sizeof(buf), - "\376%s/%s", name, proto); - key.size++; - if (db->get(db, &key, &data, 0) != 0 || - db->get(db, &data, &key, 0) != 0) { - rv = NS_NOTFOUND; - goto db_fin; + if (st->cdb == NULL) { + key.data = buf; + if (proto == NULL) + key.size = snprintf(buf, sizeof(buf), + "\376%s", name); + else + key.size = snprintf(buf, sizeof(buf), + "\376%s/%s", name, proto); + key.size++; + if (db->get(db, &key, &data, 0) != 0 || + db->get(db, &data, &key, 0) != 0) { + rv = NS_NOTFOUND; + goto db_fin; + } + rawdata = key.data; + datalen = key.size - 1; + } else { + protolen = 0; + + namelen = strlen(name); + if (namelen == 0 || namelen > 255) { + rv = NS_NOTFOUND; + goto db_fin; + } + + if (proto != NULL) { + protolen = strlen(proto); + if (protolen == 0 || protolen > 255) { + rv = NS_NOTFOUND; + goto db_fin; + } + } + + if (namelen + protolen > 255) { + rv = NS_NOTFOUND; + goto db_fin; + } + + cdbkey[0] = (uint8_t)namelen; + cdbkey[1] = (uint8_t)protolen; + + memcpy(cdbkey + 2, name, namelen); + memcpy(cdbkey + 2 + namelen, proto, protolen); + + if (cdbr_find(st->cdb, cdbkey, 2 + namelen + protolen, + &rawdata, &datalen)) { + rv = NS_NOTFOUND; + goto db_fin; + } + + cdbdata = rawdata; + cdbdata_end = cdbdata + datalen; + if (protolen) { + if (cdbdata[2] != protolen) { + rv = NS_NOTFOUND; + goto db_fin; + } + if (memcmp(cdbdata + 3, proto, protolen + 1)) { + rv = NS_NOTFOUND; + goto db_fin; + } + } + + cdbdata += 3 + cdbdata[2] + 1; + if (cdbdata > cdbdata_end) { + rv = NS_NOTFOUND; + goto db_fin; + } + + while (cdbdata != cdbdata_end) { + if (*cdbdata == '\0') { + rv = NS_NOTFOUND; + goto db_fin; + } + if (cdbdata + cdbdata[0] + 2 > cdbdata_end) { + rv = NS_NOTFOUND; + goto db_fin; + } + if (cdbdata[0] == namelen && + memcmp(cdbdata + 1, name, namelen + 1) == 0) { + break; + } + cdbdata += cdbdata[0] + 2; + } + if (cdbdata == cdbdata_end) { + rv = NS_NOTFOUND; + goto db_fin; + } } - result = &key; break; case nss_lt_id: - key.data = buf; - port = htons(port); - if (proto == NULL) - key.size = snprintf(buf, sizeof(buf), - "\377%d", port); - else - key.size = snprintf(buf, sizeof(buf), - "\377%d/%s", port, proto); - key.size++; - if (db->get(db, &key, &data, 0) != 0 || - db->get(db, &data, &key, 0) != 0) { - rv = NS_NOTFOUND; - goto db_fin; + if (st->cdb == NULL) { + key.data = buf; + port = htons(port); + if (proto == NULL) + key.size = snprintf(buf, sizeof(buf), "\377%d", port); + else + key.size = snprintf(buf, sizeof(buf), "\377%d/%s", port, proto); + key.size++; + if (db->get(db, &key, &data, 0) != 0 || + db->get(db, &data, &key, 0) != 0) { + rv = NS_NOTFOUND; + goto db_fin; + } + rawdata = key.data; + datalen = key.size - 1; + } else { + port = be16toh(port); + + protolen = 0; + + if (proto != NULL) { + protolen = strlen(proto); + if (protolen == 0 || protolen > 255) { + + rv = NS_NOTFOUND; + goto db_fin; + } + } + + if (port < 0 || port > 65536) { + rv = NS_NOTFOUND; + goto db_fin; + } + + cdbkey[0] = 0; + cdbkey[1] = (uint8_t)protolen; + + be16enc(cdbkey + 2, port); + memcpy(cdbkey + 4, proto, protolen); + + if (cdbr_find(st->cdb, cdbkey, 4 + protolen, &rawdata, &datalen)) { + + rv = NS_NOTFOUND; + goto db_fin; + } + + if (datalen < protolen + 4) { + rv = NS_NOTFOUND; + goto db_fin; + } + + cdbdata = rawdata; + if (be16dec(cdbdata) != port) { + rv = NS_NOTFOUND; + goto db_fin; + } + + if (protolen) { + if (cdbdata[2] != protolen) { + rv = NS_NOTFOUND; + goto db_fin; + } + if (memcmp(cdbdata + 3, proto, protolen + 1)) { + rv = NS_NOTFOUND; + goto db_fin; + } + } } - result = &key; break; case nss_lt_all: - key.data = buf; - key.size = snprintf(buf, sizeof(buf), "%d", - st->keynum++); - key.size++; - if (db->get(db, &key, &data, 0) != 0) { - st->keynum = -1; - rv = NS_NOTFOUND; - goto db_fin; + if (st->cdb == NULL) { + key.data = buf; + key.size = snprintf(buf, sizeof(buf), "%d", + st->keynum++); + key.size++; + if (db->get(db, &key, &data, 0) != 0) { + st->keynum = -1; + rv = NS_NOTFOUND; + goto db_fin; + } + rawdata = data.data; + datalen = data.size - 1; + } else { + if (cdbr_get(st->cdb, st->keynum++, &rawdata, &datalen)) { + st->keynum = -1; + rv = NS_NOTFOUND; + goto db_fin; + } } - result = &data; break; } - rv = parse_result(serv, buffer, bufsize, result->data, - result->size - 1, errnop); + if (st->cdb == NULL) + rv = parse_result(serv, buffer, bufsize, (char *)rawdata, + datalen, errnop); + else + rv = cdb_parse_result(serv, buffer, bufsize, rawdata, + datalen, errnop, st); } while (!(rv & NS_TERMINATE) && how == nss_lt_all); @@ -607,6 +830,11 @@ st->db = NULL; } + if (!stayopen && st->cdb != NULL) { + cdbr_close(st->cdb); + st->cdb = NULL; + } + if (rv == NS_SUCCESS && retval != NULL) *(struct servent **)retval = serv; @@ -617,6 +845,7 @@ db_setservent(void *retval, void *mdata, va_list ap) { DB *db; + struct cdbr *cdb; struct db_state *st; int rv; int f; @@ -633,10 +862,15 @@ break; case ENDSERVENT: db = st->db; + cdb = st->cdb; if (db != NULL) { db->close(db); st->db = NULL; } + if (cdb != NULL) { + cdbr_close(cdb); + st->cdb = NULL; + } st->stayopen = 0; break; default: Index: usr.sbin/services_mkdb/Makefile =================================================================== --- usr.sbin/services_mkdb/Makefile (revision 253898) +++ usr.sbin/services_mkdb/Makefile (working copy) @@ -2,9 +2,9 @@ PROG= services_mkdb MAN= services_mkdb.8 -SRCS= services_mkdb.c uniq.c extern.h +SRCS= services_mkdb.c uniq.c extern.h output_cdb.c output_db.c -DPADD+= ${LIBUTIL} -LDADD+= -lutil +DPADD+= ${LIBUTIL} ${LIBCDBRW} +LDADD+= -lutil -lcdbrw .include Index: usr.sbin/services_mkdb/extern.h =================================================================== --- usr.sbin/services_mkdb/extern.h (revision 253898) +++ usr.sbin/services_mkdb/extern.h (working copy) @@ -29,6 +29,12 @@ * $FreeBSD$ */ -extern const HASHINFO hinfo; +#include -void uniq(const char *); +int cdb_open(const char *); +void cdb_add(StringList *, size_t, const char *, size_t *, int); +int cdb_close(void); +int db_open(const char *); +void db_add(StringList *, size_t, const char *, size_t *, int); +int db_close(void); +void uniq(const char *); Index: usr.sbin/services_mkdb/Makefile =================================================================== --- usr.sbin/services_mkdb/Makefile (revision 253898) +++ usr.sbin/services_mkdb/Makefile (working copy) @@ -2,9 +2,9 @@ PROG= services_mkdb MAN= services_mkdb.8 -SRCS= services_mkdb.c uniq.c extern.h +SRCS= services_mkdb.c uniq.c extern.h output_cdb.c output_db.c -DPADD+= ${LIBUTIL} -LDADD+= -lutil +DPADD+= ${LIBUTIL} ${LIBCDBRW} +LDADD+= -lutil -lcdbrw .include Index: usr.sbin/services_mkdb/extern.h =================================================================== --- usr.sbin/services_mkdb/extern.h (revision 253898) +++ usr.sbin/services_mkdb/extern.h (working copy) @@ -29,6 +29,12 @@ * $FreeBSD$ */ -extern const HASHINFO hinfo; +#include -void uniq(const char *); +int cdb_open(const char *); +void cdb_add(StringList *, size_t, const char *, size_t *, int); +int cdb_close(void); +int db_open(const char *); +void db_add(StringList *, size_t, const char *, size_t *, int); +int db_close(void); +void uniq(const char *); Index: usr.sbin/services_mkdb/output_cdb.c =================================================================== --- usr.sbin/services_mkdb/output_cdb.c (revision 0) +++ usr.sbin/services_mkdb/output_cdb.c (working copy) @@ -0,0 +1,166 @@ +/* $NetBSD: output_cdb.c,v 1.1 2010/04/25 00:54:46 joerg Exp $ */ + +/*- + * Copyright (c) 2010 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "extern.h" + +static struct cdbw *cdbw; +static int cdbw_fd = -1; + +int +cdb_open(const char *tname) +{ + + if ((cdbw = cdbw_open()) == NULL) + return -1; + + if ((cdbw_fd = open(tname, O_WRONLY | O_CREAT | O_TRUNC, 0666)) == -1) { + cdbw_close(cdbw); + cdbw = NULL; + return -1; + } + return 0; +} + +void +cdb_add(StringList *sl, size_t port, const char *proto, size_t *cnt __unused, + int warndup) +{ + uint8_t key[255 * 2 + 2]; + uint8_t *data, *data_iter; + size_t len, protolen, datalen, keylen; + uint32_t idx; + size_t i; + + protolen = strlen(proto); + if (protolen == 0 || protolen > 255) + errx(1, "Invalid protocol ``%s'', entry skipped", proto); + + datalen = 4 + protolen; + for (i = 0; i < sl->sl_cur; ++i) { + len = strlen(sl->sl_str[i]); + if (len == 0 || len > 255) + errx(1, "Service alias ``%s'' invalid", sl->sl_str[i]); + datalen += len + 2; + } + + data = malloc(datalen); + if (data == NULL) + err(1, "malloc failed"); + be16enc(data, port); + data[2] = protolen; + data_iter = data + 3; + memcpy(data_iter, proto, protolen + 1); + data_iter += protolen + 1; + for (i = 0; i < sl->sl_cur; ++i) { + len = strlen(sl->sl_str[i]); + *data_iter++ = len; + memcpy(data_iter, sl->sl_str[i], len + 1); + data_iter += len + 1; + } + + if (cdbw_put_data(cdbw, data, datalen, &idx)) + err(1, "cdbw_put_data failed"); + + free(data); + + key[0] = 0; + key[1] = protolen; + be16enc(key + 2, port); + memcpy(key + 4, proto, protolen); + keylen = 4 + protolen; + if (cdbw_put_key(cdbw, key, keylen, idx) && warndup) + warnx("duplicate service: `%zu/%s'", port, proto); + + key[1] = 0; + keylen = 4; + if (cdbw_put_key(cdbw, key, keylen, idx) && warndup) + warnx("duplicate service: `%zu'", port); + + /* add references for service and all aliases */ + for (i = 0; i < sl->sl_cur; i++) { + len = strlen(sl->sl_str[i]); + key[0] = len; + key[1] = protolen; + memcpy(key + 2, sl->sl_str[i], len); + memcpy(key + 2 + len, proto, protolen); + keylen = 2 + len + protolen; + if (cdbw_put_key(cdbw, key, keylen, idx) && warndup) + warnx("duplicate service: `%s/%s'", sl->sl_str[i], proto); + + key[1] = 0; + keylen = 2 + len; + if (cdbw_put_key(cdbw, key, keylen, idx) && warndup) + warnx("duplicate service: `%s'", sl->sl_str[i]); + } + + sl_free(sl, 1); +} + +int +cdb_close(void) +{ + int rv, serrno; + + rv = 0; + serrno = errno; + + if (cdbw_output(cdbw, cdbw_fd, "services(5)", NULL)) { + rv = -1; + serrno = errno; + } + + cdbw_close(cdbw); + cdbw = NULL; + + if (close(cdbw_fd)) { + if (rv == 0) + serrno = errno; + rv = -1; + } + cdbw_fd = -1; + + errno = serrno; + return rv; +} Index: usr.sbin/services_mkdb/output_db.c =================================================================== --- usr.sbin/services_mkdb/output_db.c (revision 0) +++ usr.sbin/services_mkdb/output_db.c (working copy) @@ -0,0 +1,188 @@ +/* $NetBSD: output_db.c,v 1.1 2010/04/25 00:54:46 joerg Exp $ */ + +/*- + * Copyright (c) 1999 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Luke Mewburn and Christos Zoulas. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "extern.h" + +static DB *db; + +static const HASHINFO hinfo = { + .bsize = 256, + .ffactor = 4, + .nelem = 32768, + .cachesize = 1024, + .hash = NULL, + .lorder = 0 +}; + +static void store(DBT *, DBT *, int); +static void killproto(DBT *); +static const char *mkaliases(StringList *, char *, size_t); + +int +db_open(const char *tname) +{ + db = dbopen(tname, O_RDWR | O_CREAT | O_EXCL, + (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH), DB_HASH, &hinfo); + + return db != NULL ? 0 : -1; +} + +int +db_close(void) +{ + int rv; + + rv = (db->close)(db); + db = NULL; + + return rv; +} + +void +db_add(StringList *sl, size_t port, const char *proto, size_t *cnt, + int warndup) +{ + size_t i; + char keyb[BUFSIZ], datab[BUFSIZ], abuf[BUFSIZ]; + DBT data, key; + key.data = keyb; + data.data = datab; + + /* key `indirect key', data `full line' */ + data.size = snprintf(datab, sizeof(datab), "%zu", (*cnt)++) + 1; + key.size = snprintf(keyb, sizeof(keyb), "%s %zu/%s %s", + sl->sl_str[0], port, proto, mkaliases(sl, abuf, sizeof(abuf))) + 1; + store(&data, &key, warndup); + + /* key `\377port/proto', data = `indirect key' */ + key.size = snprintf(keyb, sizeof(keyb), "\377%zu/%s", + port, proto) + 1; + store(&key, &data, warndup); + + /* key `\377port', data = `indirect key' */ + killproto(&key); + store(&key, &data, warndup); + + /* add references for service and all aliases */ + for (i = 0; i < sl->sl_cur; i++) { + /* key `\376service/proto', data = `indirect key' */ + key.size = snprintf(keyb, sizeof(keyb), "\376%s/%s", + sl->sl_str[i], proto) + 1; + store(&key, &data, warndup); + + /* key `\376service', data = `indirect key' */ + killproto(&key); + store(&key, &data, warndup); + } + sl_free(sl, 1); +} + +static void +killproto(DBT *key) +{ + char *p, *d = key->data; + + if ((p = strchr(d, '/')) == NULL) + abort(); + *p++ = '\0'; + key->size = p - d; +} + +static void +store(DBT *key, DBT *data, int warndup) +{ +#ifdef DEBUG + int k = key->size - 1; + int d = data->size - 1; + (void)printf("store [%*.*s] [%*.*s]\n", + k, k, (char *)key->data + 1, + d, d, (char *)data->data + 1); +#endif + switch ((db->put)(db, key, data, R_NOOVERWRITE)) { + case 0: + break; + case 1: + if (warndup) + warnx("duplicate service `%s'", + &((char *)key->data)[1]); + break; + case -1: + err(1, "put"); + break; + default: + abort(); + break; + } +} + +static const char * +mkaliases(StringList *sl, char *buf, size_t len) +{ + size_t nc, i, pos; + + buf[0] = 0; + for (i = 1, pos = 0; i < sl->sl_cur; i++) { + nc = strlcpy(buf + pos, sl->sl_str[i], len); + if (nc >= len) + goto out; + pos += nc; + len -= nc; + nc = strlcpy(buf + pos, " ", len); + if (nc >= len) + goto out; + pos += nc; + len -= nc; + } + return buf; +out: + warn("aliases for `%s' truncated", sl->sl_str[0]); + return buf; +} Index: usr.sbin/services_mkdb/services_mkdb.c =================================================================== --- usr.sbin/services_mkdb/services_mkdb.c (revision 253898) +++ usr.sbin/services_mkdb/services_mkdb.c (working copy) @@ -1,4 +1,4 @@ -/* $NetBSD: services_mkdb.c,v 1.14 2008/04/28 20:24:17 martin Exp $ */ +/* $NetBSD: services_mkdb.c,v 1.18 2010/10/07 01:28:50 christos Exp $ */ /*- * Copyright (c) 1999 The NetBSD Foundation, Inc. @@ -33,10 +33,8 @@ __FBSDID("$FreeBSD$"); #include -#include #include -#include #include #include #include @@ -54,59 +52,58 @@ static char tname[MAXPATHLEN]; #define PMASK 0xffff -#define PROTOMAX 5 +#define PROTOMAX 6 -static void add(DB *, StringList *, size_t, const char *, size_t *, int); static StringList ***parseservices(const char *, StringList *); static void cleanup(void); -static void store(DB *, DBT *, DBT *, int); -static void killproto(DBT *); static char *getstring(const char *, size_t, char **, const char *); static size_t getprotoindex(StringList *, const char *); static const char *getprotostr(StringList *, size_t); -static const char *mkaliases(StringList *, char *, size_t); static void usage(void); -const HASHINFO hinfo = { - .bsize = 256, - .ffactor = 4, - .nelem = 32768, - .cachesize = 1024, - .hash = NULL, - .lorder = 0 -}; - - int main(int argc, char *argv[]) { - DB *db; int ch; const char *fname = _PATH_SERVICES; - const char *dbname = _PATH_SERVICES_DB; - int warndup = 1; + const char *dbname = NULL; + int use_db = 0; + int warndup = 0; int unique = 0; int otherflag = 0; size_t cnt = 0; StringList *sl, ***svc; size_t port, proto; + void (*addfn)(StringList *, size_t, const char *, size_t *, int); + int (*closefn)(void); setprogname(argv[0]); - while ((ch = getopt(argc, argv, "qo:u")) != -1) + while ((ch = getopt(argc, argv, "o:quV:v")) != -1) switch (ch) { + case 'o': + otherflag = 1; + dbname = optarg; + break; case 'q': otherflag = 1; warndup = 0; break; - case 'o': - otherflag = 1; - dbname = optarg; - break; case 'u': unique++; break; - case '?': + case 'V': + if (strcmp(optarg, "db") == 0) + use_db = 1; + else if (strcmp(optarg, "cdb") == 0) + use_db = 0; + else + usage(); + break; + case 'v': + otherflag = 1; + warndup = 1; + break; default: usage(); } @@ -122,6 +119,9 @@ if (unique) uniq(fname); + if (dbname == NULL) + dbname = use_db ? _PATH_SERVICES_DB : _PATH_SERVICES_CDB; + svc = parseservices(fname, sl = sl_init()); if (atexit(cleanup)) @@ -128,11 +128,18 @@ err(1, "Cannot install exit handler"); (void)snprintf(tname, sizeof(tname), "%s.tmp", dbname); - db = dbopen(tname, O_RDWR | O_CREAT | O_EXCL, - (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH), DB_HASH, &hinfo); - if (!db) - err(1, "Error opening temporary database `%s'", tname); + if (use_db) { + if (db_open(tname)) + err(1, "Error opening temporary database `%s'", tname); + addfn = db_add; + closefn = db_close; + } else { + if (cdb_open(tname)) + err(1, "Error opening temporary database `%s'", tname); + addfn = cdb_add; + closefn = cdb_close; + } for (port = 0; port < PMASK + 1; port++) { if (svc[port] == NULL) @@ -142,7 +149,7 @@ StringList *s; if ((s = svc[port][proto]) == NULL) continue; - add(db, s, port, getprotostr(sl, proto), &cnt, warndup); + (addfn)(s, port, getprotostr(sl, proto), &cnt, warndup); } free(svc[port]); @@ -151,8 +158,8 @@ free(svc); sl_free(sl, 1); - if ((db->close)(db)) - err(1, "Error closing temporary database `%s'", tname); + if ((closefn)()) + err(1, "Error writing temporary database `%s'", tname); if (rename(tname, dbname) == -1) err(1, "Cannot rename `%s' to `%s'", tname, dbname); @@ -160,52 +167,6 @@ return 0; } -static void -add(DB *db, StringList *sl, size_t port, const char *proto, size_t *cnt, - int warndup) -{ - size_t i; - char keyb[BUFSIZ], datab[BUFSIZ], abuf[BUFSIZ]; - DBT data, key; - key.data = keyb; - data.data = datab; - -#ifdef DEBUG - (void)printf("add %s %zu %s [ ", sl->sl_str[0], port, proto); - for (i = 1; i < sl->sl_cur; i++) - (void)printf("%s ", sl->sl_str[i]); - (void)printf("]\n"); -#endif - - /* key `indirect key', data `full line' */ - data.size = snprintf(datab, sizeof(datab), "%zu", (*cnt)++) + 1; - key.size = snprintf(keyb, sizeof(keyb), "%s %zu/%s %s", - sl->sl_str[0], port, proto, mkaliases(sl, abuf, sizeof(abuf))) + 1; - store(db, &data, &key, warndup); - - /* key `\377port/proto', data = `indirect key' */ - key.size = snprintf(keyb, sizeof(keyb), "\377%zu/%s", - port, proto) + 1; - store(db, &key, &data, warndup); - - /* key `\377port', data = `indirect key' */ - killproto(&key); - store(db, &key, &data, warndup); - - /* add references for service and all aliases */ - for (i = 0; i < sl->sl_cur; i++) { - /* key `\376service/proto', data = `indirect key' */ - key.size = snprintf(keyb, sizeof(keyb), "\376%s/%s", - sl->sl_str[i], proto) + 1; - store(db, &key, &data, warndup); - - /* key `\376service', data = `indirect key' */ - killproto(&key); - store(db, &key, &data, warndup); - } - sl_free(sl, 1); -} - static StringList *** parseservices(const char *fname, StringList *sl) { @@ -280,27 +241,28 @@ s = svc[pnum][pindex] = sl_init(); else s = svc[pnum][pindex]; - + + if (strlen(name) > 255) { + warnx("%s, %zu: invalid name too long `%s'", fname, + line, name); + continue; + } + /* build list of aliases */ - if (sl_find(s, name) == NULL) { - char *p2; + if (sl_find(s, name) == NULL) + (void)sl_add(s, strdup(name)); - if ((p2 = strdup(name)) == NULL) - err(1, "Cannot copy string"); - (void)sl_add(s, p2); - } - if (aliases) { while ((alias = strsep(&aliases, " \t")) != NULL) { if (alias[0] == '\0') continue; - if (sl_find(s, alias) == NULL) { - char *p2; - - if ((p2 = strdup(alias)) == NULL) - err(1, "Cannot copy string"); - (void)sl_add(s, p2); + if (strlen(alias) > 255) { + warnx("%s, %zu: alias name too long `%s'", + fname, line, alias); + continue; } + if (sl_find(s, alias) == NULL) + (void)sl_add(s, strdup(alias)); } } } @@ -332,49 +294,10 @@ return str; } -static void -killproto(DBT *key) -{ - char *p, *d = key->data; - - if ((p = strchr(d, '/')) == NULL) - abort(); - *p++ = '\0'; - key->size = p - d; -} - -static void -store(DB *db, DBT *key, DBT *data, int warndup) -{ -#ifdef DEBUG - int k = key->size - 1; - int d = data->size - 1; - (void)printf("store [%*.*s] [%*.*s]\n", - k, k, (char *)key->data + 1, - d, d, (char *)data->data + 1); -#endif - switch ((db->put)(db, key, data, R_NOOVERWRITE)) { - case 0: - break; - case 1: - if (warndup) - warnx("duplicate service `%s'", - &((char *)key->data)[1]); - break; - case -1: - err(1, "put"); - break; - default: - abort(); - break; - } -} - static size_t getprotoindex(StringList *sl, const char *str) { size_t i; - char *p; for (i= 0; i < sl->sl_cur; i++) if (strcmp(sl->sl_str[i], str) == 0) @@ -383,9 +306,7 @@ if (i == PROTOMAX) errx(1, "Ran out of protocols adding `%s';" " recompile with larger PROTOMAX", str); - if ((p = strdup(str)) == NULL) - err(1, "Cannot copy string"); - (void)sl_add(sl, p); + (void)sl_add(sl, strdup(str)); return i; } @@ -396,34 +317,10 @@ return sl->sl_str[i]; } -static const char * -mkaliases(StringList *sl, char *buf, size_t len) -{ - size_t nc, i, pos; - - buf[0] = 0; - for (i = 1, pos = 0; i < sl->sl_cur; i++) { - nc = strlcpy(buf + pos, sl->sl_str[i], len); - if (nc >= len) - goto out; - pos += nc; - len -= nc; - nc = strlcpy(buf + pos, " ", len); - if (nc >= len) - goto out; - pos += nc; - len -= nc; - } - return buf; -out: - warn("aliases for `%s' truncated", sl->sl_str[0]); - return buf; -} - static void usage(void) { - (void)fprintf(stderr, "Usage:\t%s [-q] [-o ] []\n" + (void)fprintf(stderr, "Usage:\t%s [-q] [-o ] [-V cdb|db] []\n" "\t%s -u []\n", getprogname(), getprogname()); exit(1); } Index: usr.sbin/services_mkdb/uniq.c =================================================================== --- usr.sbin/services_mkdb/uniq.c (revision 253898) +++ usr.sbin/services_mkdb/uniq.c (working copy) @@ -1,4 +1,4 @@ -/* $NetBSD: uniq.c,v 1.4 2008/04/28 20:24:17 martin Exp $ */ +/* $NetBSD: uniq.c,v 1.5 2010/04/25 00:54:46 joerg Exp $ */ /*- * Copyright (c) 2007 The NetBSD Foundation, Inc. @@ -42,6 +42,15 @@ #include "extern.h" +static const HASHINFO hinfo = { + .bsize = 256, + .ffactor = 4, + .nelem = 32768, + .cachesize = 1024, + .hash = NULL, + .lorder = 0 +}; + static int comp(const char *, char **, size_t *); /* Index: contrib/libcdbrw/cdb.5 =================================================================== --- contrib/libcdbrw/cdb.5 (revision 0) +++ contrib/libcdbrw/cdb.5 (working copy) @@ -0,0 +1,104 @@ +.\" $NetBSD: cdb.5,v 1.4 2013/07/20 21:39:56 wiz Exp $ +.\" +.\" Copyright (c) 2010 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" This code is derived from software contributed to The NetBSD Foundation +.\" by Joerg Sonnenberger. +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in +.\" the documentation and/or other materials provided with the +.\" distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, +.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.Dd April 27, 2010 +.Dt CDB 5 +.Os +.Sh NAME +.Nm cdb +.Nd format of the constant database +.Sh DESCRIPTION +The +.Nm +database format provides a space-efficient (key,value) database. +The format doesn't allow updates in any convenient form. +The file overhead is around 5 bytes per key and 5 bytes per entry. +Keys are not stored and it is the responsibility of the caller +to validate matches. +The index structure is based on a minimal perfect hash table, so exactly +one entry has to be checked for a match. +.Ss General Format +The header record of a +.Nm +database consists of the following: +.Bd -literal -offset indent +struct header_cdb { + uint8_t magic[7]; + uint8_t version; + uint8_t description[16]; + uint32_t data_size; + uint32_t entries; + uint32_t entries_index; + uint32_t seed; +}; +.Ed +.Pp +All fields are in Little Endian byte order. +.Pp +This is followed by a description of the hash function of +.Va entries_index +records. +The size of each index entry is the logarithm of +.Va entries +to base 256, rounded up. +.Pp +The index records are followed by the start offsets of the entries, +followed by +.Va data_size . +The offsets are relative to the end of the offset record table and are +monotically increasing. +The size of each offset record is the logarithm of +.Va data_size +to base 256, rounded up. +.Pp +The offset table is followed by the entries in order. +No separation or padding is added. +.Ss Limitations +The +.Nm +file format is by design intended for a database that can be +mapped into memory. +The hard limit for the number of entries and keys is 3435973836. +The total size of all values must be smaller than 4GiB. +.Sh SEE ALSO +.Xr cdbr 3 , +.Xr cdbw 3 +.Sh HISTORY +Support for the +.Nm cdb +format first appeared in +.Nx 6.0 . +.Sh AUTHORS +The +.Nm cdbr +and +.Nm cdbw +functions have been written by +.An Joerg Sonnenberger Aq Mt joerg@NetBSD.org . Index: contrib/libcdbrw/cdbr.3 =================================================================== --- contrib/libcdbrw/cdbr.3 (revision 0) +++ contrib/libcdbrw/cdbr.3 (working copy) @@ -0,0 +1,119 @@ +.\" $NetBSD: cdbr.3,v 1.3 2013/07/20 21:39:56 wiz Exp $ +.\" +.\" Copyright (c) 2010 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" This code is derived from software contributed to The NetBSD Foundation +.\" by Joerg Sonnenberger. +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in +.\" the documentation and/or other materials provided with the +.\" distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, +.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.Dd March 3, 2010 +.Dt CDBR 3 +.Os +.Sh NAME +.Nm cdbr +.Nm cdbr_open , +.Nm cdbr_entries , +.Nm cdbr_get , +.Nm cdbr_find , +.Nm cdbr_close , +.Nm cdbr_write +.Nd constant database access methods +.Sh SYNOPSIS +.Ft "struct cdbr *" +.Fn cdbr_open "const char *path" "int flags" +.Ft uint32_t +.Fn cdbr_entries "struct cdbr *cdbr" +.Ft int +.Fn cdbr_get "struct cdbr *cdbr" "uint32_t index" "const void **data" "size_t *datalen" +.Ft int +.Fo cdbr_find +.Fa "struct cdbr *cdbr" +.Fa "const void *key" +.Fa "size_t keylen" +.Fa "const void **data" +.Fa "size_t *datalen" +.Fc +.Ft void +.Fn cdbr_close "struct cdbr *cdbr" +.Sh DESCRIPTION +The +.Nm +library provides a space efficient (key,value) database based +on perfect hashing. +.Pp +A cdb database is opened for reading by calling +.Fn cdbr_open . +The only supported value for +.Va flags +is +.Dv CDBR_DEFAULT . +The function returns a handle to pass to the other functions. +The database is closed by invoking +.Fn cdbr_close . +All resources associated with the handle are freed and the memory +returned by +.Fn cdbr_get +and +.Fn cdbr_find +is invalidated. +.Pp +The number of records in the database can be obtained by calling +.Fn cdbr_entries . +Records can be obtained by record number using +.Fn cdbr_get +or by key using +.Fn cdbr_find . +Both functions return 0 on success and update +.Va data +and +.Va datalen +accordingly. +The location +.Va *data +remains valid until +.Fn cdbr_close +is called. +It is the responsibility of the caller of +.Fn cdbr_find +to ensure that the key matches the returned data. +The function returns the only possible match, but the database doesn't store +the keys to minimize overhead. +.Sh SEE ALSO +.Xr nbperf 1 , +.Xr cdbw 3 , +.Xr db 3 , +.Xr cdb 5 +.Sh HISTORY +Support for the +.Nm cdb +format first appeared in +.Nx 6.0 . +.Sh AUTHORS +The +.Nm cdbr +and +.Nm cdbw +functions have been written by +.An Joerg Sonnenberger Aq Mt joerg@NetBSD.org . Index: contrib/libcdbrw/cdbr.c =================================================================== --- contrib/libcdbrw/cdbr.c (revision 0) +++ contrib/libcdbrw/cdbr.c (working copy) @@ -0,0 +1,257 @@ +/* $NetBSD: cdbr.c,v 1.4 2012/09/27 00:37:43 joerg Exp $ */ +/*- + * Copyright (c) 2010 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if HAVE_NBTOOL_CONFIG_H +#include "nbtool_config.h" +#endif + +#include +__RCSID("$NetBSD: cdbr.c,v 1.4 2012/09/27 00:37:43 joerg Exp $"); + +#ifndef __FreeBSD__ +#include "namespace.h" +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef __weak_alias +__weak_alias(cdbr_close,_cdbr_close) +__weak_alias(cdbr_find,_cdbr_find) +__weak_alias(cdbr_get,_cdbr_get) +__weak_alias(cdbr_open,_cdbr_open) +#endif + +#define fast_divide32_prepare(d,m,s1,s2) (void)0 +#define fast_remainder32(v,d,m,s1,s2) (v%d) + +struct cdbr { + uint8_t *mmap_base; + size_t mmap_size; + + uint8_t *hash_base; + uint8_t *offset_base; + uint8_t *data_base; + + uint32_t data_size; + uint32_t entries; + uint32_t entries_index; + uint32_t seed; + + uint8_t offset_size; + uint8_t index_size; + + uint32_t entries_m; + uint32_t entries_index_m; + uint8_t entries_s1, entries_s2; + uint8_t entries_index_s1, entries_index_s2; +}; + +/* ARGSUSED */ +struct cdbr * +cdbr_open(const char *path, int flags) +{ + uint8_t buf[40]; + int fd; + struct cdbr *cdbr; + struct stat sb; + + if ((fd = open(path, O_RDONLY)) == -1) + return NULL; + + errno = EINVAL; + if (fstat(fd, &sb) == -1 || + read(fd, buf, sizeof(buf)) != sizeof(buf) || + memcmp(buf, "NBCDB\n\0\001", 8) || + (cdbr = malloc(sizeof(*cdbr))) == NULL) { + close(fd); + return NULL; + } + + cdbr->data_size = le32dec(buf + 24); + cdbr->entries = le32dec(buf + 28); + cdbr->entries_index = le32dec(buf + 32); + cdbr->seed = le32dec(buf + 36); + + if (cdbr->data_size < 0x100) + cdbr->offset_size = 1; + else if (cdbr->data_size < 0x10000) + cdbr->offset_size = 2; + else + cdbr->offset_size = 4; + + if (cdbr->entries_index < 0x100) + cdbr->index_size = 1; + else if (cdbr->entries_index < 0x10000) + cdbr->index_size = 2; + else + cdbr->index_size = 4; + + cdbr->mmap_size = (size_t)sb.st_size; + cdbr->mmap_base = mmap(NULL, cdbr->mmap_size, PROT_READ, MAP_FILE|MAP_SHARED, fd, 0); + close(fd); + + if (cdbr->mmap_base == MAP_FAILED) { + free(cdbr); + return NULL; + } + + cdbr->hash_base = cdbr->mmap_base + 40; + cdbr->offset_base = cdbr->hash_base + cdbr->entries_index * cdbr->index_size; + if (cdbr->entries_index * cdbr->index_size % cdbr->offset_size) + cdbr->offset_base += cdbr->offset_size - + cdbr->entries_index * cdbr->index_size % cdbr->offset_size; + cdbr->data_base = cdbr->offset_base + (cdbr->entries + 1) * cdbr->offset_size; + + if (cdbr->hash_base < cdbr->mmap_base || + cdbr->offset_base < cdbr->mmap_base || + cdbr->data_base < cdbr->mmap_base || + cdbr->data_base + cdbr->data_size < cdbr->mmap_base || + cdbr->data_base + cdbr->data_size > + cdbr->mmap_base + cdbr->mmap_size) { + errno = EINVAL; + cdbr_close(cdbr); + return NULL; + } + + if (cdbr->entries) { + fast_divide32_prepare(cdbr->entries, &cdbr->entries_m, + &cdbr->entries_s1, &cdbr->entries_s2); + } + if (cdbr->entries_index) { + fast_divide32_prepare(cdbr->entries_index, + &cdbr->entries_index_m, + &cdbr->entries_index_s1, &cdbr->entries_index_s2); + } + + return cdbr; +} + +static inline uint32_t +get_uintX(const uint8_t *addr, uint32_t idx, int size) +{ + addr += idx * size; + + if (size == 4) + return /* LINTED */le32toh(*(const uint32_t *)addr); + else if (size == 2) + return /* LINTED */le16toh(*(const uint16_t *)addr); + else + return *addr; +} + +uint32_t +cdbr_entries(struct cdbr *cdbr) +{ + + return cdbr->entries; +} + +int +cdbr_get(struct cdbr *cdbr, uint32_t idx, const void **data, size_t *data_len) +{ + uint32_t start, end; + + if (idx >= cdbr->entries) { + errno = EINVAL; + return -1; + } + + start = get_uintX(cdbr->offset_base, idx, cdbr->offset_size); + end = get_uintX(cdbr->offset_base, idx + 1, cdbr->offset_size); + + if (start > end) { + errno = EIO; + return -1; + } + + if (end > cdbr->data_size) { + errno = EIO; + return -1; + } + + *data = cdbr->data_base + start; + *data_len = end - start; + + return 0; +} + +int +cdbr_find(struct cdbr *cdbr, const void *key, size_t key_len, + const void **data, size_t *data_len) +{ + uint32_t hashes[3], idx; + + if (cdbr->entries_index == 0) { + errno = EINVAL; + return -1; + } + + mi_vector_hash(key, key_len, cdbr->seed, hashes); + + hashes[0] = fast_remainder32(hashes[0], cdbr->entries_index, + cdbr->entries_index_m, cdbr->entries_index_s1, + cdbr->entries_index_s2); + hashes[1] = fast_remainder32(hashes[1], cdbr->entries_index, + cdbr->entries_index_m, cdbr->entries_index_s1, + cdbr->entries_index_s2); + hashes[2] = fast_remainder32(hashes[2], cdbr->entries_index, + cdbr->entries_index_m, cdbr->entries_index_s1, + cdbr->entries_index_s2); + + idx = get_uintX(cdbr->hash_base, hashes[0], cdbr->index_size); + idx += get_uintX(cdbr->hash_base, hashes[1], cdbr->index_size); + idx += get_uintX(cdbr->hash_base, hashes[2], cdbr->index_size); + + return cdbr_get(cdbr, fast_remainder32(idx, cdbr->entries, + cdbr->entries_m, cdbr->entries_s1, cdbr->entries_s2), data, + data_len); +} + +void +cdbr_close(struct cdbr *cdbr) +{ + munmap(cdbr->mmap_base, cdbr->mmap_size); + free(cdbr); +} Index: contrib/libcdbrw/cdbr.h =================================================================== --- contrib/libcdbrw/cdbr.h (revision 0) +++ contrib/libcdbrw/cdbr.h (working copy) @@ -0,0 +1,56 @@ +/* $NetBSD: cdbr.h,v 1.1 2010/04/25 00:54:45 joerg Exp $ */ +/*- + * Copyright (c) 2010 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _CDBR_H +#define _CDBR_H + +#include +#include +#include + +#define CDBR_DEFAULT 0 + +struct cdbr; + +__BEGIN_DECLS + +struct cdbr *cdbr_open(const char *, int); +uint32_t cdbr_entries(struct cdbr *); +int cdbr_get(struct cdbr *, uint32_t, const void **, size_t *); +int cdbr_find(struct cdbr *, const void *, size_t, + const void **, size_t *); +void cdbr_close(struct cdbr *); + +__END_DECLS + +#endif /* _CDBR_H */ Index: contrib/libcdbrw/cdbw.3 =================================================================== --- contrib/libcdbrw/cdbw.3 (revision 0) +++ contrib/libcdbrw/cdbw.3 (working copy) @@ -0,0 +1,140 @@ +.\" $NetBSD: cdbw.3,v 1.6 2013/07/20 21:39:56 wiz Exp $ +.\" +.\" Copyright (c) 2010 The NetBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" This code is derived from software contributed to The NetBSD Foundation +.\" by Joerg Sonnenberger. +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in +.\" the documentation and/or other materials provided with the +.\" distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, +.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.Dd June 3, 2012 +.Dt CDBW 3 +.Os +.Sh NAME +.Nm cdbw_open , +.Nm cdbw_put , +.Nm cdbw_put_data , +.Nm cdbw_put_key , +.Nm cdbw_stable_seeder , +.Nm cdbw_output , +.Nm cdbw_close +.Nd create constant databases +.Sh SYNOPSIS +.In archive_entry.h +.Ft "struct cdbw *" +.Fn cdbw_open "void" +.Ft int +.Fo cdbw_put +.Fa "struct cdbw *cdbw" +.Fa "const void *key" +.Fa "size_t keylen" +.Fa "const void *data" +.Fa "size_t datalen" +.Fc +.Ft int +.Fo cdbw_put_data +.Fa "struct cdbw *cdbw" +.Fa "const void *data" +.Fa "size_t datalen" +.Fa "uint32_t *index" +.Fc +.Ft int +.Fo cdbw_put_key +.Fa "struct cdbw *cdbw" +.Fa "const void *key" +.Fa "size_t keylen" +.Fa "uint32_t index" +.Fc +.Ft uint32_t +.Fo cdbw_stable_seeder +.Fa "void" +.Fc +.Ft int +.Fo cdbw_output +.Fa "struct cdbw *cdbw" +.Fa "int output" +.Fa "const char descr[16]" +.Fa "uint32_t (*seedgen)(void)" +.Fc +.Ft void +.Fn cdbw_close "struct cdbw *cdbw" +.Sh DESCRIPTION +The +.Nm cdbw +functions are used to create a constant databases for use with +.Xr cdbr 3 . +Details about the file format, including overhead and limitations, +can be found in +.Xr cdb 5 . +.Pp +.Fn cdbw_open +prepares a new +.Nm cdb +writer. +The function returns a handle to pass to the other functions. +.Pp +.Fn cdbw_close +frees all resources associated with the handle. +.Pp +.Fn cdbw_put +adds the given (key,value) pair after checking for a duplicate key. +.Fn cdbw_put_data +adds the given value to the writer without adding a key reference. +The returned index can be used in subsequent calls to +.Fn cdbw_put_key +to add one or more keys pointing to this value. +.Fn cdbw_put_key +checks for duplicate keys and valid index arguments. +On success it adds the given key. +.Pp +.Fn cdbw_output +computes the database file and writes it to the given descriptor. +The function returns an error if the file cannot be written correctly. +The +.Fn descr +parameter provides a human readable description of the database content. +The +.Fn seedgen +parameter can be used to override the default PRNG. +The bitwise layout of the output depends on the chosen seed. +The function should return a different value for each invocation. +The +.Fn cdbw_stable_seeder +can be used to create reproducible output. +It may be slower than the default. +.Sh SEE ALSO +.Xr cdbr 3 , +.Xr cdb 5 +.Sh HISTORY +Support for the +.Nm cdb +format first appeared in +.Nx 6.0 . +.Sh AUTHORS +The +.Nm cdbr +and +.Nm cdbw +functions have been written by +.An Joerg Sonnenberger Aq Mt joerg@NetBSD.org . Index: contrib/libcdbrw/cdbw.c =================================================================== --- contrib/libcdbrw/cdbw.c (revision 0) +++ contrib/libcdbrw/cdbw.c (working copy) @@ -0,0 +1,624 @@ +/* $NetBSD: cdbw.c,v 1.5 2012/07/21 22:49:37 joerg Exp $ */ +/*- + * Copyright (c) 2009, 2010 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if HAVE_NBTOOL_CONFIG_H +#include "nbtool_config.h" +#endif + +#include +__RCSID("$NetBSD: cdbw.c,v 1.5 2012/07/21 22:49:37 joerg Exp $"); + +#ifndef __FreeBSD__ +#include "namespace.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __weak_alias +__weak_alias(cdbw_close,_cdbw_close) +__weak_alias(cdbw_open,_cdbw_open) +__weak_alias(cdbw_output,_cdbw_output) +__weak_alias(cdbw_put,_cdbw_put) +__weak_alias(cdbw_put_data,_cdbw_put_data) +__weak_alias(cdbw_put_key,_cdbw_put_key) +#endif + +struct key_hash { + SLIST_ENTRY(key_hash) link; + uint32_t hashes[3]; + uint32_t idx; + void *key; + size_t keylen; +}; + +SLIST_HEAD(key_hash_head, key_hash); + +struct cdbw { + size_t data_counter; + size_t data_allocated; + size_t data_size; + size_t *data_len; + void **data_ptr; + + size_t hash_size; + struct key_hash_head *hash; + size_t key_counter; +}; + + /* Max. data counter that allows the index size to be 32bit. */ +static const uint32_t max_data_counter = 0xccccccccU; + +struct cdbw * +cdbw_open(void) +{ + struct cdbw *cdbw; + size_t i; + + cdbw = calloc(sizeof(*cdbw), 1); + if (cdbw == NULL) + return NULL; + + cdbw->hash_size = 1024; + cdbw->hash = calloc(cdbw->hash_size, sizeof(*cdbw->hash)); + if (cdbw->hash == NULL) { + free(cdbw); + return NULL; + } + + for (i = 0; i < cdbw->hash_size; ++i) + SLIST_INIT(cdbw->hash + i); + + return cdbw; +} + +int +cdbw_put(struct cdbw *cdbw, const void *key, size_t keylen, + const void *data, size_t datalen) +{ + uint32_t idx; + int rv; + + rv = cdbw_put_data(cdbw, data, datalen, &idx); + if (rv) + return rv; + rv = cdbw_put_key(cdbw, key, keylen, idx); + if (rv) { + --cdbw->data_counter; + free(cdbw->data_ptr[cdbw->data_counter]); + cdbw->data_size -= datalen; + return rv; + } + return 0; +} + +int +cdbw_put_data(struct cdbw *cdbw, const void *data, size_t datalen, + uint32_t *idx) +{ + + if (cdbw->data_counter == max_data_counter) + return -1; + + if (cdbw->data_size + datalen < cdbw->data_size || + cdbw->data_size + datalen > 0xffffffffU) + return -1; /* Overflow */ + + if (cdbw->data_allocated == cdbw->data_counter) { + void **new_data_ptr; + size_t *new_data_len; + size_t new_allocated; + + if (cdbw->data_allocated == 0) + new_allocated = 256; + else + new_allocated = cdbw->data_allocated * 2; + + new_data_ptr = realloc(cdbw->data_ptr, + sizeof(*cdbw->data_ptr) * new_allocated); + if (new_data_ptr == NULL) + return -1; + cdbw->data_ptr = new_data_ptr; + + new_data_len = realloc(cdbw->data_len, + sizeof(*cdbw->data_len) * new_allocated); + if (new_data_len == NULL) + return -1; + cdbw->data_len = new_data_len; + + cdbw->data_allocated = new_allocated; + } + + cdbw->data_ptr[cdbw->data_counter] = malloc(datalen); + if (cdbw->data_ptr[cdbw->data_counter] == NULL) + return -1; + memcpy(cdbw->data_ptr[cdbw->data_counter], data, datalen); + cdbw->data_len[cdbw->data_counter] = datalen; + cdbw->data_size += datalen; + *idx = cdbw->data_counter++; + return 0; +} + +int +cdbw_put_key(struct cdbw *cdbw, const void *key, size_t keylen, uint32_t idx) +{ + uint32_t hashes[3]; + struct key_hash_head *head, *head2, *new_head; + struct key_hash *key_hash; + size_t new_hash_size, i; + + if (idx >= cdbw->data_counter || + cdbw->key_counter == max_data_counter) + return -1; + + mi_vector_hash(key, keylen, 0, hashes); + + head = cdbw->hash + (hashes[0] & (cdbw->hash_size - 1)); + SLIST_FOREACH(key_hash, head, link) { + if (key_hash->keylen != keylen) + continue; + if (key_hash->hashes[0] != hashes[0]) + continue; + if (key_hash->hashes[1] != hashes[1]) + continue; + if (key_hash->hashes[2] != hashes[2]) + continue; + if (memcmp(key, key_hash->key, keylen)) + continue; + return -1; + } + key_hash = malloc(sizeof(*key_hash)); + if (key_hash == NULL) + return -1; + key_hash->key = malloc(keylen); + if (key_hash->key == NULL) { + free(key_hash); + return -1; + } + memcpy(key_hash->key, key, keylen); + key_hash->hashes[0] = hashes[0]; + key_hash->hashes[1] = hashes[1]; + key_hash->hashes[2] = hashes[2]; + key_hash->keylen = keylen; + key_hash->idx = idx; + SLIST_INSERT_HEAD(head, key_hash, link); + ++cdbw->key_counter; + + if (cdbw->key_counter <= cdbw->hash_size) + return 0; + + /* Try to resize the hash table, but ignore errors. */ + new_hash_size = cdbw->hash_size * 2; + new_head = calloc(sizeof(*new_head), new_hash_size); + if (new_head == NULL) + return 0; + + head = &cdbw->hash[hashes[0] & (cdbw->hash_size - 1)]; + for (i = 0; i < new_hash_size; ++i) + SLIST_INIT(new_head + i); + + for (i = 0; i < cdbw->hash_size; ++i) { + head = cdbw->hash + i; + + while ((key_hash = SLIST_FIRST(head)) != NULL) { + SLIST_REMOVE_HEAD(head, link); + head2 = new_head + + (key_hash->hashes[0] & (new_hash_size - 1)); + SLIST_INSERT_HEAD(head2, key_hash, link); + } + } + free(cdbw->hash); + cdbw->hash_size = new_hash_size; + cdbw->hash = new_head; + + return 0; +} + +void +cdbw_close(struct cdbw *cdbw) +{ + struct key_hash_head *head; + struct key_hash *key_hash; + size_t i; + + for (i = 0; i < cdbw->hash_size; ++i) { + head = cdbw->hash + i; + while ((key_hash = SLIST_FIRST(head)) != NULL) { + SLIST_REMOVE_HEAD(head, link); + free(key_hash->key); + free(key_hash); + } + } + + for (i = 0; i < cdbw->data_counter; ++i) + free(cdbw->data_ptr[i]); + free(cdbw->data_ptr); + free(cdbw->data_len); + free(cdbw->hash); + free(cdbw); +} + +uint32_t +cdbw_stable_seeder(void) +{ + return 0; +} + +#define unused 0xffffffffU + +struct vertex { + uint32_t l_edge, m_edge, r_edge; +}; + +struct edge { + uint32_t idx; + + uint32_t left, middle, right; + uint32_t l_prev, m_prev, l_next; + uint32_t r_prev, m_next, r_next; +}; + +struct state { + uint32_t data_entries; + uint32_t entries; + uint32_t keys; + uint32_t seed; + + uint32_t *g; + char *visited; + + struct vertex *verts; + struct edge *edges; + uint32_t output_index; + uint32_t *output_order; +}; + +static void +remove_vertex(struct state *state, struct vertex *v) +{ + struct edge *e; + struct vertex *vl, *vm, *vr; + + if (v->l_edge != unused && v->m_edge != unused) + return; + if (v->l_edge != unused && v->r_edge != unused) + return; + if (v->m_edge != unused && v->r_edge != unused) + return; + if (v->l_edge == unused && v->m_edge == unused && v->r_edge == unused) + return; + + if (v->l_edge != unused) { + e = &state->edges[v->l_edge]; + if (e->l_next != unused) + return; + } else if (v->m_edge != unused) { + e = &state->edges[v->m_edge]; + if (e->m_next != unused) + return; + } else { + if (v->r_edge == unused) + abort(); + e = &state->edges[v->r_edge]; + if (e->r_next != unused) + return; + } + + state->output_order[--state->output_index] = e - state->edges; + + vl = &state->verts[e->left]; + vm = &state->verts[e->middle]; + vr = &state->verts[e->right]; + + if (e->l_prev == unused) + vl->l_edge = e->l_next; + else + state->edges[e->l_prev].l_next = e->l_next; + if (e->l_next != unused) + state->edges[e->l_next].l_prev = e->l_prev; + + if (e->m_prev == unused) + vm->m_edge = e->m_next; + else + state->edges[e->m_prev].m_next = e->m_next; + if (e->m_next != unused) + state->edges[e->m_next].m_prev = e->m_prev; + + if (e->r_prev == unused) + vr->r_edge = e->r_next; + else + state->edges[e->r_prev].r_next = e->r_next; + if (e->r_next != unused) + state->edges[e->r_next].r_prev = e->r_prev; +} + +static int +build_graph(struct cdbw *cdbw, struct state *state) +{ + struct key_hash_head *head; + struct key_hash *key_hash; + struct vertex *v; + struct edge *e; + uint32_t hashes[3]; + size_t i; + + e = state->edges; + for (i = 0; i < cdbw->hash_size; ++i) { + head = &cdbw->hash[i]; + SLIST_FOREACH(key_hash, head, link) { + e->idx = key_hash->idx; + mi_vector_hash(key_hash->key, key_hash->keylen, + state->seed, hashes); + e->left = hashes[0] % state->entries; + e->middle = hashes[1] % state->entries; + e->right = hashes[2] % state->entries; + + if (e->left == e->middle) + return -1; + if (e->left == e->right) + return -1; + if (e->middle == e->right) + return -1; + + ++e; + } + } + + for (i = 0; i < state->entries; ++i) { + v = state->verts + i; + v->l_edge = unused; + v->m_edge = unused; + v->r_edge = unused; + } + + for (i = 0; i < state->keys; ++i) { + e = state->edges + i; + v = state->verts + e->left; + if (v->l_edge != unused) + state->edges[v->l_edge].l_prev = i; + e->l_next = v->l_edge; + e->l_prev = unused; + v->l_edge = i; + + v = &state->verts[e->middle]; + if (v->m_edge != unused) + state->edges[v->m_edge].m_prev = i; + e->m_next = v->m_edge; + e->m_prev = unused; + v->m_edge = i; + + v = &state->verts[e->right]; + if (v->r_edge != unused) + state->edges[v->r_edge].r_prev = i; + e->r_next = v->r_edge; + e->r_prev = unused; + v->r_edge = i; + } + + state->output_index = state->keys; + for (i = 0; i < state->entries; ++i) + remove_vertex(state, state->verts + i); + + i = state->keys; + while (i > 0 && i > state->output_index) { + --i; + e = state->edges + state->output_order[i]; + remove_vertex(state, state->verts + e->left); + remove_vertex(state, state->verts + e->middle); + remove_vertex(state, state->verts + e->right); + } + + return state->output_index == 0 ? 0 : -1; +} + +static void +assign_nodes(struct state *state) +{ + struct edge *e; + size_t i; + + for (i = 0; i < state->keys; ++i) { + e = state->edges + state->output_order[i]; + + if (!state->visited[e->left]) { + state->g[e->left] = + (2 * state->data_entries + e->idx + - state->g[e->middle] - state->g[e->right]) + % state->data_entries; + } else if (!state->visited[e->middle]) { + state->g[e->middle] = + (2 * state->data_entries + e->idx + - state->g[e->left] - state->g[e->right]) + % state->data_entries; + } else { + state->g[e->right] = + (2 * state->data_entries + e->idx + - state->g[e->left] - state->g[e->middle]) + % state->data_entries; + } + state->visited[e->left] = 1; + state->visited[e->middle] = 1; + state->visited[e->right] = 1; + } +} + +static size_t +compute_size(uint32_t size) +{ + if (size < 0x100) + return 1; + else if (size < 0x10000) + return 2; + else + return 4; +} + +#define COND_FLUSH_BUFFER(n) do { \ + if (__predict_false(cur_pos + (n) >= sizeof(buf))) { \ + ret = write(fd, buf, cur_pos); \ + if (ret == -1 || (size_t)ret != cur_pos) \ + return -1; \ + cur_pos = 0; \ + } \ +} while (/* CONSTCOND */ 0) + +static int +print_hash(struct cdbw *cdbw, struct state *state, int fd, const char *descr) +{ + uint32_t data_size; + uint8_t buf[90000]; + size_t i, size, size2, cur_pos; + ssize_t ret; + + memcpy(buf, "NBCDB\n\0", 7); + buf[7] = 1; + strncpy((char *)buf + 8, descr, 16); + le32enc(buf + 24, cdbw->data_size); + le32enc(buf + 28, cdbw->data_counter); + le32enc(buf + 32, state->entries); + le32enc(buf + 36, state->seed); + cur_pos = 40; + + size = compute_size(state->entries); + for (i = 0; i < state->entries; ++i) { + COND_FLUSH_BUFFER(4); + le32enc(buf + cur_pos, state->g[i]); + cur_pos += size; + } + size2 = compute_size(cdbw->data_size); + size = size * state->entries % size2; + if (size != 0) { + size = size2 - size; + COND_FLUSH_BUFFER(4); + le32enc(buf + cur_pos, 0); + cur_pos += size; + } + for (data_size = 0, i = 0; i < cdbw->data_counter; ++i) { + COND_FLUSH_BUFFER(4); + le32enc(buf + cur_pos, data_size); + cur_pos += size2; + data_size += cdbw->data_len[i]; + } + COND_FLUSH_BUFFER(4); + le32enc(buf + cur_pos, data_size); + cur_pos += size2; + + for (i = 0; i < cdbw->data_counter; ++i) { + COND_FLUSH_BUFFER(cdbw->data_len[i]); + if (cdbw->data_len[i] < sizeof(buf)) { + memcpy(buf + cur_pos, cdbw->data_ptr[i], + cdbw->data_len[i]); + cur_pos += cdbw->data_len[i]; + } else { + ret = write(fd, cdbw->data_ptr[i], cdbw->data_len[i]); + if (ret == -1 || (size_t)ret != cdbw->data_len[i]) + return -1; + } + } + if (cur_pos != 0) { + ret = write(fd, buf, cur_pos); + if (ret == -1 || (size_t)ret != cur_pos) + return -1; + } + return 0; +} + +int +cdbw_output(struct cdbw *cdbw, int fd, const char descr[16], + uint32_t (*seedgen)(void)) +{ + struct state state; + int rv; + + if (cdbw->data_counter == 0 || cdbw->key_counter == 0) { + state.entries = 0; + state.seed = 0; + print_hash(cdbw, &state, fd, descr); + return 0; + } + +#if HAVE_NBTOOL_CONFIG_H + if (seedgen == NULL) + seedgen = cdbw_stable_seeder; +#else + if (seedgen == NULL) + seedgen = arc4random; +#endif + + rv = 0; + + state.keys = cdbw->key_counter; + state.data_entries = cdbw->data_counter; + state.entries = state.keys + (state.keys + 3) / 4; + if (state.entries < 10) + state.entries = 10; + +#define NALLOC(var, n) var = calloc(sizeof(*var), n) + NALLOC(state.g, state.entries); + NALLOC(state.visited, state.entries); + NALLOC(state.verts, state.entries); + NALLOC(state.edges, state.entries); + NALLOC(state.output_order, state.keys); +#undef NALLOC + + if (state.g == NULL || state.visited == NULL || state.verts == NULL || + state.edges == NULL || state.output_order == NULL) { + rv = -1; + goto release; + } + + state.seed = 0; + do { + if (seedgen == cdbw_stable_seeder) + ++state.seed; + else + state.seed = (*seedgen)(); + } while (build_graph(cdbw, &state)); + + assign_nodes(&state); + rv = print_hash(cdbw, &state, fd, descr); + +release: + free(state.g); + free(state.visited); + free(state.verts); + free(state.edges); + free(state.output_order); + + return rv; +} Index: contrib/libcdbrw/cdbw.h =================================================================== --- contrib/libcdbrw/cdbw.h (revision 0) +++ contrib/libcdbrw/cdbw.h (working copy) @@ -0,0 +1,59 @@ +/* $NetBSD: cdbw.h,v 1.2 2012/06/03 21:21:45 joerg Exp $ */ +/*- + * Copyright (c) 2010 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _CDBW_H +#define _CDBW_H + +#include +#include +#include + +struct cdbw; + +__BEGIN_DECLS + +struct cdbw *cdbw_open(void); +int cdbw_put(struct cdbw *, const void *, size_t, + const void *, size_t); +int cdbw_put_data(struct cdbw *, const void *, size_t, + uint32_t *); +int cdbw_put_key(struct cdbw *, const void *, size_t, + uint32_t); +uint32_t cdbw_stable_seeder(void); +int cdbw_output(struct cdbw *, int, const char[16], + uint32_t (*)(void)); +void cdbw_close(struct cdbw *); + +__END_DECLS + +#endif /* _CDBW_H */ Index: contrib/libcdbrw/mi_vector_hash.c =================================================================== --- contrib/libcdbrw/mi_vector_hash.c (revision 0) +++ contrib/libcdbrw/mi_vector_hash.c (working copy) @@ -0,0 +1,175 @@ +/* $NetBSD: mi_vector_hash.c,v 1.4 2011/10/21 23:45:56 joerg Exp $ */ +/*- + * Copyright (c) 2009 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Joerg Sonnenberger. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * See http://burtleburtle.net/bob/hash/doobs.html for the full description + * and the original version of the code. This version differs by exposing + * the full internal state and avoiding byte operations in the inner loop + * if the key is aligned correctly. + */ + +#if HAVE_NBTOOL_CONFIG_H +#include "nbtool_config.h" +#endif + +#include +__RCSID("$NetBSD: mi_vector_hash.c,v 1.4 2011/10/21 23:45:56 joerg Exp $"); + +#ifndef __FreeBSD__ +#include "namespace.h" +#endif + +#include +#include +#include + +#include + +#define mix(a, b, c) do { \ + a -= b; a -= c; a ^= (c >> 13); \ + b -= c; b -= a; b ^= (a << 8); \ + c -= a; c -= b; c ^= (b >> 13); \ + a -= b; a -= c; a ^= (c >> 12); \ + b -= c; b -= a; b ^= (a << 16); \ + c -= a; c -= b; c ^= (b >> 5); \ + a -= b; a -= c; a ^= (c >> 3); \ + b -= c; b -= a; b ^= (a << 10); \ + c -= a; c -= b; c ^= (b >> 15); \ +} while (/* CONSTCOND */0) + +#define FIXED_SEED 0x9e3779b9 /* Golden ratio, arbitrary constant */ + +#ifdef __weak_alias +__weak_alias(mi_vector_hash, _mi_vector_hash) +#endif + +void +mi_vector_hash(const void * __restrict key, size_t len, uint32_t seed, + uint32_t hashes[3]) +{ + static const uint32_t mask[4] = { + 0x000000ff, 0x0000ffff, 0x00ffffff, 0xffffffff + }; + uint32_t orig_len, a, b, c; + const uint8_t *k; + + orig_len = (uint32_t)len; + + a = b = FIXED_SEED; + c = seed; + + if ((uintptr_t)key & 3) { + k = key; + while (len >= 12) { + a += le32dec(k); + b += le32dec(k + 4); + c += le32dec(k + 8); + mix(a, b, c); + k += 12; + len -= 12; + } + c += orig_len; + + if (len > 8) { + switch (len) { + case 11: + c += (uint32_t)k[10] << 24; + /* FALLTHROUGH */ + case 10: + c += (uint32_t)k[9] << 16; + /* FALLTHROUGH */ + case 9: + c += (uint32_t)k[8] << 8; + /* FALLTHROUGH */ + } + b += le32dec(k + 4); + a += le32dec(k); + } else if (len > 4) { + switch (len) { + case 8: + b += (uint32_t)k[7] << 24; + /* FALLTHROUGH */ + case 7: + b += (uint32_t)k[6] << 16; + /* FALLTHROUGH */ + case 6: + b += (uint32_t)k[5] << 8; + /* FALLTHROUGH */ + case 5: + b += k[4]; + /* FALLTHROUGH */ + } + a += le32dec(k); + } else if (len) { + switch (len) { + case 4: + a += (uint32_t)k[3] << 24; + /* FALLTHROUGH */ + case 3: + a += (uint32_t)k[2] << 16; + /* FALLTHROUGH */ + case 2: + a += (uint32_t)k[1] << 8; + /* FALLTHROUGH */ + case 1: + a += k[0]; + /* FALLTHROUGH */ + } + } + } else { + const uint32_t *key32 = key; + while (len >= 12) { + a += le32toh(key32[0]); + b += le32toh(key32[1]); + c += le32toh(key32[2]); + mix(a, b, c); + key32 += 3; + len -= 12; + } + c += orig_len; + + if (len > 8) { + c += (le32toh(key32[2]) & mask[len - 9]) << 8; + b += le32toh(key32[1]); + a += le32toh(key32[0]); + } else if (len > 4) { + b += le32toh(key32[1]) & mask[len - 5]; + a += le32toh(key32[0]); + } else if (len) + a += le32toh(key32[0]) & mask[len - 1]; + } + mix(a, b, c); + hashes[0] = a; + hashes[1] = b; + hashes[2] = c; +} Index: contrib/libcdbrw/mi_vector_hash.h =================================================================== --- contrib/libcdbrw/mi_vector_hash.h (revision 0) +++ contrib/libcdbrw/mi_vector_hash.h (working copy) @@ -0,0 +1 @@ +void mi_vector_hash(const void * restrict key, size_t len, uint32_t seed, uint32_t hashes[3]);