Index: sbin/hastctl/Makefile =================================================================== --- sbin/hastctl/Makefile (wersja 218208) +++ sbin/hastctl/Makefile (kopia robocza) @@ -6,8 +6,10 @@ PROG= hastctl SRCS= activemap.c +SRCS+= crc32.c SRCS+= ebuf.c -SRCS+= hast_proto.c hastctl.c +SRCS+= hast_checksum.c hast_compression.c hast_proto.c hastctl.c +SRCS+= lzf.c SRCS+= metadata.c SRCS+= nv.c SRCS+= parse.y pjdlog.c Index: sbin/hastd/parse.y =================================================================== --- sbin/hastd/parse.y (wersja 218190) +++ sbin/hastd/parse.y (kopia robocza) @@ -1,6 +1,7 @@ %{ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation + * Copyright (c) 2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -60,6 +61,8 @@ static char depth0_control[HAST_ADDRSIZE]; static char depth0_listen[HAST_ADDRSIZE]; static int depth0_replication; +static int depth0_checksum; +static int depth0_compression; static int depth0_timeout; static char depth0_exec[PATH_MAX]; @@ -167,6 +170,8 @@ depth0_timeout = HAST_TIMEOUT; depth0_replication = HAST_REPLICATION_MEMSYNC; + depth0_checksum = HAST_CHECKSUM_NONE; + depth0_compression = HAST_COMPRESSION_HOLE; strlcpy(depth0_control, HAST_CONTROL, sizeof(depth0_control)); strlcpy(depth0_listen, HASTD_LISTEN, sizeof(depth0_listen)); depth0_exec[0] = '\0'; @@ -223,6 +228,20 @@ */ curres->hr_replication = depth0_replication; } + if (curres->hr_checksum == -1) { + /* + * Checksum is not set at resource-level. + * Use global or default setting. + */ + curres->hr_checksum = depth0_checksum; + } + if (curres->hr_compression == -1) { + /* + * Compression is not set at resource-level. + * Use global or default setting. + */ + curres->hr_compression = depth0_compression; + } if (curres->hr_timeout == -1) { /* * Timeout is not set at resource-level. @@ -256,11 +275,14 @@ } %} -%token CONTROL LISTEN PORT REPLICATION TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON -%token FULLSYNC MEMSYNC ASYNC +%token CONTROL LISTEN PORT REPLICATION CHECKSUM COMPRESSION +%token TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON +%token FULLSYNC MEMSYNC ASYNC NONE CRC32 SHA256 HOLE LZF %token NUM STR OB CB %type replication_type +%type checksum_type +%type compression_type %union { @@ -285,6 +307,10 @@ | replication_statement | + checksum_statement + | + compression_statement + | timeout_statement | exec_statement @@ -378,6 +404,54 @@ ASYNC { $$ = HAST_REPLICATION_ASYNC; } ; +checksum_statement: CHECKSUM checksum_type + { + switch (depth) { + case 0: + depth0_checksum = $2; + break; + case 1: + if (curres != NULL) + curres->hr_checksum = $2; + break; + default: + assert(!"checksum at wrong depth level"); + } + } + ; + +checksum_type: + NONE { $$ = HAST_CHECKSUM_NONE; } + | + CRC32 { $$ = HAST_CHECKSUM_CRC32; } + | + SHA256 { $$ = HAST_CHECKSUM_SHA256; } + ; + +compression_statement: COMPRESSION compression_type + { + switch (depth) { + case 0: + depth0_compression = $2; + break; + case 1: + if (curres != NULL) + curres->hr_compression = $2; + break; + default: + assert(!"compression at wrong depth level"); + } + } + ; + +compression_type: + NONE { $$ = HAST_COMPRESSION_NONE; } + | + HOLE { $$ = HAST_COMPRESSION_HOLE; } + | + LZF { $$ = HAST_COMPRESSION_LZF; } + ; + timeout_statement: TIMEOUT NUM { switch (depth) { @@ -570,6 +644,8 @@ curres->hr_role = HAST_ROLE_INIT; curres->hr_previous_role = HAST_ROLE_INIT; curres->hr_replication = -1; + curres->hr_checksum = -1; + curres->hr_compression = -1; curres->hr_timeout = -1; curres->hr_exec[0] = '\0'; curres->hr_provname[0] = '\0'; @@ -588,6 +664,10 @@ resource_entry: replication_statement | + checksum_statement + | + compression_statement + | timeout_statement | exec_statement Index: sbin/hastd/hastd.c =================================================================== --- sbin/hastd/hastd.c (wersja 218376) +++ sbin/hastd/hastd.c (kopia robocza) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation - * Copyright (c) 2010-2011 Pawel Jakub Dawidek + * Copyright (c) 2010-2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -361,6 +361,10 @@ return (true); if (res0->hr_replication != res1->hr_replication) return (true); + if (res0->hr_checksum != res1->hr_checksum) + return (true); + if (res0->hr_compression != res1->hr_compression) + return (true); if (res0->hr_timeout != res1->hr_timeout) return (true); if (strcmp(res0->hr_exec, res1->hr_exec) != 0) @@ -385,6 +389,10 @@ return (true); if (res0->hr_replication != res1->hr_replication) return (true); + if (res0->hr_checksum != res1->hr_checksum) + return (true); + if (res0->hr_compression != res1->hr_compression) + return (true); if (res0->hr_timeout != res1->hr_timeout) return (true); if (strcmp(res0->hr_exec, res1->hr_exec) != 0) @@ -404,6 +412,8 @@ nv_add_uint8(nvout, HASTCTL_RELOAD, "cmd"); nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr"); nv_add_int32(nvout, (int32_t)res->hr_replication, "replication"); + nv_add_int32(nvout, (int32_t)res->hr_checksum, "checksum"); + nv_add_int32(nvout, (int32_t)res->hr_compression, "compression"); nv_add_int32(nvout, (int32_t)res->hr_timeout, "timeout"); nv_add_string(nvout, res->hr_exec, "exec"); if (nv_error(nvout) != 0) { @@ -562,6 +572,8 @@ strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr, sizeof(cres->hr_remoteaddr)); cres->hr_replication = nres->hr_replication; + cres->hr_checksum = nres->hr_checksum; + cres->hr_compression = nres->hr_compression; cres->hr_timeout = nres->hr_timeout; strlcpy(cres->hr_exec, nres->hr_exec, sizeof(cres->hr_exec)); Index: sbin/hastd/crc32.c =================================================================== --- sbin/hastd/crc32.c (wersja 0) +++ sbin/hastd/crc32.c (wersja 0) @@ -0,0 +1,115 @@ +/*- + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + */ + +/* + * First, the polynomial itself and its table of feedback terms. The + * polynomial is + * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 + * + * Note that we take it "backwards" and put the highest-order term in + * the lowest-order bit. The X^32 term is "implied"; the LSB is the + * X^31 term, etc. The X^0 term (usually shown as "+1") results in + * the MSB being 1 + * + * Note that the usual hardware shift register implementation, which + * is what we're using (we're merely optimizing it by doing eight-bit + * chunks at a time) shifts bits into the lowest-order term. In our + * implementation, that means shifting towards the right. Why do we + * do it this way? Because the calculated CRC must be transmitted in + * order from highest-order term to lowest-order term. UARTs transmit + * characters in order from LSB to MSB. By storing the CRC this way + * we hand it to the UART in the order low-byte to high-byte; the UART + * sends each low-bit to hight-bit; and the result is transmission bit + * by bit from highest- to lowest-order term without requiring any bit + * shuffling on our part. Reception works similarly + * + * The feedback terms table consists of 256, 32-bit entries. Notes + * + * The table can be generated at runtime if desired; code to do so + * is shown later. It might not be obvious, but the feedback + * terms simply represent the results of eight shift/xor opera + * tions for all combinations of data and CRC register values + * + * The values must be right-shifted by eight bits by the "updcrc + * logic; the shift must be unsigned (bring in zeroes). On some + * hardware you could probably optimize the shift in assembler by + * using byte-swap instructions + * polynomial $edb88320 + * + * + * CRC32 code derived from work by Gary S. Brown. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#include + +uint32_t crc32_tab[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +/* + * A function that calculates the CRC-32 based on the table above is + * given below for documentation purposes. An equivalent implementation + * of this function that's actually used in the kernel can be found + * in sys/libkern.h, where it can be inlined. + * + * uint32_t + * crc32(const void *buf, size_t size) + * { + * const uint8_t *p = buf; + * uint32_t crc; + * + * crc = ~0U; + * while (size--) + * crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + * return crc ^ ~0U; + * } + */ Zmiany atrybutów dla: sbin/hastd/crc32.c ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/control.c =================================================================== --- sbin/hastd/control.c (wersja 218218) +++ sbin/hastd/control.c (kopia robocza) @@ -43,6 +43,8 @@ #include "hast.h" #include "hastd.h" +#include "hast_checksum.h" +#include "hast_compression.h" #include "hast_proto.h" #include "hooks.h" #include "nv.h" @@ -246,6 +248,10 @@ nv_add_string(nvout, "unknown", "replication%u", no); break; } + nv_add_string(nvout, checksum_name(res->hr_checksum), + "checksum%u", no); + nv_add_string(nvout, compression_name(res->hr_compression), + "compression%u", no); nv_add_string(nvout, role2str(res->hr_role), "role%u", no); switch (res->hr_role) { Index: sbin/hastd/hast_compression.c =================================================================== --- sbin/hastd/hast_compression.c (wersja 0) +++ sbin/hastd/hast_compression.c (wersja 0) @@ -0,0 +1,283 @@ +/*- + * Copyright (c) 2011 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "hast_compression.h" + +static bool +allzeros(const void *data, size_t size) +{ + const uint64_t *p = data; + unsigned int i; + uint64_t v; + + PJDLOG_ASSERT((size % sizeof(*p)) == 0); + + /* + * This is the fastest method I found for checking if the given + * buffer contain all zeros. + * Because inside the loop we don't check at every step, we would + * get an answer only after walking through entire buffer. + * To return early if the buffer doesn't contain all zeros, we probe + * 8 bytes at the begining, in the middle and at the end of the buffer + * first. + */ + + size >>= 3; /* divide by 8 */ + if ((p[0] | p[size >> 1] | p[size - 1]) != 0) + return (false); + v = 0; + for (i = 0; i < size; i++) + v |= *p++; + return (v == 0); +} + +static void * +hast_hole_compress(const unsigned char *data, size_t *sizep) +{ + uint32_t size; + void *newbuf; + + if (!allzeros(data, *sizep)) + return (NULL); + + newbuf = malloc(sizeof(size)); + if (newbuf == NULL) { + pjdlog_warning("Unable to compress (no memory: %zu).", + (size_t)*sizep); + return (NULL); + } + size = htole32((uint32_t)*sizep); + bcopy(&size, newbuf, sizeof(size)); + *sizep = sizeof(size); + + return (newbuf); +} + +static void * +hast_hole_decompress(const unsigned char *data, size_t *sizep) +{ + uint32_t size; + void *newbuf; + + if (*sizep != sizeof(size)) { + pjdlog_error("Unable to decompress (invalid size: %zu).", + *sizep); + return (NULL); + } + + bcopy(data, &size, sizeof(size)); + size = le32toh(size); + + newbuf = malloc(size); + if (newbuf == NULL) { + pjdlog_error("Unable to decompress (no memory: %zu).", + (size_t)size); + return (NULL); + } + bzero(newbuf, size); + *sizep = size; + + return (newbuf); +} + +/* Minimum block size to try to compress. */ +#define HAST_LZF_COMPRESS_MIN 1024 + +static void * +hast_lzf_compress(const unsigned char *data, size_t *sizep) +{ + unsigned char *newbuf; + uint32_t origsize; + size_t newsize; + + origsize = *sizep; + + if (origsize <= HAST_LZF_COMPRESS_MIN) + return (NULL); + + newsize = sizeof(origsize) + origsize - HAST_LZF_COMPRESS_MIN; + newbuf = malloc(newsize); + if (newbuf == NULL) { + pjdlog_warning("Unable to compress (no memory: %zu).", + newsize); + return (NULL); + } + newsize = lzf_compress(data, *sizep, newbuf + sizeof(origsize), + newsize - sizeof(origsize)); + if (newsize == 0) { + free(newbuf); + return (NULL); + } + origsize = htole32(origsize); + bcopy(&origsize, newbuf, sizeof(origsize)); + + *sizep = sizeof(origsize) + newsize; + return (newbuf); +} + +static void * +hast_lzf_decompress(const unsigned char *data, size_t *sizep) +{ + unsigned char *newbuf; + uint32_t origsize; + size_t newsize; + + PJDLOG_ASSERT(*sizep > sizeof(origsize)); + + bcopy(data, &origsize, sizeof(origsize)); + origsize = le32toh(origsize); + PJDLOG_ASSERT(origsize > HAST_LZF_COMPRESS_MIN); + + newbuf = malloc(origsize); + if (newbuf == NULL) { + pjdlog_error("Unable to decompress (no memory: %zu).", + (size_t)origsize); + return (NULL); + } + newsize = lzf_decompress(data + sizeof(origsize), + *sizep - sizeof(origsize), newbuf, origsize); + if (newsize == 0) { + free(newbuf); + pjdlog_error("Unable to decompress."); + return (NULL); + } + PJDLOG_ASSERT(newsize == origsize); + + *sizep = newsize; + return (newbuf); +} + +const char * +compression_name(int num) +{ + + switch (num) { + case HAST_COMPRESSION_NONE: + return ("none"); + case HAST_COMPRESSION_HOLE: + return ("hole"); + case HAST_COMPRESSION_LZF: + return ("lzf"); + } + return ("unknown"); +} + +int +compression_send(const struct hast_resource *res, struct nv *nv, void **datap, + size_t *sizep, bool *freedatap) +{ + unsigned char *newbuf; + int compression; + size_t size; + + size = *sizep; + compression = res->hr_compression; + + switch (compression) { + case HAST_COMPRESSION_NONE: + return (0); + case HAST_COMPRESSION_HOLE: + newbuf = hast_hole_compress(*datap, &size); + break; + case HAST_COMPRESSION_LZF: + /* Try 'hole' compression first. */ + newbuf = hast_hole_compress(*datap, &size); + if (newbuf != NULL) + compression = HAST_COMPRESSION_HOLE; + else + newbuf = hast_lzf_compress(*datap, &size); + break; + default: + PJDLOG_ABORT("Invalid compression: %d.", res->hr_compression); + } + + if (newbuf == NULL) { + /* Unable to compress the data. */ + return (0); + } + nv_add_string(nv, compression_name(compression), "compression"); + if (nv_error(nv) != 0) { + free(newbuf); + errno = nv_error(nv); + return (-1); + } + if (*freedatap) + free(*datap); + *freedatap = true; + *datap = newbuf; + *sizep = size; + + return (0); +} + +int +compression_recv(const struct hast_resource *res __unused, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap) +{ + unsigned char *newbuf; + const char *algo; + size_t size; + + algo = nv_get_string(nv, "compression"); + if (algo == NULL) + return (0); /* No compression. */ + + newbuf = NULL; + size = *sizep; + + if (strcmp(algo, "hole") == 0) + newbuf = hast_hole_decompress(*datap, &size); + else if (strcmp(algo, "lzf") == 0) + newbuf = hast_lzf_decompress(*datap, &size); + else { + pjdlog_error("Unknown compression algorithm '%s'.", algo); + return (-1); /* Unknown compression algorithm. */ + } + + if (newbuf == NULL) + return (-1); + if (*freedatap) + free(*datap); + *freedatap = true; + *datap = newbuf; + *sizep = size; + + return (0); +} Zmiany atrybutów dla: sbin/hastd/hast_compression.c ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/hast_checksum.c =================================================================== --- sbin/hastd/hast_checksum.c (wersja 0) +++ sbin/hastd/hast_checksum.c (wersja 0) @@ -0,0 +1,169 @@ +/*- + * Copyright (c) 2011 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#ifdef HAVE_CRYPTO +#include +#endif + +#include +#include +#include +#include + +#include "hast_checksum.h" + +#ifdef HAVE_CRYPTO +#define MAX_HASH_SIZE SHA256_DIGEST_LENGTH +#else +#define MAX_HASH_SIZE 4 +#endif + +static int +hast_crc32_checksum(const unsigned char *data, size_t size, + unsigned char *hash, size_t *hsizep) +{ + uint32_t crc; + + crc = crc32(data, size); + /* XXXPJD: Do we have to use htole32() on crc first? */ + bcopy(&crc, hash, sizeof(crc)); + *hsizep = sizeof(crc); + + return (0); +} + +#ifdef HAVE_CRYPTO +static int +hast_sha256_checksum(const unsigned char *data, size_t size, + unsigned char *hash, size_t *hsizep) +{ + SHA256_CTX ctx; + + SHA256_Init(&ctx); + SHA256_Update(&ctx, data, size); + SHA256_Final(hash, &ctx); + *hsizep = SHA256_DIGEST_LENGTH; + + return (0); +} +#endif /* HAVE_CRYPTO */ + +const char * +checksum_name(int num) +{ + + switch (num) { + case HAST_CHECKSUM_NONE: + return ("none"); + case HAST_CHECKSUM_CRC32: + return ("crc32"); + case HAST_CHECKSUM_SHA256: + return ("sha256"); + } + return ("unknown"); +} + +int +checksum_send(const struct hast_resource *res, struct nv *nv, void **datap, + size_t *sizep, bool *freedatap __unused) +{ + unsigned char hash[MAX_HASH_SIZE]; + size_t hsize; + int ret; + + switch (res->hr_checksum) { + case HAST_CHECKSUM_NONE: + return (0); + case HAST_CHECKSUM_CRC32: + ret = hast_crc32_checksum(*datap, *sizep, hash, &hsize); + break; +#ifdef HAVE_CRYPTO + case HAST_CHECKSUM_SHA256: + ret = hast_sha256_checksum(*datap, *sizep, hash, &hsize); + break; +#endif + default: + PJDLOG_ABORT("Invalid checksum: %d.", res->hr_checksum); + } + + if (ret != 0) + return (ret); + nv_add_string(nv, checksum_name(res->hr_checksum), "checksum"); + nv_add_uint8_array(nv, hash, hsize, "hash"); + if (nv_error(nv) != 0) { + errno = nv_error(nv); + return (-1); + } + return (0); +} + +int +checksum_recv(const struct hast_resource *res __unused, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap __unused) +{ + unsigned char chash[MAX_HASH_SIZE]; + const unsigned char *rhash; + size_t chsize, rhsize; + const char *algo; + int ret; + + algo = nv_get_string(nv, "checksum"); + if (algo == NULL) + return (0); /* No checksum. */ + rhash = nv_get_uint8_array(nv, &rhsize, "hash"); + if (rhash == NULL) { + pjdlog_error("Hash is missing."); + return (-1); /* Hash not found. */ + } + if (strcmp(algo, "crc32") == 0) + ret = hast_crc32_checksum(*datap, *sizep, chash, &chsize); +#ifdef HAVE_CRYPTO + else if (strcmp(algo, "sha256") == 0) + ret = hast_sha256_checksum(*datap, *sizep, chash, &chsize); +#endif + else { + pjdlog_error("Unknown checksum algorithm '%s'.", algo); + return (-1); /* Unknown checksum algorithm. */ + } + if (rhsize != chsize) { + pjdlog_error("Invalid hash size (%zu) for %s, should be %zu.", + rhsize, algo, chsize); + return (-1); /* Different hash size. */ + } + if (bcmp(rhash, chash, chsize) != 0) { + pjdlog_error("Hash mismatch."); + return (-1); /* Hash mismatch. */ + } + + return (0); +} Zmiany atrybutów dla: sbin/hastd/hast_checksum.c ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/crc32.h =================================================================== --- sbin/hastd/crc32.h (wersja 0) +++ sbin/hastd/crc32.h (wersja 0) @@ -0,0 +1,28 @@ +/*- + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + * + * $FreeBSD$ + */ + +#ifndef _CRC32_H_ +#define _CRC32_H_ + +#include /* uint32_t */ +#include /* size_t */ + +extern uint32_t crc32_tab[]; + +static __inline uint32_t +crc32(const void *buf, size_t size) +{ + const uint8_t *p = buf; + uint32_t crc; + + crc = ~0U; + while (size--) + crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return (crc ^ ~0U); +} + +#endif /* !_CRC32_H_ */ Zmiany atrybutów dla: sbin/hastd/crc32.h ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/hast_proto.c =================================================================== --- sbin/hastd/hast_proto.c (wersja 218371) +++ sbin/hastd/hast_proto.c (kopia robocza) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation + * Copyright (c) 2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -34,19 +35,18 @@ #include #include -#include #include -#ifdef HAVE_CRYPTO -#include -#endif - #include #include #include #include #include +#ifdef HAVE_CRYPTO +#include "hast_checksum.h" +#endif +#include "hast_compression.h" #include "hast_proto.h" struct hast_main_header { @@ -67,171 +67,11 @@ hps_recv_t *hps_recv; }; -static int compression_send(const struct hast_resource *res, struct nv *nv, - void **datap, size_t *sizep, bool *freedatap); -static int compression_recv(const struct hast_resource *res, struct nv *nv, - void **datap, size_t *sizep, bool *freedatap); -#ifdef HAVE_CRYPTO -static int checksum_send(const struct hast_resource *res, struct nv *nv, - void **datap, size_t *sizep, bool *freedatap); -static int checksum_recv(const struct hast_resource *res, struct nv *nv, - void **datap, size_t *sizep, bool *freedatap); -#endif - static struct hast_pipe_stage pipeline[] = { { "compression", compression_send, compression_recv }, -#ifdef HAVE_CRYPTO { "checksum", checksum_send, checksum_recv } -#endif }; -static int -compression_send(const struct hast_resource *res, struct nv *nv, void **datap, - size_t *sizep, bool *freedatap) -{ - unsigned char *newbuf; - - res = res; /* TODO */ - - /* - * TODO: For now we emulate compression. - * At 80% probability we succeed to compress data, which means we - * allocate new buffer, copy the data over set *freedatap to true. - */ - - if (arc4random_uniform(100) < 80) { - uint32_t *origsize; - - /* - * Compression succeeded (but we will grow by 4 bytes, not - * shrink for now). - */ - newbuf = malloc(sizeof(uint32_t) + *sizep); - if (newbuf == NULL) - return (-1); - origsize = (void *)newbuf; - *origsize = htole32((uint32_t)*sizep); - nv_add_string(nv, "null", "compression"); - if (nv_error(nv) != 0) { - free(newbuf); - errno = nv_error(nv); - return (-1); - } - bcopy(*datap, newbuf + sizeof(uint32_t), *sizep); - if (*freedatap) - free(*datap); - *freedatap = true; - *datap = newbuf; - *sizep = sizeof(uint32_t) + *sizep; - } else { - /* - * Compression failed, so we leave everything as it was. - * It is not critical for compression to succeed. - */ - } - - return (0); -} - -static int -compression_recv(const struct hast_resource *res, struct nv *nv, void **datap, - size_t *sizep, bool *freedatap) -{ - unsigned char *newbuf; - const char *algo; - size_t origsize; - - res = res; /* TODO */ - - /* - * TODO: For now we emulate compression. - */ - - algo = nv_get_string(nv, "compression"); - if (algo == NULL) - return (0); /* No compression. */ - if (strcmp(algo, "null") != 0) { - pjdlog_error("Unknown compression algorithm '%s'.", algo); - return (-1); /* Unknown compression algorithm. */ - } - - origsize = le32toh(*(uint32_t *)*datap); - newbuf = malloc(origsize); - if (newbuf == NULL) - return (-1); - bcopy((unsigned char *)*datap + sizeof(uint32_t), newbuf, origsize); - if (*freedatap) - free(*datap); - *freedatap = true; - *datap = newbuf; - *sizep = origsize; - - return (0); -} - -#ifdef HAVE_CRYPTO -static int -checksum_send(const struct hast_resource *res, struct nv *nv, void **datap, - size_t *sizep, bool *freedatap __unused) -{ - unsigned char hash[SHA256_DIGEST_LENGTH]; - SHA256_CTX ctx; - - res = res; /* TODO */ - - SHA256_Init(&ctx); - SHA256_Update(&ctx, *datap, *sizep); - SHA256_Final(hash, &ctx); - - nv_add_string(nv, "sha256", "checksum"); - nv_add_uint8_array(nv, hash, sizeof(hash), "hash"); - - return (0); -} - -static int -checksum_recv(const struct hast_resource *res, struct nv *nv, void **datap, - size_t *sizep, bool *freedatap __unused) -{ - unsigned char chash[SHA256_DIGEST_LENGTH]; - const unsigned char *rhash; - SHA256_CTX ctx; - const char *algo; - size_t size; - - res = res; /* TODO */ - - algo = nv_get_string(nv, "checksum"); - if (algo == NULL) - return (0); /* No checksum. */ - if (strcmp(algo, "sha256") != 0) { - pjdlog_error("Unknown checksum algorithm '%s'.", algo); - return (-1); /* Unknown checksum algorithm. */ - } - rhash = nv_get_uint8_array(nv, &size, "hash"); - if (rhash == NULL) { - pjdlog_error("Checksum algorithm is present, but hash is missing."); - return (-1); /* Hash not found. */ - } - if (size != sizeof(chash)) { - pjdlog_error("Invalid hash size (%zu) for %s, should be %zu.", - size, algo, sizeof(chash)); - return (-1); /* Different hash size. */ - } - - SHA256_Init(&ctx); - SHA256_Update(&ctx, *datap, *sizep); - SHA256_Final(chash, &ctx); - - if (bcmp(rhash, chash, sizeof(chash)) != 0) { - pjdlog_error("Hash mismatch."); - return (-1); /* Hash mismatch. */ - } - - return (0); -} -#endif /* HAVE_CRYPTO */ - /* * Send the given nv structure via conn. * We keep headers in nv structure and pass data in separate argument. @@ -253,18 +93,13 @@ ret = -1; if (data != NULL) { -if (false) { unsigned int ii; for (ii = 0; ii < sizeof(pipeline) / sizeof(pipeline[0]); ii++) { - ret = pipeline[ii].hps_send(res, nv, &dptr, &size, + (void)pipeline[ii].hps_send(res, nv, &dptr, &size, &freedata); - if (ret == -1) - goto end; } - ret = -1; -} nv_add_uint32(nv, size, "size"); if (nv_error(nv) != 0) { errno = nv_error(nv); @@ -359,27 +194,24 @@ else { if (proto_recv(conn, data, dsize) < 0) goto end; -if (false) { for (ii = sizeof(pipeline) / sizeof(pipeline[0]); ii > 0; ii--) { - assert(!"to be verified"); ret = pipeline[ii - 1].hps_recv(res, nv, &dptr, &dsize, &freedata); if (ret == -1) goto end; } ret = -1; - if (dsize < size) + if (dsize > size) { + errno = EINVAL; goto end; - /* TODO: 'size' doesn't seem right here. It is maximum data size. */ + } if (dptr != data) bcopy(dptr, data, dsize); -} } ret = 0; end: -if (ret < 0) printf("%s:%u %s\n", __func__, __LINE__, strerror(errno)); if (freedata) free(dptr); return (ret); Index: sbin/hastd/hast_compression.h =================================================================== --- sbin/hastd/hast_compression.h (wersja 0) +++ sbin/hastd/hast_compression.h (wersja 0) @@ -0,0 +1,44 @@ +/*- + * Copyright (c) 2011 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _HAST_COMPRESSION_H_ +#define _HAST_COMPRESSION_H_ + +#include /* size_t */ + +#include +#include + +const char *compression_name(int num); + +int compression_send(const struct hast_resource *res, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap); +int compression_recv(const struct hast_resource *res, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap); + +#endif /* !_HAST_COMPRESSION_H_ */ Zmiany atrybutów dla: sbin/hastd/hast_compression.h ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/hast_checksum.h =================================================================== --- sbin/hastd/hast_checksum.h (wersja 0) +++ sbin/hastd/hast_checksum.h (wersja 0) @@ -0,0 +1,44 @@ +/*- + * Copyright (c) 2011 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _HAST_CHECKSUM_H_ +#define _HAST_CHECKSUM_H_ + +#include /* size_t */ + +#include +#include + +const char *checksum_name(int num); + +int checksum_send(const struct hast_resource *res, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap); +int checksum_recv(const struct hast_resource *res, struct nv *nv, + void **datap, size_t *sizep, bool *freedatap); + +#endif /* !_HAST_CHECKSUM_H_ */ Zmiany atrybutów dla: sbin/hastd/hast_checksum.h ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/lzf.c =================================================================== --- sbin/hastd/lzf.c (wersja 0) +++ sbin/hastd/lzf.c (wersja 0) @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#include "lzf.h" + +#define HSIZE (1 << (HLOG)) + +/* + * don't play with this unless you benchmark! + * decompression is not dependent on the hash function + * the hashing function might seem strange, just believe me + * it works ;) + */ +#ifndef FRST +# define FRST(p) (((p[0]) << 8) | p[1]) +# define NEXT(v,p) (((v) << 8) | p[2]) +# if ULTRA_FAST +# define IDX(h) ((( h >> (3*8 - HLOG)) - h ) & (HSIZE - 1)) +# elif VERY_FAST +# define IDX(h) ((( h >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) +# else +# define IDX(h) ((((h ^ (h << 5)) >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) +# endif +#endif +/* + * IDX works because it is very similar to a multiplicative hash, e.g. + * ((h * 57321 >> (3*8 - HLOG)) & (HSIZE - 1)) + * the latter is also quite fast on newer CPUs, and compresses similarly. + * + * the next one is also quite good, albeit slow ;) + * (int)(cos(h & 0xffffff) * 1e6) + */ + +#if 0 +/* original lzv-like hash function, much worse and thus slower */ +# define FRST(p) (p[0] << 5) ^ p[1] +# define NEXT(v,p) ((v) << 5) ^ p[2] +# define IDX(h) ((h) & (HSIZE - 1)) +#endif + +#define MAX_LIT (1 << 5) +#define MAX_OFF (1 << 13) +#define MAX_REF ((1 << 8) + (1 << 3)) + +#if __GNUC__ >= 3 +# define expect(expr,value) __builtin_expect ((expr),(value)) +# define inline inline +#else +# define expect(expr,value) (expr) +# define inline static +#endif + +#define expect_false(expr) expect ((expr) != 0, 0) +#define expect_true(expr) expect ((expr) != 0, 1) + +/* + * compressed format + * + * 000LLLLL ; literal + * LLLooooo oooooooo ; backref L + * 111ooooo LLLLLLLL oooooooo ; backref L+7 + * + */ + +unsigned int +lzf_compress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len +#if LZF_STATE_ARG + , LZF_STATE htab +#endif + ) +{ +#if !LZF_STATE_ARG + LZF_STATE htab; +#endif + const u8 **hslot; + const u8 *ip = (const u8 *)in_data; + u8 *op = (u8 *)out_data; + const u8 *in_end = ip + in_len; + u8 *out_end = op + out_len; + const u8 *ref; + + /* off requires a type wide enough to hold a general pointer difference. + * ISO C doesn't have that (size_t might not be enough and ptrdiff_t only + * works for differences within a single object). We also assume that no + * no bit pattern traps. Since the only platform that is both non-POSIX + * and fails to support both assumptions is windows 64 bit, we make a + * special workaround for it. + */ +#if defined (WIN32) && defined (_M_X64) + unsigned _int64 off; /* workaround for missing POSIX compliance */ +#else + unsigned long off; +#endif + unsigned int hval; + int lit; + + if (!in_len || !out_len) + return 0; + +#if INIT_HTAB + memset (htab, 0, sizeof (htab)); +# if 0 + for (hslot = htab; hslot < htab + HSIZE; hslot++) + *hslot++ = ip; +# endif +#endif + + lit = 0; op++; /* start run */ + + hval = FRST (ip); + while (ip < in_end - 2) + { + hval = NEXT (hval, ip); + hslot = htab + IDX (hval); + ref = *hslot; *hslot = ip; + + if (1 +#if INIT_HTAB + && ref < ip /* the next test will actually take care of this, but this is faster */ +#endif + && (off = ip - ref - 1) < MAX_OFF + && ip + 4 < in_end + && ref > (const u8 *)in_data +#if STRICT_ALIGN + && ref[0] == ip[0] + && ref[1] == ip[1] + && ref[2] == ip[2] +#else + && *(const u16 *)ref == *(const u16 *)ip + && ref[2] == ip[2] +#endif + ) + { + /* match found at *ref++ */ + unsigned int len = 2; + unsigned int maxlen = in_end - ip - len; + maxlen = maxlen > MAX_REF ? MAX_REF : maxlen; + + if (expect_false (op + 3 + 1 >= out_end)) /* first a faster conservative test */ + if (op - !lit + 3 + 1 >= out_end) /* second the exact but rare test */ + return 0; + + op [- lit - 1] = lit - 1; /* stop run */ + op -= !lit; /* undo run if length is zero */ + + for (;;) + { + if (expect_true (maxlen > 16)) + { + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + } + + do + len++; + while (len < maxlen && ref[len] == ip[len]); + + break; + } + + len -= 2; /* len is now #octets - 1 */ + ip++; + + if (len < 7) + { + *op++ = (off >> 8) + (len << 5); + } + else + { + *op++ = (off >> 8) + ( 7 << 5); + *op++ = len - 7; + } + + *op++ = off; + lit = 0; op++; /* start run */ + + ip += len + 1; + + if (expect_false (ip >= in_end - 2)) + break; + +#if ULTRA_FAST || VERY_FAST + --ip; +# if VERY_FAST && !ULTRA_FAST + --ip; +# endif + hval = FRST (ip); + + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; + +# if VERY_FAST && !ULTRA_FAST + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; +# endif +#else + ip -= len + 1; + + do + { + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; + } + while (len--); +#endif + } + else + { + /* one more literal byte we must copy */ + if (expect_false (op >= out_end)) + return 0; + + lit++; *op++ = *ip++; + + if (expect_false (lit == MAX_LIT)) + { + op [- lit - 1] = lit - 1; /* stop run */ + lit = 0; op++; /* start run */ + } + } + } + + if (op + 3 > out_end) /* at most 3 bytes can be missing here */ + return 0; + + while (ip < in_end) + { + lit++; *op++ = *ip++; + + if (expect_false (lit == MAX_LIT)) + { + op [- lit - 1] = lit - 1; /* stop run */ + lit = 0; op++; /* start run */ + } + } + + op [- lit - 1] = lit - 1; /* end run */ + op -= !lit; /* undo run if length is zero */ + + return op - (u8 *)out_data; +} + +#if AVOID_ERRNO +# define SET_ERRNO(n) +#else +# include +# define SET_ERRNO(n) errno = (n) +#endif + +#if (__i386 || __amd64) && __GNUC__ >= 3 +# define lzf_movsb(dst, src, len) \ + asm ("rep movsb" \ + : "=D" (dst), "=S" (src), "=c" (len) \ + : "0" (dst), "1" (src), "2" (len)); +#endif + +unsigned int +lzf_decompress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len) +{ + u8 const *ip = (const u8 *)in_data; + u8 *op = (u8 *)out_data; + u8 const *const in_end = ip + in_len; + u8 *const out_end = op + out_len; + + do + { + unsigned int ctrl = *ip++; + + if (ctrl < (1 << 5)) /* literal run */ + { + ctrl++; + + if (op + ctrl > out_end) + { + SET_ERRNO (E2BIG); + return 0; + } + +#if CHECK_INPUT + if (ip + ctrl > in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + +#ifdef lzf_movsb + lzf_movsb (op, ip, ctrl); +#else + do + *op++ = *ip++; + while (--ctrl); +#endif + } + else /* back reference */ + { + unsigned int len = ctrl >> 5; + + u8 *ref = op - ((ctrl & 0x1f) << 8) - 1; + +#if CHECK_INPUT + if (ip >= in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + if (len == 7) + { + len += *ip++; +#if CHECK_INPUT + if (ip >= in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + } + + ref -= *ip++; + + if (op + len + 2 > out_end) + { + SET_ERRNO (E2BIG); + return 0; + } + + if (ref < (u8 *)out_data) + { + SET_ERRNO (EINVAL); + return 0; + } + +#ifdef lzf_movsb + len += 2; + lzf_movsb (op, ref, len); +#else + *op++ = *ref++; + *op++ = *ref++; + + do + *op++ = *ref++; + while (--len); +#endif + } + } + while (ip < in_end); + + return op - (u8 *)out_data; +} + Zmiany atrybutów dla: sbin/hastd/lzf.c ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/Makefile =================================================================== --- sbin/hastd/Makefile (wersja 218190) +++ sbin/hastd/Makefile (kopia robocza) @@ -4,9 +4,10 @@ PROG= hastd SRCS= activemap.c -SRCS+= control.c +SRCS+= control.c crc32.c SRCS+= ebuf.c event.c -SRCS+= hast_proto.c hastd.c hooks.c +SRCS+= hast_checksum.c hast_compression.c hast_proto.c hastd.c hooks.c +SRCS+= lzf.c SRCS+= metadata.c SRCS+= nv.c SRCS+= secondary.c Index: sbin/hastd/lzf.h =================================================================== --- sbin/hastd/lzf.h (wersja 0) +++ sbin/hastd/lzf.h (wersja 0) @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#ifndef LZF_H +#define LZF_H + +/*********************************************************************** +** +** lzf -- an extremely fast/free compression/decompression-method +** http://liblzf.plan9.de/ +** +** This algorithm is believed to be patent-free. +** +***********************************************************************/ + +#define LZF_VERSION 0x0105 /* 1.5, API version */ + +/* + * Compress in_len bytes stored at the memory block starting at + * in_data and write the result to out_data, up to a maximum length + * of out_len bytes. + * + * If the output buffer is not large enough or any error occurs return 0, + * otherwise return the number of bytes used, which might be considerably + * more than in_len (but less than 104% of the original size), so it + * makes sense to always use out_len == in_len - 1), to ensure _some_ + * compression, and store the data uncompressed otherwise (with a flag, of + * course. + * + * lzf_compress might use different algorithms on different systems and + * even different runs, thus might result in different compressed strings + * depending on the phase of the moon or similar factors. However, all + * these strings are architecture-independent and will result in the + * original data when decompressed using lzf_decompress. + * + * The buffers must not be overlapping. + * + * If the option LZF_STATE_ARG is enabled, an extra argument must be + * supplied which is not reflected in this header file. Refer to lzfP.h + * and lzf_c.c. + * + */ +unsigned int +lzf_compress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len); + +/* + * Decompress data compressed with some version of the lzf_compress + * function and stored at location in_data and length in_len. The result + * will be stored at out_data up to a maximum of out_len characters. + * + * If the output buffer is not large enough to hold the decompressed + * data, a 0 is returned and errno is set to E2BIG. Otherwise the number + * of decompressed bytes (i.e. the original length of the data) is + * returned. + * + * If an error in the compressed data is detected, a zero is returned and + * errno is set to EINVAL. + * + * This function is very fast, about as fast as a copying loop. + */ +unsigned int +lzf_decompress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len); + +/* + * Size of hashtable is (1 << HLOG) * sizeof (char *) + * decompression is independent of the hash table size + * the difference between 15 and 14 is very small + * for small blocks (and 14 is usually a bit faster). + * For a low-memory/faster configuration, use HLOG == 13; + * For best compression, use 15 or 16 (or more, up to 23). + */ +#ifndef HLOG +# define HLOG 16 +#endif + +/* + * Sacrifice very little compression quality in favour of compression speed. + * This gives almost the same compression as the default code, and is + * (very roughly) 15% faster. This is the preferred mode of operation. + */ +#ifndef VERY_FAST +# define VERY_FAST 1 +#endif + +/* + * Sacrifice some more compression quality in favour of compression speed. + * (roughly 1-2% worse compression for large blocks and + * 9-10% for small, redundant, blocks and >>20% better speed in both cases) + * In short: when in need for speed, enable this for binary data, + * possibly disable this for text data. + */ +#ifndef ULTRA_FAST +# define ULTRA_FAST 0 +#endif + +/* + * Unconditionally aligning does not cost very much, so do it if unsure + */ +#ifndef STRICT_ALIGN +# define STRICT_ALIGN !(defined(__i386) || defined (__amd64)) +#endif + +/* + * You may choose to pre-set the hash table (might be faster on some + * modern cpus and large (>>64k) blocks, and also makes compression + * deterministic/repeatable when the configuration otherwise is the same). + */ +#ifndef INIT_HTAB +# define INIT_HTAB 1 +#endif + +/* + * Avoid assigning values to errno variable? for some embedding purposes + * (linux kernel for example), this is neccessary. NOTE: this breaks + * the documentation in lzf.h. + */ +#ifndef AVOID_ERRNO +# define AVOID_ERRNO 0 +#endif + +/* + * Wether to pass the LZF_STATE variable as argument, or allocate it + * on the stack. For small-stack environments, define this to 1. + * NOTE: this breaks the prototype in lzf.h. + */ +#ifndef LZF_STATE_ARG +# define LZF_STATE_ARG 0 +#endif + +/* + * Wether to add extra checks for input validity in lzf_decompress + * and return EINVAL if the input stream has been corrupted. This + * only shields against overflowing the input buffer and will not + * detect most corrupted streams. + * This check is not normally noticable on modern hardware + * (<1% slowdown), but might slow down older cpus considerably. + */ +#ifndef CHECK_INPUT +# define CHECK_INPUT 1 +#endif + +/*****************************************************************************/ +/* nothing should be changed below */ + +typedef unsigned char u8; + +typedef const u8 *LZF_STATE[1 << (HLOG)]; + +#if !STRICT_ALIGN +/* for unaligned accesses we need a 16 bit datatype. */ +# include +# if USHRT_MAX == 65535 + typedef unsigned short u16; +# elif UINT_MAX == 65535 + typedef unsigned int u16; +# else +# undef STRICT_ALIGN +# define STRICT_ALIGN 1 +# endif +#endif + +#if ULTRA_FAST +# if defined(VERY_FAST) +# undef VERY_FAST +# endif +#endif + +#if INIT_HTAB +# ifdef __cplusplus +# include +# else +# include +# endif +#endif + +#endif Zmiany atrybutów dla: sbin/hastd/lzf.h ___________________________________________________________________ Dodane: svn:mime-type + text/plain Dodane: svn:keywords + FreeBSD=%H Dodane: svn:eol-style + native Index: sbin/hastd/hast.h =================================================================== --- sbin/hastd/hast.h (wersja 218218) +++ sbin/hastd/hast.h (kopia robocza) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation + * Copyright (c) 2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -116,6 +117,14 @@ #define HAST_REPLICATION_MEMSYNC 1 #define HAST_REPLICATION_ASYNC 2 +#define HAST_COMPRESSION_NONE 0 +#define HAST_COMPRESSION_HOLE 1 +#define HAST_COMPRESSION_LZF 2 + +#define HAST_CHECKSUM_NONE 0 +#define HAST_CHECKSUM_CRC32 1 +#define HAST_CHECKSUM_SHA256 2 + /* * Structure that describes single resource. */ @@ -132,6 +141,10 @@ int hr_keepdirty; /* Path to a program to execute on various events. */ char hr_exec[PATH_MAX]; + /* Compression algorithm. */ + int hr_compression; + /* Checksum algorithm. */ + int hr_checksum; /* Path to local component. */ char hr_localpath[PATH_MAX]; Index: sbin/hastd/hast.conf.5 =================================================================== --- sbin/hastd/hast.conf.5 (wersja 218190) +++ sbin/hastd/hast.conf.5 (kopia robocza) @@ -1,5 +1,5 @@ .\" Copyright (c) 2010 The FreeBSD Foundation -.\" Copyright (c) 2010 Pawel Jakub Dawidek +.\" Copyright (c) 2010-2011 Pawel Jakub Dawidek .\" All rights reserved. .\" .\" This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -59,6 +59,8 @@ control listen replication +checksum +compression timeout exec @@ -77,6 +79,8 @@ resource { # Resource section replication + checksum + compression name local timeout @@ -201,6 +205,36 @@ .Ic async replication mode is currently not implemented. .El +.It Ic checksum Aq algorithm +.Pp +Checksum algorithm should be one of the following: +.Bl -tag -width ".Ic sha256" +.It Ic none +No checksum will be calculated for the data being send over the network. +This is the default setting. +.It Ic crc32 +CRC32 checksum will be calculated. +.It Ic sha256 +SHA256 checksum will be calculated. +.El +.It Ic compression Aq algorithm +.Pp +Compression algorithm should be one of the following: +.Bl -tag -width ".Ic none" +.It Ic none +Data send over the network will not be compressed. +.It Ic hole +Only blocks that contain all zeros will be compressed. +This is very useful for initial synchronization where potentially many blocks +are still all zeros. +There should be no measurable performance overhead when this algorithm is being +used. +This is the default setting. +.It Ic lzf +The LZF algorithm by Marc Alexander Lehmann will be used to compress the data +send over the network. +LZF is very fast, general purpose compression algorithm. +.El .It Ic timeout Aq seconds .Pp Connection timeout in seconds. Index: sbin/hastd/primary.c =================================================================== --- sbin/hastd/primary.c (wersja 218218) +++ sbin/hastd/primary.c (kopia robocza) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2009 The FreeBSD Foundation - * Copyright (c) 2010 Pawel Jakub Dawidek + * Copyright (c) 2010-2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -1909,15 +1909,19 @@ PJDLOG_ASSERT(gres == res); nv_assert(nv, "remoteaddr"); nv_assert(nv, "replication"); + nv_assert(nv, "checksum"); + nv_assert(nv, "compression"); nv_assert(nv, "timeout"); nv_assert(nv, "exec"); ncomps = HAST_NCOMPONENTS; -#define MODIFIED_REMOTEADDR 0x1 -#define MODIFIED_REPLICATION 0x2 -#define MODIFIED_TIMEOUT 0x4 -#define MODIFIED_EXEC 0x8 +#define MODIFIED_REMOTEADDR 0x01 +#define MODIFIED_REPLICATION 0x02 +#define MODIFIED_CHECKSUM 0x04 +#define MODIFIED_COMPRESSION 0x08 +#define MODIFIED_TIMEOUT 0x10 +#define MODIFIED_EXEC 0x20 modified = 0; vstr = nv_get_string(nv, "remoteaddr"); @@ -1934,6 +1938,16 @@ gres->hr_replication = vint; modified |= MODIFIED_REPLICATION; } + vint = nv_get_int32(nv, "checksum"); + if (gres->hr_checksum != vint) { + gres->hr_checksum = vint; + modified |= MODIFIED_CHECKSUM; + } + vint = nv_get_int32(nv, "compression"); + if (gres->hr_compression != vint) { + gres->hr_compression = vint; + modified |= MODIFIED_COMPRESSION; + } vint = nv_get_int32(nv, "timeout"); if (gres->hr_timeout != vint) { gres->hr_timeout = vint; @@ -1946,10 +1960,11 @@ } /* - * If only timeout was modified we only need to change it without - * reconnecting. + * Change timeout for connected sockets. + * Don't bother if we need to reconnect. */ - if (modified == MODIFIED_TIMEOUT) { + if ((modified & MODIFIED_TIMEOUT) != 0 && + (modified & (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) == 0) { for (ii = 0; ii < ncomps; ii++) { if (!ISREMOTE(ii)) continue; @@ -1970,8 +1985,8 @@ "Unable to set connection timeout"); } } - } else if ((modified & - (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) != 0) { + } + if ((modified & (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) != 0) { for (ii = 0; ii < ncomps; ii++) { if (!ISREMOTE(ii)) continue; @@ -1985,6 +2000,8 @@ } #undef MODIFIED_REMOTEADDR #undef MODIFIED_REPLICATION +#undef MODIFIED_CHECKSUM +#undef MODIFIED_COMPRESSION #undef MODIFIED_TIMEOUT #undef MODIFIED_EXEC Index: sbin/hastd/token.l =================================================================== --- sbin/hastd/token.l (wersja 218190) +++ sbin/hastd/token.l (kopia robocza) @@ -1,6 +1,7 @@ %{ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation + * Copyright (c) 2011 Pawel Jakub Dawidek * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from @@ -48,6 +49,8 @@ listen { DP; return LISTEN; } port { DP; return PORT; } replication { DP; return REPLICATION; } +checksum { DP; return CHECKSUM; } +compression { DP; return COMPRESSION; } timeout { DP; return TIMEOUT; } exec { DP; return EXEC; } resource { DP; return RESOURCE; } @@ -58,6 +61,11 @@ fullsync { DP; return FULLSYNC; } memsync { DP; return MEMSYNC; } async { DP; return ASYNC; } +none { DP; return NONE; } +crc32 { DP; return CRC32; } +sha256 { DP; return SHA256; } +hole { DP; return HOLE; } +lzf { DP; return LZF; } [0-9]+ { DP; yylval.num = atoi(yytext); return NUM; } [a-zA-Z0-9\.\-_/\:]+ { DP; yylval.str = strdup(yytext); return STR; } \{ { DP; depth++; return OB; }