Line data Source code
1 : /*-
2 : * Copyright (c) 2011-2013 Baptiste Daroussin <bapt@FreeBSD.org>
3 : * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
4 : * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>
5 : * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org>
6 : * Copyright (c) 2014 Vsevolod Stakhov <vsevolod@FreeBSD.org>
7 : *
8 : * All rights reserved.
9 : *
10 : * Redistribution and use in source and binary forms, with or without
11 : * modification, are permitted provided that the following conditions
12 : * are met:
13 : * 1. Redistributions of source code must retain the above copyright
14 : * notice, this list of conditions and the following disclaimer
15 : * in this position and unchanged.
16 : * 2. Redistributions in binary form must reproduce the above copyright
17 : * notice, this list of conditions and the following disclaimer in the
18 : * documentation and/or other materials provided with the distribution.
19 : *
20 : * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
21 : * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 : * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 : * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
24 : * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 : * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 : * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 : * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 : * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 : * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 : */
31 : #include "pkg_config.h"
32 :
33 : #include <sys/types.h>
34 : #include <sys/stat.h>
35 : #include <sys/sysctl.h>
36 : #include <sys/wait.h>
37 : #include <sys/socket.h>
38 : #include <sys/file.h>
39 : #include <sys/time.h>
40 :
41 : #include <archive_entry.h>
42 : #include <assert.h>
43 : #include <fts.h>
44 : #include <libgen.h>
45 : #include <sqlite3.h>
46 : #include <string.h>
47 : #define _WITH_GETLINE
48 : #include <stdio.h>
49 : #include <stdbool.h>
50 : #include <sysexits.h>
51 : #include <unistd.h>
52 : #include <errno.h>
53 : #include <fcntl.h>
54 : #include <math.h>
55 : #include <poll.h>
56 : #include <sys/uio.h>
57 :
58 : #include "pkg.h"
59 : #include "private/event.h"
60 : #include "private/utils.h"
61 : #include "private/pkg.h"
62 : #include "private/pkgdb.h"
63 :
64 :
65 : struct digest_list_entry {
66 : char *origin;
67 : char *digest;
68 : long manifest_pos;
69 : long files_pos;
70 : long manifest_length;
71 : char *checksum;
72 : struct digest_list_entry *prev, *next;
73 : };
74 :
75 : struct pkg_conflict_bulk {
76 : struct pkg_conflict *conflicts;
77 : char *file;
78 : UT_hash_handle hh;
79 : };
80 :
81 : static int
82 31 : pkg_digest_sort_compare_func(struct digest_list_entry *d1,
83 : struct digest_list_entry *d2)
84 : {
85 31 : return strcmp(d1->origin, d2->origin);
86 : }
87 :
88 : static void
89 0 : pkg_repo_new_conflict(const char *uniqueid, struct pkg_conflict_bulk *bulk)
90 : {
91 : struct pkg_conflict *new;
92 :
93 0 : pkg_conflict_new(&new);
94 0 : new->uid = strdup(uniqueid);
95 :
96 0 : HASH_ADD_KEYPTR(hh, bulk->conflicts, new->uid, strlen(new->uid), new);
97 0 : }
98 :
99 : static void
100 0 : pkg_repo_write_conflicts (struct pkg_conflict_bulk *bulk, FILE *out)
101 : {
102 0 : struct pkg_conflict_bulk *pkg_bulk = NULL, *cur, *tmp, *s;
103 : struct pkg_conflict *c1, *c1tmp, *c2, *c2tmp, *ctmp;
104 :
105 : /*
106 : * Here we reorder bulk hash from hash by file
107 : * to hash indexed by a package, so we iterate over the
108 : * original hash and create a new hash indexed by package name
109 : */
110 :
111 0 : HASH_ITER (hh, bulk, cur, tmp) {
112 0 : HASH_ITER (hh, cur->conflicts, c1, c1tmp) {
113 0 : HASH_FIND_STR(pkg_bulk, c1->uid, s);
114 0 : if (s == NULL) {
115 : /* New entry required */
116 0 : s = malloc(sizeof(struct pkg_conflict_bulk));
117 0 : if (s == NULL) {
118 0 : pkg_emit_errno("malloc", "struct pkg_conflict_bulk");
119 0 : goto out;
120 : }
121 0 : memset(s, 0, sizeof(struct pkg_conflict_bulk));
122 0 : s->file = c1->uid;
123 0 : HASH_ADD_KEYPTR(hh, pkg_bulk, s->file, strlen(s->file), s);
124 : }
125 : /* Now add all new entries from this file to this conflict structure */
126 0 : HASH_ITER (hh, cur->conflicts, c2, c2tmp) {
127 0 : if (strcmp(c1->uid, c2->uid) == 0)
128 0 : continue;
129 :
130 0 : HASH_FIND_STR(s->conflicts, c2->uid, ctmp);
131 0 : if (ctmp == NULL)
132 0 : pkg_repo_new_conflict(c2->uid, s);
133 : }
134 : }
135 : }
136 :
137 0 : HASH_ITER (hh, pkg_bulk, cur, tmp) {
138 0 : fprintf(out, "%s:", cur->file);
139 0 : HASH_ITER (hh, cur->conflicts, c1, c1tmp) {
140 0 : if (c1->hh.next != NULL)
141 0 : fprintf(out, "%s,", c1->uid);
142 : else
143 0 : fprintf(out, "%s\n", c1->uid);
144 : }
145 : }
146 : out:
147 0 : HASH_ITER (hh, pkg_bulk, cur, tmp) {
148 0 : HASH_ITER (hh, cur->conflicts, c1, c1tmp) {
149 0 : HASH_DEL(cur->conflicts, c1);
150 0 : free(c1->uid);
151 0 : free(c1);
152 : }
153 0 : HASH_DEL(pkg_bulk, cur);
154 0 : free(cur);
155 : }
156 0 : return;
157 : }
158 :
159 : struct pkg_fts_item {
160 : char *fts_accpath;
161 : char *pkg_path;
162 : char *fts_name;
163 : off_t fts_size;
164 : int fts_info;
165 : struct pkg_fts_item *next;
166 : };
167 :
168 : static struct pkg_fts_item*
169 91 : pkg_create_repo_fts_new(FTSENT *fts, const char *root_path)
170 : {
171 : struct pkg_fts_item *item;
172 : char *pkg_path;
173 :
174 91 : item = malloc(sizeof(*item));
175 91 : if (item != NULL) {
176 91 : item->fts_accpath = strdup(fts->fts_accpath);
177 91 : item->fts_name = strdup(fts->fts_name);
178 91 : item->fts_size = fts->fts_statp->st_size;
179 91 : item->fts_info = fts->fts_info;
180 :
181 91 : pkg_path = fts->fts_path;
182 91 : pkg_path += strlen(root_path);
183 273 : while (pkg_path[0] == '/')
184 91 : pkg_path++;
185 :
186 91 : item->pkg_path = strdup(pkg_path);
187 : }
188 : else {
189 0 : pkg_emit_errno("malloc", "struct pkg_fts_item");
190 : }
191 :
192 91 : return (item);
193 : }
194 :
195 : static void
196 31 : pkg_create_repo_fts_free(struct pkg_fts_item *item)
197 : {
198 31 : free(item->fts_accpath);
199 31 : free(item->pkg_path);
200 31 : free(item->fts_name);
201 31 : free(item);
202 31 : }
203 :
204 : static int
205 31 : pkg_create_repo_read_fts(struct pkg_fts_item **items, FTS *fts,
206 : const char *repopath, size_t *plen, struct pkg_repo_meta *meta)
207 : {
208 : FTSENT *fts_ent;
209 : struct pkg_fts_item *fts_cur;
210 : char *ext;
211 :
212 31 : errno = 0;
213 :
214 613 : while ((fts_ent = fts_read(fts)) != NULL) {
215 : /*
216 : * Skip directories starting with '.' to avoid Poudriere
217 : * symlinks.
218 : */
219 1040 : if ((fts_ent->fts_info == FTS_D ||
220 613 : fts_ent->fts_info == FTS_DP) &&
221 192 : fts_ent->fts_namelen > 2 &&
222 68 : fts_ent->fts_name[0] == '.') {
223 0 : fts_set(fts, fts_ent, FTS_SKIP);
224 0 : continue;
225 : }
226 : /*
227 : * Ignore 'Latest' directory as it is just symlinks back to
228 : * already-processed packages.
229 : */
230 1040 : if ((fts_ent->fts_info == FTS_D ||
231 916 : fts_ent->fts_info == FTS_DP ||
232 557 : fts_ent->fts_info == FTS_SL) &&
233 130 : strcmp(fts_ent->fts_name, "Latest") == 0) {
234 6 : fts_set(fts, fts_ent, FTS_SKIP);
235 6 : continue;
236 : }
237 : /* Follow symlinks. */
238 545 : if (fts_ent->fts_info == FTS_SL) {
239 6 : fts_set(fts, fts_ent, FTS_FOLLOW);
240 : /* Restart. Next entry will be the resolved file. */
241 6 : continue;
242 : }
243 : /* Skip everything that is not a file */
244 539 : if (fts_ent->fts_info != FTS_F)
245 118 : continue;
246 :
247 421 : ext = strrchr(fts_ent->fts_name, '.');
248 :
249 421 : if (ext == NULL)
250 115 : continue;
251 :
252 306 : if (strcmp(ext + 1, packing_format_to_string(meta->packing_format)) != 0)
253 197 : continue;
254 :
255 109 : *ext = '\0';
256 :
257 212 : if (strcmp(fts_ent->fts_name, "meta") == 0 ||
258 103 : pkg_repo_meta_is_special_file(fts_ent->fts_name, meta)) {
259 18 : *ext = '.';
260 18 : continue;
261 : }
262 :
263 91 : *ext = '.';
264 91 : fts_cur = pkg_create_repo_fts_new(fts_ent, repopath);
265 91 : if (fts_cur == NULL)
266 0 : return (EPKG_FATAL);
267 :
268 91 : LL_PREPEND(*items, fts_cur);
269 91 : (*plen) ++;
270 : }
271 :
272 31 : if (errno != 0) {
273 0 : pkg_emit_errno("fts_read", "pkg_create_repo_read_fts");
274 0 : return (EPKG_FATAL);
275 : }
276 :
277 31 : return (EPKG_OK);
278 : }
279 :
280 : static int
281 49 : pkg_create_repo_worker(struct pkg_fts_item *start, size_t nelts,
282 : const char *mlfile, const char *flfile, int pip,
283 : struct pkg_repo_meta *meta)
284 : {
285 : pid_t pid;
286 49 : int mfd, ffd = -1;
287 49 : bool read_files = (flfile != NULL);
288 49 : bool legacy = (meta == NULL);
289 49 : int flags, ret = EPKG_OK;
290 49 : size_t cur_job = 0;
291 : struct pkg_fts_item *cur;
292 49 : struct pkg *pkg = NULL;
293 49 : struct pkg_manifest_key *keys = NULL;
294 49 : char *mdigest = NULL;
295 : char digestbuf[1024];
296 : struct iovec iov[2];
297 : struct msghdr msg;
298 :
299 49 : struct sbuf *b = sbuf_new_auto();
300 :
301 49 : mfd = open(mlfile, O_APPEND|O_CREAT|O_WRONLY, 00644);
302 49 : if (mfd == -1) {
303 0 : pkg_emit_errno("pkg_create_repo_worker", "open");
304 0 : return (EPKG_FATAL);
305 : }
306 :
307 49 : if (read_files) {
308 0 : ffd = open(flfile, O_APPEND|O_CREAT|O_WRONLY, 00644);
309 0 : if (ffd == -1) {
310 0 : close(mfd);
311 0 : pkg_emit_errno("pkg_create_repo_worker", "open");
312 0 : return (EPKG_FATAL);
313 : }
314 : }
315 :
316 49 : pid = fork();
317 49 : switch(pid) {
318 : case -1:
319 0 : pkg_emit_errno("pkg_create_repo_worker", "fork");
320 0 : close(mfd);
321 0 : if (read_files)
322 0 : close(ffd);
323 0 : return (EPKG_FATAL);
324 : break;
325 : case 0:
326 20 : break;
327 : default:
328 : /* Parent */
329 29 : close(mfd);
330 29 : if (read_files)
331 0 : close(ffd);
332 :
333 29 : return (EPKG_OK);
334 : break;
335 : }
336 :
337 20 : pkg_manifest_keys_new(&keys);
338 20 : pkg_debug(1, "start worker to parse %d packages", nelts);
339 :
340 20 : if (read_files)
341 0 : flags = PKG_OPEN_MANIFEST_ONLY;
342 : else
343 20 : flags = PKG_OPEN_MANIFEST_ONLY | PKG_OPEN_MANIFEST_COMPACT;
344 :
345 20 : if (read(pip, digestbuf, 1) == -1) {
346 0 : pkg_emit_errno("pkg_create_repo_worker", "read");
347 0 : goto cleanup;
348 : }
349 :
350 51 : LL_FOREACH(start, cur) {
351 40 : if (cur_job >= nelts)
352 9 : break;
353 :
354 31 : if (pkg_open(&pkg, cur->fts_accpath, keys, flags) == EPKG_OK) {
355 : int r;
356 31 : off_t mpos, fpos = 0;
357 : size_t mlen;
358 :
359 31 : pkg->sum = pkg_checksum_file(cur->fts_accpath,
360 : PKG_HASH_TYPE_SHA256_HEX);
361 31 : pkg->pkgsize = cur->fts_size;
362 31 : pkg->repopath = strdup(cur->pkg_path);
363 :
364 : /*
365 : * TODO: use pkg_checksum for new manifests
366 : */
367 31 : sbuf_clear(b);
368 31 : if (legacy)
369 0 : pkg_emit_manifest_sbuf(pkg, b, PKG_MANIFEST_EMIT_COMPACT, &mdigest);
370 : else {
371 31 : mdigest = malloc(pkg_checksum_type_size(meta->digest_format));
372 :
373 31 : pkg_emit_manifest_sbuf(pkg, b, PKG_MANIFEST_EMIT_COMPACT, NULL);
374 31 : if (pkg_checksum_generate(pkg, mdigest,
375 : pkg_checksum_type_size(meta->digest_format),
376 : meta->digest_format) != EPKG_OK) {
377 0 : pkg_emit_error("Cannot generate digest for a package");
378 0 : ret = EPKG_FATAL;
379 :
380 0 : goto cleanup;
381 : }
382 : }
383 31 : mlen = sbuf_len(b);
384 31 : sbuf_finish(b);
385 :
386 31 : if (flock(mfd, LOCK_EX) == -1) {
387 0 : pkg_emit_errno("pkg_create_repo_worker", "flock");
388 0 : ret = EPKG_FATAL;
389 0 : goto cleanup;
390 : }
391 :
392 31 : mpos = lseek(mfd, 0, SEEK_END);
393 :
394 31 : iov[0].iov_base = sbuf_data(b);
395 31 : iov[0].iov_len = sbuf_len(b);
396 31 : iov[1].iov_base = (void *)"\n";
397 31 : iov[1].iov_len = 1;
398 :
399 31 : if (writev(mfd, iov, 2) == -1) {
400 0 : pkg_emit_errno("pkg_create_repo_worker", "write");
401 0 : ret = EPKG_FATAL;
402 0 : flock(mfd, LOCK_UN);
403 0 : goto cleanup;
404 : }
405 :
406 31 : flock(mfd, LOCK_UN);
407 :
408 31 : if (read_files) {
409 : FILE *fl;
410 :
411 0 : if (flock(ffd, LOCK_EX) == -1) {
412 0 : pkg_emit_errno("pkg_create_repo_worker", "flock");
413 0 : ret = EPKG_FATAL;
414 0 : goto cleanup;
415 : }
416 0 : fpos = lseek(ffd, 0, SEEK_END);
417 0 : fl = fdopen(dup(ffd), "a");
418 0 : pkg_emit_filelist(pkg, fl);
419 0 : fclose(fl);
420 :
421 0 : flock(ffd, LOCK_UN);
422 : }
423 :
424 62 : r = snprintf(digestbuf, sizeof(digestbuf), "%s:%s:%ld:%ld:%ld:%s\n",
425 31 : pkg->origin,
426 : mdigest,
427 : (long)mpos,
428 : (long)fpos,
429 : (long)mlen,
430 31 : pkg->sum);
431 :
432 31 : free(mdigest);
433 31 : mdigest = NULL;
434 31 : iov[0].iov_base = digestbuf;
435 31 : iov[0].iov_len = r;
436 31 : memset(&msg, 0, sizeof(msg));
437 31 : msg.msg_iov = iov;
438 31 : msg.msg_iovlen = 1;
439 31 : sendmsg(pip, &msg, MSG_EOR);
440 : }
441 31 : cur_job ++;
442 : }
443 :
444 : cleanup:
445 20 : pkg_manifest_keys_free(keys);
446 :
447 20 : write(pip, ".\n", 2);
448 20 : close(pip);
449 20 : close(mfd);
450 20 : if (read_files)
451 0 : close(ffd);
452 20 : free(mdigest);
453 :
454 20 : pkg_debug(1, "worker done");
455 20 : exit(ret);
456 : }
457 :
458 : static int
459 35 : pkg_create_repo_read_pipe(int fd, struct digest_list_entry **dlist)
460 : {
461 35 : struct digest_list_entry *dig = NULL;
462 : char buf[1024];
463 : int r, i, start;
464 : enum {
465 : s_set_origin = 0,
466 : s_set_digest,
467 : s_set_mpos,
468 : s_set_fpos,
469 : s_set_mlen,
470 : s_set_checksum
471 35 : } state = 0;
472 :
473 : for (;;) {
474 66 : r = read(fd, buf, sizeof(buf));
475 :
476 66 : if (r == -1) {
477 15 : if (errno == EINTR)
478 0 : continue;
479 15 : else if (errno == ECONNRESET) {
480 : /* Treat it as the end of a connection */
481 0 : return (EPKG_END);
482 : }
483 15 : else if (errno == EAGAIN || errno == EWOULDBLOCK)
484 15 : return (EPKG_OK);
485 :
486 0 : pkg_emit_errno("pkg_create_repo_read_pipe", "read");
487 0 : return (EPKG_FATAL);
488 : }
489 51 : else if (r == 0)
490 0 : return (EPKG_END);
491 :
492 : /*
493 : * XXX: can parse merely full lines
494 : */
495 51 : start = 0;
496 4461 : for (i = 0; i < r; i ++) {
497 4461 : if (buf[i] == ':') {
498 155 : switch(state) {
499 : case s_set_origin:
500 31 : dig = calloc(1, sizeof(*dig));
501 31 : dig->origin = malloc(i - start + 1);
502 31 : strlcpy(dig->origin, &buf[start], i - start + 1);
503 31 : state = s_set_digest;
504 31 : break;
505 : case s_set_digest:
506 31 : dig->digest = malloc(i - start + 1);
507 31 : strlcpy(dig->digest, &buf[start], i - start + 1);
508 31 : state = s_set_mpos;
509 31 : break;
510 : case s_set_mpos:
511 31 : dig->manifest_pos = strtol(&buf[start], NULL, 10);
512 31 : state = s_set_fpos;
513 31 : break;
514 : case s_set_fpos:
515 31 : dig->files_pos = strtol(&buf[start], NULL, 10);
516 31 : state = s_set_mlen;
517 31 : break;
518 : case s_set_mlen:
519 31 : dig->manifest_length = strtol(&buf[start], NULL, 10);
520 31 : state = s_set_checksum;
521 31 : break;
522 : case s_set_checksum:
523 0 : dig->checksum = malloc(i - start + 1);
524 0 : strlcpy(dig->digest, &buf[start], i - start + 1);
525 0 : state = s_set_origin;
526 0 : break;
527 : }
528 155 : start = i + 1;
529 : }
530 4306 : else if (buf[i] == '\n') {
531 31 : if (state == s_set_mlen) {
532 0 : dig->manifest_length = strtol(&buf[start], NULL, 10);
533 : }
534 31 : else if (state == s_set_checksum) {
535 31 : dig->checksum = malloc(i - start + 1);
536 31 : strlcpy(dig->checksum, &buf[start], i - start + 1);
537 : }
538 31 : assert(dig->origin != NULL);
539 31 : assert(dig->digest != NULL);
540 31 : DL_APPEND(*dlist, dig);
541 31 : state = s_set_origin;
542 31 : start = i + 1;
543 31 : break;
544 : }
545 4275 : else if (buf[i] == '.' && buf[i + 1] == '\n') {
546 20 : return (EPKG_END);
547 : }
548 : }
549 31 : }
550 :
551 : /*
552 : * Never reached
553 : */
554 : return (EPKG_OK);
555 : }
556 :
557 : int
558 31 : pkg_create_repo(char *path, const char *output_dir, bool filelist,
559 : const char *metafile, bool legacy)
560 : {
561 31 : FTS *fts = NULL;
562 31 : struct pkg_fts_item *fts_items = NULL, *fts_cur, *fts_start;
563 :
564 : struct pkg_conflict *c, *ctmp;
565 31 : struct pkg_conflict_bulk *conflicts = NULL, *curcb, *tmpcb;
566 : int num_workers, i, remaining_workers, remain, cur_jobs, remain_jobs, nworker;
567 : size_t len, tasks_per_worker, ntask;
568 31 : struct digest_list_entry *dlist = NULL, *cur_dig, *dtmp;
569 31 : struct pollfd *pfd = NULL;
570 : int cur_pipe[2], fd;
571 31 : struct pkg_repo_meta *meta = NULL;
572 31 : int retcode = EPKG_OK;
573 :
574 : char *repopath[2];
575 : char packagesite[MAXPATHLEN],
576 : filesite[MAXPATHLEN],
577 : repodb[MAXPATHLEN];
578 31 : FILE *mandigests = NULL;
579 :
580 31 : if (!is_dir(path)) {
581 0 : pkg_emit_error("%s is not a directory", path);
582 0 : return (EPKG_FATAL);
583 : }
584 :
585 31 : errno = 0;
586 31 : if (!is_dir(output_dir)) {
587 : /* Try to create dir */
588 0 : if (errno == ENOENT) {
589 0 : if (mkdir(output_dir, 00755) == -1) {
590 0 : pkg_emit_error("cannot create output directory %s: %s",
591 0 : output_dir, strerror(errno));
592 0 : return (EPKG_FATAL);
593 : }
594 : }
595 : else {
596 0 : pkg_emit_error("%s is not a directory", output_dir);
597 0 : return (EPKG_FATAL);
598 : }
599 : }
600 :
601 31 : if (metafile != NULL) {
602 0 : if (pkg_repo_meta_load(metafile, &meta) != EPKG_OK) {
603 0 : pkg_emit_error("meta loading error while trying %s", metafile);
604 0 : return (EPKG_FATAL);
605 : }
606 : }
607 : else {
608 31 : meta = pkg_repo_meta_default();
609 : }
610 :
611 31 : repopath[0] = path;
612 31 : repopath[1] = NULL;
613 :
614 31 : num_workers = pkg_object_int(pkg_config_get("WORKERS_COUNT"));
615 31 : if (num_workers <= 0) {
616 31 : len = sizeof(num_workers);
617 : #ifdef HAVE_SYSCTLBYNAME
618 31 : if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1)
619 0 : num_workers = 6;
620 : #else
621 : num_workers = 6;
622 : #endif
623 : }
624 :
625 31 : if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) {
626 0 : pkg_emit_errno("fts_open", path);
627 0 : retcode = EPKG_FATAL;
628 0 : goto cleanup;
629 : }
630 :
631 31 : snprintf(packagesite, sizeof(packagesite), "%s/%s", output_dir,
632 31 : meta->manifests);
633 31 : if ((fd = open(packagesite, O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) {
634 0 : retcode = EPKG_FATAL;
635 0 : goto cleanup;
636 : }
637 31 : close(fd);
638 31 : if (filelist) {
639 0 : snprintf(filesite, sizeof(filesite), "%s/%s", output_dir,
640 0 : meta->filesite);
641 0 : if ((fd = open(filesite, O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) {
642 0 : retcode = EPKG_FATAL;
643 0 : goto cleanup;
644 : }
645 0 : close(fd);
646 : }
647 31 : snprintf(repodb, sizeof(repodb), "%s/%s", output_dir,
648 31 : meta->digests);
649 31 : if ((mandigests = fopen(repodb, "w")) == NULL) {
650 0 : retcode = EPKG_FATAL;
651 0 : goto cleanup;
652 : }
653 :
654 31 : len = 0;
655 :
656 31 : pkg_create_repo_read_fts(&fts_items, fts, path, &len, meta);
657 :
658 31 : if (len == 0) {
659 : /* Nothing to do */
660 0 : pkg_emit_error("No package files have been found");
661 0 : retcode = EPKG_FATAL;
662 0 : goto cleanup;
663 : }
664 :
665 : /* Split items over all workers */
666 31 : num_workers = MIN(num_workers, len);
667 31 : tasks_per_worker = len / num_workers;
668 : /* How much extra tasks should be distributed over the workers */
669 31 : remain = len % num_workers;
670 31 : assert(tasks_per_worker > 0);
671 :
672 : /* Launch workers */
673 31 : pkg_emit_progress_start("Creating repository in %s", output_dir);
674 :
675 31 : pfd = calloc(num_workers, sizeof(struct pollfd));
676 31 : ntask = 0;
677 31 : cur_jobs = (remain > 0) ? tasks_per_worker + 1 : tasks_per_worker;
678 31 : remain_jobs = cur_jobs;
679 31 : fts_start = fts_items;
680 31 : nworker = 0;
681 :
682 89 : LL_FOREACH(fts_items, fts_cur) {
683 78 : if (--remain_jobs == 0) {
684 : /* Create new worker */
685 : int ofl;
686 49 : int st = SOCK_DGRAM;
687 :
688 : #ifdef HAVE_SEQPACKET
689 49 : st = SOCK_SEQPACKET;
690 : #endif
691 49 : if (socketpair(AF_UNIX, st, 0, cur_pipe) == -1) {
692 0 : pkg_emit_errno("pkg_create_repo", "pipe");
693 0 : retcode = EPKG_FATAL;
694 0 : goto cleanup;
695 : }
696 :
697 49 : if (pkg_create_repo_worker(fts_start, cur_jobs,
698 : packagesite, (filelist ? filesite : NULL), cur_pipe[1],
699 : (legacy ? NULL : meta)) == EPKG_FATAL) {
700 0 : close(cur_pipe[0]);
701 0 : close(cur_pipe[1]);
702 0 : retcode = EPKG_FATAL;
703 0 : goto cleanup;
704 : }
705 :
706 29 : pfd[nworker].fd = cur_pipe[0];
707 29 : pfd[nworker].events = POLLIN;
708 29 : close(cur_pipe[1]);
709 : /* Make our end of the pipe non-blocking */
710 29 : ofl = fcntl(cur_pipe[0], F_GETFL, 0);
711 29 : fcntl(cur_pipe[0], F_SETFL, ofl | O_NONBLOCK);
712 :
713 29 : if (--remain > 0)
714 0 : cur_jobs = tasks_per_worker + 1;
715 : else
716 29 : cur_jobs = tasks_per_worker;
717 :
718 29 : remain_jobs = cur_jobs;
719 29 : fts_start = fts_cur->next;
720 29 : nworker ++;
721 : }
722 58 : ntask ++;
723 : }
724 :
725 : /* Send start marker to all workers */
726 31 : for (i = 0; i < num_workers; i ++) {
727 20 : if (write(pfd[i].fd, ".", 1) == -1)
728 0 : pkg_emit_errno("pkg_create_repo", "write");
729 : }
730 :
731 11 : ntask = 0;
732 11 : remaining_workers = num_workers;
733 56 : while(remaining_workers > 0) {
734 : int st;
735 :
736 34 : pkg_debug(1, "checking for %d workers", remaining_workers);
737 34 : retcode = poll(pfd, num_workers, -1);
738 34 : if (retcode == -1) {
739 0 : if (errno == EINTR) {
740 0 : continue;
741 : }
742 : else {
743 0 : retcode = EPKG_FATAL;
744 0 : goto cleanup;
745 : }
746 : }
747 34 : else if (retcode > 0) {
748 100 : for (i = 0; i < num_workers; i ++) {
749 123 : if (pfd[i].fd != -1 &&
750 57 : (pfd[i].revents & (POLLIN|POLLHUP|POLLERR))) {
751 35 : if (pkg_create_repo_read_pipe(pfd[i].fd, &dlist) != EPKG_OK) {
752 : /*
753 : * Wait for the worker finished
754 : */
755 :
756 40 : while (wait(&st) == -1) {
757 0 : if (errno == EINTR)
758 0 : continue;
759 :
760 0 : pkg_emit_errno("pkg_create_repo", "wait");
761 0 : break;
762 : }
763 :
764 20 : remaining_workers --;
765 20 : pkg_debug(1, "finished worker, %d remaining",
766 : remaining_workers);
767 20 : pfd[i].events = 0;
768 20 : pfd[i].revents = 0;
769 20 : close(pfd[i].fd);
770 20 : pfd[i].fd = -1;
771 : }
772 : else {
773 15 : pkg_emit_progress_tick(ntask++, len);
774 : }
775 : }
776 : }
777 : }
778 : }
779 :
780 11 : pkg_emit_progress_tick(len, len);
781 11 : retcode = EPKG_OK;
782 :
783 : /* Now sort all digests */
784 11 : DL_SORT(dlist, pkg_digest_sort_compare_func);
785 :
786 : /*
787 : * XXX: it is not used actually
788 : */
789 : #if 0
790 : pkg_repo_write_conflicts(conflicts, fconflicts);
791 : #endif
792 :
793 : /* Write metafile */
794 11 : if (!legacy) {
795 : ucl_object_t *meta_dump;
796 : FILE *mfile;
797 :
798 11 : snprintf(repodb, sizeof(repodb), "%s/%s", output_dir,
799 : "meta");
800 11 : if ((mfile = fopen(repodb, "w")) != NULL) {
801 11 : meta_dump = pkg_repo_meta_to_ucl(meta);
802 11 : ucl_object_emit_file(meta_dump, UCL_EMIT_CONFIG, mfile);
803 11 : ucl_object_unref(meta_dump);
804 11 : fclose(mfile);
805 : }
806 : else {
807 0 : pkg_emit_notice("cannot create metafile at %s", repodb);
808 : }
809 : }
810 : cleanup:
811 11 : HASH_ITER (hh, conflicts, curcb, tmpcb) {
812 0 : HASH_ITER (hh, curcb->conflicts, c, ctmp) {
813 0 : free(c->uid);
814 0 : HASH_DEL(curcb->conflicts, c);
815 0 : free(c);
816 : }
817 0 : HASH_DEL(conflicts, curcb);
818 0 : free(curcb);
819 : }
820 :
821 11 : if (pfd != NULL)
822 11 : free(pfd);
823 11 : if (fts != NULL)
824 11 : fts_close(fts);
825 :
826 11 : LL_FREE(fts_items, pkg_create_repo_fts_free);
827 42 : LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
828 31 : if (cur_dig->checksum != NULL)
829 31 : fprintf(mandigests, "%s:%s:%ld:%ld:%ld:%s\n", cur_dig->origin,
830 : cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
831 : cur_dig->manifest_length, cur_dig->checksum);
832 : else
833 0 : fprintf(mandigests, "%s:%s:%ld:%ld:%ld\n", cur_dig->origin,
834 : cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
835 : cur_dig->manifest_length);
836 :
837 31 : free(cur_dig->digest);
838 31 : free(cur_dig->origin);
839 31 : free(cur_dig);
840 : }
841 :
842 11 : pkg_repo_meta_free(meta);
843 :
844 11 : if (mandigests != NULL)
845 11 : fclose(mandigests);
846 :
847 11 : return (retcode);
848 : }
849 :
850 :
851 : static int
852 0 : pkg_repo_sign(char *path, char **argv, int argc, struct sbuf **sig, struct sbuf **cert)
853 : {
854 : FILE *fp;
855 : char *sha256;
856 0 : struct sbuf *cmd = NULL;
857 0 : struct sbuf *buf = NULL;
858 0 : char *line = NULL;
859 0 : size_t linecap = 0;
860 : ssize_t linelen;
861 0 : int i, ret = EPKG_OK;
862 :
863 0 : sha256 = pkg_checksum_file(path, PKG_HASH_TYPE_SHA256_HEX);
864 0 : if (!sha256)
865 0 : return (EPKG_FATAL);
866 :
867 0 : cmd = sbuf_new_auto();
868 :
869 0 : for (i = 0; i < argc; i++) {
870 0 : if (strspn(argv[i], " \t\n") > 0)
871 0 : sbuf_printf(cmd, " \"%s\" ", argv[i]);
872 : else
873 0 : sbuf_printf(cmd, " %s ", argv[i]);
874 : }
875 0 : sbuf_finish(cmd);
876 :
877 0 : if ((fp = popen(sbuf_data(cmd), "r+")) == NULL) {
878 0 : ret = EPKG_FATAL;
879 0 : goto done;
880 : }
881 :
882 0 : fprintf(fp, "%s\n", sha256);
883 :
884 0 : if (*sig == NULL)
885 0 : *sig = sbuf_new_auto();
886 0 : if (*cert == NULL)
887 0 : *cert = sbuf_new_auto();
888 :
889 0 : while ((linelen = getline(&line, &linecap, fp)) > 0 ) {
890 0 : if (strcmp(line, "SIGNATURE\n") == 0) {
891 0 : buf = *sig;
892 0 : continue;
893 0 : } else if (strcmp(line, "CERT\n") == 0) {
894 0 : buf = *cert;
895 0 : continue;
896 0 : } else if (strcmp(line, "END\n") == 0) {
897 0 : break;
898 : }
899 0 : if (buf != NULL)
900 0 : sbuf_bcat(buf, line, linelen);
901 : }
902 :
903 0 : if (pclose(fp) != 0) {
904 0 : ret = EPKG_FATAL;
905 0 : goto done;
906 : }
907 :
908 0 : if (sbuf_data(*sig)[sbuf_len(*sig) -1 ] == '\n')
909 0 : sbuf_setpos(*sig, sbuf_len(*sig) -1);
910 :
911 0 : sbuf_finish(*sig);
912 0 : sbuf_finish(*cert);
913 : done:
914 0 : free(sha256);
915 0 : if (cmd)
916 0 : sbuf_delete(cmd);
917 :
918 0 : return (ret);
919 : }
920 :
921 : static int
922 33 : pkg_repo_pack_db(const char *name, const char *archive, char *path,
923 : struct rsa_key *rsa, struct pkg_repo_meta *meta,
924 : char **argv, int argc)
925 : {
926 : struct packing *pack;
927 33 : unsigned char *sigret = NULL;
928 33 : unsigned int siglen = 0;
929 : char fname[MAXPATHLEN];
930 : struct sbuf *sig, *pub;
931 :
932 33 : sig = NULL;
933 33 : pub = NULL;
934 :
935 33 : if (packing_init(&pack, archive, meta->packing_format, false) != EPKG_OK)
936 0 : return (EPKG_FATAL);
937 :
938 33 : if (rsa != NULL) {
939 0 : if (rsa_sign(path, rsa, &sigret, &siglen) != EPKG_OK) {
940 0 : packing_finish(pack);
941 0 : unlink(path);
942 0 : return (EPKG_FATAL);
943 : }
944 :
945 0 : if (packing_append_buffer(pack, sigret, "signature", siglen + 1) != EPKG_OK) {
946 0 : free(sigret);
947 0 : free(pack);
948 0 : unlink(path);
949 0 : return (EPKG_FATAL);
950 : }
951 :
952 0 : free(sigret);
953 33 : } else if (argc >= 1) {
954 0 : if (pkg_repo_sign(path, argv, argc, &sig, &pub) != EPKG_OK) {
955 0 : packing_finish(pack);
956 0 : unlink(path);
957 0 : return (EPKG_FATAL);
958 : }
959 :
960 0 : snprintf(fname, sizeof(fname), "%s.sig", name);
961 0 : if (packing_append_buffer(pack, sbuf_data(sig), fname, sbuf_len(sig)) != EPKG_OK) {
962 0 : packing_finish(pack);
963 0 : sbuf_delete(sig);
964 0 : sbuf_delete(pub);
965 0 : unlink(path);
966 0 : return (EPKG_FATAL);
967 : }
968 :
969 0 : snprintf(fname, sizeof(fname), "%s.pub", name);
970 0 : if (packing_append_buffer(pack, sbuf_data(pub), fname, sbuf_len(pub)) != EPKG_OK) {
971 0 : packing_finish(pack);
972 0 : unlink(path);
973 0 : sbuf_delete(sig);
974 0 : sbuf_delete(pub);
975 0 : return (EPKG_FATAL);
976 : }
977 :
978 : }
979 33 : packing_append_file_attr(pack, path, name, "root", "wheel", 0644, 0);
980 :
981 33 : packing_finish(pack);
982 33 : unlink(path);
983 33 : if (sig != NULL)
984 0 : sbuf_delete(sig);
985 33 : if (pub != NULL)
986 0 : sbuf_delete(pub);
987 :
988 33 : return (EPKG_OK);
989 : }
990 :
991 : int
992 11 : pkg_finish_repo(const char *output_dir, pem_password_cb *password_cb,
993 : char **argv, int argc, bool filelist)
994 : {
995 : char repo_path[MAXPATHLEN];
996 : char repo_archive[MAXPATHLEN];
997 11 : struct rsa_key *rsa = NULL;
998 : struct pkg_repo_meta *meta;
999 : struct stat st;
1000 11 : int ret = EPKG_OK, nfile = 0;
1001 11 : const int files_to_pack = 4;
1002 11 : bool legacy = false;
1003 :
1004 11 : if (!is_dir(output_dir)) {
1005 0 : pkg_emit_error("%s is not a directory", output_dir);
1006 0 : return (EPKG_FATAL);
1007 : }
1008 :
1009 11 : if (argc == 1) {
1010 0 : rsa_new(&rsa, password_cb, argv[0]);
1011 : }
1012 :
1013 11 : if (argc > 1 && strcmp(argv[0], "signing_command:") != 0)
1014 0 : return (EPKG_FATAL);
1015 :
1016 11 : if (argc > 1) {
1017 0 : argc--;
1018 0 : argv++;
1019 : }
1020 :
1021 11 : pkg_emit_progress_start("Packing files for repository");
1022 11 : pkg_emit_progress_tick(nfile++, files_to_pack);
1023 :
1024 11 : snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
1025 : repo_meta_file);
1026 : /*
1027 : * If no meta is defined, then it is a legacy repo
1028 : */
1029 11 : if (access(repo_path, R_OK) != -1) {
1030 11 : if (pkg_repo_meta_load(repo_path, &meta) != EPKG_OK) {
1031 0 : pkg_emit_error("meta loading error while trying %s", repo_path);
1032 0 : rsa_free(rsa);
1033 0 : return (EPKG_FATAL);
1034 : }
1035 : else {
1036 11 : meta = pkg_repo_meta_default();
1037 : }
1038 11 : if (pkg_repo_pack_db(repo_meta_file, repo_path, repo_path, rsa, meta,
1039 : argv, argc) != EPKG_OK) {
1040 0 : ret = EPKG_FATAL;
1041 0 : goto cleanup;
1042 : }
1043 : }
1044 : else {
1045 0 : legacy = true;
1046 0 : meta = pkg_repo_meta_default();
1047 : }
1048 :
1049 11 : snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
1050 11 : meta->manifests);
1051 11 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
1052 11 : meta->manifests_archive);
1053 11 : if (pkg_repo_pack_db(meta->manifests, repo_archive, repo_path, rsa, meta,
1054 : argv, argc) != EPKG_OK) {
1055 0 : ret = EPKG_FATAL;
1056 0 : goto cleanup;
1057 : }
1058 :
1059 11 : pkg_emit_progress_tick(nfile++, files_to_pack);
1060 :
1061 11 : if (filelist) {
1062 0 : snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
1063 0 : meta->filesite);
1064 0 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s",
1065 0 : output_dir, meta->filesite_archive);
1066 0 : if (pkg_repo_pack_db(meta->filesite, repo_archive, repo_path, rsa, meta,
1067 : argv, argc) != EPKG_OK) {
1068 0 : ret = EPKG_FATAL;
1069 0 : goto cleanup;
1070 : }
1071 : }
1072 :
1073 11 : pkg_emit_progress_tick(nfile++, files_to_pack);
1074 :
1075 11 : snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
1076 11 : meta->digests);
1077 11 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
1078 11 : meta->digests_archive);
1079 11 : if (pkg_repo_pack_db(meta->digests, repo_archive, repo_path, rsa, meta,
1080 : argv, argc) != EPKG_OK) {
1081 0 : ret = EPKG_FATAL;
1082 0 : goto cleanup;
1083 : }
1084 :
1085 11 : pkg_emit_progress_tick(nfile++, files_to_pack);
1086 :
1087 : #if 0
1088 : snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
1089 : meta->conflicts);
1090 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
1091 : meta->conflicts_archive);
1092 : if (pkg_repo_pack_db(meta->conflicts, repo_archive, repo_path, rsa, meta,
1093 : argv, argc) != EPKG_OK) {
1094 : ret = EPKG_FATAL;
1095 : goto cleanup;
1096 : }
1097 : #endif
1098 :
1099 : /* Now we need to set the equal mtime for all archives in the repo */
1100 11 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
1101 : output_dir, repo_meta_file);
1102 11 : if (stat(repo_archive, &st) == 0) {
1103 22 : struct timeval ftimes[2] = {
1104 : {
1105 11 : .tv_sec = st.st_mtime,
1106 : .tv_usec = 0
1107 : },
1108 : {
1109 11 : .tv_sec = st.st_mtime,
1110 : .tv_usec = 0
1111 : }
1112 : };
1113 11 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
1114 11 : output_dir, meta->manifests_archive);
1115 11 : utimes(repo_archive, ftimes);
1116 11 : snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
1117 11 : output_dir, meta->digests_archive);
1118 11 : utimes(repo_archive, ftimes);
1119 11 : if (filelist) {
1120 0 : snprintf(repo_archive, sizeof(repo_archive),
1121 0 : "%s/%s.txz", output_dir, meta->filesite_archive);
1122 0 : utimes(repo_archive, ftimes);
1123 : }
1124 11 : if (!legacy) {
1125 11 : snprintf(repo_archive, sizeof(repo_archive),
1126 : "%s/%s.txz", output_dir, repo_meta_file);
1127 11 : utimes(repo_archive, ftimes);
1128 : }
1129 : }
1130 :
1131 : cleanup:
1132 11 : pkg_emit_progress_tick(files_to_pack, files_to_pack);
1133 11 : pkg_repo_meta_free(meta);
1134 :
1135 11 : rsa_free(rsa);
1136 :
1137 11 : return (ret);
1138 : }
|
