Print this page
| Split |
Close |
| Expand all |
| Collapse all |
--- old/post.c
+++ new/post.c
1 1 /*
2 2 * Copyright (c) 2009-2020 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
3 + * Copyright 2020, Kebe Software & Services
3 4 *
4 5 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 6 * of this software and associated documentation files (the "Software"), to deal
6 7 * in the Software without restriction, including without limitation the rights
7 8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 9 * copies of the Software, and to permit persons to whom the Software is
9 10 * furnished to do so, subject to the following conditions:
10 11 *
11 12 * The above copyright notice and this permission notice shall be included in
12 13 * all copies or substantial portions of the Software.
13 14 *
14 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 21 * SOFTWARE.
21 22 */
22 23
23 24 #include <stdlib.h>
24 25 #include <stddef.h>
25 26 #include <stdio.h>
26 27 #include <limits.h>
27 28 #include <string.h>
28 29 #include <sys/mman.h>
29 30 #include <sys/types.h>
30 31 #include <sys/stat.h>
31 32 #include <unistd.h>
32 33 #include <fcntl.h>
33 34 #include <time.h>
34 35 #include <dirent.h>
35 36
36 37 #include <jeffpc/taskq.h>
37 38 #include <jeffpc/error.h>
38 39 #include <jeffpc/io.h>
39 40 #include <jeffpc/mem.h>
40 41 #include <jeffpc/file-cache.h>
41 42
42 43 #include "post.h"
43 44 #include "vars.h"
44 45 #include "req.h"
45 46 #include "parse.h"
46 47 #include "utils.h"
47 48 #include "debug.h"
48 49
49 50 static struct mem_cache *post_cache;
50 51 static struct mem_cache *comment_cache;
51 52
52 53 static LOCK_CLASS(post_lc);
53 54
54 55 static void post_remove_all_tags(struct rb_tree *taglist);
55 56 static void post_remove_all_comments(struct post *post);
56 57
57 58 static int tag_cmp(const void *va, const void *vb)
58 59 {
59 60 const struct post_tag *a = va;
60 61 const struct post_tag *b = vb;
61 62 int ret;
62 63
63 64 ret = strcasecmp(str_cstr(a->tag), str_cstr(b->tag));
64 65
65 66 if (ret < 0)
66 67 return -1;
67 68 if (ret > 0)
68 69 return 1;
69 70 return 0;
70 71 }
71 72
72 73 void init_post_subsys(void)
73 74 {
74 75 post_cache = mem_cache_create("post-cache", sizeof(struct post), 0);
75 76 ASSERT(!IS_ERR(post_cache));
76 77
77 78 comment_cache = mem_cache_create("comment-cache",
78 79 sizeof(struct comment), 0);
79 80 ASSERT(!IS_ERR(comment_cache));
80 81
81 82 init_post_index();
82 83 }
83 84
84 85 struct str *post_get_cached_file(struct post *post, const char *path)
85 86 {
86 87 struct str *out;
87 88 uint64_t rev;
88 89 int err;
89 90
90 91 out = file_cache_get(path, &rev);
91 92 if (IS_ERR(out))
92 93 return out;
93 94
94 95 err = nvl_set_int(post->files, path, rev);
95 96 if (err) {
96 97 str_putref(out);
97 98 out = ERR_PTR(err);
98 99 }
99 100
100 101 return out;
101 102 }
102 103
103 104 static void post_remove_all_filenames(struct post *post)
104 105 {
105 106 const struct nvpair *pair;
106 107
107 108 while ((pair = nvl_iter_start(post->files)) != NULL) {
108 109 struct str *name = nvpair_name_str(pair);
109 110
110 111 VERIFY0(nvl_unset(post->files, str_cstr(name)));
111 112
112 113 str_putref(name);
113 114 }
114 115 }
115 116
116 117 /* consumes the struct val reference */
117 118 static void post_add_tags(struct rb_tree *taglist, struct val *list)
118 119 {
119 120 struct val *tagval;
120 121 struct val *tmp;
121 122
122 123 sexpr_for_each_noref(tagval, tmp, list) {
123 124 struct post_tag *tag;
124 125
125 126 /* sanity check */
126 127 ASSERT3U(tagval->type, ==, VT_STR);
127 128
128 129 tag = malloc(sizeof(struct post_tag));
129 130 ASSERT(tag);
130 131
131 132 tag->tag = val_getref_str(tagval);
132 133
133 134 if (rb_insert(taglist, tag)) {
134 135 /* found a duplicate */
135 136 str_putref(tag->tag);
136 137 free(tag);
137 138 }
138 139 }
139 140
140 141 val_putref(list);
141 142 }
142 143
143 144 static void post_remove_all_comments(struct post *post)
144 145 {
145 146 struct comment *com;
146 147
147 148 while ((com = list_remove_head(&post->comments))) {
148 149 str_putref(com->author);
149 150 str_putref(com->email);
150 151 str_putref(com->ip);
151 152 str_putref(com->url);
152 153 str_putref(com->body);
153 154 mem_cache_free(comment_cache, com);
154 155 }
155 156
156 157 post->numcom = 0;
157 158 }
158 159
159 160 static struct str *load_comment(struct post *post, int commid)
160 161 {
161 162 char path[FILENAME_MAX];
162 163 struct str *out;
163 164
164 165 snprintf(path, FILENAME_MAX, "%s/posts/%d/comments/%d/text.txt",
165 166 str_cstr(config.data_dir), post->id, commid);
166 167
167 168 out = post_get_cached_file(post, path);
168 169 if (IS_ERR(out))
169 170 out = STATIC_STR("Error: could not load comment text.");
170 171
171 172 return out;
172 173 }
173 174
174 175 static void post_add_comment(struct post *post, int commid)
175 176 {
176 177 char path[FILENAME_MAX];
177 178 struct comment *comm;
178 179 struct str *meta;
179 180 struct val *lv;
180 181 struct val *v;
181 182
182 183 snprintf(path, FILENAME_MAX, "%s/posts/%d/comments/%d/meta.lisp",
183 184 str_cstr(config.data_dir), post->id, commid);
184 185
185 186 meta = post_get_cached_file(post, path);
186 187 ASSERT(!IS_ERR(meta));
187 188
188 189 lv = sexpr_parse_str(meta);
189 190 ASSERT(!IS_ERR(lv));
190 191
191 192 v = sexpr_cdr(sexpr_assoc(lv, "moderated"));
192 193 if (!v || (v->type != VT_BOOL) || !v->b)
193 194 goto done;
194 195
195 196 comm = mem_cache_alloc(comment_cache);
196 197 ASSERT(comm);
197 198
198 199 comm->id = commid;
199 200 comm->author = sexpr_alist_lookup_str(lv, "author");
200 201 comm->email = sexpr_alist_lookup_str(lv, "email");
201 202 comm->time = parse_time_str(sexpr_alist_lookup_str(lv, "time"));
202 203 comm->ip = sexpr_alist_lookup_str(lv, "ip");
203 204 comm->url = sexpr_alist_lookup_str(lv, "url");
204 205 comm->body = load_comment(post, comm->id);
205 206
206 207 if (!comm->author)
207 208 comm->author = STATIC_STR("[unknown]");
208 209
209 210 list_insert_tail(&post->comments, comm);
210 211
211 212 post->numcom++;
212 213
213 214 done:
214 215 val_putref(v);
215 216 val_putref(lv);
216 217 str_putref(meta);
217 218 }
218 219
219 220 /* consumes the struct val reference */
220 221 static void post_add_comments(struct post *post, struct val *list)
221 222 {
222 223 struct val *val;
223 224 struct val *tmp;
224 225
225 226 sexpr_for_each_noref(val, tmp, list) {
226 227 /* sanity check */
227 228 ASSERT3U(val->type, ==, VT_INT);
228 229
229 230 /* add the comment */
230 231 post_add_comment(post, val->i);
231 232 }
232 233
233 234 val_putref(list);
234 235 }
|
↓ open down ↓ |
222 lines elided |
↑ open up ↑ |
235 236
236 237 static int __do_load_post_body_fmt2(struct post *post, struct str *html)
237 238 {
238 239 str_putref(post->body); /* free the previous */
239 240 post->body = str_getref(html);
240 241 ASSERT(post->body);
241 242
242 243 return 0;
243 244 }
244 245
246 +static int
247 +__do_load_post_body_fmt4(struct post *post, const struct str *md)
248 +{
249 + /*
250 + * Like HTML, assume the .lisp file contains all of the other post
251 + * metadata. If that assumption changes, update here.
252 + *
253 + * Unlike HTML, we plan to provide a clean error string if MD
254 + * parsing fails.
255 + */
256 + str_putref(post->body); /* Free the previous text. */
257 + post->body = fmt4_md_to_html(md);
258 + if (post->body == NULL) {
259 + /*
260 + * XXX KEBE SAYS do something clever here with a small HTML
261 + * string. For now, panic.
262 + */
263 + ASSERT(post->body != NULL);
264 + }
265 +
266 + return (0);
267 +}
268 +
245 269 static int __do_load_post_body_fmt3(struct post *post, const struct str *input)
246 270 {
247 271 struct parser_output x;
248 272 int ret;
249 273
250 274 x.req = NULL;
251 275 x.post = post;
252 276 x.input = str_cstr(input);
253 277 x.len = str_len(input);
254 278 x.pos = 0;
255 279 x.lineno = 0;
256 280 x.table_nesting = 0;
257 281 x.texttt_nesting = 0;
258 282 x.sc_title = NULL;
259 283 x.sc_pub = NULL;
260 284 x.sc_tags = NULL;
261 285 x.sc_twitter_img = NULL;
262 286
263 287 fmt3_lex_init(&x.scanner);
264 288 fmt3_set_extra(&x, x.scanner);
265 289
266 290 ret = fmt3_parse(&x);
267 291 if (ret)
268 292 panic("failed to parse post id %u", post->id);
269 293
270 294 fmt3_lex_destroy(x.scanner);
271 295
272 296 /*
273 297 * Now update struct post based on what we got from the .tex file.
274 298 * The struct is already populated by data from the metadata file.
275 299 * For the simple string values, we merely override whatever was
276 300 * there. For tags we use the union.
277 301 */
278 302
279 303 if (x.sc_title) {
280 304 str_putref(post->title);
281 305 post->title = str_getref(x.sc_title);
282 306 }
283 307
284 308 if (x.sc_pub)
285 309 post->time = parse_time_str(str_getref(x.sc_pub));
286 310
287 311 if (x.sc_twitter_img) {
288 312 str_putref(post->twitter_img);
289 313 post->twitter_img = str_getref(x.sc_twitter_img);
290 314 }
291 315
292 316 post_add_tags(&post->tags, x.sc_tags);
293 317
294 318 str_putref(x.sc_title);
295 319 str_putref(x.sc_pub);
296 320 str_putref(x.sc_twitter_img);
|
↓ open down ↓ |
42 lines elided |
↑ open up ↑ |
297 321
298 322 str_putref(post->body); /* free the previous */
299 323 post->body = x.stroutput;
300 324 ASSERT(post->body);
301 325
302 326 return 0;
303 327 }
304 328
305 329 static int __load_post_body(struct post *post)
306 330 {
307 - static const char *exts[4] = {
331 + static const char *exts[5] = {
308 332 [2] = "html",
309 333 [3] = "tex",
334 + [4] = "md",
310 335 };
311 336
312 337 char path[FILENAME_MAX];
313 338 struct str *raw;
314 339 int ret;
315 340
316 341 ASSERT3U(post->fmt, >=, 2);
317 - ASSERT3U(post->fmt, <=, 3);
342 + ASSERT3U(post->fmt, <=, 4);
318 343
319 344 snprintf(path, FILENAME_MAX, "%s/posts/%d/post.%s",
320 345 str_cstr(config.data_dir), post->id, exts[post->fmt]);
321 346
322 347 raw = post_get_cached_file(post, path);
323 348 if (IS_ERR(raw))
324 349 return PTR_ERR(raw);
325 350
326 351 switch (post->fmt) {
327 352 case 2:
328 353 ret = __do_load_post_body_fmt2(post, raw);
329 354 break;
330 355 case 3:
331 356 ret = __do_load_post_body_fmt3(post, raw);
332 357 break;
358 + case 4:
359 + ret = __do_load_post_body_fmt4(post, raw);
360 + break;
333 361 }
334 362
335 363 str_putref(raw);
336 364
337 365 return ret;
338 366 }
339 367
340 368 static void __refresh_published_prop(struct post *post, struct val *lv)
341 369 {
342 370 /* update the time */
343 371 post->time = parse_time_str(sexpr_alist_lookup_str(lv, "time"));
344 372
345 373 /* update the title */
346 374 post->title = sexpr_alist_lookup_str(lv, "title");
347 375
348 376 /* update the format */
349 377 post->fmt = sexpr_alist_lookup_int(lv, "fmt", NULL);
350 378
351 379 /* update the listed bool */
352 380 post->listed = sexpr_alist_lookup_bool(lv, "listed", true, NULL);
353 381 }
354 382
355 383 static int __refresh_published(struct post *post)
356 384 {
357 385 char path[FILENAME_MAX];
358 386 struct str *meta;
359 387 struct val *lv;
360 388
361 389 snprintf(path, FILENAME_MAX, "%s/posts/%d/post.lisp",
362 390 str_cstr(config.data_dir), post->id);
363 391
364 392 meta = post_get_cached_file(post, path);
365 393 if (IS_ERR(meta))
366 394 return PTR_ERR(meta);
367 395
368 396 lv = sexpr_parse_str(meta);
369 397 if (IS_ERR(lv)) {
370 398 str_putref(meta);
371 399 return PTR_ERR(lv);
372 400 }
373 401
374 402 __refresh_published_prop(post, lv);
375 403
376 404 /* empty out the tags/comments lists */
377 405 post_remove_all_tags(&post->tags);
378 406 post_remove_all_comments(post);
379 407
380 408 /* populate the tags/comments lists */
381 409 post_add_tags(&post->tags, sexpr_alist_lookup_list(lv, "tags"));
382 410 post_add_comments(post, sexpr_alist_lookup_list(lv, "comments"));
383 411
384 412 val_putref(lv);
385 413 str_putref(meta);
386 414
387 415 return 0;
388 416 }
389 417
390 418 static bool must_refresh(struct post *post)
391 419 {
392 420 const struct nvpair *pair;
393 421
394 422 if (post->preview)
395 423 return true; /* always refresh previews */
396 424
397 425 if (nvl_iter_start(post->files) == NULL)
398 426 return true; /* no files means we have no idea what is needed */
399 427
400 428 nvl_for_each(pair, post->files) {
401 429 struct str *name = nvpair_name_str(pair);
402 430 uint64_t file_rev;
403 431
404 432 ASSERT0(nvpair_value_int(pair, &file_rev));
405 433
406 434 if (!file_cache_has_newer(str_cstr(name), file_rev)) {
407 435 str_putref(name);
408 436 continue;
409 437 }
410 438
411 439 cmn_err(CE_DEBUG, "post %u needs a refresh "
412 440 "('%s' changed, old rev %"PRIu64")", post->id,
413 441 str_cstr(name), file_rev);
414 442
415 443 str_putref(name);
416 444
417 445 return true; /* no need to check oher files, we are refreshing */
418 446 }
419 447
420 448 return false;
421 449 }
422 450
423 451 int post_refresh(struct post *post)
424 452 {
425 453 int ret;
426 454
427 455 if (!must_refresh(post))
428 456 return 0;
429 457
430 458 post_remove_all_filenames(post);
431 459
432 460 str_putref(post->title);
433 461 post->title = NULL;
434 462
435 463 if (post->preview) {
436 464 post->title = STATIC_STR("PREVIEW");
437 465 post->time = time(NULL);
438 466 post->fmt = 3;
439 467 } else {
440 468 ret = __refresh_published(post);
441 469 if (ret)
442 470 return ret;
443 471 }
444 472
445 473 if ((ret = __load_post_body(post)))
446 474 return ret;
447 475
448 476 /* No title set at all? Set it to something non-NULL. */
449 477 if (!post->title)
450 478 post->title = STATIC_STR("Untitled");
451 479
452 480 return 0;
453 481 }
454 482
455 483 struct post *load_post(int postid, bool preview)
456 484 {
457 485 struct post *post;
458 486 int err;
459 487
460 488 /*
461 489 * If it is *not* a preview, try to get it from the cache.
462 490 */
463 491 if (!preview) {
464 492 post = index_lookup_post(postid);
465 493 if (post)
466 494 return post;
467 495 }
468 496
469 497 post = mem_cache_alloc(post_cache);
470 498 if (!post) {
471 499 err = -ENOMEM;
472 500 goto err;
473 501 }
474 502
475 503 memset(post, 0, sizeof(struct post));
476 504
477 505 post->id = postid;
478 506 post->title = NULL;
479 507 post->body = NULL;
480 508 post->numcom = 0;
481 509 post->preview = preview;
482 510
483 511 rb_create(&post->tags, tag_cmp, sizeof(struct post_tag),
484 512 offsetof(struct post_tag, node));
485 513 list_create(&post->comments, sizeof(struct comment),
486 514 offsetof(struct comment, list));
487 515 refcnt_init(&post->refcnt, 1);
488 516 MXINIT(&post->lock, &post_lc);
489 517
490 518 post->files = nvl_alloc();
491 519 if (IS_ERR(post->files)) {
492 520 err = PTR_ERR(post->files);
493 521 post->files = NULL;
494 522 goto err_free;
495 523 }
496 524
497 525 if ((err = post_refresh(post)))
498 526 goto err_free;
499 527
500 528 if (!post->preview)
501 529 ASSERT0(index_insert_post(post));
502 530
503 531 return post;
504 532
505 533 err_free:
506 534 post_destroy(post);
507 535
508 536 err:
509 537 cmn_err(CE_ERROR, "Failed to load post id %u: %s", postid,
510 538 xstrerror(err));
511 539 return NULL;
512 540 }
513 541
514 542 static void post_remove_all_tags(struct rb_tree *taglist)
515 543 {
516 544 struct post_tag *tag;
517 545 struct rb_cookie cookie;
518 546
519 547 memset(&cookie, 0, sizeof(cookie));
520 548 while ((tag = rb_destroy_nodes(taglist, &cookie))) {
521 549 str_putref(tag->tag);
522 550 free(tag);
523 551 }
524 552
525 553 rb_create(taglist, tag_cmp, sizeof(struct post_tag),
526 554 offsetof(struct post_tag, node));
527 555 }
528 556
529 557 void post_destroy(struct post *post)
530 558 {
531 559 post_remove_all_tags(&post->tags);
532 560 post_remove_all_comments(post);
533 561
534 562 nvl_putref(post->files);
535 563
536 564 str_putref(post->title);
537 565 str_putref(post->body);
538 566
539 567 MXDESTROY(&post->lock);
540 568
541 569 mem_cache_free(post_cache, post);
542 570 }
543 571
544 572 static void __tq_load_post(void *arg)
545 573 {
546 574 int postid = (uintptr_t) arg;
547 575
548 576 /* load the post, but then free it since we don't need it */
549 577 post_putref(load_post(postid, false));
550 578 }
551 579
552 580 int load_all_posts(void)
553 581 {
554 582 const char *data_dir = str_cstr(config.data_dir);
555 583 char path[FILENAME_MAX];
556 584 struct stat statbuf;
557 585 struct dirent *de;
558 586 uint32_t postid;
559 587 uint64_t start_ts, end_ts;
560 588 unsigned nposts;
561 589 struct taskq *tq;
562 590 DIR *dir;
563 591 int ret;
564 592
565 593 snprintf(path, sizeof(path), "%s/posts", data_dir);
566 594 dir = opendir(path);
567 595 if (!dir)
568 596 return -errno;
569 597
570 598 tq = taskq_create_fixed("load-all-posts", -1);
571 599 if (IS_ERR(tq)) {
572 600 closedir(dir);
573 601 return PTR_ERR(tq);
574 602 }
575 603
576 604 nposts = 0;
577 605 start_ts = gettime();
578 606
579 607 while ((de = readdir(dir))) {
580 608 if (!strcmp(de->d_name, ".") ||
581 609 !strcmp(de->d_name, ".."))
582 610 continue;
583 611
584 612 ret = str2u32(de->d_name, &postid);
585 613 if (ret) {
586 614 cmn_err(CE_INFO, "skipping '%s/%s' - not a number",
587 615 data_dir, de->d_name);
588 616 continue;
589 617 }
590 618
591 619 snprintf(path, FILENAME_MAX, "%s/posts/%u", data_dir, postid);
592 620
593 621 /* check that it is a directory */
594 622 ret = xlstat(path, &statbuf);
595 623 if (ret) {
596 624 cmn_err(CE_INFO, "skipping '%s' - failed to xlstat: %s",
597 625 path, xstrerror(ret));
598 626 continue;
599 627 }
600 628
601 629 if (!S_ISDIR(statbuf.st_mode)) {
602 630 cmn_err(CE_INFO, "skipping '%s' - not a directory; "
603 631 "mode = %o", path,
604 632 (unsigned int) statbuf.st_mode);
605 633 continue;
606 634 }
607 635
608 636 /* load the post asynchronously */
609 637 if (taskq_dispatch(tq, __tq_load_post, (void *)(uintptr_t) postid))
610 638 __tq_load_post((void *)(uintptr_t) postid);
611 639
612 640 nposts++;
613 641 }
614 642
615 643 taskq_wait(tq);
616 644 taskq_destroy(tq);
617 645
618 646 end_ts = gettime();
619 647
620 648 cmn_err(CE_INFO, "Loaded %u posts in %"PRIu64".%09"PRIu64" seconds",
621 649 nposts,
622 650 (end_ts - start_ts) / 1000000000UL,
623 651 (end_ts - start_ts) % 1000000000UL);
624 652
625 653 closedir(dir);
626 654
627 655 return 0;
628 656 }
|
↓ open down ↓ |
286 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX