From ec8746b44dc85fd3e3b42835f779890684a9e90a Mon Sep 17 00:00:00 2001 From: Chip Black Date: Tue, 22 Feb 2011 04:34:11 -0600 Subject: [PATCH] Add subscription feed fetching, clean up a bit Add a /feed URL for fetching subscription feeds. Genericized "struct tag" into "struct blergref" -- no actual change, but the same thing is now used by subscription indexes. --- cgi/cgi_blerg.c | 6 +- common/stringbucket.c | 3 + common/stringbucket.h | 3 + database/blergref.h | 12 ++++ database/database.c | 6 +- database/subscription.c | 78 +++++++++++++++++++++++-- database/subscription.h | 11 ++-- database/tags.c | 22 +++---- database/tags.h | 8 +-- http/http_blerg.c | 126 +++++++++++++++++++++++++++------------- tools/blerglatest.c | 3 + tools/blergtool.c | 2 +- 12 files changed, 209 insertions(+), 71 deletions(-) create mode 100644 database/blergref.h diff --git a/cgi/cgi_blerg.c b/cgi/cgi_blerg.c index b6dcb3a..c004a7d 100644 --- a/cgi/cgi_blerg.c +++ b/cgi/cgi_blerg.c @@ -45,7 +45,7 @@ void respond_for_range(struct blerg *b, uint64_t from, uint64_t to) { yajl_gen_free(g); } -void respond_taglist(struct tag *results, int i) { +void respond_blergref_list(struct blergref * results, int i) { const unsigned char *ybuf; unsigned int len; struct blerg *b; @@ -170,12 +170,12 @@ int main(int argc, char *argv[]) { } int recs = 50; - struct tag *taglist = tag_list(info.name, 0, &recs, -1); + struct blergref *taglist = tag_list(info.name, 0, &recs, -1); if (recs == 0) { respond_simple_data("[]", 2); } else { - respond_taglist(taglist, recs); + respond_blergref_list(taglist, recs); } } else if (strncmp(path, "/put", 4) == 0) { if (strcmp(request_method, "POST") != 0) { diff --git a/common/stringbucket.c b/common/stringbucket.c index ec827d1..d892541 100644 --- a/common/stringbucket.c +++ b/common/stringbucket.c @@ -1,3 +1,6 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ #include #include #include diff --git a/common/stringbucket.h b/common/stringbucket.h index 5594bd8..b7eb282 100644 --- a/common/stringbucket.h +++ b/common/stringbucket.h @@ -1,3 +1,6 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ #ifndef _STRINGBUCKET_H #define _STRINGBUCKET_H diff --git a/database/blergref.h b/database/blergref.h new file mode 100644 index 0000000..57b472b --- /dev/null +++ b/database/blergref.h @@ -0,0 +1,12 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ +#ifndef _BLERGREF_H +#define _BLERGREF_H + +struct blergref { + char author[32]; + uint64_t record; +}; + +#endif /* _BLERGREF_H */ diff --git a/database/database.c b/database/database.c index cb5cf0c..a45c92b 100644 --- a/database/database.c +++ b/database/database.c @@ -13,6 +13,7 @@ #include #include #include "database.h" +#include "subscription.h" #include "util.h" #include "config.h" @@ -246,11 +247,12 @@ int blerg_store(struct blerg *blerg, const char *data, int len) { blerg->index[seg_rec].length = len; blerg->index[seg_rec].timestamp = time(NULL); - tag_scan(blerg->name, data, len, record); - flock(blerg->data_fd, LOCK_UN); flock(blerg->index_fd, LOCK_UN); + tag_scan(blerg->name, data, len, record); + subscription_notify(blerg->name, record); + return record; } diff --git a/database/subscription.c b/database/subscription.c index e6ae94a..d67d31f 100644 --- a/database/subscription.c +++ b/database/subscription.c @@ -1,7 +1,12 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ #include #include -#include +#include +#include #include +#include #include #include "subscription.h" #include "stringbucket.h" @@ -41,14 +46,14 @@ void subscription_notify_add_item(char *to, void *stuff) { char filename[512]; snprintf(filename, 512, "%s/%s/subscription_feed", DATA_PATH, to); - int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT); - write(fd, stuff, sizeof(struct subscription_record)); + int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0600); + write(fd, stuff, sizeof(struct blergref)); close(fd); } int subscription_notify(const char *author, uint64_t record) { char filename[512]; - struct subscription_record r; + struct blergref r; strncpy(r.author, author, 32); r.record = record; @@ -58,3 +63,68 @@ int subscription_notify(const char *author, uint64_t record) { stringbucket_iterate(sb, subscription_notify_add_item, &r); stringbucket_close(sb); } + +struct blergref * subscription_list(const char *author, uint64_t offset, int *count, int direction) { + char filename[512]; + struct stat st; + struct blergref * slist; + struct blergref * retlist; + uint64_t n_subscription_records; + + if (!valid_name(author)) + return NULL; + + snprintf(filename, 512, "%s/%s/subscription_feed", DATA_PATH, author); + + int feed_fd = open(filename, O_RDONLY); + if (feed_fd == -1) { + perror("Could not open subscription feed"); + goto subscription_list_map_failed; + } + + fstat(feed_fd, &st); + if (st.st_size == 0) { + close(feed_fd); + goto subscription_list_map_failed; + } + n_subscription_records = st.st_size / sizeof(struct blergref); + if (*count > n_subscription_records - offset) + *count = n_subscription_records - offset; + if (offset > n_subscription_records) { + fprintf(stderr, "Cannot access subscription record beyond end\n"); + goto subscription_list_map_failed; + close(feed_fd); + return NULL; + } + + slist = (struct blergref *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, feed_fd, 0); + if (slist == MAP_FAILED) { + perror("Could not mmap tag_file"); + goto subscription_list_map_failed; + } + retlist = (struct blergref *) malloc(sizeof(struct blergref) * *count); + if (retlist == NULL) { + perror("Could not allocate memory for subscription feed list"); + goto subscription_list_malloc_failed; + } + + switch(direction) { + case 1: + memcpy(retlist, slist + offset, sizeof(struct blergref) * *count); + break; + case -1: + memcpy(retlist, slist + (n_subscription_records - *count - offset), sizeof(struct blergref) * *count); + } + + munmap(slist, st.st_size); + close(feed_fd); + return retlist; + +subscription_list_malloc_failed: + munmap(slist, st.st_size); +subscription_list_map_failed: + close(feed_fd); +subscription_list_open_failed: + *count = 0; + return NULL; +} diff --git a/database/subscription.h b/database/subscription.h index 3d8ce4f..a7cf567 100644 --- a/database/subscription.h +++ b/database/subscription.h @@ -1,14 +1,15 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ #ifndef _SUBSCRIPTION_H #define _SUBSCRIPTION_H #include - -struct subscription_record { - char author[32]; - uint64_t record; -}; +#include "blergref.h" int subscription_add(const char *from, const char *to); int subscription_remove(const char *from, const char *to); +int subscription_notify(const char *author, uint64_t record); +struct blergref * subscription_list(const char *author, uint64_t offset, int *count, int direction); #endif /* _SUBSCRIPTION_H */ diff --git a/database/tags.c b/database/tags.c index 5ec12fa..dc6bac8 100644 --- a/database/tags.c +++ b/database/tags.c @@ -62,7 +62,7 @@ tag_scan_start: int tag_add(const char *author, const char *tag, uint64_t record) { char filename[512]; - struct tag t; + struct blergref t; memset(t.author, 0, 32); strncpy(t.author, author, 32); @@ -86,8 +86,8 @@ int tag_add(const char *author, const char *tag, uint64_t record) { perror("Could not open tag file"); return 0; } - int n = write(tag_fd, &t, sizeof(struct tag)); - if (n < sizeof(struct tag)) { + int n = write(tag_fd, &t, sizeof(struct blergref)); + if (n < sizeof(struct blergref)) { perror("Could not write new tag"); return 0; } @@ -97,11 +97,11 @@ int tag_add(const char *author, const char *tag, uint64_t record) { return 1; } -struct tag * tag_list(const char *tag, uint64_t offset, int *count, int direction) { +struct blergref * tag_list(const char *tag, uint64_t offset, int *count, int direction) { char filename[512]; struct stat st; - struct tag *taglist; - struct tag *retlist; + struct blergref *taglist; + struct blergref *retlist; uint64_t n_tag_records; if (!valid_tag_name(tag + 1)) @@ -129,7 +129,7 @@ struct tag * tag_list(const char *tag, uint64_t offset, int *count, int directio if (st.st_size == 0) { goto tag_list_map_failed; } - n_tag_records = st.st_size / sizeof(struct tag); + n_tag_records = st.st_size / sizeof(struct blergref); if (*count > n_tag_records - offset) *count = n_tag_records - offset; if (offset > n_tag_records) { @@ -137,22 +137,22 @@ struct tag * tag_list(const char *tag, uint64_t offset, int *count, int directio goto tag_list_map_failed; } - taglist = (struct tag *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, tag_fd, 0); + taglist = (struct blergref *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, tag_fd, 0); if (taglist == MAP_FAILED) { perror("Could not mmap tag file"); goto tag_list_map_failed; } - retlist = (struct tag *) malloc(sizeof(struct tag) * *count); + retlist = (struct blergref *) malloc(sizeof(struct blergref) * *count); if (retlist == NULL) { perror("Could not allocate memory for tag list"); goto tag_list_malloc_failed; } switch(direction) { case 1: - memcpy(retlist, taglist + offset, sizeof(struct tag) * *count); + memcpy(retlist, taglist + offset, sizeof(struct blergref) * *count); break; case -1: - memcpy(retlist, taglist + (n_tag_records - *count - offset), sizeof(struct tag) * *count); + memcpy(retlist, taglist + (n_tag_records - *count - offset), sizeof(struct blergref) * *count); break; } diff --git a/database/tags.h b/database/tags.h index a75047b..9c6f78f 100644 --- a/database/tags.h +++ b/database/tags.h @@ -5,15 +5,11 @@ #define _TAGS_H #include - -struct tag { - char author[32]; - uint64_t record; -}; +#include "blergref.h" int tag_scan(const char *, const char *, int, uint64_t); int tag_add(const char *, const char *, uint64_t); -struct tag * tag_list(const char *, uint64_t, int *count, int direction); +struct blergref * tag_list(const char *, uint64_t, int *count, int direction); int tag_exists(const char *tag); #endif //_TAGS_H diff --git a/http/http_blerg.c b/http/http_blerg.c index a6d3a5f..e205749 100644 --- a/http/http_blerg.c +++ b/http/http_blerg.c @@ -8,6 +8,7 @@ #include #include "database.h" #include "tags.h" +#include "subscription.h" #include "auth.h" #include "canned_responses.h" #include "app.h" @@ -43,10 +44,10 @@ struct get_state { int done; }; -struct tag_state { +struct blergref_state { yajl_gen g; unsigned int yoff; - struct tag *results; + struct blergref *results; uint64_t i; int done; }; @@ -111,66 +112,66 @@ void GET_generate_list_free(void *cls) { free(gs); } -ssize_t GET_generate_taglist(void *cls, uint64_t pos, char *buf, size_t max) { - struct tag_state *ts = cls; +ssize_t GET_generate_blergref_list(void *cls, uint64_t pos, char *buf, size_t max) { + struct blergref_state *bs = cls; struct blerg *b; const unsigned char *ybuf; unsigned int len; - if (ts->yoff > 0) { - yajl_gen_get_buf(ts->g, &ybuf, &len); - size_t bytes_remaining = len - ts->yoff; + if (bs->yoff > 0) { + yajl_gen_get_buf(bs->g, &ybuf, &len); + size_t bytes_remaining = len - bs->yoff; if (bytes_remaining > max) { - memcpy(buf, ybuf + ts->yoff, max); - ts->yoff += max; + memcpy(buf, ybuf + bs->yoff, max); + bs->yoff += max; return max; } else { - memcpy(buf, ybuf + ts->yoff, bytes_remaining); - ts->yoff = 0; - yajl_gen_clear(ts->g); + memcpy(buf, ybuf + bs->yoff, bytes_remaining); + bs->yoff = 0; + yajl_gen_clear(bs->g); return bytes_remaining; } } - if (ts->done) + if (bs->done) return -1; if (pos == 0) { /* Start iterating */ - yajl_gen_array_open(ts->g); + yajl_gen_array_open(bs->g); } /* Snarf one record */ - b = blerg_open(ts->results[ts->i].author); + b = blerg_open(bs->results[bs->i].author); if (b != NULL) { - json_generate_one_record(ts->g, ts->results[ts->i].author, b, ts->results[ts->i].record, 0); + json_generate_one_record(bs->g, bs->results[bs->i].author, b, bs->results[bs->i].record, 0); blerg_close(b); } - if (ts->i == 0) { - yajl_gen_array_close(ts->g); - ts->done = 1; + if (bs->i == 0) { + yajl_gen_array_close(bs->g); + bs->done = 1; } - ts->i--; + bs->i--; - yajl_gen_get_buf(ts->g, &ybuf, &len); + yajl_gen_get_buf(bs->g, &ybuf, &len); if (len > max) { memcpy(buf, ybuf, max); - ts->yoff = max; + bs->yoff = max; return max; } else { memcpy(buf, ybuf, len); - yajl_gen_clear(ts->g); + yajl_gen_clear(bs->g); return len; } } -void GET_generate_taglist_free(void *cls) { - struct tag_state *ts = cls; +void GET_generate_blergref_list_free(void *cls) { + struct blergref_state *bs = cls; - yajl_gen_free(ts->g); - free(ts->results); - free(ts); + yajl_gen_free(bs->g); + free(bs->results); + free(bs); } int POST_auth_iterator(void *cls, enum MHD_ValueKind kind, const char *key, const char *filename, const char *content_type, const char *transfer_encoding, const char *data, uint64_t off, size_t size) { @@ -255,14 +256,14 @@ struct MHD_Response *create_response_for_range(struct blerg *b, uint64_t from, u return response; } -struct MHD_Response *create_tag_response(struct tag *results, uint64_t len) { - struct tag_state *ts = malloc(sizeof(struct tag_state)); - ts->g = yajl_gen_alloc(&yajl_c, NULL); - ts->results = results; - ts->i = len - 1; - ts->yoff = ts->done = 0; +struct MHD_Response *create_blergref_response(struct blergref *results, uint64_t len) { + struct blergref_state *bs = malloc(sizeof(struct blergref_state)); + bs->g = yajl_gen_alloc(&yajl_c, NULL); + bs->results = results; + bs->i = len - 1; + bs->yoff = bs->done = 0; - return MHD_create_response_from_callback(-1, 262144, &GET_generate_taglist, ts, &GET_generate_taglist_free); + return MHD_create_response_from_callback(-1, 262144, &GET_generate_blergref_list, bs, &GET_generate_blergref_list_free); } static int @@ -342,16 +343,18 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c if ((ret & URL_INFO_NAME) == 0) return respond_404(connection); + if (info.name[0] == 'H') + info.name[0] = '#'; if (!tag_exists(info.name)) return respond_404(connection); int recs = 50; - struct tag *taglist = tag_list(info.name, 0, &recs, -1); + struct blergref *taglist = tag_list(info.name, 0, &recs, -1); if (recs == 0) { response = MHD_create_response_from_data(2, "[]", MHD_NO, MHD_NO); } else { - response = create_tag_response(taglist, recs); + response = create_blergref_response(taglist, recs); } if (response == NULL) @@ -557,7 +560,7 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c } else { return respond_JSON_Failure(connection); } - } else if (strncmp(url, "/subscribe", 11) == 0) { + } else if (strncmp(url, "/subscribe", 11) == 0 || strncmp(url, "/unsubscribe", 13) == 0) { struct subscribe_state *ss = (struct subscribe_state *) *ptr; if (ss == NULL) { @@ -579,11 +582,56 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c const char *given_token = MHD_lookup_connection_value(connection, MHD_COOKIE_KIND, "auth"); if (auth_check_token(ss->username, given_token)) { - subscription_add(ss->username, ss->to); + if (url[1] == 'u') { + subscription_remove(ss->username, ss->to); + } else { + subscription_add(ss->username, ss->to); + } return respond_JSON_Success(connection); } else { return respond_JSON_Failure(connection); } + } else if (strncmp(url, "/feed", 6) == 0) { + struct auth_state *as = (struct auth_state *) *ptr; + + if (as == NULL) { + if (strcmp(method, MHD_HTTP_METHOD_POST) != 0) + return respond_405(connection); + + struct auth_state *as = malloc(sizeof(struct auth_state)); + as->username[0] = as->password[0] = 0; + as->pp = MHD_create_post_processor(connection, 1024, &POST_auth_iterator, as); + *ptr = as; + return MHD_YES; + } + + if (*upload_data_size) { + MHD_post_process(as->pp, upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + const char *given_token = MHD_lookup_connection_value(connection, MHD_COOKIE_KIND, "auth"); + if (auth_check_token(as->username, given_token)) { + int recs = 50; + struct blergref *feedlist = subscription_list(as->username, 0, &recs, -1); + + if (recs == 0) { + response = MHD_create_response_from_data(2, "[]", MHD_NO, MHD_NO); + } else { + response = create_blergref_response(feedlist, recs); + } + + if (response == NULL) + return respond_JSON_Failure(connection); + + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); + + return ret; + } else { + return respond_JSON_Failure(connection); + } } else { return respond_404(connection); } diff --git a/tools/blerglatest.c b/tools/blerglatest.c index ed30419..1fe133d 100644 --- a/tools/blerglatest.c +++ b/tools/blerglatest.c @@ -1,3 +1,6 @@ +/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a + * BSD-style license. Please see the COPYING file for details. + */ #include #include #include diff --git a/tools/blergtool.c b/tools/blergtool.c index 0531858..b0e7368 100644 --- a/tools/blergtool.c +++ b/tools/blergtool.c @@ -57,7 +57,7 @@ int main(int argc, char *argv[]) { } else if (strncmp(argv[1], "list", 4) == 0) { char *tag = argv[2]; int count = 50; - struct tag *list = tag_list(tag, 0, &count, -1); + struct blergref *list = tag_list(tag, 0, &count, -1); if (list == NULL) { printf("No entries"); } else { -- 2.25.1