Add subscription feed fetching, clean up a bit

Add a /feed URL for fetching subscription feeds.  Genericized "struct
tag" into "struct blergref" -- no actual change, but the same thing is
now used by subscription indexes.

diff --git a/cgi/cgi_blerg.c b/cgi/cgi_blerg.c
index b6dcb3a..c004a7d 100644 (file)
--- a/cgi/cgi_blerg.c
+++ b/cgi/cgi_blerg.c
@@ -45,7 +45,7 @@ void respond_for_range(struct blerg *b, uint64_t from, uint64_t to) {
        yajl_gen_free(g);
 }
 
-void respond_taglist(struct tag *results, int i) {
+void respond_blergref_list(struct blergref * results, int i) {
        const unsigned char *ybuf;
        unsigned int len;
        struct blerg *b;
@@ -170,12 +170,12 @@ int main(int argc, char *argv[]) {
                }
 
                int recs = 50;
-               struct tag *taglist = tag_list(info.name, 0, &recs, -1);
+               struct blergref *taglist = tag_list(info.name, 0, &recs, -1);
 
                if (recs == 0) {
                        respond_simple_data("[]", 2);
                } else {
-                       respond_taglist(taglist, recs);
+                       respond_blergref_list(taglist, recs);
                }
        } else if (strncmp(path, "/put", 4) == 0) {
                if (strcmp(request_method, "POST") != 0) {

diff --git a/common/stringbucket.c b/common/stringbucket.c
index ec827d1..d892541 100644 (file)
--- a/common/stringbucket.c
+++ b/common/stringbucket.c
@@ -1,3 +1,6 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/mman.h>

diff --git a/common/stringbucket.h b/common/stringbucket.h
index 5594bd8..b7eb282 100644 (file)
--- a/common/stringbucket.h
+++ b/common/stringbucket.h
@@ -1,3 +1,6 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
 #ifndef _STRINGBUCKET_H
 #define _STRINGBUCKET_H
 

diff --git a/database/blergref.h b/database/blergref.h
new file mode 100644 (file)
index 0000000..57b472b
--- /dev/null
+++ b/database/blergref.h
@@ -0,0 +1,12 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
+#ifndef _BLERGREF_H
+#define _BLERGREF_H
+
+struct blergref {
+       char author[32];
+       uint64_t record;
+};
+
+#endif /* _BLERGREF_H */

diff --git a/database/database.c b/database/database.c
index cb5cf0c..a45c92b 100644 (file)
--- a/database/database.c
+++ b/database/database.c
@@ -13,6 +13,7 @@
 #include <sys/file.h>
 #include <fcntl.h>
 #include "database.h"
+#include "subscription.h"
 #include "util.h"
 #include "config.h"
 
@@ -246,11 +247,12 @@ int blerg_store(struct blerg *blerg, const char *data, int len) {
        blerg->index[seg_rec].length = len;
        blerg->index[seg_rec].timestamp = time(NULL);
 
-       tag_scan(blerg->name, data, len, record);
-
        flock(blerg->data_fd, LOCK_UN);
        flock(blerg->index_fd, LOCK_UN);
 
+       tag_scan(blerg->name, data, len, record);
+       subscription_notify(blerg->name, record);
+
        return record;
 }
 

diff --git a/database/subscription.c b/database/subscription.c
index e6ae94a..d67d31f 100644 (file)
--- a/database/subscription.c
+++ b/database/subscription.c
@@ -1,7 +1,12 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/mman.h>
+#include <stdlib.h>
 #include <stdio.h>
+#include <fcntl.h>
 #include <string.h>
 #include "subscription.h"
 #include "stringbucket.h"
@@ -41,14 +46,14 @@ void subscription_notify_add_item(char *to, void *stuff) {
        char filename[512];
 
        snprintf(filename, 512, "%s/%s/subscription_feed", DATA_PATH, to);
-       int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT);
-       write(fd, stuff, sizeof(struct subscription_record));
+       int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0600);
+       write(fd, stuff, sizeof(struct blergref));
        close(fd);
 }
 
 int subscription_notify(const char *author, uint64_t record) {
        char filename[512];
-       struct subscription_record r;
+       struct blergref r;
 
        strncpy(r.author, author, 32);
        r.record = record;
@@ -58,3 +63,68 @@ int subscription_notify(const char *author, uint64_t record) {
        stringbucket_iterate(sb, subscription_notify_add_item, &r);
        stringbucket_close(sb);
 }
+
+struct blergref * subscription_list(const char *author, uint64_t offset, int *count, int direction) {
+       char filename[512];
+       struct stat st;
+       struct blergref * slist;
+       struct blergref * retlist;
+       uint64_t n_subscription_records;
+
+       if (!valid_name(author))
+               return NULL;
+
+       snprintf(filename, 512, "%s/%s/subscription_feed", DATA_PATH, author);
+
+       int feed_fd = open(filename, O_RDONLY);
+       if (feed_fd == -1) {
+               perror("Could not open subscription feed");
+               goto subscription_list_map_failed;
+       }
+
+       fstat(feed_fd, &st);
+       if (st.st_size == 0) {
+               close(feed_fd);
+               goto subscription_list_map_failed;
+       }
+       n_subscription_records = st.st_size / sizeof(struct blergref);
+       if (*count > n_subscription_records - offset)
+               *count = n_subscription_records - offset;
+       if (offset > n_subscription_records) {
+               fprintf(stderr, "Cannot access subscription record beyond end\n");
+               goto subscription_list_map_failed;
+               close(feed_fd);
+               return NULL;
+       }
+
+       slist = (struct blergref *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, feed_fd, 0);
+       if (slist == MAP_FAILED) {
+               perror("Could not mmap tag_file");
+               goto subscription_list_map_failed;
+       }
+       retlist = (struct blergref *) malloc(sizeof(struct blergref) * *count);
+       if (retlist == NULL) {
+               perror("Could not allocate memory for subscription feed list");
+               goto subscription_list_malloc_failed;
+       }
+       
+       switch(direction) {
+       case 1:
+               memcpy(retlist, slist + offset, sizeof(struct blergref) * *count);
+               break;
+       case -1:
+               memcpy(retlist, slist + (n_subscription_records - *count - offset), sizeof(struct blergref) * *count);
+       }
+
+       munmap(slist, st.st_size);
+       close(feed_fd);
+       return retlist;
+
+subscription_list_malloc_failed:
+       munmap(slist, st.st_size);
+subscription_list_map_failed:
+       close(feed_fd);
+subscription_list_open_failed:
+       *count = 0;
+       return NULL;
+}

diff --git a/database/subscription.h b/database/subscription.h
index 3d8ce4f..a7cf567 100644 (file)
--- a/database/subscription.h
+++ b/database/subscription.h
@@ -1,14 +1,15 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
 #ifndef _SUBSCRIPTION_H
 #define _SUBSCRIPTION_H
 
 #include <stdint.h>
-
-struct subscription_record {
-       char author[32];
-       uint64_t record;
-};
+#include "blergref.h"
 
 int subscription_add(const char *from, const char *to);
 int subscription_remove(const char *from, const char *to);
+int subscription_notify(const char *author, uint64_t record);
+struct blergref * subscription_list(const char *author, uint64_t offset, int *count, int direction);
 
 #endif /* _SUBSCRIPTION_H */

diff --git a/database/tags.c b/database/tags.c
index 5ec12fa..dc6bac8 100644 (file)
--- a/database/tags.c
+++ b/database/tags.c
@@ -62,7 +62,7 @@ tag_scan_start:
 
 int tag_add(const char *author, const char *tag, uint64_t record) {
        char filename[512];
-       struct tag t;
+       struct blergref t;
 
        memset(t.author, 0, 32);
        strncpy(t.author, author, 32);
@@ -86,8 +86,8 @@ int tag_add(const char *author, const char *tag, uint64_t record) {
                perror("Could not open tag file");
                return 0;
        }
-       int n = write(tag_fd, &t, sizeof(struct tag));
-       if (n < sizeof(struct tag)) {
+       int n = write(tag_fd, &t, sizeof(struct blergref));
+       if (n < sizeof(struct blergref)) {
                perror("Could not write new tag");
                return 0;
        }
@@ -97,11 +97,11 @@ int tag_add(const char *author, const char *tag, uint64_t record) {
        return 1;
 }
 
-struct tag * tag_list(const char *tag, uint64_t offset, int *count, int direction) {
+struct blergref * tag_list(const char *tag, uint64_t offset, int *count, int direction) {
        char filename[512];
        struct stat st;
-       struct tag *taglist;
-       struct tag *retlist;
+       struct blergref *taglist;
+       struct blergref *retlist;
        uint64_t n_tag_records;
 
        if (!valid_tag_name(tag + 1))
@@ -129,7 +129,7 @@ struct tag * tag_list(const char *tag, uint64_t offset, int *count, int directio
        if (st.st_size == 0) {
                goto tag_list_map_failed;
        }
-       n_tag_records = st.st_size / sizeof(struct tag);
+       n_tag_records = st.st_size / sizeof(struct blergref);
        if (*count > n_tag_records - offset)
                *count = n_tag_records - offset;
        if (offset > n_tag_records) {
@@ -137,22 +137,22 @@ struct tag * tag_list(const char *tag, uint64_t offset, int *count, int directio
                goto tag_list_map_failed;
        }
 
-       taglist = (struct tag *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, tag_fd, 0);
+       taglist = (struct blergref *) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, tag_fd, 0);
        if (taglist == MAP_FAILED) {
                perror("Could not mmap tag file");
                goto tag_list_map_failed;
        }
-       retlist = (struct tag *) malloc(sizeof(struct tag) * *count);
+       retlist = (struct blergref *) malloc(sizeof(struct blergref) * *count);
        if (retlist == NULL) {
                perror("Could not allocate memory for tag list");
                goto tag_list_malloc_failed;
        }
        switch(direction) {
        case 1:
-               memcpy(retlist, taglist + offset, sizeof(struct tag) * *count);
+               memcpy(retlist, taglist + offset, sizeof(struct blergref) * *count);
                break;
        case -1:
-               memcpy(retlist, taglist + (n_tag_records - *count - offset), sizeof(struct tag) * *count);
+               memcpy(retlist, taglist + (n_tag_records - *count - offset), sizeof(struct blergref) * *count);
                break;
        }
 

diff --git a/database/tags.h b/database/tags.h
index a75047b..9c6f78f 100644 (file)
--- a/database/tags.h
+++ b/database/tags.h
@@ -5,15 +5,11 @@
 #define _TAGS_H
 
 #include <stdint.h>
-
-struct tag {
-       char author[32];
-       uint64_t record;
-};
+#include "blergref.h"
 
 int tag_scan(const char *, const char *, int, uint64_t);
 int tag_add(const char *, const char *, uint64_t);
-struct tag * tag_list(const char *, uint64_t, int *count, int direction);
+struct blergref * tag_list(const char *, uint64_t, int *count, int direction);
 int tag_exists(const char *tag);
 
 #endif //_TAGS_H

diff --git a/http/http_blerg.c b/http/http_blerg.c
index a6d3a5f..e205749 100644 (file)
--- a/http/http_blerg.c
+++ b/http/http_blerg.c
@@ -8,6 +8,7 @@
 #include <yajl/yajl_gen.h>
 #include "database.h"
 #include "tags.h"
+#include "subscription.h"
 #include "auth.h"
 #include "canned_responses.h"
 #include "app.h"
@@ -43,10 +44,10 @@ struct get_state {
        int done;
 };
 
-struct tag_state {
+struct blergref_state {
        yajl_gen g;
        unsigned int yoff;
-       struct tag *results;
+       struct blergref *results;
        uint64_t i;
        int done;
 };
@@ -111,66 +112,66 @@ void GET_generate_list_free(void *cls) {
        free(gs);
 }
 
-ssize_t GET_generate_taglist(void *cls, uint64_t pos, char *buf, size_t max) {
-       struct tag_state *ts = cls;
+ssize_t GET_generate_blergref_list(void *cls, uint64_t pos, char *buf, size_t max) {
+       struct blergref_state *bs = cls;
        struct blerg *b;
        const unsigned char *ybuf;
        unsigned int len;
 
-       if (ts->yoff > 0) {
-               yajl_gen_get_buf(ts->g, &ybuf, &len);
-               size_t bytes_remaining = len - ts->yoff;
+       if (bs->yoff > 0) {
+               yajl_gen_get_buf(bs->g, &ybuf, &len);
+               size_t bytes_remaining = len - bs->yoff;
                if (bytes_remaining > max) {
-                       memcpy(buf, ybuf + ts->yoff, max);
-                       ts->yoff += max;
+                       memcpy(buf, ybuf + bs->yoff, max);
+                       bs->yoff += max;
                        return max;
                } else {
-                       memcpy(buf, ybuf + ts->yoff, bytes_remaining);
-                       ts->yoff = 0;
-                       yajl_gen_clear(ts->g);
+                       memcpy(buf, ybuf + bs->yoff, bytes_remaining);
+                       bs->yoff = 0;
+                       yajl_gen_clear(bs->g);
                        return bytes_remaining;
                }
        }
 
-       if (ts->done)
+       if (bs->done)
                return -1;
 
        if (pos == 0) { /* Start iterating */
-               yajl_gen_array_open(ts->g);
+               yajl_gen_array_open(bs->g);
        }
 
        /* Snarf one record */
-       b = blerg_open(ts->results[ts->i].author);
+       b = blerg_open(bs->results[bs->i].author);
        if (b != NULL) {
-               json_generate_one_record(ts->g, ts->results[ts->i].author, b, ts->results[ts->i].record, 0);
+               json_generate_one_record(bs->g, bs->results[bs->i].author, b, bs->results[bs->i].record, 0);
                blerg_close(b);
        }
 
-       if (ts->i == 0) {
-               yajl_gen_array_close(ts->g);
-               ts->done = 1;
+       if (bs->i == 0) {
+               yajl_gen_array_close(bs->g);
+               bs->done = 1;
        }
 
-       ts->i--;
+       bs->i--;
 
-       yajl_gen_get_buf(ts->g, &ybuf, &len);
+       yajl_gen_get_buf(bs->g, &ybuf, &len);
        if (len > max) {
                memcpy(buf, ybuf, max);
-               ts->yoff = max;
+               bs->yoff = max;
                return max;
        } else {
                memcpy(buf, ybuf, len);
-               yajl_gen_clear(ts->g);
+               yajl_gen_clear(bs->g);
                return len;
        }
 }
 
-void GET_generate_taglist_free(void *cls) {
-       struct tag_state *ts = cls;
+void GET_generate_blergref_list_free(void *cls) {
+       struct blergref_state *bs = cls;
 
-       yajl_gen_free(ts->g);
-       free(ts->results);
-       free(ts);
+       yajl_gen_free(bs->g);
+       free(bs->results);
+       free(bs);
 }
 
 int POST_auth_iterator(void *cls, enum MHD_ValueKind kind, const char *key, const char *filename, const char *content_type, const char *transfer_encoding, const char *data, uint64_t off, size_t size) {
@@ -255,14 +256,14 @@ struct MHD_Response *create_response_for_range(struct blerg *b, uint64_t from, u
        return response;
 }
 
-struct MHD_Response *create_tag_response(struct tag *results, uint64_t len) {
-       struct tag_state *ts = malloc(sizeof(struct tag_state));
-       ts->g = yajl_gen_alloc(&yajl_c, NULL);
-       ts->results = results;
-       ts->i = len - 1;
-       ts->yoff = ts->done = 0;
+struct MHD_Response *create_blergref_response(struct blergref *results, uint64_t len) {
+       struct blergref_state *bs = malloc(sizeof(struct blergref_state));
+       bs->g = yajl_gen_alloc(&yajl_c, NULL);
+       bs->results = results;
+       bs->i = len - 1;
+       bs->yoff = bs->done = 0;
 
-       return MHD_create_response_from_callback(-1, 262144, &GET_generate_taglist, ts, &GET_generate_taglist_free);
+       return MHD_create_response_from_callback(-1, 262144, &GET_generate_blergref_list, bs, &GET_generate_blergref_list_free);
 }
 
 static int
@@ -342,16 +343,18 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c
                if ((ret & URL_INFO_NAME) == 0)
                        return respond_404(connection);
 
+               if (info.name[0] == 'H')
+                       info.name[0] = '#';
                if (!tag_exists(info.name))
                        return respond_404(connection);
 
                int recs = 50;
-               struct tag *taglist = tag_list(info.name, 0, &recs, -1);
+               struct blergref *taglist = tag_list(info.name, 0, &recs, -1);
 
                if (recs == 0) {
                        response = MHD_create_response_from_data(2, "[]", MHD_NO, MHD_NO);
                } else {
-                       response = create_tag_response(taglist, recs);
+                       response = create_blergref_response(taglist, recs);
                }
 
                if (response == NULL)
@@ -557,7 +560,7 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c
                } else {
                        return respond_JSON_Failure(connection);
                }
-       } else if (strncmp(url, "/subscribe", 11) == 0) {
+       } else if (strncmp(url, "/subscribe", 11) == 0 || strncmp(url, "/unsubscribe", 13) == 0) {
                struct subscribe_state *ss = (struct subscribe_state *) *ptr;
 
                if (ss == NULL) {
@@ -579,11 +582,56 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c
 
                const char *given_token = MHD_lookup_connection_value(connection, MHD_COOKIE_KIND, "auth");
                if (auth_check_token(ss->username, given_token)) {
-                       subscription_add(ss->username, ss->to);
+                       if (url[1] == 'u') {
+                               subscription_remove(ss->username, ss->to);
+                       } else {
+                               subscription_add(ss->username, ss->to);
+                       }
                        return respond_JSON_Success(connection);
                } else {
                        return respond_JSON_Failure(connection);
                }
+       } else if (strncmp(url, "/feed", 6) == 0) {
+               struct auth_state *as = (struct auth_state *) *ptr;
+
+               if (as == NULL) {
+                       if (strcmp(method, MHD_HTTP_METHOD_POST) != 0)
+                               return respond_405(connection);
+
+                       struct auth_state *as = malloc(sizeof(struct auth_state));
+                       as->username[0] = as->password[0] = 0;
+                       as->pp = MHD_create_post_processor(connection, 1024, &POST_auth_iterator, as);
+                       *ptr = as;
+                       return MHD_YES;
+               }
+
+               if (*upload_data_size) {
+                       MHD_post_process(as->pp, upload_data, *upload_data_size);
+                       *upload_data_size = 0;
+                       return MHD_YES;
+               }
+
+               const char *given_token = MHD_lookup_connection_value(connection, MHD_COOKIE_KIND, "auth");
+               if (auth_check_token(as->username, given_token)) {
+                       int recs = 50;
+                       struct blergref *feedlist = subscription_list(as->username, 0, &recs, -1);
+
+                       if (recs == 0) {
+                               response = MHD_create_response_from_data(2, "[]", MHD_NO, MHD_NO);
+                       } else {
+                               response = create_blergref_response(feedlist, recs);
+                       }
+
+                       if (response == NULL)
+                               return respond_JSON_Failure(connection);
+
+                       ret = MHD_queue_response(connection, MHD_HTTP_OK, response);
+                       MHD_destroy_response(response);
+
+                       return ret;
+               } else {
+                       return respond_JSON_Failure(connection);
+               }
        } else {
                return respond_404(connection);
        }

diff --git a/tools/blerglatest.c b/tools/blerglatest.c
index ed30419..1fe133d 100644 (file)
--- a/tools/blerglatest.c
+++ b/tools/blerglatest.c
@@ -1,3 +1,6 @@
+/* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
+ * BSD-style license.  Please see the COPYING file for details.
+ */
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>

diff --git a/tools/blergtool.c b/tools/blergtool.c
index 0531858..b0e7368 100644 (file)
--- a/tools/blergtool.c
+++ b/tools/blergtool.c
@@ -57,7 +57,7 @@ int main(int argc, char *argv[]) {
        } else if (strncmp(argv[1], "list", 4) == 0) {
                char *tag = argv[2];
                int count = 50;
-               struct tag *list = tag_list(tag, 0, &count, -1);
+               struct blergref *list = tag_list(tag, 0, &count, -1);
                if (list == NULL) {
                        printf("No entries");
                } else {