From: Chip Black Date: Mon, 27 Dec 2010 10:49:25 +0000 (-0600) Subject: Add tag searching to web API X-Git-Tag: ohfuckreddit~37 X-Git-Url: http://git.bytex64.net/?a=commitdiff_plain;h=97d379a62944d0be7d760f744a7efff7acfc2b4b;p=blerg.git Add tag searching to web API --- diff --git a/config.h b/config.h index 80327ca..f732009 100644 --- a/config.h +++ b/config.h @@ -3,7 +3,7 @@ #define DATA_PATH "data" #define HASH_TAGS_PATH "hash_tags" -#define AUTHOR_TAGS_PATH "author_tags" +#define REF_TAGS_PATH "ref_tags" #define RECORDS_PER_SEGMENT 65536 #define MAX_RECORD_SIZE 65535 /* No greater than 65535 */ diff --git a/http_blerg.c b/http_blerg.c index 845f312..93d0984 100644 --- a/http_blerg.c +++ b/http_blerg.c @@ -38,6 +38,14 @@ struct get_state { int done; }; +struct tag_state { + yajl_gen g; + unsigned int yoff; + struct tag *results; + uint64_t i; + int done; +}; + int parse_url_info(const char *url, struct url_info *info) { const char *c; int len; @@ -83,6 +91,33 @@ uint64_t *make_sequential_list(uint64_t from, uint64_t to) { return list; } +void json_generate_one_record(yajl_gen g, const char *author, struct blerg *b, uint64_t record) { + char *data; + char number[21]; + int len; + + if (!blerg_fetch(b, record, &data, &len)) { + fprintf(stderr, "Could not fetch record\n"); + return; + } + + yajl_gen_map_open(g); + if (author != NULL) { + yajl_gen_string(g, "author", 6); + yajl_gen_string(g, author, strlen(author)); + } + yajl_gen_string(g, "record", 6); + snprintf(number, 21, "%llu", record); + yajl_gen_string(g, number, strlen(number)); + yajl_gen_string(g, "timestamp", 9); + yajl_gen_integer(g, blerg_get_timestamp(b, record)); + yajl_gen_string(g, "data", 4); + yajl_gen_string(g, data, len); + yajl_gen_map_close(g); + + free(data); +} + ssize_t GET_generate_list(void *cls, uint64_t pos, char *buf, size_t max) { struct get_state *gs = cls; const unsigned char *ybuf; @@ -113,19 +148,8 @@ ssize_t GET_generate_list(void *cls, uint64_t pos, char *buf, size_t max) { } /* Snarf one record */ - blerg_fetch(gs->b, gs->entries[gs->i], &data, &len); - - yajl_gen_map_open(gs->g); - yajl_gen_string(gs->g, "record", 6); - snprintf(number, 21, "%llu", gs->entries[gs->i]); - yajl_gen_string(gs->g, number, strlen(number)); - yajl_gen_string(gs->g, "timestamp", 9); - yajl_gen_integer(gs->g, blerg_get_timestamp(gs->b, gs->entries[gs->i])); - yajl_gen_string(gs->g, "data", 4); - yajl_gen_string(gs->g, data, len); - yajl_gen_map_close(gs->g); + json_generate_one_record(gs->g, NULL, gs->b, gs->entries[gs->i]); - free(data); if (gs->i == 0) { yajl_gen_array_close(gs->g); gs->done = 1; @@ -154,6 +178,71 @@ void GET_generate_list_free(void *cls) { free(gs); } +ssize_t GET_generate_taglist(void *cls, uint64_t pos, char *buf, size_t max) { + struct tag_state *ts = cls; + struct blerg *b; + const unsigned char *ybuf; + unsigned int len; + + if (ts->yoff > 0) { + yajl_gen_get_buf(ts->g, &ybuf, &len); + size_t bytes_remaining = len - ts->yoff; + if (bytes_remaining > max) { + memcpy(buf, ybuf + ts->yoff, max); + ts->yoff += max; + return max; + } else { + memcpy(buf, ybuf + ts->yoff, bytes_remaining); + ts->yoff = 0; + yajl_gen_clear(ts->g); + return bytes_remaining; + } + } + + if (ts->done) + return -1; + + if (pos == 0) { /* Start iterating */ + yajl_gen_array_open(ts->g); + } + + /* Snarf one record */ + b = blerg_open(ts->results[ts->i].author); + if (b == NULL) + goto skip_bad_blerg; + + json_generate_one_record(ts->g, ts->results[ts->i].author, b, ts->results[ts->i].record); + + blerg_close(b); + +skip_bad_blerg: + if (ts->i == 0) { + yajl_gen_array_close(ts->g); + ts->done = 1; + } + + ts->i--; + + yajl_gen_get_buf(ts->g, &ybuf, &len); + if (len > max) { + memcpy(buf, ybuf, max); + ts->yoff = max; + return max; + } else { + memcpy(buf, ybuf, len); + yajl_gen_clear(ts->g); + return len; + } +} + +void GET_generate_taglist_free(void *cls) { + struct tag_state *ts = cls; + + yajl_gen_free(ts->g); + free(ts->results); + free(ts); +} + int POST_create_iterator(void *cls, enum MHD_ValueKind kind, const char *key, const char *filename, const char *content_type, const char *transfer_encoding, const char *data, uint64_t off, size_t size) { struct create_state *cs = cls; @@ -216,6 +305,16 @@ struct MHD_Response *create_response_for_range(struct blerg *b, uint64_t from, u return response; } +struct MHD_Response *create_tag_response(struct tag *results, uint64_t len) { + struct tag_state *ts = malloc(sizeof(struct tag_state)); + ts->g = yajl_gen_alloc(&yajl_c, NULL); + ts->results = results; + ts->i = len - 1; + ts->yoff = ts->done = 0; + + return MHD_create_response_from_callback(-1, 262144, &GET_generate_taglist, ts, &GET_generate_taglist_free); +} + static int ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **ptr) { @@ -273,6 +372,41 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c ret = MHD_queue_response(connection, MHD_HTTP_OK, response); MHD_destroy_response(response); + return ret; + } else if (strncmp(url, "/tag", 4) == 0 && strlen(url) > 4) { + if (*ptr == NULL) { + if (strcmp(method, MHD_HTTP_METHOD_GET) != 0) + return respond_405(connection); + + *ptr = (void *) 1; + return MHD_YES; + } + + if (url[4] != '/') + return respond_404(connection); + + ret = parse_url_info(url + 5, &info); + if ((ret & URL_INFO_AUTHOR) == 0) + return respond_404(connection); + + if (!tag_exists(info.author)) + return respond_404(connection); + + uint64_t recs = 50; + struct tag *taglist = tag_list(info.author, 0, &recs, -1); + + if (recs == 0) { + response = MHD_create_response_from_data(2, "[]", MHD_NO, MHD_NO); + } else { + response = create_tag_response(taglist, recs); + } + + if (response == NULL) + return respond_JSON_Failure(connection); + + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); + return ret; } else if (strncmp(url, "/put", 4) == 0) { struct put_state *ps = (struct put_state *) *ptr; diff --git a/tags.c b/tags.c index 4705b87..1130d57 100644 --- a/tags.c +++ b/tags.c @@ -68,7 +68,7 @@ int tag_add(const char *author, const char *tag, uint64_t record) { snprintf(filename, 512, "%s/%s", HASH_TAGS_PATH, tag + 1); break; case '@': - snprintf(filename, 512, "%s/%s", AUTHOR_TAGS_PATH, tag + 1); + snprintf(filename, 512, "%s/%s", REF_TAGS_PATH, tag + 1); break; default: fprintf(stderr, "Invalid tag type: %s\n", tag); @@ -104,7 +104,7 @@ struct tag * tag_list(const char *tag, uint64_t offset, uint64_t *count, int dir snprintf(filename, 512, "%s/%s", HASH_TAGS_PATH, tag + 1); break; case '@': - snprintf(filename, 512, "%s/%s", AUTHOR_TAGS_PATH, tag + 1); + snprintf(filename, 512, "%s/%s", REF_TAGS_PATH, tag + 1); break; default: fprintf(stderr, "Invalid tag type: %s\n", tag); @@ -161,3 +161,36 @@ tag_list_map_failed: tag_list_open_failed: return NULL; } + +int tag_exists(const char *tag) { + int taglen = strlen(tag); + char filename[512]; + + if (taglen < 2) { + fprintf(stderr, "Tag too short\n"); + return 0; + } else if (taglen > 33) { + fprintf(stderr, "Tag too long\n"); + return 0; + } + if (!(tag[0] == '@' || tag[0] == '#')) { + fprintf(stderr, "Invalid tag: %s\n", tag); + return 0; + } + + switch(tag[0]) { + case '#': + snprintf(filename, 512, "%s/%s", HASH_TAGS_PATH, tag + 1); + break; + case '@': + snprintf(filename, 512, "%s/%s", REF_TAGS_PATH, tag + 1); + break; + default: + fprintf(stderr, "Invalid tag type: %s\n", tag); + return 0; + } + if (access(filename, F_OK) == -1) + return 0; + else + return 1; +} diff --git a/tags.h b/tags.h index df0cf98..18e7587 100644 --- a/tags.h +++ b/tags.h @@ -11,5 +11,6 @@ struct tag { int tag_scan(const char *, const char *, int, uint64_t); int tag_add(const char *, const char *, uint64_t); struct tag * tag_list(const char *, uint64_t, uint64_t *, int); +int tag_exists(const char *tag); #endif //_TAGS_H