Properly iterate on /put and /get

diff --git a/http_blerg.c b/http_blerg.c
index e6eddad..58dd115 100644 (file)
--- a/http_blerg.c
+++ b/http_blerg.c
@@ -6,6 +6,7 @@
 #include "database.h"
 #include "tags.h"
 #include "auth.h"
+#include "config.h"
 
 #define URL_INFO_AUTHOR 0x1
 #define URL_INFO_RECORD 0x2
@@ -23,6 +24,15 @@ struct put_state {
        int data_size;
 };
 
+struct get_state {
+       struct blerg *b;
+       yajl_gen g;
+       unsigned int yoff;
+       uint64_t from;
+       uint64_t to;
+       int done;
+};
+
 struct MHD_Response *response_401;
 struct MHD_Response *response_404;
 struct MHD_Response *response_405;
@@ -93,6 +103,80 @@ int parse_url_info(const char *url, char *author, uint64_t *record) {
        return ret;
 }
 
+ssize_t GET_generate_list(void *cls, uint64_t pos, char *buf, size_t max) {
+       struct get_state *gs = cls;
+       const unsigned char *ybuf;
+       char *data;
+       unsigned int len;
+
+       if (gs->yoff > 0) {
+               yajl_gen_get_buf(gs->g, &ybuf, &len);
+               size_t bytes_remaining = len - gs->yoff;
+               if (bytes_remaining > max) {
+                       memcpy(buf, ybuf + gs->yoff, max);
+                       gs->yoff += max;
+                       printf("generate list: gave %d bytes\n", max);
+                       return max;
+               } else {
+                       memcpy(buf, ybuf + gs->yoff, bytes_remaining);
+                       gs->yoff = 0;
+                       yajl_gen_clear(gs->g);
+                       printf("generate list: gave %d bytes\n", bytes_remaining);
+                       return bytes_remaining;
+               }
+       }
+
+       if (gs->done)
+               return -1;
+
+       if (pos == 0) { /* Start iterating */
+               gs->yoff = 0;
+               yajl_gen_array_open(gs->g);
+               uint64_t record_count = blerg_get_record_count(gs->b);
+               if (record_count == 0) {
+                       gs->from = 1;
+                       gs->to = 0;
+               } else {
+                       gs->to = record_count - 1;
+                       gs->from = (record_count > 50 ? gs->to - 49 : 0);
+               }
+       }
+
+       if (gs->from > gs->to) { /* Done iterating */
+               yajl_gen_array_close(gs->g);
+               gs->done = 1;
+       } else {
+               /* Snarf one record */
+               blerg_fetch(gs->b, gs->from, &data, &len);
+               yajl_gen_string(gs->g, data, len);
+               free(data);
+               gs->from++;
+       }
+
+
+       yajl_gen_get_buf(gs->g, &ybuf, &len);
+       printf("%d bytes in yajl buffer\n", len);
+       if (len > max) {
+               memcpy(buf, ybuf, max);
+               gs->yoff = max;
+               printf("generate list: gave %d bytes\n", max);
+               return max;
+       } else {
+               memcpy(buf, ybuf, len);
+               yajl_gen_clear(gs->g);
+               printf("generate list: gave %d bytes\n", len);
+               return len;
+       }
+}
+
+void GET_generate_list_free(void *cls) {
+       struct get_state *gs = cls;
+
+       blerg_close(gs->b);
+       yajl_gen_free(gs->g);
+       free(gs);
+}
+
 int POST_create_iterator(void *cls, enum MHD_ValueKind kind, const char *key, const char *filename, const char *content_type, const char *transfer_encoding, const char *data, uint64_t off, size_t size) {
        struct create_state *cs = cls;
 
@@ -113,10 +197,20 @@ int POST_put_iterator(void *cls, enum MHD_ValueKind kind, const char *key, const
        struct put_state *ps = cls;
 
        if (strncmp(key, "data", 5) == 0) {
-               ps->data_size = size;
-               ps->data = malloc(size);
-               memcpy(ps->data, data, size);
-               return MHD_NO;
+               if (ps->data == NULL) {
+                       ps->data_size = size;
+                       ps->data = malloc(size);
+               } else {
+                       if (ps->data_size + size > MAX_RECORD_SIZE) {
+                               size = MAX_RECORD_SIZE - ps->data_size;
+                       }
+                       ps->data_size += size;
+                       ps->data = realloc(ps->data, ps->data_size);
+               }
+               printf("Copying %d bytes, total size %d\n", size, ps->data_size);
+               memcpy(ps->data + off, data, size);
+               if (ps->data_size == MAX_RECORD_SIZE)
+                       return MHD_NO;
        }
 
        return MHD_YES;
@@ -166,9 +260,19 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c
                        MHD_destroy_response(response);
                        return ret;
                } else {
-                       response = MHD_create_response_from_data(strlen(DERP), DERP, MHD_NO, MHD_NO);
+                       yajl_gen_config c;
+                       c.beautify = 0;
+
+                       struct get_state *gs = malloc(sizeof(struct get_state));
+                       gs->b = blerg_open(author);
+                       gs->g = yajl_gen_alloc(&c, NULL);
+                       gs->yoff = 0;
+                       gs->done = 0;
+
+                       response = MHD_create_response_from_callback(-1, 262144, &GET_generate_list, gs, &GET_generate_list_free);
                        ret = MHD_queue_response(connection, MHD_HTTP_OK, response);
                        MHD_destroy_response(response);
+
                        return ret;
                }
        } else if (strncmp(url, "/put", 4) == 0) {
@@ -198,7 +302,8 @@ ahc_derp (void *cls, struct MHD_Connection *connection, const char *url, const c
 
                        struct put_state *ps = malloc(sizeof(struct put_state));
                        ps->data = NULL;
-                       ps->pp = MHD_create_post_processor(connection, 4096, &POST_put_iterator, ps);
+                       ps->data_size = 0;
+                       ps->pp = MHD_create_post_processor(connection, 16384, &POST_put_iterator, ps);
                        *ptr = ps;
                        return MHD_YES;
                }