database/database.c

   1 /* Blerg is (C) 2011 The Dominion of Awesome, and is distributed under a
   2  * BSD-style license.  Please see the COPYING file for details.
   3  */
   4 #include <stdint.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <time.h>
   9 #include <unistd.h>
  10 #include <sys/stat.h>
  11 #include <sys/types.h>
  12 #include <sys/mman.h>
  13 #include <sys/file.h>
  14 #include <fcntl.h>
  15 #include "database.h"
  16 #include "configuration.h"
  17 #include "subscription.h"
  18 #include "tags.h"
  19 #include "util.h"
  20 #include "config.h"
  21
  22 #define CHECK_VALID_BLERG(r)                               \
  23         if (blerg == NULL) {                               \
  24                 fprintf(stderr, "Invalid struct blerg\n"); \
  25                 return r;                                  \
  26         }
  27
  28 int blerg_init() {
  29         if (!blerg_configuration_init()) {
  30                 return 0;
  31         }
  32         return 1;
  33 }
  34
  35 uint64_t blerg_get_record_count(struct blerg *blerg) {
  36         uint64_t count;
  37         flock(blerg->meta_fd, LOCK_SH);
  38         count = blerg->meta->sequence;
  39         flock(blerg->meta_fd, LOCK_UN);
  40         return count;
  41 }
  42
  43 /* Returns last usable record */
  44 uint64_t blerg_increment_record_count(struct blerg *blerg) {
  45         uint64_t count;
  46         flock(blerg->meta_fd, LOCK_EX);
  47         count = blerg->meta->sequence++;
  48         flock(blerg->meta_fd, LOCK_UN);
  49         return count;
  50 }
  51
  52 void blerg_segment_close(struct blerg *blerg) {
  53         if (blerg->data != NULL)
  54                 munmap((void *)blerg->data, blerg->data_size);
  55         if (blerg->data_fd != -1)
  56                 close(blerg->data_fd);
  57         if (blerg->index != NULL)
  58                 munmap((void *)blerg->index, RECORDS_PER_SEGMENT * sizeof(struct record));
  59         if (blerg->index_fd != -1)
  60                 close(blerg->index_fd);
  61 }
  62
  63 int blerg_remap_data(struct blerg *blerg) {
  64         struct stat st;
  65
  66         if (blerg->data != NULL)
  67                 munmap(blerg->data, blerg->data_size);
  68         fstat(blerg->data_fd, &st);
  69         blerg->data_size = st.st_size;
  70         if (blerg->data_size == 0) {
  71                 /* Can't map an empty data file. */
  72                 return 1;
  73         }
  74         blerg->data = (char *) mmap(NULL, blerg->data_size, PROT_READ, MAP_SHARED, blerg->data_fd, 0);
  75         if (blerg->data == MAP_FAILED) {
  76                 perror("Could not remap data");
  77                 return 0;
  78         }
  79         return 1;
  80 }
  81
  82 int blerg_segment_switch(struct blerg *blerg, int new_segment) {
  83         char filename[FILENAME_MAX];
  84         uint64_t max_sequence_no = blerg_get_record_count(blerg);
  85         struct stat st;
  86
  87         if (blerg->index != NULL && blerg->data != NULL && new_segment == blerg->current_segment) {
  88                 return 1;
  89         }
  90
  91         if (new_segment > max_sequence_no / RECORDS_PER_SEGMENT) {
  92                 fprintf(stderr, "Cannot switch to sequence beyond last record\n");
  93                 return 0;
  94         }
  95         if (new_segment < 0) {
  96                 fprintf(stderr, "Cannot switch to negative segment\n");
  97                 return 0;
  98         }
  99
 100         blerg_segment_close(blerg);
 101
 102         /* Load and map the index */
 103         snprintf(filename, FILENAME_MAX, "%s/index%d", blerg->base_path, new_segment);
 104         blerg->index_fd = open(filename, O_RDWR | O_CREAT, 0600);
 105         if (blerg->index_fd == -1) {
 106                 perror("Could not open index");
 107                 goto open_failed_index_open;
 108         }
 109         flock(blerg->index_fd, LOCK_EX);
 110         fstat(blerg->index_fd, &st);
 111         if (st.st_size == 0) {
 112                 /* ftruncate() means never having to say you're sorry.  Sorry
 113                    in this case meaning "allocating disk space for a 1MB file
 114                    full or zeroes". */
 115                 ftruncate(blerg->index_fd, RECORDS_PER_SEGMENT * sizeof(struct record));
 116         }
 117         flock(blerg->index_fd, LOCK_UN);
 118
 119         blerg->index = (struct record *) mmap(NULL, RECORDS_PER_SEGMENT * sizeof(struct record), PROT_READ | PROT_WRITE, MAP_SHARED, blerg->index_fd, 0);
 120         if (blerg->index == MAP_FAILED) {
 121                 perror("Could not mmap index");
 122                 goto open_failed_index_mmap;
 123         }
 124
 125         /* Load data file */
 126         snprintf(filename, FILENAME_MAX, "%s/data%d", blerg->base_path, new_segment);
 127         blerg->data_fd = open(filename, O_RDWR | O_APPEND | O_CREAT, 0600);
 128         if (blerg->data_fd == -1) {
 129                 perror("Could not open data");
 130                 goto open_failed_data_open;
 131         }
 132
 133         if (!blerg_remap_data(blerg)) {
 134                 goto open_failed_data_mmap;
 135         }
 136
 137         blerg->current_segment = new_segment;
 138
 139         return 1;
 140
 141 open_failed_data_mmap:
 142         close(blerg->data_fd);
 143 open_failed_data_open:
 144         munmap((void *)blerg->index, RECORDS_PER_SEGMENT * sizeof(struct record));
 145 open_failed_index_mmap:
 146         close(blerg->index_fd);
 147 open_failed_index_open:
 148         return 0;
 149 }
 150
 151 int blerg_exists(const char *name) {
 152         char filename[FILENAME_MAX];
 153
 154         if (!valid_name(name)) {
 155                 fprintf(stderr, "Invalid name\n");
 156                 return 0;
 157         }
 158
 159         snprintf(filename, FILENAME_MAX, "%s/%s", blergconf.data_path, name);
 160         if (access(filename, F_OK) == -1)
 161                 return 0;
 162         else
 163                 return 1;
 164 }
 165
 166 struct blerg *blerg_open(const char *name) {
 167         int namelen = strlen(name);
 168         char filename[FILENAME_MAX];
 169         struct stat st;
 170         uint64_t sequence;
 171
 172         if (!valid_name(name)) {
 173                 fprintf(stderr, "Invalid name\n");
 174                 return NULL;
 175         }
 176         struct blerg *blerg = malloc(sizeof(struct blerg));
 177         if (!blerg) {
 178                 perror("Cannot allocate memory for blerg");
 179                 goto open_failed_blerg_malloc;
 180         }
 181         blerg->name = malloc(namelen + 1);
 182         memcpy(blerg->name, name, namelen + 1);
 183         blerg->meta_fd = blerg->index_fd = blerg->data_fd = -1;
 184         blerg->meta = NULL;
 185         blerg->index = NULL;
 186         blerg->data = NULL;
 187
 188         /* Make the directory if it doesn't exist */
 189         blerg->base_path = malloc(FILENAME_MAX);
 190         snprintf(blerg->base_path, FILENAME_MAX, "%s/%s", blergconf.data_path, name);
 191         if (access(blerg->base_path, F_OK) == -1)
 192                 mkdir(blerg->base_path, 0755);
 193
 194         /* Open and map metadata */
 195         snprintf(filename, FILENAME_MAX, "%s/meta", blerg->base_path);
 196         blerg->meta_fd = open(filename, O_RDWR | O_CREAT, 0600);
 197         if (blerg->meta_fd == -1) {
 198                 perror("Could not open metadata");
 199                 goto open_failed_meta_open;
 200         }
 201         fstat(blerg->meta_fd, &st);
 202         if (st.st_size < sizeof(struct meta)) {
 203                 /* Extend the file if sizeof(struct meta) is larger than the
 204                    file. This allows seamless upgrades as long as struct meta
 205                    only adds members. */
 206                 posix_fallocate(blerg->meta_fd, 0, sizeof(struct meta));
 207         }
 208         blerg->meta = (struct meta *) mmap(NULL, sizeof(struct meta), PROT_READ | PROT_WRITE, MAP_SHARED, blerg->meta_fd, 0);
 209         if (blerg->meta == MAP_FAILED) {
 210                 perror("Could not map metadata");
 211                 goto open_failed_meta_mmap;
 212         }
 213
 214         /* Open and map index and data for the current segment */
 215         blerg->current_segment = blerg_get_record_count(blerg) / RECORDS_PER_SEGMENT;
 216         if (!blerg_segment_switch(blerg, blerg->current_segment)) {
 217                 fprintf(stderr, "Could not switch segment\n");
 218                 goto open_failed_segment_switch;
 219         }
 220
 221         return blerg;
 222
 223 open_failed_segment_switch:
 224         munmap((void *)blerg->meta, sizeof(struct meta));
 225 open_failed_meta_mmap:
 226         close(blerg->meta_fd);
 227 open_failed_meta_open:
 228         free(blerg->name);
 229         free(blerg);
 230 open_failed_blerg_malloc:
 231         return NULL;
 232 }
 233
 234 int blerg_close(struct blerg *blerg) {
 235         CHECK_VALID_BLERG(0)
 236         blerg_segment_close(blerg);
 237         munmap((void *)blerg->meta, sizeof(struct meta));
 238         close(blerg->meta_fd);
 239         free(blerg->base_path);
 240         free(blerg->name);
 241         free(blerg);
 242         return 1;
 243 }
 244
 245 uint64_t blerg_store(struct blerg *blerg, const char *data, int length) {
 246         struct stat st;
 247         int n;
 248
 249         CHECK_VALID_BLERG(BLERG_INVALID_RECORD)
 250
 251         if (length > MAX_RECORD_SIZE || length <= 0) {
 252                 fprintf(stderr, "length out of bounds\n");
 253                 return BLERG_INVALID_RECORD;
 254         }
 255
 256         flock(blerg->index_fd, LOCK_EX);
 257         flock(blerg->data_fd, LOCK_EX);
 258
 259         uint64_t record = blerg_get_record_count(blerg);
 260         if (record == -1) {  /* Intentional signed-unsigned coercion */
 261                 fprintf(stderr, "Could not find free record\n");
 262                 return BLERG_INVALID_RECORD;
 263         }
 264         int segment = record / RECORDS_PER_SEGMENT;
 265         if (segment != blerg->current_segment)
 266                 blerg_segment_switch(blerg, segment);
 267         int seg_rec = record % RECORDS_PER_SEGMENT;
 268
 269         /* Get the position for the new data */
 270         fstat(blerg->data_fd, &st);
 271         int curpos = st.st_size;
 272
 273         /* Write data to the data log */
 274         n = write(blerg->data_fd, data, length);
 275         if (n < length) {
 276                 perror("Could not write data");
 277                 /* Truncate anything we may have written */
 278                 ftruncate(blerg->data_fd, curpos);
 279                 return BLERG_INVALID_RECORD;
 280         }
 281
 282         /* Update the index */
 283         blerg->index[seg_rec].flags = 0x0001;
 284         blerg->index[seg_rec].offset = curpos;
 285         blerg->index[seg_rec].length = length;
 286         blerg->index[seg_rec].timestamp = time(NULL);
 287
 288         /* And finally increment the record count */
 289         blerg_increment_record_count(blerg);
 290
 291         flock(blerg->data_fd, LOCK_UN);
 292         flock(blerg->index_fd, LOCK_UN);
 293
 294         if (!blerg_get_status(blerg, BLERGSTATUS_MUTED)) {
 295                 /* Now do those dirty microblogging deeds */
 296                 tag_scan(blerg->name, data, length, record);
 297                 subscription_notify(blerg->name, record);
 298         }
 299
 300         return record;
 301 }
 302
 303 int blerg_fetch(struct blerg *blerg, uint64_t record, char **data, int *length) {
 304         CHECK_VALID_BLERG(0)
 305         if (record == BLERG_INVALID_RECORD || record >= blerg_get_record_count(blerg)) {
 306                 fprintf(stderr, "Invalid record\n");
 307                 return 0;
 308         }
 309         if (data == NULL || length == NULL) {
 310                 fprintf(stderr, "data or length is null\n");
 311                 return 0;
 312         }
 313
 314         int segment = record / RECORDS_PER_SEGMENT;
 315         if (segment != blerg->current_segment)
 316                 blerg_segment_switch(blerg, segment);
 317         int seg_rec = record % RECORDS_PER_SEGMENT;
 318
 319         if ((blerg->index[seg_rec].flags & 0x1) == 0) {
 320                 fprintf(stderr, "Invalid record\n");
 321                 return 0;
 322         }
 323
 324         int rec_offset = blerg->index[seg_rec].offset;
 325         int rec_length = blerg->index[seg_rec].length;
 326         if (rec_offset >= blerg->data_size) {
 327                 /* We're accessing an out-of-bounds record in our mmap.  Remap
 328                    and recheck. */
 329                 if (!blerg_remap_data(blerg)) {
 330                         return 0;
 331                 }
 332                 if (rec_offset >= blerg->data_size) {
 333                         fprintf(stderr, "Record offset outside of data!?");
 334                         return 0;
 335                 }
 336         }
 337
 338         *data = malloc(rec_length);
 339         if (*data == NULL) {
 340                 perror("Could not allocate string in fetch");
 341                 return 0;
 342         }
 343
 344         memcpy(*data, blerg->data + rec_offset, rec_length);
 345
 346         *length = rec_length;
 347
 348         return 1;
 349 }
 350
 351 time_t blerg_get_timestamp(struct blerg *blerg, uint64_t record) {
 352         CHECK_VALID_BLERG(0)
 353         if (record == BLERG_INVALID_RECORD || record >= blerg_get_record_count(blerg)) {
 354                 fprintf(stderr, "Invalid record\n");
 355                 return 0;
 356         }
 357
 358         int segment = record / RECORDS_PER_SEGMENT;
 359         if (segment != blerg->current_segment)
 360                 blerg_segment_switch(blerg, segment);
 361         int seg_rec = record % RECORDS_PER_SEGMENT;
 362
 363         if ((blerg->index[seg_rec].flags & 0x1) == 0) {
 364                 fprintf(stderr, "Invalid record\n");
 365                 return 0;
 366         }
 367
 368         return blerg->index[seg_rec].timestamp;
 369 }
 370
 371 int blerg_set_subscription_mark(struct blerg *blerg) {
 372         CHECK_VALID_BLERG(0)
 373         blerg->meta->subscription_mark = subscription_count_items(blerg->name);
 374         return 1;
 375 }
 376
 377 uint64_t blerg_get_subscription_mark(struct blerg *blerg) {
 378         CHECK_VALID_BLERG(0)
 379         return blerg->meta->subscription_mark;
 380 }
 381
 382 int blerg_set_status(struct blerg *blerg, uint32_t status, int v) {
 383         CHECK_VALID_BLERG(0)
 384         if (v) {
 385                 blerg->meta->status |= status;
 386         } else {
 387                 blerg->meta->status &= ~status;
 388         }
 389         return 1;
 390 }
 391
 392 int blerg_get_status(struct blerg *blerg, uint32_t status) {
 393         CHECK_VALID_BLERG(0)
 394         return (blerg->meta->status & status) > 0;
 395 }