ccnr_store.c

Go to the documentation of this file.
00001 /**
00002  * @file ccnr_store.c
00003  * 
00004  * Part of ccnr -  CCNx Repository Daemon.
00005  *
00006  */
00007 
00008 /*
00009  * Copyright (C) 2011 Palo Alto Research Center, Inc.
00010  *
00011  * This work is free software; you can redistribute it and/or modify it under
00012  * the terms of the GNU General Public License version 2 as published by the
00013  * Free Software Foundation.
00014  * This work is distributed in the hope that it will be useful, but WITHOUT ANY
00015  * WARRANTY; without even the implied warranty of MERCHANTABILITY or
00016  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
00017  * for more details. You should have received a copy of the GNU General Public
00018  * License along with this program; if not, write to the
00019  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
00020  * Boston, MA 02110-1301, USA.
00021  */
00022  
00023 #include <errno.h>
00024 #include <fcntl.h>
00025 #include <limits.h>
00026 #include <netdb.h>
00027 #include <poll.h>
00028 #include <signal.h>
00029 #include <stddef.h>
00030 #include <stdint.h>
00031 #include <stdio.h>
00032 #include <stdlib.h>
00033 #include <string.h>
00034 #include <time.h>
00035 #include <unistd.h>
00036 #include <arpa/inet.h>
00037 #include <sys/time.h>
00038 #include <sys/socket.h>
00039 #include <sys/stat.h>
00040 #include <sys/types.h>
00041 #include <sys/un.h>
00042 #include <netinet/in.h>
00043 
00044 #include <ccn/bloom.h>
00045 #include <ccn/btree_content.h>
00046 #include <ccn/ccn.h>
00047 #include <ccn/ccn_private.h>
00048 #include <ccn/charbuf.h>
00049 #include <ccn/face_mgmt.h>
00050 #include <ccn/hashtb.h>
00051 #include <ccn/indexbuf.h>
00052 #include <ccn/schedule.h>
00053 #include <ccn/reg_mgmt.h>
00054 #include <ccn/uri.h>
00055 
00056 #include "ccnr_private.h"
00057 
00058 #include "ccnr_stats.h"
00059 #include "ccnr_store.h"
00060 #include "ccnr_init.h"
00061 #include "ccnr_link.h"
00062 #include "ccnr_util.h"
00063 #include "ccnr_proto.h"
00064 #include "ccnr_msg.h"
00065 #include "ccnr_sync.h"
00066 #include "ccnr_match.h"
00067 #include "ccnr_sendq.h"
00068 #include "ccnr_io.h"
00069 
00070 struct content_entry {
00071     ccnr_accession accession;   /**< permanent repository id */
00072     ccnr_cookie cookie;         /**< for in-memory references */
00073     int flags;                  /**< see below - use accessor functions */
00074     int size;                   /**< size of ContentObject */
00075     struct ccn_charbuf *flatname; /**< for skiplist, et. al. */
00076     struct ccn_charbuf *cob;    /**< may contain ContentObject, or be NULL */
00077 };
00078 
00079 static const unsigned char *bogon = NULL;
00080 const ccnr_accession r_store_mark_repoFile1 = ((ccnr_accession)1) << 48;
00081 
00082 static int
00083 r_store_set_flatname(struct ccnr_handle *h, struct content_entry *content,
00084                      struct ccn_parsed_ContentObject *pco);
00085 static int
00086 r_store_content_btree_insert(struct ccnr_handle *h,
00087                              struct content_entry *content,
00088                              struct ccn_parsed_ContentObject *pco,
00089                              ccnr_accession *accession);
00090 
00091 #define FAILIF(cond) do {} while ((cond) && r_store_fatal(h, __func__, __LINE__))
00092 #define CHKSYS(res) FAILIF((res) == -1)
00093 #define CHKRES(res) FAILIF((res) < 0)
00094 #define CHKPTR(p)   FAILIF((p) == NULL)
00095 
00096 static int
00097 r_store_fatal(struct ccnr_handle *h, const char *fn, int lineno)
00098 {
00099     if (h != NULL) {
00100         ccnr_msg(h,
00101                  "fatal error in %s, line %d, errno %d%s",
00102                  fn, lineno, errno, strerror(errno));
00103     }
00104     abort();
00105     return(0);
00106 }
00107 
00108 PUBLIC ccnr_accession
00109 r_store_content_accession(struct ccnr_handle *h, struct content_entry *content)
00110 {
00111     return(content->accession);
00112 }
00113 
00114 PUBLIC ccnr_cookie
00115 r_store_content_cookie(struct ccnr_handle *h, struct content_entry *content)
00116 {
00117     return(content->cookie);
00118 }
00119 
00120 PUBLIC size_t
00121 r_store_content_size(struct ccnr_handle *h, struct content_entry *content)
00122 {
00123     return(content->size);
00124 }
00125 
00126 static off_t
00127 r_store_offset_from_accession(struct ccnr_handle *h, ccnr_accession a)
00128 {
00129     return(a & ((((ccnr_accession)1) << 48) - 1));
00130 }
00131 
00132 static unsigned
00133 r_store_repofile_from_accession(struct ccnr_handle *h, ccnr_accession a)
00134 {
00135     /* Initially this should always be 1 */
00136     return(a >> 48);
00137 }
00138 
00139 
00140 static const unsigned char *
00141 r_store_content_mapped(struct ccnr_handle *h, struct content_entry *content)
00142 {
00143     return(NULL);
00144 }
00145 
00146 static const unsigned char *
00147 r_store_content_read(struct ccnr_handle *h, struct content_entry *content)
00148 {
00149     unsigned repofile;
00150     off_t offset;
00151     struct ccn_charbuf *cob = NULL;
00152     ssize_t rres = 0;
00153     int fd = -1;
00154     unsigned char buf[8800];
00155     struct ccn_skeleton_decoder decoder = {0};
00156     struct ccn_skeleton_decoder *d = &decoder;
00157     ssize_t dres;
00158     
00159     repofile = r_store_repofile_from_accession(h, content->accession);
00160     offset = r_store_offset_from_accession(h, content->accession);
00161     if (repofile != 1)
00162         goto Bail;
00163     if (content->cob != NULL)
00164         goto Bail;
00165     fd = r_io_repo_data_file_fd(h, repofile, 0);
00166     if (fd == -1)
00167         goto Bail;
00168     cob = ccn_charbuf_create_n(content->size);
00169     if (cob == NULL)
00170         goto Bail;
00171     if (content->size > 0) {
00172         rres = pread(fd, cob->buf, content->size, offset);
00173         if (rres == content->size) {
00174             cob->length = content->size;
00175             content->cob = cob;
00176             h->cob_count++;
00177             return(cob->buf);
00178         }
00179         if (rres == -1)
00180             ccnr_msg(h, "r_store_content_read %u :%s (errno = %d)",
00181                      fd, strerror(errno), errno);
00182         else
00183             ccnr_msg(h, "r_store_content_read %u expected %d bytes, but got %d",
00184                      fd, (int)content->size, (int)rres);
00185     } else {
00186         rres = pread(fd, buf, 8800, offset); // XXX - should be symbolic
00187         if (rres == -1) {
00188             ccnr_msg(h, "r_store_content_read %u :%s (errno = %d)",
00189                      fd, strerror(errno), errno);
00190             goto Bail;
00191         }
00192         dres = ccn_skeleton_decode(d, buf, rres);
00193         if (d->state != 0) {
00194             ccnr_msg(h, "r_store_content_read %u : error parsing cob", fd);
00195             goto Bail;
00196         }
00197         content->size = dres;
00198         if (ccn_charbuf_append(cob, buf, dres) < 0)
00199             goto Bail;
00200         content->cob = cob;
00201         h->cob_count++;
00202         return(cob->buf);        
00203     }
00204 Bail:
00205     ccn_charbuf_destroy(&cob);
00206     return(NULL);
00207 }
00208 
00209 /**
00210  *  If the content appears to be safely stored in the repository,
00211  *  removes any buffered copy.
00212  * @returns 0 if buffer was removed, -1 if not.
00213  */
00214 PUBLIC int
00215 r_store_content_trim(struct ccnr_handle *h, struct content_entry *content)
00216 {
00217     if (content->accession != CCNR_NULL_ACCESSION && content->cob != NULL) {
00218         ccn_charbuf_destroy(&content->cob);
00219         h->cob_count--;
00220         return(0);
00221     }
00222     return(-1);
00223 }
00224 
00225 /**
00226  *  Evict recoverable content from in-memory buffers
00227  */
00228 PUBLIC void
00229 r_store_trim(struct ccnr_handle *h, unsigned long limit)
00230 {
00231     struct content_entry *content = NULL;
00232     int checklimit;
00233     unsigned before;
00234     unsigned rover;
00235     unsigned mask;
00236     
00237     r_store_index_needs_cleaning(h);
00238     before = h->cob_count;
00239     if (before <= limit)
00240         return;
00241     checklimit = h->cookie_limit;
00242     mask = h->cookie_limit - 1;
00243     for (rover = (h->trim_rover & mask);
00244          checklimit > 0 && h->cob_count > limit;
00245          checklimit--, rover = (rover + 1) & mask) {
00246         content = h->content_by_cookie[rover];
00247         if (content != NULL)
00248             r_store_content_trim(h, content);
00249     }
00250     h->trim_rover = rover;
00251     if (CCNSHOULDLOG(h, sdf, CCNL_FINER))
00252         ccnr_msg(h, "trimmed %u cobs", before - h->cob_count);
00253 }
00254 
00255 /**
00256  *  Get the base address of the content object
00257  *
00258  * This may involve reading the object in.  Caller should not assume that
00259  * the address will stay valid after it relinquishes control, either by
00260  * returning or by calling routines that might invalidate objects.
00261  *
00262  */
00263 PUBLIC const unsigned char *
00264 r_store_content_base(struct ccnr_handle *h, struct content_entry *content)
00265 {
00266     const unsigned char *ans = NULL;
00267     
00268     if (content->cob != NULL && content->cob->length == content->size) {
00269         ans = content->cob->buf;
00270         goto Finish;
00271     }
00272     if (content->accession == CCNR_NULL_ACCESSION)
00273         goto Finish;
00274     ans = r_store_content_mapped(h, content);
00275     if (ans != NULL)
00276         goto Finish;
00277     ans = r_store_content_read(h, content);
00278 Finish:
00279     if (ans != NULL) {
00280         /* Sanity check - make sure first 2 and last 2 bytes are good */
00281         if (content->size < 5 || ans[0] != 0x04 || ans[1] != 0x82 ||
00282             ans[content->size - 1] != 0 || ans[content->size - 2] != 0) {
00283             bogon = ans; /* for debugger */
00284             ans = NULL;
00285         }
00286     }
00287     if (ans == NULL || CCNSHOULDLOG(h, xxxx, CCNL_FINEST))
00288         ccnr_msg(h, "r_store_content_base.%d returning %p (acc=0x%jx, cookie=%u)",
00289                  __LINE__,
00290                  ans,
00291                  ccnr_accession_encode(h, content->accession),
00292                  (unsigned)content->cookie);
00293     return(ans);
00294 }
00295 
00296 PUBLIC int
00297 r_store_name_append_components(struct ccn_charbuf *dst,
00298                                struct ccnr_handle *h,
00299                                struct content_entry *content,
00300                                int skip,
00301                                int count)
00302 {
00303     int res;
00304     
00305     res = ccn_name_append_flatname(dst,
00306                                    content->flatname->buf,
00307                                    content->flatname->length, skip, count);
00308     return(res);
00309 }
00310 
00311 PUBLIC int
00312 r_store_content_flags(struct content_entry *content)
00313 {
00314     return(content->flags);
00315 }
00316 
00317 PUBLIC int
00318 r_store_content_change_flags(struct content_entry *content, int set, int clear)
00319 {
00320     int old = content->flags;
00321     content->flags |= set;
00322     content->flags &= ~clear;
00323     return(old);
00324 }
00325 
00326 /**
00327  * Write a file named index/stable that contains the size of
00328  * repoFile1 when the repository is shut down.
00329  */
00330 static int
00331 r_store_write_stable_point(struct ccnr_handle *h)
00332 {
00333     struct ccn_charbuf *path = NULL;
00334     struct ccn_charbuf *cb = NULL;
00335     int fd, res;
00336     
00337     path = ccn_charbuf_create();
00338     cb = ccn_charbuf_create();
00339     if (path == NULL || cb == NULL) {
00340         ccnr_msg(h, "memory allocation failure writing stable mark");
00341         goto Bail;
00342     }
00343     ccn_charbuf_putf(path, "%s/index/stable", h->directory);
00344     unlink(ccn_charbuf_as_string(path)); /* Should not exist, but just in case. */
00345     fd = open(ccn_charbuf_as_string(path),
00346               O_CREAT | O_EXCL | O_WRONLY | O_TRUNC, 0666);
00347     if (fd == -1) {
00348         ccnr_msg(h, "cannot write stable mark %s: %s",
00349                  ccn_charbuf_as_string(path), strerror(errno));
00350         unlink(ccn_charbuf_as_string(path));
00351     }
00352     else {
00353         ccn_charbuf_putf(cb, "%ju", (uintmax_t)(h->stable));
00354         res = write(fd, cb->buf, cb->length);
00355         close(fd);
00356         if (res != cb->length) {
00357             unlink(ccn_charbuf_as_string(path));
00358             ccnr_msg(h, "cannot write stable mark %s: unexpected write result %d",
00359                      ccn_charbuf_as_string(path), res);
00360         }
00361         if (CCNSHOULDLOG(h, dfsdf, CCNL_INFO))
00362             ccnr_msg(h, "Index marked stable - %s", ccn_charbuf_as_string(cb));
00363     }
00364 Bail:
00365     ccn_charbuf_destroy(&path);
00366     ccn_charbuf_destroy(&cb);
00367     return(0);
00368 }
00369 
00370 /**
00371  * Read the former size of repoFile1 from index/stable, and remove
00372  * the latter.
00373  */
00374 static void
00375 r_store_read_stable_point(struct ccnr_handle *h)
00376 {
00377     struct ccn_charbuf *path = NULL;
00378     struct ccn_charbuf *cb = NULL;
00379     int fd;
00380     int i;
00381     ssize_t rres;
00382     uintmax_t val;
00383     unsigned char c;
00384     
00385     path = ccn_charbuf_create();
00386     cb = ccn_charbuf_create();
00387     ccn_charbuf_putf(path, "%s/index/stable", h->directory);
00388     fd = open(ccn_charbuf_as_string(path), O_RDONLY, 0666);
00389     if (fd != -1) {
00390         rres = read(fd, ccn_charbuf_reserve(cb, 80), 80);
00391         if (rres > 0)
00392             cb->length = rres;
00393         close(fd);
00394         if (CCNSHOULDLOG(h, dfsdf, CCNL_INFO))
00395             ccnr_msg(h, "Last stable at %s", ccn_charbuf_as_string(cb));
00396     }
00397     for (val = 0, i = 0; i < cb->length; i++) {
00398         c = cb->buf[i];
00399         if ('0' <= c && c <= '9')
00400             val = val * 10 + (c - '0');
00401         else
00402             break;
00403     }
00404     if (i == 0 || i < cb->length) {
00405         ccnr_msg(h, "Bad stable mark - %s", ccn_charbuf_as_string(cb));
00406         h->stable = 0;
00407     }
00408     else {
00409         h->stable = val;
00410         unlink(ccn_charbuf_as_string(path));
00411     }
00412     ccn_charbuf_destroy(&path);
00413     ccn_charbuf_destroy(&cb);
00414 }
00415 
00416 /**
00417  * Log a bit if we are taking a while to re-index.
00418  */
00419 static int
00420 r_store_reindexing(struct ccn_schedule *sched,
00421                    void *clienth,
00422                    struct ccn_scheduled_event *ev,
00423                    int flags)
00424 {
00425     struct ccnr_handle *h = clienth;
00426     struct fdholder *in = NULL;
00427     unsigned pct;
00428     
00429     if ((flags & CCN_SCHEDULE_CANCEL) != 0)
00430         return(0);
00431     in = r_io_fdholder_from_fd(h, h->active_in_fd);
00432     if (in == NULL)
00433         return(0);
00434     pct = ccnr_meter_total(in->meter[FM_BYTI]) / ((h->startupbytes / 100) + 1);
00435     if (pct >= 100)
00436         return(0);
00437     ccnr_msg(h, "indexing %u%% complete", pct);
00438     return(2000000);
00439 }
00440 
00441 /**
00442  * Select power of 2 between l and m + 1 (if possible).
00443  */
00444 static unsigned
00445 choose_limit(unsigned l, unsigned m)
00446 {
00447     unsigned k;
00448     
00449     for (k = 0; k < l; k = 2 * k + 1)
00450         continue;
00451     while (k > (m | 1) || k + 1 < k)
00452         k >>= 1;
00453     return(k + 1);
00454 }
00455 
00456 PUBLIC void
00457 r_store_init(struct ccnr_handle *h)
00458 {
00459     struct ccn_btree *btree = NULL;
00460     struct ccn_btree_node *node = NULL;
00461     struct hashtb_param param = {0};
00462     int i;
00463     int j;
00464     int res;
00465     struct ccn_charbuf *path = NULL;
00466     struct ccn_charbuf *msgs = NULL;
00467     off_t offset;
00468     
00469     path = ccn_charbuf_create();
00470     param.finalize_data = h;
00471     param.finalize = 0;
00472     
00473     h->cob_limit = r_init_confval(h, "CCNR_CONTENT_CACHE", 16, 2000000, 4201);
00474     h->cookie_limit = choose_limit(h->cob_limit, (ccnr_cookie)(~0U));
00475     h->content_by_cookie = calloc(h->cookie_limit, sizeof(h->content_by_cookie[0]));
00476     CHKPTR(h->content_by_cookie);
00477     h->content_by_accession_tab = hashtb_create(sizeof(struct content_by_accession_entry), NULL);
00478     CHKPTR(h->content_by_accession_tab);
00479     h->btree = btree = ccn_btree_create();
00480     CHKPTR(btree);
00481     FAILIF(btree->nextnodeid != 1);
00482     ccn_charbuf_putf(path, "%s/index", h->directory);
00483     res = mkdir(ccn_charbuf_as_string(path), 0700);
00484     if (res != 0 && errno != EEXIST)
00485         r_init_fail(h, __LINE__, ccn_charbuf_as_string(path), errno);
00486     else {
00487         msgs = ccn_charbuf_create();
00488         btree->io = ccn_btree_io_from_directory(ccn_charbuf_as_string(path), msgs);
00489         if (btree->io == NULL)
00490             res = errno;
00491         if (msgs->length != 0 && CCNSHOULDLOG(h, sffdsdf, CCNL_WARNING)) {
00492             ccnr_msg(h, "while initializing %s - %s",
00493                      ccn_charbuf_as_string(path),
00494                      ccn_charbuf_as_string(msgs));
00495         }
00496         ccn_charbuf_destroy(&msgs);
00497         if (btree->io == NULL)
00498             r_init_fail(h, __LINE__, ccn_charbuf_as_string(path), res);
00499     }
00500     node = ccn_btree_getnode(btree, 1, 0);
00501     if (btree->io != NULL)
00502         btree->nextnodeid = btree->io->maxnodeid + 1;
00503     CHKPTR(node);
00504     if (node->buf->length == 0) {
00505         res = ccn_btree_init_node(node, 0, 'R', 0);
00506         CHKSYS(res);
00507     }
00508     ccn_charbuf_destroy(&path);
00509     if (h->running == -1)
00510         return;
00511     r_store_read_stable_point(h);
00512     h->active_in_fd = -1;
00513     h->active_out_fd = r_io_open_repo_data_file(h, "repoFile1", 1); /* output */
00514     offset = lseek(h->active_out_fd, 0, SEEK_END);
00515     h->startupbytes = offset;
00516     if (offset != h->stable || node->corrupt != 0) {
00517         ccnr_msg(h, "Index not current - resetting");
00518         ccn_btree_init_node(node, 0, 'R', 0);
00519         node = NULL;
00520         ccn_btree_destroy(&h->btree);
00521         path = ccn_charbuf_create();
00522         /* Remove old index files to avoid confusion */
00523         for (i = 1, j = 0; i > 0 && j < 3; i++) {
00524             path->length = 0;
00525             res = ccn_charbuf_putf(path, "%s/index/%d", h->directory, i);
00526             if (res >= 0)
00527                 res = unlink(ccn_charbuf_as_string(path));
00528             if (res < 0)
00529                 j++;
00530         }
00531         h->btree = btree = ccn_btree_create();
00532         path->length = 0;
00533         ccn_charbuf_putf(path, "%s/index", h->directory);
00534         btree->io = ccn_btree_io_from_directory(ccn_charbuf_as_string(path), msgs);
00535         CHKPTR(btree->io);
00536         btree->io->maxnodeid = 0;
00537         btree->nextnodeid = 1;
00538         node = ccn_btree_getnode(btree, 1, 0);
00539         btree->nextnodeid = btree->io->maxnodeid + 1;
00540         ccn_btree_init_node(node, 0, 'R', 0);
00541         h->stable = 0;
00542         h->active_in_fd = r_io_open_repo_data_file(h, "repoFile1", 0); /* input */
00543         ccn_charbuf_destroy(&path);
00544         if (CCNSHOULDLOG(h, dfds, CCNL_INFO))
00545             ccn_schedule_event(h->sched, 50000, r_store_reindexing, NULL, 0);
00546     }
00547     if (CCNSHOULDLOG(h, weuyg, CCNL_FINEST)) {
00548         FILE *dumpfile = NULL;
00549         
00550         path = ccn_charbuf_create();
00551         ccn_charbuf_putf(path, "%s/index/btree_check.out", h->directory);
00552         dumpfile = fopen(ccn_charbuf_as_string(path), "w");
00553         res = ccn_btree_check(btree, dumpfile);
00554         if (dumpfile != NULL) {
00555             fclose(dumpfile);
00556             dumpfile = NULL;
00557         }
00558         else
00559             path->length = 0;
00560         ccnr_msg(h, "ccn_btree_check returned %d (%s)",
00561                     res, ccn_charbuf_as_string(path));
00562         ccn_charbuf_destroy(&path);
00563         if (res < 0)
00564             r_init_fail(h, __LINE__, "index is corrupt", res);
00565     }
00566     btree->full = r_init_confval(h, "CCNR_BTREE_MAX_FANOUT", 4, 9999, 1999);
00567     btree->full0 = r_init_confval(h, "CCNR_BTREE_MAX_LEAF_ENTRIES", 4, 9999, 1999);
00568     btree->nodebytes = r_init_confval(h, "CCNR_BTREE_MAX_NODE_BYTES", 1024, 8388608, 2097152);
00569     btree->nodepool = r_init_confval(h, "CCNR_BTREE_NODE_POOL", 16, 2000000, 512);
00570     if (h->running != -1)
00571         r_store_index_needs_cleaning(h);
00572 }
00573 
00574 PUBLIC int
00575 r_store_final(struct ccnr_handle *h, int stable) {
00576     int res;
00577     
00578     res = ccn_btree_destroy(&h->btree);
00579     if (res < 0)
00580         ccnr_msg(h, "r_store_final.%d-%d Errors while closing index", __LINE__, res);
00581     if (res >= 0 && stable)
00582         res = r_store_write_stable_point(h);
00583     return(res);
00584 }
00585     
00586 PUBLIC struct content_entry *
00587 r_store_content_from_accession(struct ccnr_handle *h, ccnr_accession accession)
00588 {
00589     struct ccn_parsed_ContentObject obj = {0};
00590     struct content_entry *content = NULL;
00591     struct content_by_accession_entry *entry;
00592     const unsigned char *content_base = NULL;
00593     int res;
00594     ccnr_accession acc;
00595     
00596     if (accession == CCNR_NULL_ACCESSION)
00597         return(NULL);
00598     entry = hashtb_lookup(h->content_by_accession_tab,
00599                           &accession, sizeof(accession));
00600     if (entry != NULL) {
00601         h->content_from_accession_hits++;
00602         return(entry->content);
00603     }
00604     h->content_from_accession_misses++;
00605     content = calloc(1, sizeof(*content));
00606     CHKPTR(content);
00607     content->cookie = 0;
00608     content->accession = accession;
00609     content->cob = NULL;
00610     content->size = 0;
00611     content_base = r_store_content_base(h, content);
00612     if (content_base == NULL || content->size == 0)
00613         goto Bail;
00614     res = r_store_set_flatname(h, content, &obj);
00615     if (res < 0) goto Bail;
00616     r_store_enroll_content(h, content);
00617     res = r_store_content_btree_insert(h, content, &obj, &acc);
00618     if (res < 0) goto Bail;
00619     if (res == 1 || CCNSHOULDLOG(h, sdf, CCNL_FINEST))
00620         ccnr_debug_content(h, __LINE__, "content/accession", NULL, content);
00621     return(content);
00622 Bail:
00623     ccnr_msg(h, "r_store_content_from_accession.%d failed 0x%jx",
00624              __LINE__, ccnr_accession_encode(h, accession));
00625     r_store_forget_content(h, &content);
00626     return(content);
00627 }
00628 
00629 PUBLIC struct content_entry *
00630 r_store_content_from_cookie(struct ccnr_handle *h, ccnr_cookie cookie)
00631 {
00632     struct content_entry *ans = NULL;
00633     
00634     ans = h->content_by_cookie[cookie & (h->cookie_limit - 1)];
00635     if (ans != NULL && ans->cookie != cookie)
00636         ans = NULL;
00637     return(ans);
00638 }
00639 
00640 /**
00641  * This makes a cookie for content, and, if it has an accession number already,
00642  * enters it into the content_by_accession_tab.  Does not index by name.
00643  */
00644 PUBLIC ccnr_cookie
00645 r_store_enroll_content(struct ccnr_handle *h, struct content_entry *content)
00646 {
00647     ccnr_cookie cookie;
00648     unsigned mask;
00649     
00650     mask = h->cookie_limit - 1;
00651     cookie = ++(h->cookie);
00652     if (cookie == 0)
00653         cookie = ++(h->cookie); /* Cookie numbers may wrap */
00654     // XXX - check for persistence here, if we add that
00655     r_store_forget_content(h, &(h->content_by_cookie[cookie & mask]));
00656     content->cookie = cookie;
00657     h->content_by_cookie[cookie & mask] = content;
00658     if (content->accession != CCNR_NULL_ACCESSION) {
00659         struct hashtb_enumerator ee;
00660         struct hashtb_enumerator *e = &ee;
00661         ccnr_accession accession = content->accession;
00662         struct content_by_accession_entry *entry = NULL;
00663         hashtb_start(h->content_by_accession_tab, e);
00664         hashtb_seek(e, &accession, sizeof(accession), 0);
00665         entry = e->data;
00666         if (entry != NULL)
00667             entry->content = content;
00668         hashtb_end(e);
00669         content->flags |= CCN_CONTENT_ENTRY_STABLE;
00670     }
00671     return(cookie);
00672 }
00673 
00674 /** @returns 2 if content was added to index, 1 if it was there but had no accession, 0 if it was already there, -1 for error */
00675 static int
00676 r_store_content_btree_insert(struct ccnr_handle *h,
00677                              struct content_entry *content,
00678                              struct ccn_parsed_ContentObject *pco,
00679                              ccnr_accession *accp)
00680 {
00681     const unsigned char *content_base = NULL;
00682     struct ccn_btree *btree = NULL;
00683     struct ccn_btree_node *leaf = NULL;
00684     struct ccn_btree_node *node = NULL;
00685     struct ccn_charbuf *flat = NULL;
00686     int i;
00687     int limit;
00688     int res;
00689 
00690     btree = h->btree;
00691     if (btree == NULL)
00692         return(-1);
00693     flat = content->flatname;
00694     if (flat == NULL)
00695         return(-1);
00696     res = ccn_btree_lookup(h->btree, flat->buf, flat->length, &leaf);
00697     if (res < 0)
00698         return(-1);
00699     i = CCN_BT_SRCH_INDEX(res);
00700     if (CCN_BT_SRCH_FOUND(res)) {
00701         *accp = ccnr_accession_decode(h, ccn_btree_content_cobid(leaf, i));
00702         return(*accp == CCNR_NULL_ACCESSION);
00703     }
00704     else {
00705         content_base = r_store_content_base(h, content);
00706         if (content_base == NULL)
00707             return(-1);
00708         res = ccn_btree_prepare_for_update(h->btree, leaf);
00709         if (res < 0)
00710             return(-1);
00711         res = ccn_btree_insert_content(leaf, i,
00712                                        ccnr_accession_encode(h, content->accession),
00713                                        content_base,
00714                                        pco,
00715                                        content->flatname);
00716         if (res < 0)
00717             return(-1);
00718         if (ccn_btree_oversize(btree, leaf)) {
00719             res = ccn_btree_split(btree, leaf);
00720             for (limit = 100; res >= 0 && btree->nextsplit != 0; limit--) {
00721                 if (limit == 0) abort();
00722                 node = ccn_btree_getnode(btree, btree->nextsplit, 0);
00723                 if (node == NULL)
00724                     return(-1);
00725                 res = ccn_btree_split(btree, node);
00726             }
00727         }
00728         r_store_index_needs_cleaning(h);
00729         
00730         *accp = content->accession;
00731         return(2);
00732     }
00733 }
00734 
00735 /**
00736  *  Remove internal representation of a content object
00737  */
00738 PUBLIC void
00739 r_store_forget_content(struct ccnr_handle *h, struct content_entry **pentry)
00740 {
00741     unsigned i;
00742     struct content_entry *entry = *pentry;
00743     
00744     if (entry == NULL)
00745         return;
00746     *pentry = NULL;
00747     if ((entry->flags & CCN_CONTENT_ENTRY_STALE) != 0)
00748         h->n_stale--;
00749     if (CCNSHOULDLOG(h, LM_4, CCNL_FINER))
00750         ccnr_debug_content(h, __LINE__, "remove", NULL, entry);
00751     /* Remove the cookie reference */
00752     i = entry->cookie & (h->cookie_limit - 1);
00753     if (h->content_by_cookie[i] == entry)
00754         h->content_by_cookie[i] = NULL;
00755     entry->cookie = 0;
00756     /* Remove the accession reference */
00757     if (entry->accession != CCNR_NULL_ACCESSION) {
00758         struct hashtb_enumerator ee;
00759         struct hashtb_enumerator *e = &ee;
00760         hashtb_start(h->content_by_accession_tab, e);
00761         if (hashtb_seek(e, &entry->accession, sizeof(entry->accession), 0) ==
00762             HT_NEW_ENTRY) {
00763             ccnr_msg(h, "orphaned content %llu",
00764                      (unsigned long long)(entry->accession));
00765             hashtb_delete(e);
00766             hashtb_end(e);
00767             return;
00768         }
00769         hashtb_delete(e);
00770         hashtb_end(e);
00771         entry->accession = CCNR_NULL_ACCESSION;
00772     }
00773     /* Clean up allocated subfields */
00774     ccn_charbuf_destroy(&entry->flatname);
00775     if (entry->cob != NULL) {
00776         h->cob_count--;
00777         ccn_charbuf_destroy(&entry->cob);
00778     }
00779     free(entry);
00780 }
00781 
00782 /**
00783  *  Get a handle on the content object that matches key, or if there is
00784  * no match, the one that would come just after it.
00785  *
00786  * The key is in flatname format.
00787  */
00788 static struct content_entry *    
00789 r_store_look(struct ccnr_handle *h, const unsigned char *key, size_t size)
00790 {
00791     struct content_entry *content = NULL;
00792     struct ccn_btree_node *leaf = NULL;
00793     ccnr_accession accession;
00794     int ndx;
00795     int res;
00796 
00797     res = ccn_btree_lookup(h->btree, key, size, &leaf);
00798     if (res >= 0) {
00799         ndx = CCN_BT_SRCH_INDEX(res);
00800         if (ndx == ccn_btree_node_nent(leaf)) {
00801             res = ccn_btree_next_leaf(h->btree, leaf, &leaf);
00802             if (res <= 0)
00803                 return(NULL);
00804             ndx = 0;
00805         }
00806         accession = ccnr_accession_decode(h, ccn_btree_content_cobid(leaf, ndx));
00807         if (accession != CCNR_NULL_ACCESSION) {
00808             struct content_by_accession_entry *entry;
00809             entry = hashtb_lookup(h->content_by_accession_tab,
00810                                     &accession, sizeof(accession));
00811             if (entry != NULL)
00812                 content = entry->content;
00813             if (content == NULL) {
00814                 /* Construct handle without actually reading the cob */
00815                 res = ccn_btree_content_cobsz(leaf, ndx);
00816                 content = calloc(1, sizeof(*content));
00817                 if (res > 0 && content != NULL) {
00818                     content->accession = accession;
00819                     content->cob = NULL;
00820                     content->size = res;
00821                     content->flatname = ccn_charbuf_create();
00822                     CHKPTR(content->flatname);
00823                     res = ccn_btree_key_fetch(content->flatname, leaf, ndx);
00824                     CHKRES(res);
00825                     r_store_enroll_content(h, content);
00826                 }
00827             }
00828         }
00829     }
00830     return(content);
00831 }
00832 
00833 /**
00834  * Extract the flatname representations of the bounds for the
00835  * next component after the name prefix of the interest.
00836  * These are exclusive bounds.  The results are appended to
00837  * lower and upper (when not NULL).  If there is
00838  * no lower bound, lower will be unchanged.
00839  * If there is no upper bound, a sentinel value is appended to upper.
00840  *
00841  * @returns on success the number of Components in Exclude.
00842  *          A negative value indicates an error.
00843  */
00844 static int
00845 ccn_append_interest_bounds(const unsigned char *interest_msg,
00846                            const struct ccn_parsed_interest *pi,
00847                            struct ccn_charbuf *lower,
00848                            struct ccn_charbuf *upper)
00849 {
00850     struct ccn_buf_decoder decoder;
00851     struct ccn_buf_decoder *d = NULL;
00852     size_t xstart = 0;
00853     size_t xend = 0;
00854     int atlower = 0;
00855     int atupper = 0;
00856     int res = 0;
00857     int nexcl = 0;
00858     
00859     if (pi->offset[CCN_PI_B_Exclude] < pi->offset[CCN_PI_E_Exclude]) {
00860         d = ccn_buf_decoder_start(&decoder,
00861                                   interest_msg + pi->offset[CCN_PI_B_Exclude],
00862                                   pi->offset[CCN_PI_E_Exclude] -
00863                                   pi->offset[CCN_PI_B_Exclude]);
00864         ccn_buf_advance(d);
00865         if (ccn_buf_match_dtag(d, CCN_DTAG_Any)) {
00866             ccn_buf_advance(d);
00867             ccn_buf_check_close(d);
00868             atlower = 1; /* look for <Exclude><Any/><Component>... case */
00869         }
00870         else if (ccn_buf_match_dtag(d, CCN_DTAG_Bloom))
00871             ccn_buf_advance_past_element(d);
00872         while (ccn_buf_match_dtag(d, CCN_DTAG_Component)) {
00873             nexcl++;
00874             xstart = pi->offset[CCN_PI_B_Exclude] + d->decoder.token_index;
00875             ccn_buf_advance_past_element(d);
00876             xend = pi->offset[CCN_PI_B_Exclude] + d->decoder.token_index;
00877             if (atlower && lower != NULL && d->decoder.state >= 0) {
00878                 res = ccn_flatname_append_from_ccnb(lower,
00879                         interest_msg + xstart, xend - xstart, 0, 1);
00880                 if (res < 0)
00881                     d->decoder.state = - __LINE__;
00882             }
00883             atlower = 0;
00884             atupper = 0;
00885             if (ccn_buf_match_dtag(d, CCN_DTAG_Any)) {
00886                 atupper = 1; /* look for ...</Component><Any/></Exclude> case */
00887                 ccn_buf_advance(d);
00888                 ccn_buf_check_close(d);
00889             }
00890             else if (ccn_buf_match_dtag(d, CCN_DTAG_Bloom))
00891                 ccn_buf_advance_past_element(d);
00892         }
00893         ccn_buf_check_close(d);
00894         res = d->decoder.state;
00895     }
00896     if (upper != NULL) {
00897         if (atupper && res >= 0)
00898             res = ccn_flatname_append_from_ccnb(upper,
00899                      interest_msg + xstart, xend - xstart, 0, 1);
00900         else
00901             ccn_charbuf_append(upper, "\377\377\377", 3);
00902     }
00903     return (res < 0 ? res : 0);
00904 }
00905 
00906 static struct content_entry *
00907 r_store_lookup_backwards(struct ccnr_handle *h,
00908                          const unsigned char *interest_msg,
00909                          const struct ccn_parsed_interest *pi,
00910                          struct ccn_indexbuf *comps)
00911 {
00912     struct content_entry *content = NULL;
00913     struct ccn_btree_node *leaf = NULL;
00914     struct ccn_charbuf *lower = NULL;
00915     struct ccn_charbuf *f = NULL;
00916     size_t size;
00917     size_t fsz;
00918     int errline = 0;
00919     int try = 0;
00920     int ndx;
00921     int res;
00922     int rnc;
00923     
00924     size = pi->offset[CCN_PI_E];
00925     f = ccn_charbuf_create_n(pi->offset[CCN_PI_E_Name]);
00926     lower = ccn_charbuf_create();
00927     if (f == NULL || lower == NULL) { errline = __LINE__; goto Done; };
00928     rnc = ccn_flatname_from_ccnb(f, interest_msg, size);
00929     fsz = f->length;
00930     res = ccn_charbuf_append_charbuf(lower, f);
00931     if (rnc < 0 || res < 0) { errline = __LINE__; goto Done; };
00932     res = ccn_append_interest_bounds(interest_msg, pi, lower, f);
00933     if (res < 0) { errline = __LINE__; goto Done; };
00934     /* Now f is beyond any we care about */
00935     res = ccn_btree_lookup(h->btree, f->buf, f->length, &leaf);
00936     if (res < 0) { errline = __LINE__; goto Done; };
00937     ndx = CCN_BT_SRCH_INDEX(res);
00938     for (try = 1;; try++) {
00939         if (ndx == 0) {
00940             res = ccn_btree_prev_leaf(h->btree, leaf, &leaf);
00941             if (res != 1) goto Done;
00942             ndx = ccn_btree_node_nent(leaf);
00943             if (ndx <= 0) goto Done;
00944         }
00945         ndx -= 1;
00946         res = ccn_btree_compare(lower->buf, lower->length, leaf, ndx);
00947         if (res > 0 || (res == 0 && lower->length > fsz))
00948             goto Done;
00949         f->length = 0;
00950         res = ccn_btree_key_fetch(f, leaf, ndx);
00951         if (res < 0) { errline = __LINE__; goto Done; }
00952         if (f->length > fsz) {
00953             rnc = ccn_flatname_next_comp(f->buf + fsz, f->length - fsz);
00954             if (rnc < 0) { errline = __LINE__; goto Done; };
00955             f->length = fsz + CCNFLATDELIMSZ(rnc) + CCNFLATDATASZ(rnc);
00956             res = ccn_btree_lookup(h->btree, f->buf, f->length, &leaf);
00957             if (res < 0) { errline = __LINE__; goto Done; };
00958             ndx = CCN_BT_SRCH_INDEX(res);
00959         }
00960         else if (f->length < fsz) { errline = __LINE__; goto Done; }
00961         res = ccn_btree_match_interest(leaf, ndx, interest_msg, pi, f);
00962         if (res == 1) {
00963             res = ccn_btree_key_fetch(f, leaf, ndx);
00964             if (res < 0) { errline = __LINE__; goto Done; }
00965             content = r_store_look(h, f->buf, f->length);
00966             goto Done;
00967         }
00968         else if (res != 0) { errline = __LINE__; goto Done; }
00969     }
00970 Done:
00971     if (errline != 0)
00972         ccnr_debug_ccnb(h, errline, "match_error", NULL, interest_msg, size);
00973     else {
00974         if (content != NULL) {
00975             h->count_rmc_found += 1;
00976             h->count_rmc_found_iters += try;
00977         }
00978         else {
00979             h->count_rmc_notfound += 1;
00980             h->count_rmc_notfound_iters += try;
00981         }
00982     }
00983     ccn_charbuf_destroy(&lower);
00984     ccn_charbuf_destroy(&f);
00985     return(content);
00986 }
00987 
00988 PUBLIC struct content_entry *
00989 r_store_find_first_match_candidate(struct ccnr_handle *h,
00990                                    const unsigned char *interest_msg,
00991                                    const struct ccn_parsed_interest *pi)
00992 {
00993     struct ccn_charbuf *flatname = NULL;
00994     struct content_entry *content = NULL;
00995     
00996     flatname = ccn_charbuf_create_n(pi->offset[CCN_PI_E]);
00997     ccn_flatname_from_ccnb(flatname, interest_msg, pi->offset[CCN_PI_E]);
00998     ccn_append_interest_bounds(interest_msg, pi, flatname, NULL);
00999     content = r_store_look(h, flatname->buf, flatname->length);
01000     ccn_charbuf_destroy(&flatname);
01001     return(content);
01002 }
01003 
01004 PUBLIC int
01005 r_store_content_matches_interest_prefix(struct ccnr_handle *h,
01006                                 struct content_entry *content,
01007                                 const unsigned char *interest_msg,
01008                                 size_t interest_size)
01009 {
01010     struct ccn_charbuf *flatname = ccn_charbuf_create_n(interest_size);
01011     int ans;
01012     int cmp;
01013 
01014     ccn_flatname_from_ccnb(flatname, interest_msg, interest_size);
01015     cmp = ccn_flatname_charbuf_compare(flatname, content->flatname);
01016     ans = (cmp == 0 || cmp == -9999);
01017     ccn_charbuf_destroy(&flatname);
01018     return(ans);
01019 }
01020 
01021 PUBLIC struct content_entry *
01022 r_store_content_next(struct ccnr_handle *h, struct content_entry *content)
01023 {
01024     if (content == NULL)
01025         return(0);
01026     /* We need to go past the current name, so make sure there is a 0 byte */
01027     ccn_charbuf_as_string(content->flatname);
01028     content = r_store_look(h, content->flatname->buf, content->flatname->length + 1);
01029     return(content);
01030 }
01031 
01032 PUBLIC struct content_entry *
01033 r_store_next_child_at_level(struct ccnr_handle *h,
01034                     struct content_entry *content, int level)
01035 {
01036     struct content_entry *next = NULL;
01037     struct ccn_charbuf *name;
01038     struct ccn_charbuf *flatname = NULL;
01039     int res;
01040     
01041     if (content == NULL)
01042         return(NULL);
01043     name = ccn_charbuf_create();
01044     ccn_name_init(name);
01045     res = ccn_name_append_flatname(name,
01046                                    content->flatname->buf,
01047                                    content->flatname->length, 0, level + 1);
01048     if (res < level)
01049         goto Bail;
01050     if (res == level)
01051         res = ccn_name_append(name, NULL, 0);
01052     else if (res == level + 1)
01053         res = ccn_name_next_sibling(name); // XXX - would be nice to have a flatname version of this
01054     if (res < 0)
01055         goto Bail;
01056     if (CCNSHOULDLOG(h, LM_8, CCNL_FINER))
01057         ccnr_debug_ccnb(h, __LINE__, "child_successor", NULL,
01058                         name->buf, name->length);
01059     flatname = ccn_charbuf_create();
01060     ccn_flatname_from_ccnb(flatname, name->buf, name->length);
01061     next = r_store_look(h, flatname->buf, flatname->length);
01062     if (next == content) {
01063         // XXX - I think this case should not occur, but just in case, avoid a loop.
01064         ccnr_debug_content(h, __LINE__, "urp", NULL, next);
01065         next = NULL;
01066     }
01067 Bail:
01068     ccn_charbuf_destroy(&name);
01069     ccn_charbuf_destroy(&flatname);
01070     return(next);
01071 }
01072 
01073 PUBLIC struct content_entry *
01074 r_store_lookup(struct ccnr_handle *h,
01075                const unsigned char *msg,
01076                const struct ccn_parsed_interest *pi,
01077                struct ccn_indexbuf *comps)
01078 {
01079     struct content_entry *content = NULL;
01080     struct ccn_btree_node *leaf = NULL;
01081     ccnr_cookie last_match = 0;
01082     ccnr_accession last_match_acc = CCNR_NULL_ACCESSION;
01083     struct ccn_charbuf *scratch = NULL;
01084     size_t size = pi->offset[CCN_PI_E];
01085     int ndx;
01086     int res;
01087     int try;
01088     
01089     if ((pi->orderpref & 1) == 1) {
01090         content = r_store_lookup_backwards(h, msg, pi, comps);
01091         return(content);
01092     }
01093     
01094     content = r_store_find_first_match_candidate(h, msg, pi);
01095     if (content != NULL && CCNSHOULDLOG(h, LM_8, CCNL_FINER))
01096         ccnr_debug_content(h, __LINE__, "first_candidate", NULL,
01097                            content);
01098     if (content != NULL &&
01099         !r_store_content_matches_interest_prefix(h, content, msg, size)) {
01100             if (CCNSHOULDLOG(h, LM_8, CCNL_FINER))
01101                 ccnr_debug_ccnb(h, __LINE__, "prefix_mismatch", NULL,
01102                                 msg, size);
01103             content = NULL;
01104         }
01105     scratch = ccn_charbuf_create();
01106     for (try = 1; content != NULL; try++) {
01107         res = ccn_btree_lookup(h->btree,
01108                                content->flatname->buf,
01109                                content->flatname->length,
01110                                &leaf);
01111         if (CCN_BT_SRCH_FOUND(res) == 0) {
01112             ccnr_debug_content(h, __LINE__, "impossible", NULL, content);
01113             content = NULL;
01114             break;
01115         }
01116         ndx = CCN_BT_SRCH_INDEX(res);
01117         res = ccn_btree_match_interest(leaf, ndx, msg, pi, scratch);
01118         if (res == -1) {
01119             ccnr_debug_ccnb(h, __LINE__, "match_error", NULL, msg, size);
01120             content = NULL;
01121             break;
01122         }
01123         if (res == 1) {
01124             if ((pi->orderpref & 1) == 0) // XXX - should be symbolic
01125                 break;
01126             last_match = content->cookie;
01127             last_match_acc = content->accession;
01128             content = r_store_next_child_at_level(h, content, comps->n - 1);
01129         }
01130         else
01131             content = r_store_content_next(h, content);
01132         if (content != NULL &&
01133             !r_store_content_matches_interest_prefix(h, content, msg, size))
01134                 content = NULL;
01135     }
01136     if (last_match != 0) {
01137         content = r_store_content_from_cookie(h, last_match);
01138         if (content == NULL)
01139             content = r_store_content_from_accession(h, last_match_acc);
01140     }
01141     ccn_charbuf_destroy(&scratch);
01142     if (content != NULL) {
01143         h->count_lmc_found += 1;
01144         h->count_lmc_found_iters += try;
01145     }
01146     else {
01147         h->count_lmc_notfound += 1;
01148         h->count_lmc_notfound_iters += try;
01149     }
01150     return(content);
01151 }
01152 
01153 /**
01154  * Find the first content handle that matches the prefix given by the namish,
01155  * which may be a Name, Interest, ContentObject, ...
01156  *
01157  * Does not check the other parts of namish, in particular, does not generate
01158  * the digest component of a ContentObject.
01159  */
01160 PUBLIC struct content_entry *
01161 r_store_lookup_ccnb(struct ccnr_handle *h,
01162                     const unsigned char *namish, size_t size)
01163 {
01164     struct content_entry *content = NULL;
01165     struct ccn_charbuf *flatname = NULL;
01166     int res;
01167     
01168     flatname = ccn_charbuf_create();
01169     if (flatname == NULL)
01170         goto Bail;
01171     res = ccn_flatname_from_ccnb(flatname, namish, size);
01172     if (res < 0)
01173         goto Bail;
01174     content = r_store_look(h, flatname->buf, flatname->length);
01175     if (content != NULL) {
01176         res = ccn_flatname_charbuf_compare(flatname, content->flatname);
01177         if (res == 0 || res == -9999) {
01178             /* prefix matches */
01179         }
01180         else
01181             content = NULL;
01182     }
01183 Bail:
01184     ccn_charbuf_destroy(&flatname);
01185     return(content);
01186 }
01187 
01188 /**
01189  * Mark content as stale
01190  */
01191 PUBLIC void
01192 r_store_mark_stale(struct ccnr_handle *h, struct content_entry *content)
01193 {
01194     ccnr_cookie cookie = content->cookie;
01195     if ((content->flags & CCN_CONTENT_ENTRY_STALE) != 0)
01196         return;
01197     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01198             ccnr_debug_content(h, __LINE__, "stale", NULL, content);
01199     content->flags |= CCN_CONTENT_ENTRY_STALE;
01200     h->n_stale++;
01201     if (cookie < h->min_stale)
01202         h->min_stale = cookie;
01203     if (cookie > h->max_stale)
01204         h->max_stale = cookie;
01205 }
01206 
01207 /**
01208  * Scheduled event that makes content stale when its FreshnessSeconds
01209  * has expired.
01210  */
01211 static int
01212 expire_content(struct ccn_schedule *sched,
01213                void *clienth,
01214                struct ccn_scheduled_event *ev,
01215                int flags)
01216 {
01217     struct ccnr_handle *h = clienth;
01218     ccnr_cookie cookie = ev->evint;
01219     struct content_entry *content = NULL;
01220     if ((flags & CCN_SCHEDULE_CANCEL) != 0)
01221         return(0);
01222     content = r_store_content_from_cookie(h, cookie);
01223     if (content != NULL)
01224         r_store_mark_stale(h, content);
01225     return(0);
01226 }
01227 
01228 /**
01229  * Schedules content expiration based on its FreshnessSeconds.
01230  *
01231  */
01232 PUBLIC void
01233 r_store_set_content_timer(struct ccnr_handle *h, struct content_entry *content,
01234                   struct ccn_parsed_ContentObject *pco)
01235 {
01236     int seconds = 0;
01237     int microseconds = 0;
01238     size_t start = pco->offset[CCN_PCO_B_FreshnessSeconds];
01239     size_t stop  = pco->offset[CCN_PCO_E_FreshnessSeconds];
01240     const unsigned char *content_msg = NULL;
01241     if (start == stop)
01242         return;
01243     content_msg = r_store_content_base(h, content);
01244     if (content_msg == NULL) {
01245         ccnr_debug_content(h, __LINE__, "Missing_content_base", NULL,
01246                            content);
01247         return;        
01248     }
01249     seconds = ccn_fetch_tagged_nonNegativeInteger(
01250                 CCN_DTAG_FreshnessSeconds,
01251                 content_msg,
01252                 start, stop);
01253     if (seconds <= 0)
01254         return;
01255     if (seconds > ((1U<<31) / 1000000)) {
01256         ccnr_debug_content(h, __LINE__, "FreshnessSeconds_too_large", NULL,
01257                            content);
01258         return;
01259     }
01260     microseconds = seconds * 1000000;
01261     ccn_schedule_event(h->sched, microseconds,
01262                        &expire_content, NULL, content->cookie);
01263 }
01264 
01265 /**
01266  * Parses content object and sets content->flatname
01267  */
01268 static int
01269 r_store_set_flatname(struct ccnr_handle *h, struct content_entry *content,
01270                      struct ccn_parsed_ContentObject *pco)
01271 {
01272     int res;
01273     struct ccn_charbuf *flatname = NULL;
01274     const unsigned char *msg = NULL;
01275     size_t size;
01276     
01277     msg = r_store_content_base(h, content);
01278     size = content->size;
01279     if (msg == NULL)
01280         goto Bail;
01281     flatname = ccn_charbuf_create();
01282     if (flatname == NULL)
01283         goto Bail;    
01284     res = ccn_parse_ContentObject(msg, size, pco, NULL);
01285     if (res < 0) {
01286         ccnr_msg(h, "error parsing ContentObject - code %d", res);
01287         goto Bail;
01288     }
01289     ccn_digest_ContentObject(msg, pco);
01290     if (pco->digest_bytes != 32)
01291         goto Bail;
01292     res = ccn_flatname_from_ccnb(flatname, msg, size);
01293     if (res < 0) goto Bail;
01294     res = ccn_flatname_append_component(flatname, pco->digest, pco->digest_bytes);
01295     if (res < 0) goto Bail;
01296     content->flatname = flatname;
01297     flatname = NULL;
01298     return(0);
01299 Bail:
01300     ccn_charbuf_destroy(&flatname);
01301     return(-1);
01302 }
01303 
01304 /**
01305  *  Get the flatname associated with content
01306  *
01307  * @returns flatname in a charbuf, which should be treated as read-only.
01308  */
01309 PUBLIC struct ccn_charbuf *
01310 r_store_content_flatname(struct ccnr_handle *h, struct content_entry *content)
01311 {
01312     return(content->flatname);
01313 }
01314 
01315 PUBLIC struct content_entry *
01316 process_incoming_content(struct ccnr_handle *h, struct fdholder *fdholder,
01317                          unsigned char *msg, size_t size, off_t *offsetp)
01318 {
01319     struct ccn_parsed_ContentObject obj = {0};
01320     int res;
01321     struct content_entry *content = NULL;
01322     ccnr_accession accession = CCNR_NULL_ACCESSION;
01323     
01324     content = calloc(1, sizeof(*content));
01325     if (content == NULL)
01326         goto Bail;    
01327     content->cob = ccn_charbuf_create();
01328     if (content->cob == NULL)
01329         goto Bail;    
01330     res = ccn_charbuf_append(content->cob, msg, size);
01331     if (res < 0) goto Bail;
01332     content->size = size;
01333     res = r_store_set_flatname(h, content, &obj);
01334     if (res < 0) goto Bail;
01335     ccnr_meter_bump(h, fdholder->meter[FM_DATI], 1);
01336     content->accession = CCNR_NULL_ACCESSION;
01337     if (fdholder->filedesc == h->active_in_fd && offsetp != NULL) {
01338         // if we are reading from repoFile1 to rebuild the index we already know
01339         // the accession number
01340         content->accession = ((ccnr_accession)*offsetp) | r_store_mark_repoFile1;
01341     }
01342     r_store_enroll_content(h, content);
01343     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01344         ccnr_debug_content(h, __LINE__, "content_from", fdholder, content);
01345     res = r_store_content_btree_insert(h, content, &obj, &accession);
01346     if (res < 0) goto Bail;
01347     if (res == 0) {
01348         /* Content was there, with an accession */
01349         if (CCNSHOULDLOG(h, LM_4, CCNL_FINER))
01350             ccnr_debug_content(h, __LINE__, "content_duplicate",
01351                                fdholder, content);
01352         h->content_dups_recvd++;
01353         r_store_forget_content(h, &content);
01354         content = r_store_content_from_accession(h, accession);
01355         if (content == NULL)
01356             goto Bail;
01357     }
01358     r_store_set_content_timer(h, content, &obj);
01359     r_match_match_interests(h, content, &obj, NULL, fdholder);
01360     return(content);
01361 Bail:
01362     r_store_forget_content(h, &content);
01363     return(content);
01364 }
01365 
01366 PUBLIC int
01367 r_store_content_field_access(struct ccnr_handle *h,
01368                              struct content_entry *content,
01369                              enum ccn_dtag dtag,
01370                              const unsigned char **bufp, size_t *sizep)
01371 {
01372     int res = -1;
01373     const unsigned char *content_msg;
01374     struct ccn_parsed_ContentObject pco = {0};
01375     
01376     content_msg = r_store_content_base(h, content);
01377     if (content_msg == NULL)
01378         return(-1);
01379     res = ccn_parse_ContentObject(content_msg, content->size, &pco, NULL);
01380     if (res < 0)
01381         return(-1);
01382     if (dtag == CCN_DTAG_Content)
01383         res = ccn_ref_tagged_BLOB(CCN_DTAG_Content, content_msg,
01384                                   pco.offset[CCN_PCO_B_Content],
01385                                   pco.offset[CCN_PCO_E_Content],
01386                                   bufp, sizep);
01387     return(res);
01388 }
01389 
01390 
01391 PUBLIC int
01392 r_store_set_accession_from_offset(struct ccnr_handle *h,
01393                                   struct content_entry *content,
01394                                   struct fdholder *fdholder, off_t offset)
01395 {
01396     struct ccn_btree_node *leaf = NULL;
01397     uint_least64_t cobid;
01398     int ndx;
01399     int res = -1;
01400     
01401     if (offset != (off_t)-1 && content->accession == CCNR_NULL_ACCESSION) {
01402         struct hashtb_enumerator ee;
01403         struct hashtb_enumerator *e = &ee;
01404         struct content_by_accession_entry *entry = NULL;
01405         
01406         content->flags |= CCN_CONTENT_ENTRY_STABLE;
01407         content->accession = ((ccnr_accession)offset) | r_store_mark_repoFile1;
01408         hashtb_start(h->content_by_accession_tab, e);
01409         hashtb_seek(e, &content->accession, sizeof(content->accession), 0);
01410         entry = e->data;
01411         if (entry != NULL) {
01412             entry->content = content;
01413             if (content->cob != NULL)
01414                 h->cob_count++;
01415         }
01416         hashtb_end(e);
01417         if (content->flatname != NULL) {
01418             res = ccn_btree_lookup(h->btree,
01419                                    content->flatname->buf,
01420                                    content->flatname->length, &leaf);
01421             if (res >= 0 && CCN_BT_SRCH_FOUND(res)) {
01422                 ndx = CCN_BT_SRCH_INDEX(res);
01423                 cobid = ccnr_accession_encode(h, content->accession);
01424                 ccn_btree_prepare_for_update(h->btree, leaf);
01425                 res = ccn_btree_content_set_cobid(leaf, ndx, cobid);
01426             }
01427             else
01428                 res = -1;
01429         }
01430         if (res >= 0 && content->accession >= h->notify_after) 
01431             r_sync_notify_content(h, 0, content);
01432     }
01433     return(res);
01434 }
01435 
01436 PUBLIC void
01437 r_store_send_content(struct ccnr_handle *h, struct fdholder *fdholder, struct content_entry *content)
01438 {
01439     const unsigned char *content_msg = NULL;
01440     off_t offset;
01441 
01442     content_msg = r_store_content_base(h, content);
01443     if (content_msg == NULL) {
01444         ccnr_debug_content(h, __LINE__, "content_missing", fdholder, content);
01445         return;        
01446     }
01447     if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01448         ccnr_debug_content(h, __LINE__, "content_to", fdholder, content);
01449     r_link_stuff_and_send(h, fdholder, content_msg, content->size, NULL, 0, &offset);
01450     if (offset != (off_t)-1 && content->accession == CCNR_NULL_ACCESSION) {
01451         int res;
01452         res = r_store_set_accession_from_offset(h, content, fdholder, offset);
01453         if (res == 0)
01454             if (CCNSHOULDLOG(h, LM_4, CCNL_FINE))
01455                 ccnr_debug_content(h, __LINE__, "content_stored",
01456                                    r_io_fdholder_from_fd(h, h->active_out_fd),
01457                                    content);
01458     }
01459 }
01460 
01461 PUBLIC int
01462 r_store_commit_content(struct ccnr_handle *h, struct content_entry *content)
01463 {
01464     struct fdholder *fdholder = r_io_fdholder_from_fd(h, h->active_out_fd);
01465     // XXX - here we need to check if this is something we *should* be storing, according to our policy
01466     if ((r_store_content_flags(content) & CCN_CONTENT_ENTRY_STABLE) == 0) {
01467         if (fdholder == NULL)
01468         {
01469             ccnr_msg(h, "Repository shutting down due to error storing content.");
01470             h->running = 0;
01471             return(-1);
01472         }
01473         r_store_send_content(h, r_io_fdholder_from_fd(h, h->active_out_fd), content);
01474         r_store_content_change_flags(content, CCN_CONTENT_ENTRY_STABLE, 0);
01475     }
01476     return(0);
01477 }
01478 
01479 PUBLIC void
01480 ccnr_debug_content(struct ccnr_handle *h,
01481                    int lineno,
01482                    const char *msg,
01483                    struct fdholder *fdholder,
01484                    struct content_entry *content)
01485 {
01486     struct ccn_charbuf *c = ccn_charbuf_create();
01487     struct ccn_charbuf *flat = content->flatname;
01488     
01489     if (c == NULL)
01490         return;
01491     ccn_charbuf_putf(c, "debug.%d %s ", lineno, msg);
01492     if (fdholder != NULL)
01493         ccn_charbuf_putf(c, "%u ", fdholder->filedesc);
01494     if (flat != NULL)
01495         ccn_uri_append_flatname(c, flat->buf, flat->length, 1);
01496     ccn_charbuf_putf(c, " (%d bytes)", content->size);
01497     ccnr_msg(h, "%s", ccn_charbuf_as_string(c));
01498     ccn_charbuf_destroy(&c);
01499 }
01500 
01501 /** Number of btree index writes to do in a batch */
01502 #define CCN_BT_CLEAN_BATCH 3
01503 /** Approximate delay between batches of btree index writes */
01504 #define CCN_BT_CLEAN_TICK_MICROS 65536
01505 static int
01506 r_store_index_cleaner(struct ccn_schedule *sched,
01507     void *clienth,
01508     struct ccn_scheduled_event *ev,
01509     int flags)
01510 {
01511     struct ccnr_handle *h = clienth;
01512     struct hashtb_enumerator ee;
01513     struct hashtb_enumerator *e = &ee;
01514     struct ccn_btree_node *node = NULL;
01515     int k;
01516     int res;
01517     int overquota;
01518     
01519     (void)(sched);
01520     (void)(ev);
01521     if ((flags & CCN_SCHEDULE_CANCEL) != 0 ||
01522          h->btree == NULL || h->btree->io == NULL) {
01523         h->index_cleaner = NULL;
01524         ccn_indexbuf_destroy(&h->toclean);
01525         return(0);
01526     }
01527     /* First, work on cleaning the things we already know need cleaning */
01528     if (h->toclean != NULL) {
01529         for (k = 0; k < CCN_BT_CLEAN_BATCH && h->toclean->n > 0; k++) {
01530             node = ccn_btree_rnode(h->btree, h->toclean->buf[--h->toclean->n]);
01531             if (node != NULL && node->iodata != NULL) {
01532                 res = ccn_btree_chknode(node); /* paranoia */
01533                 if (res < 0 || CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01534                     ccnr_msg(h, "write index node %u (err %d)",
01535                              (unsigned)node->nodeid, node->corrupt);
01536                 if (res >= 0) {
01537                     if (node->clean != node->buf->length)
01538                         res = h->btree->io->btwrite(h->btree->io, node);
01539                     if (res < 0)
01540                         ccnr_msg(h, "failed to write index node %u",
01541                                  (unsigned)node->nodeid);
01542                     else
01543                         node->clean = node->buf->length;
01544                 }
01545                 if (res >= 0 && node->iodata != NULL && node->activity == 0) {
01546                     if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01547                         ccnr_msg(h, "close index node %u",
01548                                  (unsigned)node->nodeid);
01549                     res = ccn_btree_close_node(h->btree, node);
01550                 }
01551             }
01552         }
01553         if (h->toclean->n > 0)
01554             return(nrand48(h->seed) % (2U * CCN_BT_CLEAN_TICK_MICROS) + 500);
01555     }
01556     /* Sweep though and find the nodes that still need cleaning */
01557     overquota = 0;
01558     if (h->btree->nodepool >= 16)
01559         overquota = hashtb_n(h->btree->resident) - h->btree->nodepool;
01560     hashtb_start(h->btree->resident, e);
01561     for (node = e->data; node != NULL; node = e->data) {
01562         if (overquota > 0 &&
01563               node->activity == 0 &&
01564               node->iodata == NULL &&
01565               node->clean == node->buf->length) {
01566             overquota -= 1;
01567             if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINEST))
01568                 ccnr_msg(h, "prune index node %u",
01569                          (unsigned)node->nodeid);
01570             hashtb_delete(e);
01571             continue;
01572         }
01573         node->activity /= 2; /* Age the node's activity */
01574         if (node->clean != node->buf->length ||
01575             (node->iodata != NULL && node->activity == 0)) {
01576             if (h->toclean == NULL) {
01577                 h->toclean = ccn_indexbuf_create();
01578                 if (h->toclean == NULL)
01579                     break;
01580             }
01581             ccn_indexbuf_append_element(h->toclean, node->nodeid);
01582         }
01583         hashtb_next(e);
01584     }
01585     hashtb_end(e);
01586     /* If nothing to do, shut down cleaner */
01587     if ((h->toclean == NULL || h->toclean->n == 0) && overquota <= 0 &&
01588         h->btree->io->openfds <= CCN_BT_OPEN_NODES_IDLE) {
01589         h->btree->cleanreq = 0;
01590         h->index_cleaner = NULL;
01591         ccn_indexbuf_destroy(&h->toclean);
01592         if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINE))
01593             ccnr_msg(h, "index btree nodes all clean");
01594         
01595         return(0);
01596     }
01597     return(nrand48(h->seed) % (2U * CCN_BT_CLEAN_TICK_MICROS) + 500);
01598 }
01599 
01600 PUBLIC void
01601 r_store_index_needs_cleaning(struct ccnr_handle *h)
01602 {
01603     int k;
01604     if (h->btree != NULL && h->btree->io != NULL && h->btree->cleanreq > 0) {
01605         if (h->index_cleaner == NULL) {
01606             h->index_cleaner = ccn_schedule_event(h->sched,
01607                                                   CCN_BT_CLEAN_TICK_MICROS,
01608                                                   r_store_index_cleaner, NULL, 0);
01609             if (CCNSHOULDLOG(h, sdfsdffd, CCNL_FINER))
01610                 ccnr_msg(h, "index cleaner started");
01611         }
01612         /* If necessary, clean in a hurry. */
01613         for (k = 30; /* Backstop to make sure we do not loop here */
01614              k > 0 && h->index_cleaner != NULL &&
01615              h->btree->io->openfds > CCN_BT_OPEN_NODES_LIMIT - 2; k--)
01616             r_store_index_cleaner(h->sched, h, h->index_cleaner, 0);
01617         if (k == 0)
01618             ccnr_msg(h, "index cleaner is in trouble");
01619     }
01620 }
01621 
01622 #undef FAILIF
01623 #undef CHKSYS
01624 #undef CHKRES
01625 #undef CHKPTR
Generated on Tue Aug 21 14:54:16 2012 for Content-Centric Networking in C by  doxygen 1.6.3