Merge pull request #44 from threefoldtech/development-v2-metadata

metadata: improve support and listing

libflist/database_redis.c

@@ -173,6 +173,14 @@ static int database_redis_mddel(flist_db_t *database, char *key) {
     return 1;
 }
 
+static slist_t database_redis_mdlist(flist_db_t *database) {
+    (void) database;
+    slist_t value = {.list = NULL, .length = 0};
+
+    debug("[-] libflist: database_redis_mdlist: not implemented\n");
+    return value;
+}
+
 static flist_db_t *database_redis_init_global(flist_db_t *db) {
     // setting global db
     db->type = "REDIS";
@@ -191,6 +199,7 @@ static flist_db_t *database_redis_init_global(flist_db_t *db) {
     db->mdget = database_redis_mdget;
     db->mdset = database_redis_mdset;
     db->mddel = database_redis_mddel;
+    db->mdlist = database_redis_mdlist;
 
     return db;
 }

libflist/database_sqlite.c

@@ -68,6 +68,7 @@ static int database_sqlite_optimize(flist_db_t *database) {
         {.target = &db->mdget,  .query = "SELECT value FROM metadata WHERE key = ?1"},
         {.target = &db->mdset,  .query = "REPLACE INTO metadata (key, value) VALUES (?1, ?2)"},
         {.target = &db->mddel,  .query = "DELETE FROM metadata WHERE key = ?1"},
+        {.target = &db->mdlist, .query = "SELECT key FROM metadata"},
     };
 
     for(size_t i = 0; i < sizeof(stmts) / sizeof(struct __stmtop); i++) {
@@ -290,6 +291,34 @@ static int database_sqlite_mddel(flist_db_t *database, char *key) {
     return 0;
 }
 
+static slist_t database_sqlite_mdlist(flist_db_t *database) {
+    database_sqlite_t *db = (database_sqlite_t *) database->handler;
+    int data;
+
+    slist_t value = {
+        .list = malloc(0),
+        .length = 0,
+    };
+
+    sqlite3_reset(db->mdlist);
+
+    while((data = sqlite3_step(db->mdlist)) == SQLITE_ROW) {
+        value.length += 1;
+        value.list = realloc(value.list, value.length * sizeof(char *));
+
+        size_t length = sqlite3_column_bytes(db->mdlist, 0) + 1;
+        value.list[value.length - 1] = malloc(length);
+
+        char *data = (void *) sqlite3_column_text(db->mdlist, 0);
+        strcpy(value.list[value.length - 1], data);
+    }
+
+    if(data == SQLITE_DONE)
+        return value;
+
+    libflist_set_error("get: sqlite3_step: %s", sqlite3_errmsg(db->db));
+    return value;
+}
 
 
 // poor implementation of exists
@@ -358,6 +387,7 @@ flist_db_t *libflist_db_sqlite_init(char *rootpath) {
     db->mdget = database_sqlite_mdget;
     db->mdset = database_sqlite_mdset;
     db->mddel = database_sqlite_mddel;
+    db->mdlist = database_sqlite_mdlist;
 
     return db;
 }

libflist/database_sqlite.h

@@ -16,6 +16,7 @@
         sqlite3_stmt *mdset;
         sqlite3_stmt *mdget;
         sqlite3_stmt *mddel;
+        sqlite3_stmt *mdlist;
 
     } database_sqlite_t;
 

libflist/libflist.h

@@ -103,6 +103,12 @@
     } dirnode_t;
 
 
+    typedef struct slist_t {
+        char **list;
+        size_t length;
+
+    } slist_t;
+
 
     typedef struct flist_db_value_t {
         void *handler;
@@ -133,6 +139,7 @@
         value_t* (*mdget)(struct flist_db_t *db, char *key);
         int (*mdset)(struct flist_db_t *db, char *key, char *data);
         int (*mddel)(struct flist_db_t *db, char *key);
+        slist_t (*mdlist)(struct flist_db_t *db);
 
         void (*clean)(value_t *value);
 
@@ -228,6 +235,7 @@
     char *libflist_hashhex(unsigned char *hash, int length);
     void *libflist_bufdup(void *source, size_t length);
 
+    void libflist_progress(flist_ctx_t *ctx, char *message, size_t current, size_t total);
     char *libflist_version();
 
     //
@@ -363,6 +371,8 @@
     int libflist_metadata_set(flist_db_t *database, char *metadata, char *payload);
     int libflist_metadata_remove(flist_db_t *database, char *metadata);
     char *libflist_metadata_get(flist_db_t *database, char *metadata);
+    slist_t libflist_metadata_list(flist_db_t *database);
+    void libflist_metadata_list_free(slist_t *list);
     flist_db_t *libflist_metadata_backend_database(flist_db_t *database);
     flist_db_t *libflist_metadata_backend_database_json(char *input);
 
@@ -375,6 +385,7 @@
     //   with information (and not with capnp object which are difficult to use)
     void libflist_serial_dirnode_commit(dirnode_t *root, flist_ctx_t *ctx, dirnode_t *parent);
     acl_t *libflist_serial_acl_get(flist_db_t *database, const char *aclkey);
+
     //
     // statistics.c
     //

libflist/metadata.c

@@ -45,6 +45,16 @@ int libflist_metadata_remove(flist_db_t *database, char *metadata) {
     return 1;
 }
 
+slist_t libflist_metadata_list(flist_db_t *database) {
+    slist_t raw = database->mdlist(database);
+    return raw;
+}
+
+void libflist_metadata_list_free(slist_t *list) {
+    for(size_t a = 0; a < list->length; a++)
+        free(list->list[a]);
+}
+
 flist_db_t *libflist_metadata_backend_database_json(char *input) {
     flist_db_t *backdb;
     json_error_t error;

libflist/verbose.h

@@ -17,8 +17,5 @@
     // wrap printf style error definiton
     void *libflist_set_error(const char *format, ...);
 
-    // progression
-    void libflist_progress(flist_ctx_t *ctx, char *message, size_t current, size_t total);
-
     #define debug(...) { if(libflist_debug_flag) { printf(__VA_ARGS__); } }
 #endif

zflist/README.md

@@ -172,6 +172,8 @@ Here are the list of metadata available:
 - Volumes
 - Readme
 
+In addition, any generic metadata can now be set as well.
+
 ## Backend
 
 The flist format contains only metadata about a filesystem, not the file contents. On theses metadata,
@@ -219,3 +221,10 @@ this mountpoint, but this is not needed.
 ## Readme
 
 You can provide an arbitraty text and license name to describe your flist.
+
+## Generic Metadata
+
+You can list, set and get any other value for metadata, eg:
+ - Listing metadata keys available: `./zflist metadata`
+ - Set (or replace) a metadata entry: `./zflist metadata hello world`
+ - Get a metadata entry: `./zflist metadata hello`

zflist/actions.c

@@ -411,6 +411,24 @@ int zf_find(zf_callback_t *cb) {
     return 0;
 }
 
+//
+// chunks dumps
+//
+int zf_chunks(zf_callback_t *cb) {
+    dirnode_t *dirnode;
+
+    if(!(dirnode = libflist_dirnode_get(cb->ctx->db, "/"))) {
+        zf_error(cb, "chunks", "no such root directory");
+        return 1;
+    }
+
+    zf_chunks_recursive(cb, dirnode, 0);
+
+    libflist_dirnode_free(dirnode);
+
+    return 0;
+}
+
 //
 // check
 //
@@ -474,10 +492,9 @@ int zf_stat(zf_callback_t *cb) {
 // metadata
 //
 int zf_metadata(zf_callback_t *cb) {
-    if(cb->argc < 2) {
-        zf_error(cb, "metadata", "missing metadata name");
-        return 1;
-    }
+    // list metadata if none defined
+    if(cb->argc < 2)
+        return zf_metadata_list(cb);
 
     // fetching metadata from database
     if(cb->argc == 2)
@@ -506,8 +523,7 @@ int zf_metadata(zf_callback_t *cb) {
     else if(strcmp(cb->argv[0], "readme") == 0)
         return zf_metadata_set_readme(cb);
 
-    zf_error(cb, "metadata", "unknown metadata name");
-    return 1;
+    return zf_metadata_set_generic(cb);
 }
 
 //

zflist/actions.h

@@ -12,6 +12,7 @@
     int zf_mkdir(zf_callback_t *cb);
     int zf_ls(zf_callback_t *cb);
     int zf_find(zf_callback_t *cb);
+    int zf_chunks(zf_callback_t *cb);
     int zf_check(zf_callback_t *cb);
     int zf_stat(zf_callback_t *cb);
     int zf_cat(zf_callback_t *cb);

zflist/actions_metadata.c

@@ -80,7 +80,7 @@ static int zf_metadata_get_json(zf_callback_t *cb, char *metaname, char *str) {
     json_t *value;
 
     if(!(value = json_loads(str, 0, &error)))
-        return 1;
+        value = json_string(str);
 
     json_t *metadata = json_object();
 
@@ -641,3 +641,41 @@ int zf_metadata_set_readme(zf_callback_t *cb) {
     return zf_metadata_apply(cb, "readme", root);
 }
 
+
+
+int zf_metadata_set_generic(zf_callback_t *cb) {
+    char *metadata = cb->argv[0];
+    char *value = cb->argv[1];
+
+    if(!libflist_metadata_set(cb->ctx->db, metadata, value)) {
+        zf_error(cb, "metadata", "%s", libflist_strerror());
+        return 1;
+    }
+
+    return 0;
+}
+
+int zf_metadata_list_json(zf_callback_t *cb, slist_t list) {
+    json_t *names = json_array();
+
+    for(size_t a = 0; a < list.length; a++)
+        json_array_append(names, json_string(list.list[a]));
+
+    json_object_set_new(cb->jout, "response", names);
+
+    libflist_metadata_list_free(&list);
+    return 0;
+}
+
+int zf_metadata_list(zf_callback_t *cb) {
+    slist_t list = libflist_metadata_list(cb->ctx->db);
+
+    if(cb->jout)
+        return zf_metadata_list_json(cb, list);
+
+    for(size_t a = 0; a < list.length; a++)
+        printf("%s\n", list.list[a]);
+
+    libflist_metadata_list_free(&list);
+    return 0;
+}

zflist/actions_metadata.h

@@ -9,4 +9,7 @@
     int zf_metadata_set_readme(zf_callback_t *cb);
 
     int zf_metadata_get(zf_callback_t *cb);
+
+    int zf_metadata_set_generic(zf_callback_t *cb);
+    int zf_metadata_list(zf_callback_t *cb);
 #endif

zflist/tools.c

@@ -442,6 +442,101 @@ int zf_find_finalize(zf_callback_t *cb) {
     return zf_find_finalize_text(cb);
 }
 
+//
+// chunks dumps implementation
+//
+static int zf_chunks_recursive_text(zf_callback_t *cb, dirnode_t *dirnode, int integrity) {
+    dirnode_t *subnode = NULL;
+
+    for(inode_t *inode = dirnode->inode_list; inode; inode = inode->next) {
+        // if it's a directory, let's walk inside
+        if(inode->type == INODE_DIRECTORY) {
+            libflist_stats_directory_add(cb->ctx, 1);
+
+            if(!(subnode = libflist_dirnode_get(cb->ctx->db, inode->fullpath))) {
+                zf_error(cb, "find", "recursive directory not found");
+                return 1;
+            }
+
+            zf_chunks_recursive_text(cb, subnode, integrity);
+            libflist_dirnode_free(subnode);
+        }
+
+        if(inode->type == INODE_FILE) {
+            if(!inode->chunks || inode->chunks->size == 0)
+                continue;
+
+            for(size_t i = 0; i < inode->chunks->size; i++) {
+                inode_chunk_t *ichunk = &inode->chunks->list[i];
+
+                discard char *hashstr = libflist_hashhex((unsigned char *) ichunk->entryid, ichunk->entrylen);
+
+                printf("%s\n", hashstr);
+            }
+        }
+
+    }
+
+    return 0;
+}
+
+static int zf_chunks_recursive_json(zf_callback_t *cb, dirnode_t *dirnode, int integrity) {
+    dirnode_t *subnode = NULL;
+
+    json_t *response = json_object_get(cb->jout, "response");
+    json_t *content = json_object_get(response, "content");
+
+    for(inode_t *inode = dirnode->inode_list; inode; inode = inode->next) {
+        json_t *entry = json_object();
+
+        // if it's a directory, let's walk inside
+        if(inode->type == INODE_DIRECTORY) {
+            libflist_stats_directory_add(cb->ctx, 1);
+
+            if(!(subnode = libflist_dirnode_get(cb->ctx->db, inode->fullpath))) {
+                zf_error(cb, "find", "recursive directory not found");
+                return 1;
+            }
+
+            zf_chunks_recursive_json(cb, subnode, integrity);
+            libflist_dirnode_free(subnode);
+        }
+
+        if(inode->type == INODE_FILE) {
+            if(!inode->chunks || inode->chunks->size == 0)
+                continue;
+
+            for(size_t i = 0; i < inode->chunks->size; i++) {
+                inode_chunk_t *ichunk = &inode->chunks->list[i];
+
+                discard char *hashstr = libflist_hashhex((unsigned char *) ichunk->entryid, ichunk->entrylen);
+
+                entry = json_string(hashstr);
+                json_array_append_new(content, entry);
+            }
+        }
+    }
+
+    return 0;
+}
+
+int zf_chunks_recursive(zf_callback_t *cb, dirnode_t *dirnode, int integrity) {
+    if(cb->jout) {
+        json_t *response = json_object_get(cb->jout, "response");
+        json_t *content = json_array();
+
+        json_object_set(response, "content", content);
+        cb->userptr = content;
+
+        return zf_chunks_recursive_json(cb, dirnode, integrity);
+    }
+
+    return zf_chunks_recursive_text(cb, dirnode, integrity);
+}
+
+
+
+
 //
 // error handling
 //