From 79014f2ba20f09ad58147d1a194c4ef744b4a684 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Thu, 25 Jan 2024 17:24:41 -0500
Subject: [PATCH 01/16] wip: fix epoch migration bug

---
 pkg/noun/version.h | 11 +++++-
 pkg/vere/disk.c    | 95 ++++++++++++++++++++++++++--------------------
 pkg/vere/mars.c    |  2 +-
 pkg/vere/pier.c    |  8 ++--
 pkg/vere/vere.h    |  4 +-
 5 files changed, 72 insertions(+), 48 deletions(-)

diff --git a/pkg/noun/version.h b/pkg/noun/version.h
index 5cc6acb86d..0b04502d9b 100644
--- a/pkg/noun/version.h
+++ b/pkg/noun/version.h
@@ -20,9 +20,16 @@ typedef c3_w       u3e_version;
 #define U3E_VERLAT U3E_VER1
 
 /* DISK FORMAT
+ *
+ * synchronized between:
+ * - u3_disk->ver_w
+ * - log/<0iN>/epoc.txt
+ * - version field in META table of data.mdb files
  */
 
-#define U3D_VER1   1
-#define U3D_VERLAT U3L_VER1
+#define U3D_VER1   1         // <= vere-v2
+#define U3D_VER2   2         // migrating to >= vere-v3
+#define U3D_VER3   3         // >= vere-v3
+#define U3D_VERLAT U3D_VER3
 
 #endif /* ifndef U3_VERSION_H */
diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index de141988bd..e7c4f25e22 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -645,13 +645,14 @@ _disk_save_meta(MDB_env* mdb_u, const c3_c* key_c, u3_atom dat)
 */
 c3_o
 u3_disk_save_meta(MDB_env* mdb_u,
+                  c3_w     ver_w,
                   c3_d     who_d[2],
                   c3_o     fak_o,
                   c3_w     lif_w)
 {
   u3_assert( c3y == u3a_is_cat(lif_w) );
 
-  if (  (c3n == _disk_save_meta(mdb_u, "version", 1))
+  if (  (c3n == _disk_save_meta(mdb_u, "version", ver_w))
      || (c3n == _disk_save_meta(mdb_u, "who", u3i_chubs(2, who_d)))
      || (c3n == _disk_save_meta(mdb_u, "fake", fak_o))
      || (c3n == _disk_save_meta(mdb_u, "life", lif_w)) )
@@ -688,6 +689,7 @@ _disk_meta_read_cb(void* ptr_v, ssize_t val_i, void* val_v)
 */
 c3_o
 u3_disk_read_meta(MDB_env* mdb_u,
+                  c3_w*    ver_w,
                   c3_d*    who_d,
                   c3_o*    fak_o,
                   c3_w*    lif_w)
@@ -702,11 +704,16 @@ u3_disk_read_meta(MDB_env* mdb_u,
     fprintf(stderr, "disk: read meta: no version\r\n");
     return c3n;
   }
-  else if ( (1 != val_u.hav_i) || (1 != *val_u.buf_y) ) {
+  else if ( (1 != val_u.hav_i) || (U3D_VERLAT < *val_u.buf_y) ) {
     fprintf(stderr, "disk: read meta: unknown version %u\r\n", *val_u.buf_y);
     return c3n;
   }
 
+  if ( ver_w ) {
+    c3_y* byt_y = val_u.buf_y;
+    *ver_w = (c3_w)byt_y[0];
+  }
+
   //  identity
   //
   u3_lmdb_read_meta(mdb_u, &val_u, "who", _disk_meta_read_cb);
@@ -1115,7 +1122,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
   c3_c epv_c[8193];
   snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c);
   FILE* epv_f = fopen(epv_c, "w");  // XX errors
-  fprintf(epv_f, "%d", U3D_VER1);
+  fprintf(epv_f, "%d", U3D_VER3);
   fclose(epv_f);
 
   //  create binary version file, overwriting any existing file
@@ -1126,11 +1133,12 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
   fclose(biv_f);
 
   //  get metadata from old epoch or unmigrated event log's db
+  c3_w     ver_w;
   c3_d     who_d[2];
   c3_o     fak_o;
   c3_w     lif_w;
   if ( c3y == eps_o ) {  //  skip if no epochs yet
-    if ( c3y != u3_disk_read_meta(log_u->mdb_u, who_d, &fak_o, &lif_w) ) {
+    if ( c3y != u3_disk_read_meta(log_u->mdb_u, &ver_w, who_d, &fak_o, &lif_w) ) {
       fprintf(stderr, "disk: failed to read metadata\r\n");
       goto fail3;
     }
@@ -1153,7 +1161,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
 
   // write the metadata to the database
   if ( c3y == eps_o ) {
-    if ( c3n == u3_disk_save_meta(log_u->mdb_u, who_d, fak_o, lif_w) ) {
+    if ( c3n == u3_disk_save_meta(log_u->mdb_u, log_u->ver_w, who_d, fak_o, lif_w) ) {
       fprintf(stderr, "disk: failed to save metadata\r\n");
       goto fail3;
     }
@@ -1300,6 +1308,15 @@ _disk_need_migrate(u3_disk* log_u)
   //  check if data.mdb is readable in log directory
   //
   if ( 0 == access(dut_c, F_OK)) {
+    //  check if data.mdb has version 3
+    c3_w ver_w;
+    if ( c3y == u3_disk_read_meta(log_u->mdb_u, &ver_w, 0, 0, 0) &&
+         ver_w == U3D_VER3 )
+    {
+      fprintf(stderr, "disk: failed to read metadata\r\n");
+      return c3n;
+    }
+
     return c3y;
   }
 
@@ -1339,7 +1356,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     luk_o = c3y;
   }
 
-  fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1);
+  fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER3);
 
   //  migrate existing pier which has either:
   //  - not started the migration, or
@@ -1423,7 +1440,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   }
 
   //  success
-  fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1);
+  fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER3);
 
   return c3y;
 }
@@ -1499,9 +1516,9 @@ _disk_epoc_load(u3_disk* log_u, c3_d lat_d)
       return _epoc_fail;
     }
 
-    if ( U3D_VER1 != ver_w ) {
+    if ( U3D_VER3 != ver_w ) {
       fprintf(stderr, "disk: unknown epoch version: '%s', expected '%d'\r\n",
-                      ver_c, U3D_VER1);
+                      ver_c, U3D_VER3);
       return _epoc_late;
     }
 
@@ -1533,7 +1550,7 @@ _disk_epoc_load(u3_disk* log_u, c3_d lat_d)
   if (  (c3n == u3_Host.ops_u.nuu )
      && !fir_d
      && !las_d
-     && (c3n == u3_disk_read_meta(log_u->mdb_u, 0, 0, 0)) )
+     && (c3n == u3_disk_read_meta(log_u->mdb_u, 0, 0, 0, 0)) )
   {
     u3_lmdb_exit(log_u->mdb_u);
     log_u->mdb_u = 0;
@@ -1618,7 +1635,16 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       return 0;
     }
 
-    //  if fresh boot, initialize disk v1
+    //  load the old data.mdb file
+    fprintf(stderr, "disk: loading old format\r\n");
+
+    if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
+      fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+      c3_free(log_u);
+      return 0;
+    }
+
+    //  if fresh boot, initialize disk U3D_VER3
     //
     if ( c3y == u3_Host.ops_u.nuu ) {
       //  initialize first epoch "0i0"
@@ -1626,15 +1652,17 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         fprintf(stderr, "disk: failed to initialize first epoch\r\n");
         return 0;
       }
-    }
-    else if ( c3y == _disk_need_migrate(log_u) ) {
-      fprintf(stderr, "disk: loading old format\r\n");
 
-      if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
-        fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+      //  set its meta table version key to U3D_VER3 to ensure that if our pier
+      //  is booted with <= vere-v2, it crashes instead of potentially
+      //  corrupting its event log
+      if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, pir_u->) ) {
+        fprintf(stderr, "disk: failed to save metadata version\r\n");
         c3_free(log_u);
+        return 0;
       }
-
+    }
+    else if ( c3y == _disk_need_migrate(log_u) ) {
       c3_d fir_d;
       if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) {
         fprintf(stderr, "disk: failed to load latest event from lmdb\r\n");
@@ -1643,32 +1671,17 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         return 0;
       }
 
-      //  if a top-level log exists but is empty it's safe to remove.
-      //  for extra safety, we only do this if we also have epochs.
-      //  (the top-level log was most likely created by vere downgrade)
+      //  we keep the old event log around forever
+      //  
+      //  U3D_VER2 in meta table means in-progress migration to vere-v3
+      //  U3D_VER3 in meta table means migration complete
       //
-      c3_d nop_d;
-      if (  !fir_d
-         && !log_u->dun_d
-         && (c3y == u3_disk_epoc_last(log_u, &nop_d)) )
-      {
-        c3_c luk_c[8193]; c3_c dut_c[8193];
-        snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_c);
-        snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c);
-
-        u3_lmdb_exit(log_u->mdb_u);
-        log_u->mdb_u = 0;
-
-        if ( c3_unlink(dut_c)
-           ||  (  (0 == access(luk_c, R_OK))
-               && c3_unlink(luk_c) ))
-        {
-          fprintf(stderr, "disk: failed to unlink empty, old log\r\n");
-        }
-
-        //  continue to epoc initialization
+      //  this prevents the pier from booting with <= vere-v2
+      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER2) ) {
+        fprintf(stderr, "disk: failed to save metadata version\r\n");
+        c3_free(log_u);
+        return 0;
       }
-
       else {
         log_u->sen_d = log_u->dun_d;
         log_u->ver_w = 0;
diff --git a/pkg/vere/mars.c b/pkg/vere/mars.c
index 5e940182dc..020f885d6c 100644
--- a/pkg/vere/mars.c
+++ b/pkg/vere/mars.c
@@ -248,7 +248,7 @@ u3_mars_play(u3_mars* mar_u, c3_d eve_d, c3_d sap_d)
   if ( !mar_u->dun_d ) {
     c3_w lif_w;
 
-    if ( c3n == u3_disk_read_meta(log_u->mdb_u, 0, 0, &lif_w) ) {
+    if ( c3n == u3_disk_read_meta(log_u->mdb_u, 0, 0, 0, &lif_w) ) {
       fprintf(stderr, "mars: disk read meta fail\r\n");
       //  XX exit code, cb
       //
diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c
index 5c5b047f3e..48f7483bda 100644
--- a/pkg/vere/pier.c
+++ b/pkg/vere/pier.c
@@ -1644,7 +1644,7 @@ _pier_init(c3_w wag_w, c3_c* pax_c)
       return 0;
     }
 
-    u3_assert( U3D_VER1 == pir_u->log_u->ver_w );
+    u3_assert( U3D_VER3 == pir_u->log_u->ver_w );
   }
 
   //  initialize compute
@@ -1698,7 +1698,8 @@ u3_pier_stay(c3_w wag_w, u3_noun pax)
     return 0;
   }
 
-  if ( c3n == u3_disk_read_meta(pir_u->log_u->mdb_u,  pir_u->who_d,
+  if ( c3n == u3_disk_read_meta(pir_u->log_u->mdb_u,
+                               &pir_u->log_u->ver_w,  pir_u->who_d,
                                &pir_u->fak_o,        &pir_u->lif_w) )
   {
     fprintf(stderr, "pier: disk read meta fail\r\n");
@@ -1895,7 +1896,8 @@ _pier_boot_plan(u3_pier* pir_u,
     pir_u->lif_w = u3qb_lent(bot_u.bot);
   }
 
-  if ( c3n == u3_disk_save_meta(pir_u->log_u->mdb_u, pir_u->who_d,
+  if ( c3n == u3_disk_save_meta(pir_u->log_u->mdb_u,
+                                pir_u->log_u->ver_w, pir_u->who_d,
                                 pir_u->fak_o,        pir_u->lif_w) )
   {
     //  XX dispose bot_u
diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h
index 9527246e19..d63c5f598e 100644
--- a/pkg/vere/vere.h
+++ b/pkg/vere/vere.h
@@ -542,7 +542,7 @@
           u3_dire*         urb_u;               //  urbit system data
           u3_dire*         com_u;               //  log directory
           c3_o             liv_o;               //  live
-          c3_w             ver_w;               //  pier version
+          c3_w             ver_w;               //  version (see version.h)
           void*            mdb_u;               //  lmdb env of current epoch
           c3_d             sen_d;               //  commit requested
           c3_d             dun_d;               //  committed
@@ -976,6 +976,7 @@
       */
         c3_o
         u3_disk_read_meta(MDB_env* mdb_u,
+                          c3_w*    ver_w,
                           c3_d*    who_d,
                           c3_o*    fak_o,
                           c3_w*    lif_w);
@@ -984,6 +985,7 @@
       */
         c3_o
         u3_disk_save_meta(MDB_env* mdb_u,
+                          c3_w     ver_w,
                           c3_d     who_d[2],
                           c3_o     fak_o,
                           c3_w     lif_w);

From 984bf9b1da66fe1cff9a2a05b568bdcad08f16c7 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Thu, 25 Jan 2024 21:29:53 -0500
Subject: [PATCH 02/16] wip: some things work

---
 pkg/vere/disk.c | 103 ++++++++++++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 42 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index e7c4f25e22..80b7aa7758 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -685,6 +685,34 @@ _disk_meta_read_cb(void* ptr_v, ssize_t val_i, void* val_v)
   }
 }
 
+/* _disk_meta_read_vers(): read metadata version.
+*/
+c3_o
+_disk_meta_read_vers(MDB_env* mdb_u, c3_w* ver_w)
+{
+  _mdb_val val_u;
+
+  //  version
+  //
+  u3_lmdb_read_meta(mdb_u, &val_u, "version", _disk_meta_read_cb);
+
+  if ( 0 > val_u.hav_i ) {
+    fprintf(stderr, "disk: read meta: no version\r\n");
+    return c3n;
+  }
+  else if ( (1 != val_u.hav_i) || (U3D_VERLAT < *val_u.buf_y) ) {
+    fprintf(stderr, "disk: read vers: unknown version %u\r\n", *val_u.buf_y);
+    return c3n;
+  }
+
+  if ( ver_w ) {
+    c3_y* byt_y = val_u.buf_y;
+    *ver_w = (c3_w)byt_y[0];
+  }
+
+  return c3y;
+}
+
 /* u3_disk_read_meta(): read metadata.
 */
 c3_o
@@ -705,13 +733,17 @@ u3_disk_read_meta(MDB_env* mdb_u,
     return c3n;
   }
   else if ( (1 != val_u.hav_i) || (U3D_VERLAT < *val_u.buf_y) ) {
-    fprintf(stderr, "disk: read meta: unknown version %u\r\n", *val_u.buf_y);
+    fprintf(stderr, "disk: read vers: have bytes %zd\r\n", val_u.hav_i);
+    fprintf(stderr, "disk: read vers: unknown version %u\r\n", *val_u.buf_y);
     return c3n;
   }
 
   if ( ver_w ) {
     c3_y* byt_y = val_u.buf_y;
-    *ver_w = (c3_w)byt_y[0];
+    *ver_w = (c3_w)byt_y[0]
+           | (c3_w)byt_y[1] << 8
+           | (c3_w)byt_y[2] << 16
+           | (c3_w)byt_y[3] << 24;
   }
 
   //  identity
@@ -1133,12 +1165,11 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
   fclose(biv_f);
 
   //  get metadata from old epoch or unmigrated event log's db
-  c3_w     ver_w;
   c3_d     who_d[2];
   c3_o     fak_o;
   c3_w     lif_w;
   if ( c3y == eps_o ) {  //  skip if no epochs yet
-    if ( c3y != u3_disk_read_meta(log_u->mdb_u, &ver_w, who_d, &fak_o, &lif_w) ) {
+    if ( c3y != u3_disk_read_meta(log_u->mdb_u, 0, who_d, &fak_o, &lif_w) ) {
       fprintf(stderr, "disk: failed to read metadata\r\n");
       goto fail3;
     }
@@ -1161,7 +1192,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
 
   // write the metadata to the database
   if ( c3y == eps_o ) {
-    if ( c3n == u3_disk_save_meta(log_u->mdb_u, log_u->ver_w, who_d, fak_o, lif_w) ) {
+    if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
       fprintf(stderr, "disk: failed to save metadata\r\n");
       goto fail3;
     }
@@ -1302,25 +1333,15 @@ u3_disk_epoc_list(u3_disk* log_u, c3_d* sot_d)
 static c3_o
 _disk_need_migrate(u3_disk* log_u)
 {
-  c3_c dut_c[8193];
-  snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c);
-
-  //  check if data.mdb is readable in log directory
-  //
-  if ( 0 == access(dut_c, F_OK)) {
-    //  check if data.mdb has version 3
-    c3_w ver_w;
-    if ( c3y == u3_disk_read_meta(log_u->mdb_u, &ver_w, 0, 0, 0) &&
-         ver_w == U3D_VER3 )
-    {
-      fprintf(stderr, "disk: failed to read metadata\r\n");
-      return c3n;
-    }
-
-    return c3y;
+  //  check if log/data.mdb has version 3
+  c3_w ver_w;
+  if ( c3n == _disk_meta_read_vers(log_u->mdb_u, &ver_w) ) {
+    fprintf(stderr, "disk: failed to read version\r\n");
   }
-
-  return c3n;
+  else if ( U3D_VER3 == ver_w ) {
+    return c3n;
+  }
+  return c3y;
 }
 
 /* _disk_migrate: migrates disk format.
@@ -1331,8 +1352,9 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   /*  migration steps:
    *  0. detect whether we need to migrate or not
    *     a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration
-   *     b. if data.mdb is readable in log directory -> execute migration
+   *     b. if log/data.mdb is readable and is v3 -> execute migration
    *        if not -> skip migration (returns yes)
+   *  1. set log/data.mdb to version 2 (migration in progress)
    *  1. initialize epoch 0i0 (first call to u3_disk_epoc_init())
    *     a. creates epoch directory
    *     b. creates epoch version file
@@ -1345,8 +1367,14 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
    *  3. rollover to new epoch (second call to u3_disk_epoc_init())
    *     a. same as 1a-g but also copies current snapshot between c/d steps
    *  4. delete backup snapshot (c3_unlink() and c3_rmdir() calls)
-   *  5. delete old data.mdb and lock.mdb files (c3_unlink() calls)
+   *  5. set log/data.mdb to version 3 (migration complete)
    */
+  
+  //  set version to 2 (migration in progress)
+  if ( c3n == _disk_save_meta(log_u->mdb_u, "version", (c3_w)U3D_VER2) ) {
+    fprintf(stderr, "disk: failed to set version to 2\r\n");
+    return c3n;
+  }
 
   //  check if lock.mdb is readable in log directory
   c3_o luk_o = c3n;
@@ -1428,15 +1456,11 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     }
   }
 
-  //  delete old lock.mdb and data.mdb files
-  if ( c3_unlink(luk_c) ) {
-    fprintf(stderr, "disk: failed to unlink lock.mdb: %s\r\n",
-                    strerror(errno));
-  }
-  if ( c3_unlink(dut_c) ) {
-    fprintf(stderr, "disk: failed to unlink data.mdb: %s\r\n",
-                    strerror(errno));
-    return c3n;  //  migration succeeds only if we can unlink data.mdb
+  //  set log/data.mdb to version 3 (migration complete)
+  MDB_env* mub_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i);
+  if ( c3n == _disk_save_meta(mub_u, "version", U3D_VER3) ) {
+    fprintf(stderr, "disk: failed to set version to 3\r\n");
+    return c3n;
   }
 
   //  success
@@ -1653,10 +1677,9 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         return 0;
       }
 
-      //  set its meta table version key to U3D_VER3 to ensure that if our pier
-      //  is booted with <= vere-v2, it crashes instead of potentially
-      //  corrupting its event log
-      if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, pir_u->) ) {
+      //  set version to U3D_VER3 to ensure that if our pier is booted with
+      //  <= vere-v2, it crashes instead of potentially corrupting its log
+      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER3) ) {
         fprintf(stderr, "disk: failed to save metadata version\r\n");
         c3_free(log_u);
         return 0;
@@ -1673,10 +1696,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
 
       //  we keep the old event log around forever
       //  
-      //  U3D_VER2 in meta table means in-progress migration to vere-v3
-      //  U3D_VER3 in meta table means migration complete
-      //
-      //  this prevents the pier from booting with <= vere-v2
       if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER2) ) {
         fprintf(stderr, "disk: failed to save metadata version\r\n");
         c3_free(log_u);

From 66a9ebb74e31ad4f293221c8ba7ec28e2c473405 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Fri, 26 Jan 2024 10:35:11 -0500
Subject: [PATCH 03/16] wip: support ships migrated to epoch system by
 vere-v3.0 pre-release

---
 pkg/noun/version.h |  2 +-
 pkg/vere/disk.c    | 20 ++++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/pkg/noun/version.h b/pkg/noun/version.h
index 0b04502d9b..40743c3826 100644
--- a/pkg/noun/version.h
+++ b/pkg/noun/version.h
@@ -24,7 +24,7 @@ typedef c3_w       u3e_version;
  * synchronized between:
  * - u3_disk->ver_w
  * - log/<0iN>/epoc.txt
- * - version field in META table of data.mdb files
+ * - version key in META table of data.mdb files
  */
 
 #define U3D_VER1   1         // <= vere-v2
diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index 80b7aa7758..a312b2a18c 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1659,14 +1659,30 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       return 0;
     }
 
+    //  check if old data.mdb file exists
+    c3_c dut_c[8193];
+    c3_o exs_o = c3n;
+    snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_c);
+    if ( 0 == access(dut_c, F_OK) ) {
+      exs_o = c3y;
+    }
     //  load the old data.mdb file
-    fprintf(stderr, "disk: loading old format\r\n");
-
     if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
       fprintf(stderr, "disk: failed to initialize lmdb\r\n");
       c3_free(log_u);
       return 0;
     }
+    fprintf(stderr, "disk: loaded old log\r\n");
+    //  if old data.mdb did not exist before loading, set it to v3, indicating
+    //  that it is an epoch system pier whose old event log was removed, most
+    //  likely by a development release of vere-v3 or an operator
+    if ( c3n == exs_o ) {
+      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER3) ) {
+        fprintf(stderr, "disk: failed to set version to 3\r\n");
+        c3_free(log_u);
+        return 0;
+      }
+    }
 
     //  if fresh boot, initialize disk U3D_VER3
     //

From 237df39545d37660c795ee31355d68a13037ad36 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Fri, 26 Jan 2024 16:07:01 -0500
Subject: [PATCH 04/16] disk: refactored epoch migration

---
 pkg/vere/disk.c | 420 ++++++++++++++++++++++++++++--------------------
 pkg/vere/main.c |   6 +-
 pkg/vere/vere.h |   2 +-
 3 files changed, 252 insertions(+), 176 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index a312b2a18c..b134369faa 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -685,34 +685,6 @@ _disk_meta_read_cb(void* ptr_v, ssize_t val_i, void* val_v)
   }
 }
 
-/* _disk_meta_read_vers(): read metadata version.
-*/
-c3_o
-_disk_meta_read_vers(MDB_env* mdb_u, c3_w* ver_w)
-{
-  _mdb_val val_u;
-
-  //  version
-  //
-  u3_lmdb_read_meta(mdb_u, &val_u, "version", _disk_meta_read_cb);
-
-  if ( 0 > val_u.hav_i ) {
-    fprintf(stderr, "disk: read meta: no version\r\n");
-    return c3n;
-  }
-  else if ( (1 != val_u.hav_i) || (U3D_VERLAT < *val_u.buf_y) ) {
-    fprintf(stderr, "disk: read vers: unknown version %u\r\n", *val_u.buf_y);
-    return c3n;
-  }
-
-  if ( ver_w ) {
-    c3_y* byt_y = val_u.buf_y;
-    *ver_w = (c3_w)byt_y[0];
-  }
-
-  return c3y;
-}
-
 /* u3_disk_read_meta(): read metadata.
 */
 c3_o
@@ -728,22 +700,13 @@ u3_disk_read_meta(MDB_env* mdb_u,
   //
   u3_lmdb_read_meta(mdb_u, &val_u, "version", _disk_meta_read_cb);
 
-  if ( 0 > val_u.hav_i ) {
-    fprintf(stderr, "disk: read meta: no version\r\n");
-    return c3n;
-  }
-  else if ( (1 != val_u.hav_i) || (U3D_VERLAT < *val_u.buf_y) ) {
-    fprintf(stderr, "disk: read vers: have bytes %zd\r\n", val_u.hav_i);
-    fprintf(stderr, "disk: read vers: unknown version %u\r\n", *val_u.buf_y);
+  if ( 1 != val_u.hav_i ) {
+    fprintf(stderr, "disk: read meta: strange version: %zd bytes\r\n", val_u.hav_i);
     return c3n;
   }
 
   if ( ver_w ) {
-    c3_y* byt_y = val_u.buf_y;
-    *ver_w = (c3_w)byt_y[0]
-           | (c3_w)byt_y[1] << 8
-           | (c3_w)byt_y[2] << 16
-           | (c3_w)byt_y[3] << 24;
+    *ver_w = val_u.buf_y[0];
   }
 
   //  identity
@@ -1108,14 +1071,83 @@ _disk_epoc_meta(u3_disk*    log_u,
   return c3y;
 }
 
+/* u3_disk_epoc_zero: create epoch zero.
+*/
+c3_o
+u3_disk_epoc_zero(u3_disk* log_u)
+{
+  //  create new epoch directory if it doesn't exist
+  c3_c epo_c[8193];
+  c3_i epo_i;
+  snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c);
+  c3_d ret_d = c3_mkdir(epo_c, 0700);
+  if ( ( ret_d < 0 ) && ( errno != EEXIST ) ) {
+    fprintf(stderr, "disk: epoch 0i0 mkdir failed: %s\r\n", strerror(errno));
+    return c3n;
+  }
+
+  //  open epoch directory
+  if ( -1 == (epo_i = c3_open(epo_c, O_RDONLY)) ) {
+    fprintf(stderr, "disk: open epoch dir 0i0 failed: %s\r\n", strerror(errno));
+    goto fail1;
+  }
+
+  //  sync epoch directory
+  if ( -1 == c3_sync(epo_i) ) {  //  XX fdatasync on linux?
+    fprintf(stderr, "disk: sync epoch dir 0i0 failed: %s\r\n", strerror(errno));
+    goto fail2;
+  }
+
+  //  create epoch version file, overwriting any existing file
+  c3_c epv_c[8193];
+  snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c);
+  FILE* epv_f = fopen(epv_c, "w");  // XX errors
+  fprintf(epv_f, "%d", U3D_VER3);
+  fclose(epv_f);
+
+  //  create binary version file, overwriting any existing file
+  c3_c biv_c[8193];
+  snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c);
+  FILE* biv_f = fopen(biv_c, "w");  //  XX errors
+  fprintf(biv_f, URBIT_VERSION);    //  XX append a sentinel for auto-rollover?
+  fclose(biv_f);
+
+  if ( -1 == c3_sync(epo_i) ) {  //  XX fdatasync on linux?
+    fprintf(stderr, "disk: sync epoch dir 0i0 failed: %s\r\n", strerror(errno));
+    goto fail3;
+  }
+  close(epo_i);
+
+  //  load new epoch directory and set it in log_u
+  log_u->epo_d = 0;
+  log_u->ver_w = U3D_VER3;
+
+  //  success
+  return c3y;
+
+fail3:
+  c3_unlink(epv_c);
+  c3_unlink(biv_c);
+fail2:
+  close(epo_i);
+fail1:
+  c3_rmdir(epo_c);
+  return c3n;
+}
+
 /* u3_disk_epoc_init: create new epoch.
 */
 c3_o
-u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
+u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d)
 {
+  u3_assert(epo_d);
+
   //  check if any epoch directories exist
   c3_d lat_d;
-  c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d);
+  if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) {
+    fprintf(stderr, "disk: failed to get last epoch\r\n");
+    return c3n;
+  }
 
   //  create new epoch directory if it doesn't exist
   c3_c epo_c[8193];
@@ -1129,13 +1161,11 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
   }
 
   //  copy snapshot files (skip if first epoch)
-  if ( epo_d > 0 ) {
-    c3_c chk_c[8193];
-    snprintf(chk_c, 8192, "%s/.urb/chk", u3_Host.dir_c);
-    if ( c3n == u3e_backup(chk_c, epo_c, c3y) ) {
-      fprintf(stderr, "disk: copy epoch snapshot failed\r\n");
-      goto fail1;
-    }
+  c3_c chk_c[8193];
+  snprintf(chk_c, 8192, "%s/.urb/chk", u3_Host.dir_c);
+  if ( c3n == u3e_backup(chk_c, epo_c, c3y) ) {
+    fprintf(stderr, "disk: copy epoch snapshot failed\r\n");
+    goto fail1;
   }
 
   if ( -1 == (epo_i = c3_open(epo_c, O_RDONLY)) ) {
@@ -1164,38 +1194,29 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
   fprintf(biv_f, URBIT_VERSION);
   fclose(biv_f);
 
-  //  get metadata from old epoch or unmigrated event log's db
+  //  get metadata from old log
   c3_d     who_d[2];
   c3_o     fak_o;
   c3_w     lif_w;
-  if ( c3y == eps_o ) {  //  skip if no epochs yet
-    if ( c3y != u3_disk_read_meta(log_u->mdb_u, 0, who_d, &fak_o, &lif_w) ) {
-      fprintf(stderr, "disk: failed to read metadata\r\n");
-      goto fail3;
-    }
-
-    u3_lmdb_exit(log_u->mdb_u);
-    log_u->mdb_u = 0;
+  if ( c3y != u3_disk_read_meta(log_u->mdb_u, 0, who_d, &fak_o, &lif_w) ) {
+    fprintf(stderr, "disk: failed to read metadata\r\n");
+    goto fail3;
   }
+  
+  u3_lmdb_exit(log_u->mdb_u);
+  log_u->mdb_u = 0;
 
   //  initialize db of new epoch
-  if ( c3y == u3_Host.ops_u.nuu || epo_d > 0 ) {
-    c3_c dat_c[8193];
-    snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c);
-
-    if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
-      fprintf(stderr, "disk: failed to initialize database\r\n");
-      c3_free(log_u);
-      goto fail3;
-    }
+  if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
+    fprintf(stderr, "disk: failed to initialize database\r\n");
+    c3_free(log_u);
+    goto fail3;
   }
 
   // write the metadata to the database
-  if ( c3y == eps_o ) {
-    if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
-      fprintf(stderr, "disk: failed to save metadata\r\n");
-      goto fail3;
-    }
+  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+    fprintf(stderr, "disk: failed to save metadata\r\n");
+    goto fail3;
   }
 
   if ( -1 == c3_sync(epo_i) ) {  //  XX fdatasync on linux?
@@ -1208,6 +1229,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d)
 
   //  load new epoch directory and set it in log_u
   log_u->epo_d = epo_d;
+  log_u->ver_w = U3D_VER3;
 
   //  success
   return c3y;
@@ -1328,22 +1350,6 @@ u3_disk_epoc_list(u3_disk* log_u, c3_d* sot_d)
   return len_z;
 }
 
-/* _disk_need_migrate: does the disk need to be migrated?
-*/
-static c3_o
-_disk_need_migrate(u3_disk* log_u)
-{
-  //  check if log/data.mdb has version 3
-  c3_w ver_w;
-  if ( c3n == _disk_meta_read_vers(log_u->mdb_u, &ver_w) ) {
-    fprintf(stderr, "disk: failed to read version\r\n");
-  }
-  else if ( U3D_VER3 == ver_w ) {
-    return c3n;
-  }
-  return c3y;
-}
-
 /* _disk_migrate: migrates disk format.
  */
 static c3_o
@@ -1352,10 +1358,10 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   /*  migration steps:
    *  0. detect whether we need to migrate or not
    *     a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration
-   *     b. if log/data.mdb is readable and is v3 -> execute migration
+   *     b. if log/data.mdb is readable and is not v3 -> execute migration
    *        if not -> skip migration (returns yes)
    *  1. set log/data.mdb to version 2 (migration in progress)
-   *  1. initialize epoch 0i0 (first call to u3_disk_epoc_init())
+   *  2. initialize epoch 0i0 (first call to u3_disk_epoc_init())
    *     a. creates epoch directory
    *     b. creates epoch version file
    *     c. creates binary version file
@@ -1363,19 +1369,40 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
    *     e. reads metadata from old database
    *     f. writes metadata to new database
    *     g. loads new epoch directory and sets it in log_u
-   *  2. create hard links to data.mdb and lock.mdb in 0i0/
-   *  3. rollover to new epoch (second call to u3_disk_epoc_init())
-   *     a. same as 1a-g but also copies current snapshot between c/d steps
+   *  3. create hard links to data.mdb and lock.mdb in 0i0/
    *  4. delete backup snapshot (c3_unlink() and c3_rmdir() calls)
    *  5. set log/data.mdb to version 3 (migration complete)
    */
   
+  //  NB: requires that log_u->mdb_u is initialized to log/data.mdb
+  //  XX: put old log in separate pointer (old_u?)?
+
+  //  get metadata from old log
+  c3_d     who_d[2];
+  c3_o     fak_o;
+  c3_w     lif_w;
+  if ( c3y != u3_disk_read_meta(log_u->mdb_u, 0, who_d, &fak_o, &lif_w) ) {
+    fprintf(stderr, "disk: failed to read metadata\r\n");
+    return c3n;
+  }
+
+  //  get first/last event numbers from old log
+  c3_d fir_d, las_d;
+  if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) {
+    fprintf(stderr, "disk: failed to get first/last event numbers\r\n");
+    return c3n;
+  }
+
   //  set version to 2 (migration in progress)
   if ( c3n == _disk_save_meta(log_u->mdb_u, "version", (c3_w)U3D_VER2) ) {
     fprintf(stderr, "disk: failed to set version to 2\r\n");
     return c3n;
   }
 
+  //  finish with old log
+  u3_lmdb_exit(log_u->mdb_u);
+  log_u->mdb_u = 0;
+
   //  check if lock.mdb is readable in log directory
   c3_o luk_o = c3n;
   c3_c luk_c[8193];
@@ -1386,17 +1413,6 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
 
   fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER3);
 
-  //  migrate existing pier which has either:
-  //  - not started the migration, or
-  //  - crashed before completing the migration
-
-  //  get first/last event numbers from pre-migrated lmdb
-  c3_d fir_d, las_d;
-  if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) {
-    fprintf(stderr, "disk: failed to get first/last event numbers\r\n");
-    return c3n;
-  }
-
   // ensure there's a current snapshot
   if ( eve_d != las_d ) {
     fprintf(stderr, "disk: snapshot is out of date, please "
@@ -1407,7 +1423,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   }
 
   //  initialize first epoch "0i0"
-  if ( c3n == u3_disk_epoc_init(log_u, 0) ) {
+  if ( c3n == u3_disk_epoc_zero(log_u) ) {
     fprintf(stderr, "disk: failed to initialize first epoch\r\n");
     return c3n;
   }
@@ -1419,23 +1435,17 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c);
   snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c);
 
-  if ( 0 < c3_link(dut_c, dat_c) ) {
+  if ( c3_link(dut_c, dat_c) ) {
     fprintf(stderr, "disk: failed to create data.mdb hard link\r\n");
     return c3n;
   }
   if ( c3y == luk_o ) {  //  only link lock.mdb if it exists
-    if ( 0 < c3_link(luk_c, lok_c) ) {
+    if ( c3_link(luk_c, lok_c) ) {
       fprintf(stderr, "disk: failed to create lock.mdb hard link\r\n");
       return c3n;
     }
   }
 
-  //  rollover to new epoch
-  if ( c3n == u3_disk_epoc_init(log_u, las_d) ) {
-    fprintf(stderr, "disk: failed to initialize new epoch\r\n");
-    return c3n;
-  }
-
   //  delete backup snapshot
   c3_c bhk_c[8193], nop_c[8193], sop_c[8193];
   snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c);
@@ -1456,12 +1466,56 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     }
   }
 
-  //  set log/data.mdb to version 3 (migration complete)
-  MDB_env* mub_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i);
-  if ( c3n == _disk_save_meta(mub_u, "version", U3D_VER3) ) {
-    fprintf(stderr, "disk: failed to set version to 3\r\n");
+  //  make log/tmp
+  c3_c tmp_c[8193];
+  snprintf(tmp_c, sizeof(tmp_c), "%s/tmp", log_u->com_u->pax_c);
+  if ( c3_mkdir(tmp_c, 0700) && (errno != EEXIST) ) {
+    fprintf(stderr, "disk: failed to create log/tmp: %s\r\n",
+                    strerror(errno));
     return c3n;
   }
+  //  u3_lmdb_init
+  if ( 0 == (log_u->mdb_u = u3_lmdb_init(tmp_c, siz_i)) ) {
+    fprintf(stderr, "disk: failed to initialize database at %s\r\n",
+                    tmp_c);
+    return c3n;
+  }
+  //  u3_disk_save_meta with v3
+  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+    fprintf(stderr, "disk: failed to save metadata\r\n");
+    return c3n;
+  }
+  
+  //  atomic truncation of old log
+  //
+  u3_lmdb_exit(log_u->mdb_u);
+  log_u->mdb_u = 0;
+
+  c3_c trd_c[8193];
+  snprintf(trd_c, sizeof(trd_c), "%s/data.mdb", tmp_c);
+  if ( c3_rename(trd_c, dut_c) ) {
+    fprintf(stderr, "disk: failed mv %s to %s: %s\r\n", trd_c, dut_c,
+                    strerror(errno));
+    return c3n;
+  }
+
+  snprintf(trd_c, sizeof(trd_c), "%s/lock.mdb", tmp_c);
+  if ( c3_unlink(trd_c) && (errno != ENOENT) ) {
+    fprintf(stderr, "disk: failed to delete log/tmp: %s\r\n",
+                    strerror(errno));
+  }
+  if ( c3_rmdir(tmp_c) ) {
+    fprintf(stderr, "disk: failed to delete log/tmp: %s\r\n",
+                    strerror(errno));
+  }
+
+  if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
+    fprintf(stderr, "disk: failed to initialize database at %s\r\n",
+                    epo_c);
+    return c3n;
+  }
+
+  //  XX: maybe rollover
 
   //  success
   fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER3);
@@ -1494,15 +1548,30 @@ _disk_vere_diff(u3_disk* log_u)
 void
 u3_disk_kindly(u3_disk* log_u, c3_d eve_d)
 {
-  if ( c3y == _disk_need_migrate(log_u) ) {
-    if ( c3n == _disk_migrate(log_u, eve_d) ) {
-      fprintf(stderr, "disk: failed to migrate event log\r\n");
-      exit(1);
+  switch ( log_u->ver_w ) {
+    case U3D_VER1:
+    case U3D_VER2: {
+      if ( c3n == _disk_migrate(log_u, eve_d) ) {
+        fprintf(stderr, "disk: failed to migrate event log\r\n");
+        exit(1);
+      }
+      break;
     }
-  }
-  else if ( c3y == _disk_vere_diff(log_u) ) {
-    if ( c3n == u3_disk_epoc_init(log_u, log_u->dun_d) ) {
-      fprintf(stderr, "disk: failed to initialize epoch\r\n");
+
+    case U3D_VER3: {
+      if ( (0 == log_u->epo_d) || 
+           (c3y == _disk_vere_diff(log_u)) )
+      {
+        if ( c3n == u3_disk_epoc_roll(log_u, log_u->dun_d) ) {
+          fprintf(stderr, "disk: failed to initialize epoch\r\n");
+          exit(1);
+        }
+      }
+      break;
+    }
+
+    default: {
+      fprintf(stderr, "disk: unknown disk version: %d\r\n", log_u->ver_w);
       exit(1);
     }
   }
@@ -1666,63 +1735,64 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
     if ( 0 == access(dut_c, F_OK) ) {
       exs_o = c3y;
     }
-    //  load the old data.mdb file
-    if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
-      fprintf(stderr, "disk: failed to initialize lmdb\r\n");
-      c3_free(log_u);
-      return 0;
-    }
-    fprintf(stderr, "disk: loaded old log\r\n");
-    //  if old data.mdb did not exist before loading, set it to v3, indicating
-    //  that it is an epoch system pier whose old event log was removed, most
-    //  likely by a development release of vere-v3 or an operator
-    if ( c3n == exs_o ) {
-      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER3) ) {
-        fprintf(stderr, "disk: failed to set version to 3\r\n");
-        c3_free(log_u);
-        return 0;
-      }
-    }
 
     //  if fresh boot, initialize disk U3D_VER3
     //
     if ( c3y == u3_Host.ops_u.nuu ) {
+      //  ensure old data.mdb file does not exist
+      if ( c3y == exs_o ) {
+        fprintf(stderr, "disk: old data.mdb file exists\r\n");
+        return 0;
+      }
       //  initialize first epoch "0i0"
-      if ( c3n == u3_disk_epoc_init(log_u, 0) ) {
+      if ( c3n == u3_disk_epoc_zero(log_u) ) {
         fprintf(stderr, "disk: failed to initialize first epoch\r\n");
-        return 0;
+        return log_u;  //  XX
       }
+    }
 
-      //  set version to U3D_VER3 to ensure that if our pier is booted with
-      //  <= vere-v2, it crashes instead of potentially corrupting its log
-      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER3) ) {
-        fprintf(stderr, "disk: failed to save metadata version\r\n");
-        c3_free(log_u);
-        return 0;
-      }
+    if ( c3n == exs_o ) {  //  XX handle later
+      fprintf(stderr, "disk: vere-v3.0 pre-release pier needs fixing\r\n");
+      return 0;
     }
-    else if ( c3y == _disk_need_migrate(log_u) ) {
-      c3_d fir_d;
-      if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) {
-        fprintf(stderr, "disk: failed to load latest event from lmdb\r\n");
-        // XX dispose mdb_u
-        c3_free(log_u);
-        return 0;
-      }
 
-      //  we keep the old event log around forever
-      //  
-      if ( c3n == _disk_save_meta(log_u->mdb_u, "version", U3D_VER2) ) {
-        fprintf(stderr, "disk: failed to save metadata version\r\n");
-        c3_free(log_u);
-        return 0;
-      }
-      else {
+    //  load the old data.mdb file
+    if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
+      fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+      c3_free(log_u);
+      return 0;
+    }
+
+    //  read version from old log
+    if ( c3n == u3_disk_read_meta(log_u->mdb_u, &log_u->ver_w, 0, 0, 0) ) {
+      fprintf(stderr, "disk: failed to read metadata\r\n");
+      c3_free(log_u);
+      return 0;
+    }
+
+    switch ( log_u->ver_w ) {
+      case U3D_VER1:
+      case U3D_VER2: {  //  XX maybe finish migration eagerly
+        fprintf(stderr, "disk: loaded old log\r\n");
+        c3_d fir_d;
+        if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) {
+          fprintf(stderr, "disk: failed to load latest event from lmdb\r\n");
+          c3_free(log_u);
+          return 0;
+        }
+
         log_u->sen_d = log_u->dun_d;
-        log_u->ver_w = 0;
 
         return log_u;
       }
+
+      case U3D_VER3: break;
+
+      default: {
+        fprintf(stderr, "disk: unknown log version: %d\r\n", log_u->ver_w);
+        c3_free(log_u);
+        return 0;
+      }
     }
 
     //  get latest epoch number
@@ -1766,7 +1836,9 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
             return 0;
           }
 
-          c3_z len_z = u3_disk_epoc_list(log_u, 0);
+          c3_z dir_z  = u3_disk_epoc_list(log_u, 0);
+          c3_d* sot_d = c3_malloc(dir_z * sizeof(*sot_d));
+          c3_z len_z  = u3_disk_epoc_list(log_u, sot_d);
 
           if ( len_z <= 1 ) {
             fprintf(stderr, "only epoch is bad, bailing out\r\n");
@@ -1774,13 +1846,17 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
             return 0;
           }
 
-          c3_d* sot_d = c3_malloc(len_z * sizeof(*sot_d));
-          u3_disk_epoc_list(log_u, sot_d);
-
           fprintf(stderr, "disk: latest epoch is 0i%" PRIc3_d " is bogus; "
                           "falling back to previous at 0i%" PRIc3_d "\r\n",
                           lat_d, sot_d[1]);
 
+          fprintf(stderr, "kindly: len_z: %ld\r\n", len_z);
+          for ( c3_z i_z = 0; i_z < len_z; i_z++ ) {
+            fprintf(stderr, "kindly: sot_d[%ld]: %" PRIc3_d "\r\n",
+                            i_z, sot_d[i_z]);
+          }
+          return 0;
+
           u3_disk_epoc_kill(log_u, lat_d);
 
           lat_d = sot_d[1];
diff --git a/pkg/vere/main.c b/pkg/vere/main.c
index 930d8981ad..02bbcb8083 100644
--- a/pkg/vere/main.c
+++ b/pkg/vere/main.c
@@ -2484,7 +2484,7 @@ _cw_chop(c3_i argc, c3_c* argv[])
   }
 
   //  create new epoch if latest isn't empty
-  if ( (fir_d != las_d) && (c3n == u3_disk_epoc_init(log_u, las_d)) ) {
+  if ( (fir_d != las_d) && (c3n == u3_disk_epoc_roll(log_u, las_d)) ) {
     fprintf(stderr, "chop: failed to create new epoch\r\n");
     exit(1);
   }
@@ -2576,7 +2576,7 @@ _cw_roll(c3_i argc, c3_c* argv[])
   u3_disk_kindly(log_u, u3_Host.eve_d);
 
   // check if there's a *current* snapshot
-  if ( log_u->dun_d != u3A->eve_d ) {
+  if ( log_u->dun_d != u3_Host.eve_d ) {
     fprintf(stderr, "roll: error: snapshot is out of date, please "
                     "start/shutdown your pier gracefully first\r\n");
     fprintf(stderr, "roll: eve_d: %" PRIc3_d ", dun_d: %" PRIc3_d "\r\n", \
@@ -2595,7 +2595,7 @@ _cw_roll(c3_i argc, c3_c* argv[])
     fprintf(stderr, "roll: latest epoch already empty\r\n");
     exit(0);
   }
-  else if ( c3n == u3_disk_epoc_init(log_u, las_d) ) {
+  else if ( c3n == u3_disk_epoc_roll(log_u, las_d) ) {
     fprintf(stderr, "roll: failed to create new epoch\r\n");
     exit(1);
   }
diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h
index d63c5f598e..8db7523903 100644
--- a/pkg/vere/vere.h
+++ b/pkg/vere/vere.h
@@ -1013,7 +1013,7 @@
       /* u3_disk_epoc_init(): create new epoch.
        */
         c3_o
-        u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d);
+        u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d);
 
       /* u3_disk_epoc_kill(): delete an epoch.
        */

From 1053059da648290db0951cef4e47efa94c648799 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Fri, 26 Jan 2024 16:38:16 -0500
Subject: [PATCH 05/16] disk: handle pre-release piers

---
 pkg/vere/disk.c | 79 +++++++++++++++++++++++++++++++++++++++----------
 pkg/vere/main.c |  1 +
 2 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index b134369faa..be9fde7905 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1555,6 +1555,11 @@ u3_disk_kindly(u3_disk* log_u, c3_d eve_d)
         fprintf(stderr, "disk: failed to migrate event log\r\n");
         exit(1);
       }
+
+      if ( c3n == u3_disk_epoc_roll(log_u, log_u->dun_d) ) {
+        fprintf(stderr, "disk: failed to initialize epoch\r\n");
+        exit(1);
+      }
       break;
     }
 
@@ -1615,7 +1620,7 @@ _disk_epoc_load(u3_disk* log_u, c3_d lat_d)
       return _epoc_late;
     }
 
-    log_u->ver_w = ver_w;
+    log_u->ver_w = ver_w;  //  XX conflicts with data.mdb version
   }
 
   //  set path to latest epoch
@@ -1751,23 +1756,29 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       }
     }
 
-    if ( c3n == exs_o ) {  //  XX handle later
-      fprintf(stderr, "disk: vere-v3.0 pre-release pier needs fixing\r\n");
-      return 0;
-    }
-
-    //  load the old data.mdb file
-    if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
-      fprintf(stderr, "disk: failed to initialize lmdb\r\n");
-      c3_free(log_u);
-      return 0;
+    if ( c3n == exs_o ) {
+      c3_d lat_d;
+      if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) {
+        fprintf(stderr, "disk: no event log anywhere\r\n");
+        return 0;
+      }
+      //  presume pre-release migrated pier
+      log_u->ver_w = U3D_VER3;
     }
+    else {
+      //  load the old data.mdb file
+      if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
+        fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+        c3_free(log_u);
+        return 0;
+      }
 
-    //  read version from old log
-    if ( c3n == u3_disk_read_meta(log_u->mdb_u, &log_u->ver_w, 0, 0, 0) ) {
-      fprintf(stderr, "disk: failed to read metadata\r\n");
-      c3_free(log_u);
-      return 0;
+      //  read version from old log
+      if ( c3n == u3_disk_read_meta(log_u->mdb_u, &log_u->ver_w, 0, 0, 0) ) {
+        fprintf(stderr, "disk: failed to read metadata\r\n");
+        c3_free(log_u);
+        return 0;
+      }
     }
 
     switch ( log_u->ver_w ) {
@@ -1795,6 +1806,12 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       }
     }
 
+    //  close lmdb
+    if ( c3y == exs_o ) {
+      u3_lmdb_exit(log_u->mdb_u);
+      log_u->mdb_u = 0;
+    }
+
     //  get latest epoch number
     c3_d lat_d;
     if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) {
@@ -1821,6 +1838,36 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
           u3t_trace_open(pax_c);
 #endif
 
+          if ( c3n == exs_o ) {
+            fprintf(stderr, "disk: repairing pre-release pier metadata\r\n");
+
+            //  read metadata from epoch's log
+            c3_d who_d[2];
+            c3_o fak_o;
+            c3_w lif_w;
+            if ( c3n == u3_disk_read_meta(log_u->mdb_u, 0, who_d, &fak_o, &lif_w) )
+            {
+              fprintf(stderr, "disk: failed to read metadata\r\n");
+              c3_free(log_u);
+              return 0;
+            }
+
+            MDB_env* dbm_u;
+            if ( 0 == (dbm_u = u3_lmdb_init(log_c, siz_i)) ) {
+              fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+              c3_free(log_u);
+              return 0;
+            }
+
+            if ( c3n == u3_disk_save_meta(dbm_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+              fprintf(stderr, "disk: failed to read metadata\r\n");
+              c3_free(log_u);
+              return 0;
+            }
+
+            u3_lmdb_exit(dbm_u);
+          }
+
           return log_u;
         } break;
 
diff --git a/pkg/vere/main.c b/pkg/vere/main.c
index 02bbcb8083..c6e72a3cdf 100644
--- a/pkg/vere/main.c
+++ b/pkg/vere/main.c
@@ -2222,6 +2222,7 @@ _cw_play_impl(c3_d eve_d, c3_d sap_d, c3_o mel_o, c3_o sof_o, c3_o ful_o)
   }
 
   u3_disk_exit(log_u);
+  //  NB: loom migrations without replay are not saved
   u3m_stop();
 
   return pay_d;

From 2bdd9da1ec1ac7d124bec02492c07177050e5ae1 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Fri, 26 Jan 2024 16:49:02 -0500
Subject: [PATCH 06/16] disk: prepare version macro for epoch system

---
 pkg/noun/events.c  |  6 +++---
 pkg/noun/version.h | 13 ++++++++++---
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/pkg/noun/events.c b/pkg/noun/events.c
index d4f6c92ed0..d5e0ddac0f 100644
--- a/pkg/noun/events.c
+++ b/pkg/noun/events.c
@@ -522,10 +522,10 @@ _ce_patch_verify(u3_ce_patch* pat_u)
   c3_zs ret_zs;
   c3_o  sou_o = c3n;  // south seen
 
-  if ( U3E_VERLAT != pat_u->con_u->ver_w ) {
+  if ( U3P_VERLAT != pat_u->con_u->ver_w ) {
     fprintf(stderr, "loom: patch version mismatch: have %"PRIc3_w", need %u\r\n",
                     pat_u->con_u->ver_w,
-                    U3E_VERLAT);
+                    U3P_VERLAT);
     return c3n;
   }
 
@@ -737,7 +737,7 @@ _ce_patch_compose(c3_w nor_w, c3_w sou_w)
 
     _ce_patch_create(pat_u);
     pat_u->con_u = c3_malloc(sizeof(u3e_control) + (pgs_w * sizeof(u3e_line)));
-    pat_u->con_u->ver_w = U3E_VERLAT;
+    pat_u->con_u->ver_w = U3P_VERLAT;
     pgc_w = 0;
 
     for ( i_w = 0; i_w < nor_w; i_w++ ) {
diff --git a/pkg/noun/version.h b/pkg/noun/version.h
index 40743c3826..6046e99489 100644
--- a/pkg/noun/version.h
+++ b/pkg/noun/version.h
@@ -11,13 +11,13 @@ typedef c3_w       u3v_version;
 #define U3V_VER3   3
 #define U3V_VERLAT U3V_VER3
 
-/* EVENTS
+/* PATCHES
  */
 
 typedef c3_w       u3e_version;
 
-#define U3E_VER1   1
-#define U3E_VERLAT U3E_VER1
+#define U3P_VER1   1
+#define U3P_VERLAT U3P_VER1
 
 /* DISK FORMAT
  *
@@ -32,4 +32,11 @@ typedef c3_w       u3e_version;
 #define U3D_VER3   3         // >= vere-v3
 #define U3D_VERLAT U3D_VER3
 
+/* EPOCH SYSTEM
+ * XX use this instead of U3D_VER for epoc.txt files
+*/
+
+#define U3E_VER1   1
+#define U3E_VERLAT U3E_VER3
+
 #endif /* ifndef U3_VERSION_H */

From 713d1253cdc798eb5009dad699254b49bfe533a5 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 10:13:03 -0500
Subject: [PATCH 07/16] epoch: log linking still succeeds in case of
 pre-existing file

---
 pkg/vere/disk.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index be9fde7905..893f5c029e 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1435,12 +1435,12 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c);
   snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c);
 
-  if ( c3_link(dut_c, dat_c) ) {
+  if ( c3_link(dut_c, dat_c) && (EEXIST != errno) ) {
     fprintf(stderr, "disk: failed to create data.mdb hard link\r\n");
     return c3n;
   }
   if ( c3y == luk_o ) {  //  only link lock.mdb if it exists
-    if ( c3_link(luk_c, lok_c) ) {
+    if ( c3_link(luk_c, lok_c) && (EEXIST != errno) ) {
       fprintf(stderr, "disk: failed to create lock.mdb hard link\r\n");
       return c3n;
     }

From 30f682a5a3312917d4d1819a6121dbeaf6c7d63f Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 10:28:02 -0500
Subject: [PATCH 08/16] epoch: remove debug lines

---
 pkg/vere/disk.c | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index 893f5c029e..6000297e43 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1897,13 +1897,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
                           "falling back to previous at 0i%" PRIc3_d "\r\n",
                           lat_d, sot_d[1]);
 
-          fprintf(stderr, "kindly: len_z: %ld\r\n", len_z);
-          for ( c3_z i_z = 0; i_z < len_z; i_z++ ) {
-            fprintf(stderr, "kindly: sot_d[%ld]: %" PRIc3_d "\r\n",
-                            i_z, sot_d[i_z]);
-          }
-          return 0;
-
           u3_disk_epoc_kill(log_u, lat_d);
 
           lat_d = sot_d[1];

From 6f1a09f37a5fad6a530717bfb8ce4536bfa9ecc5 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 10:42:36 -0500
Subject: [PATCH 09/16] epoch: fix roll function comment

---
 pkg/vere/disk.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index 6000297e43..b283a73508 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1135,7 +1135,7 @@ u3_disk_epoc_zero(u3_disk* log_u)
   return c3n;
 }
 
-/* u3_disk_epoc_init: create new epoch.
+/* u3_disk_epoc_roll: epoch rollover.
 */
 c3_o
 u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d)

From 0d3707cf0f090dabaea6dae5bee499cd62e907b4 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 11:16:03 -0500
Subject: [PATCH 10/16] epoch: use separate versions for disk and epoch system

---
 pkg/noun/version.h |  4 +---
 pkg/vere/disk.c    | 34 +++++++++++++++++-----------------
 pkg/vere/pier.c    |  2 +-
 3 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/pkg/noun/version.h b/pkg/noun/version.h
index 6046e99489..47e43ee336 100644
--- a/pkg/noun/version.h
+++ b/pkg/noun/version.h
@@ -23,7 +23,6 @@ typedef c3_w       u3e_version;
  *
  * synchronized between:
  * - u3_disk->ver_w
- * - log/<0iN>/epoc.txt
  * - version key in META table of data.mdb files
  */
 
@@ -33,10 +32,9 @@ typedef c3_w       u3e_version;
 #define U3D_VERLAT U3D_VER3
 
 /* EPOCH SYSTEM
- * XX use this instead of U3D_VER for epoc.txt files
 */
 
 #define U3E_VER1   1
-#define U3E_VERLAT U3E_VER3
+#define U3E_VERLAT U3E_VER1
 
 #endif /* ifndef U3_VERSION_H */
diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index b283a73508..bcfbe30977 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1102,7 +1102,7 @@ u3_disk_epoc_zero(u3_disk* log_u)
   c3_c epv_c[8193];
   snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c);
   FILE* epv_f = fopen(epv_c, "w");  // XX errors
-  fprintf(epv_f, "%d", U3D_VER3);
+  fprintf(epv_f, "%d", U3E_VERLAT);
   fclose(epv_f);
 
   //  create binary version file, overwriting any existing file
@@ -1120,7 +1120,7 @@ u3_disk_epoc_zero(u3_disk* log_u)
 
   //  load new epoch directory and set it in log_u
   log_u->epo_d = 0;
-  log_u->ver_w = U3D_VER3;
+  log_u->ver_w = U3D_VERLAT;
 
   //  success
   return c3y;
@@ -1184,7 +1184,7 @@ u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d)
   c3_c epv_c[8193];
   snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c);
   FILE* epv_f = fopen(epv_c, "w");  // XX errors
-  fprintf(epv_f, "%d", U3D_VER3);
+  fprintf(epv_f, "%d", U3E_VERLAT);
   fclose(epv_f);
 
   //  create binary version file, overwriting any existing file
@@ -1214,7 +1214,7 @@ u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d)
   }
 
   // write the metadata to the database
-  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
     fprintf(stderr, "disk: failed to save metadata\r\n");
     goto fail3;
   }
@@ -1229,7 +1229,7 @@ u3_disk_epoc_roll(u3_disk* log_u, c3_d epo_d)
 
   //  load new epoch directory and set it in log_u
   log_u->epo_d = epo_d;
-  log_u->ver_w = U3D_VER3;
+  log_u->ver_w = U3D_VERLAT;
 
   //  success
   return c3y;
@@ -1314,7 +1314,9 @@ u3_disk_epoc_list(u3_disk* log_u, c3_d* sot_d)
   c3_z     len_z = 0;
 
   while ( den_u ) {  //  count epochs
-    len_z++;
+    if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, (sot_d + len_z)) ) {
+      len_z++;
+    }
     den_u = den_u->nex_u;
   }
 
@@ -1411,7 +1413,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     luk_o = c3y;
   }
 
-  fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER3);
+  fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VERLAT);
 
   // ensure there's a current snapshot
   if ( eve_d != las_d ) {
@@ -1481,7 +1483,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     return c3n;
   }
   //  u3_disk_save_meta with v3
-  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+  if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
     fprintf(stderr, "disk: failed to save metadata\r\n");
     return c3n;
   }
@@ -1518,7 +1520,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
   //  XX: maybe rollover
 
   //  success
-  fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER3);
+  fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VERLAT);
 
   return c3y;
 }
@@ -1614,13 +1616,11 @@ _disk_epoc_load(u3_disk* log_u, c3_d lat_d)
       return _epoc_fail;
     }
 
-    if ( U3D_VER3 != ver_w ) {
+    if ( U3E_VERLAT != ver_w ) {
       fprintf(stderr, "disk: unknown epoch version: '%s', expected '%d'\r\n",
-                      ver_c, U3D_VER3);
+                      ver_c, U3E_VERLAT);
       return _epoc_late;
     }
-
-    log_u->ver_w = ver_w;  //  XX conflicts with data.mdb version
   }
 
   //  set path to latest epoch
@@ -1741,7 +1741,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       exs_o = c3y;
     }
 
-    //  if fresh boot, initialize disk U3D_VER3
+    //  if fresh boot, initialize disk U3D_VERLAT
     //
     if ( c3y == u3_Host.ops_u.nuu ) {
       //  ensure old data.mdb file does not exist
@@ -1763,7 +1763,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         return 0;
       }
       //  presume pre-release migrated pier
-      log_u->ver_w = U3D_VER3;
+      log_u->ver_w = U3D_VERLAT;
     }
     else {
       //  load the old data.mdb file
@@ -1797,7 +1797,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         return log_u;
       }
 
-      case U3D_VER3: break;
+      case U3D_VERLAT: break;
 
       default: {
         fprintf(stderr, "disk: unknown log version: %d\r\n", log_u->ver_w);
@@ -1859,7 +1859,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
               return 0;
             }
 
-            if ( c3n == u3_disk_save_meta(dbm_u, U3D_VER3, who_d, fak_o, lif_w) ) {
+            if ( c3n == u3_disk_save_meta(dbm_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
               fprintf(stderr, "disk: failed to read metadata\r\n");
               c3_free(log_u);
               return 0;
diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c
index 48f7483bda..c41729e03a 100644
--- a/pkg/vere/pier.c
+++ b/pkg/vere/pier.c
@@ -1644,7 +1644,7 @@ _pier_init(c3_w wag_w, c3_c* pax_c)
       return 0;
     }
 
-    u3_assert( U3D_VER3 == pir_u->log_u->ver_w );
+    u3_assert( U3D_VERLAT == pir_u->log_u->ver_w );
   }
 
   //  initialize compute

From 89600125d88b767dae18e6817f944ce3b10aed7b Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 11:36:53 -0500
Subject: [PATCH 11/16] epoch: update migration comment

---
 pkg/vere/disk.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index bcfbe30977..cff85e96c0 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1363,17 +1363,17 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
    *     b. if log/data.mdb is readable and is not v3 -> execute migration
    *        if not -> skip migration (returns yes)
    *  1. set log/data.mdb to version 2 (migration in progress)
-   *  2. initialize epoch 0i0 (first call to u3_disk_epoc_init())
+   *  2. initialize epoch 0i0
    *     a. creates epoch directory
    *     b. creates epoch version file
    *     c. creates binary version file
-   *     d. initializes database
-   *     e. reads metadata from old database
-   *     f. writes metadata to new database
-   *     g. loads new epoch directory and sets it in log_u
+   *     d. creates hard links to data.mdb and lock.mdb in 0i0/
+   *     e. deletes backup snapshot
    *  3. create hard links to data.mdb and lock.mdb in 0i0/
-   *  4. delete backup snapshot (c3_unlink() and c3_rmdir() calls)
-   *  5. set log/data.mdb to version 3 (migration complete)
+   *  4. use scratch space to initialize new log/data.db in log/tmp
+   *  5. save old metadata to new db in scratch space
+   *  6. clobber old log/data.mdb with new log/tmp/data.mdb
+   *  7. open epoch lmdb and set it in log_u
    */
   
   //  NB: requires that log_u->mdb_u is initialized to log/data.mdb
@@ -1468,7 +1468,9 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
     }
   }
 
-  //  make log/tmp
+  //  use scratch space to initialize new log/data.db
+  //  and clobber old log/data.db
+  //
   c3_c tmp_c[8193];
   snprintf(tmp_c, sizeof(tmp_c), "%s/tmp", log_u->com_u->pax_c);
   if ( c3_mkdir(tmp_c, 0700) && (errno != EEXIST) ) {
@@ -1476,13 +1478,13 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
                     strerror(errno));
     return c3n;
   }
-  //  u3_lmdb_init
+
   if ( 0 == (log_u->mdb_u = u3_lmdb_init(tmp_c, siz_i)) ) {
     fprintf(stderr, "disk: failed to initialize database at %s\r\n",
                     tmp_c);
     return c3n;
   }
-  //  u3_disk_save_meta with v3
+
   if ( c3n == u3_disk_save_meta(log_u->mdb_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
     fprintf(stderr, "disk: failed to save metadata\r\n");
     return c3n;

From 1329ee12a4ab0430df7dd016c5dedc2396d1fb04 Mon Sep 17 00:00:00 2001
From: Matthew LeVan <matthew@coeli.network>
Date: Mon, 29 Jan 2024 14:17:51 -0500
Subject: [PATCH 12/16] epoch: ensure old mdb *and* epochs exist before trying
 to repair metadata

---
 pkg/vere/disk.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index cff85e96c0..f0a1abba2f 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1743,6 +1743,13 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       exs_o = c3y;
     }
 
+    //  check if epochs exist
+    c3_o epx_o = c3n;
+    c3_d lax_d;
+    if ( c3y == u3_disk_epoc_last(log_u, &lax_d) ) {
+      epx_o = c3y;
+    }
+
     //  if fresh boot, initialize disk U3D_VERLAT
     //
     if ( c3y == u3_Host.ops_u.nuu ) {
@@ -1840,7 +1847,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
           u3t_trace_open(pax_c);
 #endif
 
-          if ( c3n == exs_o ) {
+          if ( c3n == exs_o && c3y == epx_o ) {
             fprintf(stderr, "disk: repairing pre-release pier metadata\r\n");
 
             //  read metadata from epoch's log

From d299a548f8805e4973048ed273742d6c2da3bf9c Mon Sep 17 00:00:00 2001
From: Joe Bryan <joemfb@gmail.com>
Date: Tue, 30 Jan 2024 10:01:26 -0500
Subject: [PATCH 13/16] Revert "epoch: ensure old mdb *and* epochs exist before
 trying to repair metadata"

This reverts commit 1329ee12a4ab0430df7dd016c5dedc2396d1fb04.
---
 pkg/vere/disk.c | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index f0a1abba2f..cff85e96c0 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1743,13 +1743,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       exs_o = c3y;
     }
 
-    //  check if epochs exist
-    c3_o epx_o = c3n;
-    c3_d lax_d;
-    if ( c3y == u3_disk_epoc_last(log_u, &lax_d) ) {
-      epx_o = c3y;
-    }
-
     //  if fresh boot, initialize disk U3D_VERLAT
     //
     if ( c3y == u3_Host.ops_u.nuu ) {
@@ -1847,7 +1840,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
           u3t_trace_open(pax_c);
 #endif
 
-          if ( c3n == exs_o && c3y == epx_o ) {
+          if ( c3n == exs_o ) {
             fprintf(stderr, "disk: repairing pre-release pier metadata\r\n");
 
             //  read metadata from epoch's log

From 59b4f034d2995b7b5de046dec530e2f69cda5505 Mon Sep 17 00:00:00 2001
From: Joe Bryan <joemfb@gmail.com>
Date: Tue, 30 Jan 2024 10:01:40 -0500
Subject: [PATCH 14/16] vere: open lmdb environment after creating epoc 0i0 on
 boot

---
 pkg/vere/disk.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index cff85e96c0..754e9d8c39 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1749,19 +1749,31 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
       //  ensure old data.mdb file does not exist
       if ( c3y == exs_o ) {
         fprintf(stderr, "disk: old data.mdb file exists\r\n");
+        c3_free(log_u);
         return 0;
       }
+
       //  initialize first epoch "0i0"
       if ( c3n == u3_disk_epoc_zero(log_u) ) {
         fprintf(stderr, "disk: failed to initialize first epoch\r\n");
-        return log_u;  //  XX
+        c3_free(log_u);
+        return 0;
+      }
+
+      if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
+        fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+        c3_free(log_u);
+        return 0;
       }
+
+      return log_u;
     }
 
     if ( c3n == exs_o ) {
       c3_d lat_d;
       if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) {
         fprintf(stderr, "disk: no event log anywhere\r\n");
+        c3_free(log_u);
         return 0;
       }
       //  presume pre-release migrated pier

From 8d97aacbb820d398b3eb999a4275093ff11719dd Mon Sep 17 00:00:00 2001
From: Joe Bryan <joemfb@gmail.com>
Date: Tue, 30 Jan 2024 10:40:02 -0500
Subject: [PATCH 15/16] disk: *correctly* open lmdb environment in epoc 0i0 on
 boot

---
 pkg/vere/disk.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index 754e9d8c39..d12d1a92c6 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -1760,7 +1760,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
         return 0;
       }
 
-      if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
+      if ( _epoc_good != _disk_epoc_load(log_u, 0) ) {
         fprintf(stderr, "disk: failed to initialize lmdb\r\n");
         c3_free(log_u);
         return 0;

From d28cbd3453cd2924ce5b8cd4c7f9d63ed70a766f Mon Sep 17 00:00:00 2001
From: Joe Bryan <joemfb@gmail.com>
Date: Tue, 30 Jan 2024 10:40:26 -0500
Subject: [PATCH 16/16] disk: save top-level metadata on boot

---
 pkg/vere/disk.c | 40 +++++++++++++++++++++++++++++-----------
 pkg/vere/pier.c |  7 +++++++
 pkg/vere/vere.h |  8 ++++++++
 3 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c
index d12d1a92c6..b510f75fb9 100644
--- a/pkg/vere/disk.c
+++ b/pkg/vere/disk.c
@@ -663,6 +663,33 @@ u3_disk_save_meta(MDB_env* mdb_u,
   return c3y;
 }
 
+
+/* u3_disk_save_meta_meta(): save meta metadata.
+*/
+c3_o
+u3_disk_save_meta_meta(c3_c* log_c,
+                       c3_d  who_d[2],
+                       c3_o  fak_o,
+                       c3_w  lif_w)
+{
+  MDB_env* dbm_u;
+
+  if ( 0 == (dbm_u = u3_lmdb_init(log_c, siz_i)) ) {
+    fprintf(stderr, "disk: failed to initialize meta-lmdb\r\n");
+    return c3n;
+  }
+
+  if ( c3n == u3_disk_save_meta(dbm_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
+    fprintf(stderr, "disk: failed to save metadata\r\n");
+    return c3n;
+  }
+
+  u3_lmdb_exit(dbm_u);
+
+  return c3y;
+}
+
+
 typedef struct {
   ssize_t hav_i;
   c3_y    buf_y[16];
@@ -1866,20 +1893,11 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
               return 0;
             }
 
-            MDB_env* dbm_u;
-            if ( 0 == (dbm_u = u3_lmdb_init(log_c, siz_i)) ) {
-              fprintf(stderr, "disk: failed to initialize lmdb\r\n");
+            if ( c3n == u3_disk_save_meta_meta(log_c, who_d, fak_o, lif_w) ) {
+              fprintf(stderr, "disk: failed to save top-level metadata\r\n");
               c3_free(log_u);
               return 0;
             }
-
-            if ( c3n == u3_disk_save_meta(dbm_u, U3D_VERLAT, who_d, fak_o, lif_w) ) {
-              fprintf(stderr, "disk: failed to read metadata\r\n");
-              c3_free(log_u);
-              return 0;
-            }
-
-            u3_lmdb_exit(dbm_u);
           }
 
           return log_u;
diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c
index c41729e03a..7514e17155 100644
--- a/pkg/vere/pier.c
+++ b/pkg/vere/pier.c
@@ -1905,6 +1905,13 @@ _pier_boot_plan(u3_pier* pir_u,
     return c3n;
   }
 
+  if ( c3n == u3_disk_save_meta_meta(pir_u->log_u->com_u->pax_c,
+                                     pir_u->who_d, pir_u->fak_o, pir_u->lif_w) )
+  {
+    fprintf(stderr, "disk: failed to save top-level metadata\r\n");
+    return c3n;
+  }
+
   //  insert boot sequence directly
   //
   //    Note that these are not ovum or (pair @da ovum) events,
diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h
index 8db7523903..d7a1439fc8 100644
--- a/pkg/vere/vere.h
+++ b/pkg/vere/vere.h
@@ -990,6 +990,14 @@
                           c3_o     fak_o,
                           c3_w     lif_w);
 
+      /* u3_disk_save_meta_meta(): save meta metadata.
+      */
+        c3_o
+        u3_disk_save_meta_meta(c3_c* log_c,
+                               c3_d  who_d[2],
+                               c3_o  fak_o,
+                               c3_w  lif_w);
+
       /* u3_disk_read(): read [len_d] events starting at [eve_d].
       */
         void