support both new style and old style bootstraping

Bootstrap need to know how to boot networks that
is not using new style tagging yet. This is because
bootstrap binary is used by ALL networks unfortunately.

.github/workflows/bin-package-18.04.yaml

@@ -25,7 +25,8 @@ jobs:
           if [ "${{ github.ref_type }}" = "tag" ]; then
             echo "reference=${ref#refs/tags/}" >> $GITHUB_OUTPUT
           else
-            echo "reference=${{ github.sha }}" >> $GITHUB_OUTPUT
+            reference="${{ github.sha }}"
+            echo "reference=${reference:0:7}" >> $GITHUB_OUTPUT
           fi
       - name: Setup basesystem
         run: |

.github/workflows/bin-package.yaml

@@ -24,7 +24,8 @@ jobs:
           if [ "${{ github.ref_type }}" = "tag" ]; then
             echo "reference=${ref#refs/tags/}" >> $GITHUB_OUTPUT
           else
-            echo "reference=${{ github.sha }}" >> $GITHUB_OUTPUT
+            reference="${{ github.sha }}"
+            echo "reference=${reference:0:7}" >> $GITHUB_OUTPUT
           fi
       - name: Setup basesystem
         run: |

.github/workflows/publish.yaml

@@ -32,7 +32,8 @@ jobs:
           if [ "${{ github.ref_type }}" = "tag" ]; then
             echo "reference=${ref#refs/tags/}" >> $GITHUB_OUTPUT
           else
-            echo "reference=${{ github.sha }}" >> $GITHUB_OUTPUT
+            reference="${{ github.sha }}"
+            echo "reference=${reference:0:7}" >> $GITHUB_OUTPUT
           fi
 
       - name: Set version of build
@@ -71,7 +72,7 @@ jobs:
           action: crosstag
           user: tf-zos
           name: development
-          target: tf-autobuilder/${{ github.sha }}
+          target: tf-autobuilder/${{ steps.tag.outputs.reference }}
       # compatibility with old release
       # this is needed so old machines that is already running on devnet
       # gets the new code to be able to use the new release tag

bootstrap/bootstrap/src/bootstrap.rs

@@ -1,3 +1,7 @@
+use crate::hub::Flist;
+use crate::hub::Kind;
+use crate::hub::Repo;
+
 use super::config;
 use super::hub;
 use super::workdir::WorkDir;
@@ -7,15 +11,17 @@ use anyhow::{Context, Result};
 use config::{RunMode, Version};
 use retry;
 
-const FLIST_REPO: &str = "tf-zos";
+const ZOS_REPO: &str = "tf-zos";
 const BIN_REPO_V2: &str = "tf-zos-bins";
 const BIN_REPO_V3: &str = "tf-zos-v3-bins";
 
 const FLIST_INFO_FILE: &str = "/tmp/flist.info";
 const FLIST_NAME_FILE: &str = "/tmp/flist.name";
+const FLIST_TAG_FILE: &str = "/tmp/tag.info";
+
 const WORKDIR: &str = "/tmp/bootstrap";
 
-fn boostrap_zos(cfg: &config::Config) -> Result<()> {
+fn bootstrap_zos(cfg: &config::Config) -> Result<()> {
     let flist = match &cfg.runmode {
         RunMode::Prod => match &cfg.version {
             Version::V3 => "zos:production-3:latest.flist",
@@ -31,8 +37,8 @@ fn boostrap_zos(cfg: &config::Config) -> Result<()> {
         },
     };
 
-    debug!("using flist: {}/{}", FLIST_REPO, flist);
-    let repo = hub::Repo::new(FLIST_REPO);
+    debug!("using flist: {}/{}", ZOS_REPO, flist);
+    let repo = hub::Repo::new(ZOS_REPO);
     let flist = retry::retry(retry::delay::Exponential::from_millis(200), || {
         info!("get flist info: {}", flist);
         let info = match repo.get(flist) {
@@ -53,22 +59,11 @@ fn boostrap_zos(cfg: &config::Config) -> Result<()> {
 
     // write down boot info for other system components (like upgraded)
     flist.write(FLIST_INFO_FILE)?;
-    std::fs::write(FLIST_NAME_FILE, format!("{}/{}", FLIST_REPO, flist.name))?;
+    std::fs::write(FLIST_NAME_FILE, format!("{}/{}", ZOS_REPO, flist.name))?;
 
     install_package(&flist)
 }
 
-/// bootstrap stage install and starts all zos daemons
-pub fn bootstrap(cfg: &config::Config) -> Result<()> {
-    debug!("runmode: {:?}", cfg.runmode);
-    let result = WorkDir::run(WORKDIR, || {
-        boostrap_zos(cfg)?;
-        Ok(())
-    })?;
-
-    result
-}
-
 /// update stage make sure we are running latest
 /// version of bootstrap
 pub fn update(cfg: &config::Config) -> Result<()> {
@@ -121,15 +116,54 @@ fn update_bootstrap(debug: bool) -> Result<()> {
 
 ///install installs all binaries from the tf-zos-bins repo
 pub fn install(cfg: &config::Config) -> Result<()> {
+    let repo = Repo::new(ZOS_REPO);
+
+    let runmode = cfg.runmode.to_string();
+    // we need to list all taglinks inside the repo
+
+    let mut tag = None;
+    for list in repo.list()? {
+        if list.kind == Kind::TagLink && list.name == runmode {
+            tag = Some(list);
+            break;
+        }
+    }
+
+    if let Some(ref tag) = tag {
+        info!("found tag {} => {:?}", tag.name, tag.target);
+    }
+
     let result = WorkDir::run(WORKDIR, || -> Result<()> {
-        install_packages(cfg)?;
-        Ok(())
+        match tag {
+            None => {
+                // old style bootstrap.
+                // we need to install binaries and zos from 2 different
+                // places
+                // we also track which binaries are installed individually
+                install_packages_old(cfg)?;
+                bootstrap_zos(cfg)
+            }
+            Some(tag) => {
+                // new style bootstrap
+                // we need then to
+                tag.write(FLIST_TAG_FILE)?;
+
+                let (repo, tag) = tag.tag_link();
+                let client = Repo::new(repo);
+                let packages = client.list_tag(tag)?.context("tag is not found")?;
+
+                // new style setup, just install every thing.
+                install_packages(&packages)
+
+                //TODO: write down which version of the tag is installed
+            }
+        }
     })?;
 
     result
 }
 
-fn install_packages(cfg: &config::Config) -> Result<()> {
+fn install_packages_old(cfg: &config::Config) -> Result<()> {
     let name = match cfg.version {
         Version::V3 => BIN_REPO_V3,
     };
@@ -180,6 +214,14 @@ fn install_packages(cfg: &config::Config) -> Result<()> {
     Ok(())
 }
 
+fn install_packages(packages: &[Flist]) -> Result<()> {
+    for package in packages {
+        install_package(&package)?;
+    }
+
+    Ok(())
+}
+
 fn install_package(flist: &hub::Flist) -> Result<()> {
     let result = retry::retry(retry::delay::Exponential::from_millis(200), || {
         info!("download flist: {}", flist.name);

bootstrap/bootstrap/src/config.rs

@@ -1,3 +1,5 @@
+use std::fmt::Display;
+
 use super::kparams;
 use anyhow::Result;
 use clap::{App, Arg};
@@ -10,6 +12,19 @@ pub enum RunMode {
     QA,
 }
 
+impl Display for RunMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let d = match self {
+            Self::Prod => "production",
+            Self::QA => "qa",
+            Self::Test => "testing",
+            Self::Dev => "development",
+        };
+
+        f.write_str(d)
+    }
+}
+
 #[derive(Debug)]
 pub enum Version {
     V3,
@@ -20,9 +35,9 @@ fn runmode() -> Result<RunMode> {
     let mode = match params.get("runmode") {
         Some(mode) => match mode {
             Some(mode) => match mode.as_ref() {
-                "prod" => RunMode::Prod,
-                "dev" => RunMode::Dev,
-                "test" => RunMode::Test,
+                "prod" | "production" => RunMode::Prod,
+                "dev" | "development" => RunMode::Dev,
+                "test" | "testing" => RunMode::Test,
                 "qa" => RunMode::QA,
                 m => {
                     bail!("unknown runmode: {}", m);

bootstrap/bootstrap/src/hub.rs

@@ -2,7 +2,6 @@ use anyhow::Result;
 use reqwest::{blocking::get, StatusCode};
 use serde::{Deserialize, Serialize};
 use serde_json;
-use std::fmt::format;
 use std::fs::{write, OpenOptions};
 use std::io::copy;
 use std::path::Path;
@@ -21,6 +20,8 @@ pub enum Kind {
     Symlink,
     #[serde(rename = "tag")]
     Tag,
+    #[serde(rename = "taglink")]
+    TagLink,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -61,6 +62,41 @@ impl Repo {
         Ok(info)
     }
 
+    pub fn list_tag<S: AsRef<str>>(&self, tag: S) -> Result<Option<Vec<Flist>>> {
+        let tag = tag.as_ref();
+
+        let url = format!("{}/api/flist/{}/tags/{}", HUB, self.name, tag);
+        let response = get(&url)?;
+        let mut info: Vec<Flist> = match response.status() {
+            StatusCode::OK => response.json()?,
+            StatusCode::NOT_FOUND => return Ok(None),
+            s => bail!("failed to get flist info: {}", s),
+        };
+
+        // when listing tags. The flists inside have target as full name
+        // so we need
+        for flist in info.iter_mut() {
+            if flist.kind == Kind::Regular {
+                // this is impossible because tags can only have symlinks
+                continue;
+            }
+
+            let target = match &flist.target {
+                None => {
+                    // this is also not possible may be we should return an error
+                    // but we can just skip for now
+                    continue;
+                }
+                Some(target) => target,
+            };
+
+            flist.url = format!("{}/{}", HUB, target);
+        }
+
+        Ok(Some(info))
+    }
+
+    /// gets flist information from name. the flist must be of type flist or symlink
     pub fn get<T>(&self, flist: T) -> Result<Flist>
     where
         T: AsRef<str>,
@@ -78,6 +114,21 @@ impl Repo {
 }
 
 impl Flist {
+    // tag_link return the target repo and tag name for
+    // a taglink flist
+    pub fn tag_link(self) -> (String, String) {
+        if self.kind != Kind::TagLink {
+            panic!("invalid flist type must be a taglink");
+        }
+
+        let target = self.target.unwrap();
+        let parts: Vec<&str> = target.split('/').collect();
+        assert_eq!(parts.len(), 3, "link must be 3 parts");
+        assert_eq!(parts[1], "tags");
+
+        (parts[0].to_owned(), parts[2].to_owned())
+    }
+
     /// download the flist to `out`
     pub fn download<T>(&self, out: T) -> Result<()>
     where
@@ -122,6 +173,22 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_list_tag() -> Result<()> {
+        let repo = Repo::new(String::from("tf-autobuilder"));
+
+        let list = repo.list_tag("7a704e4712d6b173ffb877fa5a19efe4da6d2e5c")?;
+
+        assert!(list.is_some());
+        let list = list.unwrap();
+
+        assert_ne!(list.len(), 0);
+
+        assert!(repo.list_tag("wrong")?.is_none());
+
+        Ok(())
+    }
+
     #[test]
     fn test_download_flist() -> Result<()> {
         let repo = Repo::new(String::from("azmy"));

bootstrap/bootstrap/src/main.rs

@@ -41,8 +41,6 @@ fn app() -> Result<()> {
         },
         // install all system binaries
         |cfg| -> Result<()> { bootstrap::install(cfg) },
-        // install and start all services
-        |cfg| -> Result<()> { bootstrap::bootstrap(cfg) },
     ];
 
     let index = config.stage as usize - 1;