allow to sign apt debians

buildkite/scripts/debian/promote.sh

@@ -12,6 +12,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -s|--from-component) FROM_COMPONENT="$2"; shift;;
   -t|--to-component) TO_COMPONENT="$2"; shift;;
   --new-name) NEW_NAME="$2"; shift;;
+  --repo-key) REPO_KEY="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -28,6 +29,7 @@ function usage() {
   echo "  -s, --from-component   The source channel in which package currently resides"
   echo "  -t, --to-component     The target channel for package (unstable, alpha, beta etc.)"
   echo "  -c, --codename         The Debian codename (bullseye, focal etc.)"
+  echo "  --repo-key         The Debian target repo key"
   echo ""
   echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly"
   exit 1
@@ -40,6 +42,7 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi;
 if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi;
 if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi;
+if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi;
 
 # check for AWS Creds
 if [ -z "$AWS_ACCESS_KEY_ID" ]; then
@@ -65,5 +68,6 @@ else
     --new-version $NEW_VERSION \
     --suite $FROM_COMPONENT \
     --new-suite $TO_COMPONENT \
-    --new-name $NEW_NAME
+    --new-name $NEW_NAME \
+    --sign $REPO_KEY
 fi

buildkite/scripts/debian/publish.sh

@@ -15,8 +15,17 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     exit 0
 fi
 
+if [ -z "${SIGN:-}" ]; then
+  SIGN_ARG=""
+else
+  sudo chown -R opam ~/.gnupg/
+  gpg --batch --yes --import /var/secrets/debian/key.gpg
+  SIGN_ARG="--sign $SIGN"
+fi
+
 source scripts/debian/publish.sh \
   --names "${DOWNLOAD_FOLDER}/mina-*.deb" \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \
-  --codename $MINA_DEB_CODENAME  
+  --codename $MINA_DEB_CODENAME \
+  $SIGN_ARG

buildkite/scripts/download-artifact-from-cache.sh

@@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then
 fi
 
 DOWNLOAD_BIN=gsutil
-PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID}
+PREFIX=gs://buildkite_k8s/coda/shared/0193492f-2c3f-4dde-8e38-b1c9c36ccab5
 FILE="$1"
 REMOTE_LOCATION="$2"
 OPTS=${3:-""}

buildkite/src/Command/MinaArtifact.dhall

@@ -46,6 +46,7 @@ let MinaBuildSpec =
           , mode : PipelineMode.Type
           , tags : List PipelineTag.Type
           , channel : DebianChannel.Type
+          , debianRepo : DebianRepo.Type
           }
       , default =
           { prefix = "MinaArtifact"
@@ -58,6 +59,7 @@ let MinaBuildSpec =
           , mode = PipelineMode.Type.PullRequest
           , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
           , channel = DebianChannel.Type.Unstable
+          , debianRepo = DebianRepo.Type.PackagesO1Test
           }
       }
 
@@ -112,12 +114,15 @@ let publish_to_debian_repo =
                 Toolchain.select
                   spec.toolchainSelectMode
                   spec.debVersion
-                  [ "AWS_ACCESS_KEY_ID"
-                  , "AWS_SECRET_ACCESS_KEY"
-                  , "MINA_DEB_CODENAME=${DebianVersions.lowerName
-                                           spec.debVersion}"
-                  , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}"
-                  ]
+                  (   [ "AWS_ACCESS_KEY_ID"
+                      , "AWS_SECRET_ACCESS_KEY"
+                      , "MINA_DEB_CODENAME=${DebianVersions.lowerName
+                                               spec.debVersion}"
+                      , "MINA_DEB_RELEASE=${DebianChannel.lowerName
+                                              spec.channel}"
+                      ]
+                    # DebianRepo.keyIdEnvList spec.debianRepo
+                  )
                   "./buildkite/scripts/debian/publish.sh"
             , label =
                 "Publish Mina for ${DebianVersions.capitalName

buildkite/src/Constants/DebianRepo.dhall

@@ -1,3 +1,11 @@
+let Prelude = ../External/Prelude.dhall
+
+let Optional/map = Prelude.Optional.map
+
+let Optional/default = Prelude.Optional.default
+
+let Optional/toList = Prelude.Optional.toList
+
 let DebianRepo
     : Type
     = < Local | PackagesO1Test >
@@ -10,4 +18,94 @@ let address =
             }
             repo
 
-in  { Type = DebianRepo, address = address }
+let bucket =
+          \(repo : DebianRepo)
+      ->  merge
+            { Local = None Text, PackagesO1Test = Some "packages.o1test.net" }
+            repo
+
+let bucket_or_default =
+          \(repo : DebianRepo)
+      ->  let maybeBucket =
+                Optional/map
+                  Text
+                  Text
+                  (\(bucket : Text) -> bucket)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeBucket
+
+let bucketArg =
+          \(repo : DebianRepo)
+      ->  let maybeBucket =
+                Optional/map
+                  Text
+                  Text
+                  (\(bucket : Text) -> "--bucket " ++ bucket)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeBucket
+
+let keyId =
+          \(repo : DebianRepo)
+      ->  merge { Local = None Text, PackagesO1Test = None Text } repo
+
+let keyAddress =
+          \(repo : DebianRepo)
+      ->  merge { Local = None Text, PackagesO1Test = None Text } repo
+
+let keyAddressArg =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(key : Text) -> "--key-path " ++ key)
+                  (keyAddress repo)
+
+          in  Optional/default Text "" maybeKey
+
+let keyArg =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "--sign " ++ repo)
+                  (keyId repo)
+
+          in  Optional/default Text "" maybeKey
+
+let keyIdEnvList =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "SIGN=" ++ repo)
+                  (keyId repo)
+
+          in  Optional/toList Text maybeKey
+
+let bucketEnv =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "BUCKET=" ++ repo)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeKey
+
+in  { Type = DebianRepo
+    , keyIdEnvList = keyIdEnvList
+    , keyAddressArg = keyAddressArg
+    , address = address
+    , bucket = bucket
+    , bucket_or_default = bucket_or_default
+    , bucketArg = bucketArg
+    , bucketEnv = bucketEnv
+    , keyId = keyId
+    , keyArg = keyArg
+    }

scripts/debian/publish.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eo pipefail
+set -eox pipefail
 
 CLEAR='\033[0m'
 RED='\033[0;31m'
@@ -11,6 +11,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -r|--release) DEB_RELEASE="$2"; shift;;
   -v|--version) DEB_VERSION="$2"; shift;;
   -c|--codename) DEB_CODENAME="$2"; shift;;
+  -s|--sign) SIGN="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -23,6 +24,7 @@ function usage() {
   echo "  -r, --release       The Debian release"
   echo "  -v, --version       The Debian version"
   echo "  -c, --codename      The Debian codename"
+  echo "  -s, --sign          The Debian key id used for sign"
   echo ""
   echo "Example: $0 --name mina-archive --release unstable --version 2.0.0-rc1-48efea4 --codename bullseye "
   exit 1
@@ -34,6 +36,12 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi;
 
 
+if [[ -z "${SIGN:-}" ]]; then 
+  SIGN_ARG=""
+else
+  SIGN_ARG="--sign=$SIGN"
+fi
+
 BUCKET_ARG="--bucket=packages.o1test.net"
 S3_REGION_ARG="--s3-region=us-west-2"
 # utility for publishing deb repo with commons options
@@ -47,20 +55,24 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \
   --fail-if-exists \
   --lock \
   --preserve-versions \
-  --cache-control=max-age=120"
+  --cache-control=max-age=120 \
+  $SIGN_ARG"
+
+if [[ -z "${PASSPHRASE:-}" ]]; then
+  GPG_OPTS=""
+else
+  GPG_OPTS="--gpg-options=\"--batch --pinentry-mode=loopback --yes "
+fi
+
+
 
 echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}"
 # Upload the deb files to s3.
 # If this fails, attempt to remove the lockfile and retry.
 for _ in {1..10}; do (
-  ${DEBS3_UPLOAD} \
-    --component "${DEB_RELEASE}" \
-    --codename "${DEB_CODENAME}" \
-    "${DEB_NAMES}"
+  ${DEBS3_UPLOAD} "--component ${DEB_RELEASE} --codename ${DEB_CODENAME} ${GPG_OPTS}" "${DEB_NAMES}"
 ) && break || scripts/debian/clear-s3-lockfile.sh; done
 
-debs=()
-
 for deb in $DEB_NAMES
 do
   # extracting name from debian package path. E.g:

scripts/debian/reversion.sh

@@ -3,6 +3,7 @@ set -eo pipefail
 
 CLEAR='\033[0m'
 RED='\033[0;31m'
+BUCKET=packages.o1test.net
 
 while [[ "$#" -gt 0 ]]; do case $1 in
   -d|--deb) DEB="$2"; shift;;
@@ -14,6 +15,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   --new-version) NEW_VERSION="$2"; shift;;
   --suite) SUITE="$2"; shift;;
   --new-suite) NEW_SUITE="$2"; shift;;
+  --sign) SIGN="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -30,6 +32,7 @@ function usage() {
   echo "  --new-version     The New Debian version"
   echo "  --suite           The Current Debian suite"
   echo "  --new-suite       The New Debian suite"
+  echo "  --sign            The Public Key id, which is used to sign package. Key must be stored locally"
   echo ""
   echo "Example: $0 --deb mina-archive --version 2.0.0-rc1-48efea4 --new-version 2.0.0-rc1 --codename bullseye --release unstable --new-release umt"
   exit 1
@@ -39,18 +42,21 @@ if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$DEB; fi;
 if [[ -z "$NEW_RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi;
 if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$VERSION; fi;
 if [[ -z "$NEW_SUITE" ]]; then NEW_SUITE=$SUITE; fi;
-
 if [[ -z "$DEB" ]]; then NEW_NAME=$DEB; fi;
 if [[ -z "$RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi;
 if [[ -z "$VERSION" ]]; then NEW_VERSION=$VERSION; fi;
 if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi;
-
+if [[ -z "$SIGN" ]]; then 
+  SIGN_ARG=""
+else 
+  SIGN_ARG="--sign $SIGN"
+fi
 
 function rebuild_deb() {
   rm -f "${DEB}_${VERSION}.deb"
   rm -rf "${NEW_NAME}_${NEW_VERSION}"
 
-  wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb
+  wget https://s3.us-west-2.amazonaws.com/${BUCKET}/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb
   dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}"
   sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control"
   sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control"
@@ -59,5 +65,4 @@ function rebuild_deb() {
 }
 
 rebuild_deb
-
-source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE}
+source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} --bucket ${BUCKET} ${SIGN_ARG}

scripts/debian/verify.sh

@@ -4,12 +4,14 @@ set -eox pipefail
 CHANNEL=umt-mainnet
 VERSION=3.0.0-f872d85
 CODENAME=bullseye
+BUCKET=packages.o1test.net
 
 while [[ "$#" -gt 0 ]]; do case $1 in
   -c|--channel) CHANNEL="$2"; shift;;
   -v|--version) VERSION="$2"; shift;;
   -p|--package) PACKAGE="$2"; shift;;
   -m|--codename) CODENAME="$2"; shift;;
+  -b|--bucket) BUCKET="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -29,7 +31,7 @@ SCRIPT=' set -x \
     && echo installing mina \
     && apt-get update > /dev/null \
     && apt-get install -y lsb-release ca-certificates > /dev/null \
-    && echo "deb [trusted=yes] http://packages.o1test.net '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \
+    && echo "deb [trusted=yes] https://'$BUCKET' '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \
     && apt-get update > /dev/null \
     && apt list -a '$PACKAGE' \
     && apt-get install -y --allow-downgrades '$PACKAGE'='$VERSION' \

scripts/docker/build.sh

@@ -47,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   --deb-profile) DEB_PROFILE="$2"; shift;;
   --deb-repo) INPUT_REPO="$2"; shift;;
   --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;;
+  --deb-repo-key) DEB_REPO_KEY="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done