[WIP] vmpublish.rb: publish public images

.gitignore

@@ -2,3 +2,4 @@
 /productization
 kickstarts/generated/*.ks
 scripts/Gemfile.dev.rb
+config/creds/*.json

bin/release-build.sh

@@ -12,5 +12,10 @@ if [[ $# != 1 ]]; then
 fi
 
 log_file="/build/logs/${1}.log"
-nohup time ruby ${BUILD_DIR}/scripts/vmbuild.rb --type release --upload --reference $1 --copy-dir ${1%%-*} > $log_file 2>&1 &
+copy_dir="${1%%-*}"
+
+nohup sh -c \
+        "echo time ruby ${BUILD_DIR}/scripts/vmbuild.rb --type release --upload --reference $1 --copy-dir $copy_dir \
+         && time ruby ${BUILD_DIR}/scripts/vmpublish.rb --copy-dir $copy_dir" \
+        > $log_file 2>&1 &
 echo "${1} release build kicked off, see log @ $log_file ..."

config/creds/README.creds

@@ -0,0 +1,21 @@
+This directory will hold credentials files on the build host.
+
+All JSON files in this directory are ignored via gitignore(5) to reduce the
+risk that credentials accentally get commit.
+
+The following files are currently used:
+
+* gce.json
+
+  A service account JSON file for the Google Cloud Platform. This file can be
+  generated as describe here:
+  https://cloud.google.com/storage/docs/authentication#generating-a-private-key
+
+* atlas.json
+
+  A JSON file with an Atlas token, as follows:
+
+  {
+      "username": "<username>",
+      "token": "<token"
+  }

scripts/config.rb

@@ -0,0 +1,15 @@
+require 'pathname'
+
+BUILD_BASE          = Pathname.new("/build")
+GPG_DIR             = Pathname.new("/root/.gnupg")
+CFG_DIR             = BUILD_BASE.join("config")
+CREDS_DIR           = CFG_DIR.join("creds")
+FILESHARE_DIR       = BUILD_BASE.join("fileshare")
+REFS_DIR            = BUILD_BASE.join("references")
+IMGFAC_DIR          = BUILD_BASE.join("imagefactory")
+IMGFAC_CONF         = CFG_DIR.join("imagefactory.conf")
+STORAGE_DIR         = BUILD_BASE.join("storage")
+
+FILE_SERVER         = ENV["BUILD_FILE_SERVER"]             # SSH Server to host files
+FILE_SERVER_ACCOUNT = ENV["BUILD_FILE_SERVER_ACCOUNT"]     # Account to SSH as
+FILE_SERVER_BASE    = Pathname.new(ENV["BUILD_FILE_SERVER_BASE"] || ".") # Subdirectory of Account where to store builds

scripts/vmbuild.rb

@@ -10,6 +10,7 @@
 require_relative 'cli'
 require_relative 'uploader'
 require_relative 'target'
+require_relative 'config'
 
 $log = Logger.new(STDOUT)
 
@@ -30,19 +31,6 @@
   build_label = "#{cli_options[:type]}-#{cli_options[:manageiq_ref]}"
 end
 
-BUILD_BASE          = Pathname.new("/build")
-GPG_DIR             = Pathname.new("/root/.gnupg")
-CFG_DIR             = BUILD_BASE.join("config")
-FILESHARE_DIR       = BUILD_BASE.join("fileshare")
-REFS_DIR            = BUILD_BASE.join("references")
-IMGFAC_DIR          = BUILD_BASE.join("imagefactory")
-IMGFAC_CONF         = CFG_DIR.join("imagefactory.conf")
-STORAGE_DIR         = BUILD_BASE.join("storage")
-
-FILE_SERVER         = ENV["BUILD_FILE_SERVER"]             # SSH Server to host files
-FILE_SERVER_ACCOUNT = ENV["BUILD_FILE_SERVER_ACCOUNT"]     # Account to SSH as
-FILE_SERVER_BASE    = Pathname.new(ENV["BUILD_FILE_SERVER_BASE"] || ".") # Subdirectory of Account where to store builds
-
 if !cli_options[:local] && cli_options[:build_url]
   build_repo = cli_options[:build_url]
   cfg_base = REFS_DIR.join(cli_options[:build_ref])
@@ -101,8 +89,7 @@ def verify_run(output)
 file_rdu_dir_base = FILE_SERVER_BASE.join(directory)
 file_rdu_dir      = file_rdu_dir_base.join(directory_name)
 
-fileshare_dir         = BUILD_BASE.join("fileshare")
-stream_directory      = fileshare_dir.join(directory)
+stream_directory      = FILESHARE_DIR.join(directory)
 destination_directory = stream_directory.join(build_label == "test" ? "test" : directory_name)
 
 $log.info "Creating Fileshare Directory: #{destination_directory}"
@@ -152,7 +139,7 @@ def verify_run(output)
     FileUtils.mkdir_p(destination_directory)
     file_name = "#{name}-#{target}-#{build_label}-#{timestamp}-#{manageiq_checkout.commit_sha}.#{target.file_extension}"
     destination = destination_directory.join(file_name)
-    $log.info `mv #{source} #{destination}`
+    $log.info `ln #{source} #{destination}`
 
     if !File.exist?(destination)
       $log.warn "Cannot find the target file #{destination}"

scripts/vmpublish.rb

@@ -0,0 +1,80 @@
+require 'logger'
+require 'pathname'
+require 'trollop'
+require 'json'
+
+require_relative 'config'
+
+
+def find_imgfac_image(image)
+  f = File.new(image)
+  Dir.chdir(STORAGE_DIR) do
+    Dir.glob('*.body') do |filename|
+      return filename.split('.').first if File.new(filename).stat.ino == f.stat.ino
+    end
+  end
+end
+
+
+def upload_imgfac_image(uuid, target, reference, version)
+  if target == 'vagrant'
+    creds_file = CREDS_DIR.join('atlas.json')
+    params = "atlas @ignored #{creds_file} --id #{uuid}"
+    params += " --parameter atlas_box_name #{reference} --parameter atlas_box_version #{version}"
+  elsif target == 'gce'
+    creds_file = CREDS_DIR.join('gce.json')
+    params = "gce @manageiq #{creds_file} --id #{uuid}"
+    params += " --parameter gce_object_name manageiq-#{reference}-#{version}.tar.gz"
+    params += " --parameter gce_image_name manageiq-#{reference}-#{version} --parameter gce_image_family centos-7"
+  end
+
+  if !File.exist?(creds_file)
+    $log.info "credentials file #{creds_file} does not exist, skipping #{target} publish"
+    return
+  end
+
+  Dir.chdir(IMGFAC_DIR) do
+    # Show progress on STDOUT
+    command = "./imagefactory provider_image #{params}"
+    $log.info "running command: #{command}"
+    if !system(command)
+      $log.error "imagefactory exited with status #{$?.exitstatus}"
+      exit 1
+    end
+  end
+end
+
+
+$log = Logger.new(STDOUT)
+
+dir_desc  = "Directory builds were copied to"
+type_desc = "Build type (stable or latest)"
+
+options = Trollop.options(ARGV) do
+  banner "Usage: vmpublish.rb [options]"
+  opt :copy_dir,  dir_desc,   :type => :string,   :short => 'd', :default => "master"
+  opt :type,      type_desc,  :type => :string,   :short => 't', :default => "stable"
+end
+
+copy_dir = FILESHARE_DIR.join(options[:copy_dir], options[:type])
+
+Dir.foreach(copy_dir) do |filename|
+  next if ['.', '..'].include? filename
+
+  parts = filename.split('.').first.split('-')
+  next unless parts.length >= 6        # only release builds, which are build from a tag (name-version)
+  next unless parts[0] == 'manageiq'
+  target = parts[1]
+  next unless ['vagrant', 'gce'].include? target
+  reference = parts[2]
+  version = parts[3..parts.length-3].join('-')
+
+  image = copy_dir.join(filename)
+  uuid = find_imgfac_image(image)
+
+  $log.info "processing image: #{image}"
+  $log.info "target = #{target}, reference = #{reference}, version = #{version}"
+  $log.info "imagefactory image = #{uuid}"
+
+  upload_imgfac_image(uuid, target, reference, version)
+end