pipeline(depls): online stemcells support

By default, we use offline stemcell, but it can be overridden by `shared-config.yml` or `private-config.yml`. We also rename `bosh-stemcell` resource to `((stemcell-main-name))` to be clearer.

closes #128

concourse/pipelines/template/depls-pipeline.yml.erb

@@ -14,13 +14,14 @@
     deployment_info['status'] == 'enabled' && deployment_info['bosh-deployment']
   end
 
-  uniq_releases= {}
+  uniq_releases = {}
   enabled_deployments&.sort&.each do |name, boshrelease|
     boshrelease['releases']&.each do |release, info|
-      uniq_releases[release]= info
+      uniq_releases[release] = info
     end
   end
 
+  offline_stemcells_enabled = config['offline-mode'] && config['offline-mode']['stemcells'] && true
   jobs = Hash.new {|h,k| h[k]=[]}
 %>
 ---
@@ -105,8 +106,8 @@ resources:
     skip_ssl_verification: true
 
 <% unless enabled_deployments.empty? %>
-- name: bosh-stemcell
-# name: ((stemcell-name))
+- name: ((stemcell-main-name))
+<% if offline_stemcells_enabled %>
   type: s3
   source:
     bucket: ((s3-stemcell-bucket))
@@ -117,6 +118,11 @@ resources:
     secret_access_key: ((s3-stemcell-secret-key))
     endpoint: ((s3-stemcell-endpoint))
     skip_ssl_verification: ((s3-stemcell-skip-ssl-verification))
+<% else %>
+  type: bosh-io-stemcell
+  source:
+    name: ((stemcell-name-prefix))((stemcell-main-name))
+<% end %>
 <% end %>
 
 <% enabled_deployments.sort.each do |name, boshrelease| %>
@@ -451,8 +457,12 @@ jobs:
   - aggregate:
     - get: secrets-full-writer
       params: { submodules: none}
-    - get: bosh-stemcell
+    - get: ((stemcell-main-name))
+      <% if offline_stemcells_enabled %>
       version: { path: ((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz }
+      <% else %>
+      version: { version: ((stemcell-version)) }
+      <% end %>
       trigger: true
       attempts: 2
     - get: cf-ops-automation
@@ -600,7 +610,11 @@ jobs:
       target_file: bosh-generated-config/bosh_target
   <% end %>
       stemcells:
-      - bosh-stemcell/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz
+      <% if offline_stemcells_enabled %>
+      - ((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz
+      <% else %>
+      - ((stemcell-main-name))/stemcell.tgz
+      <% end %>
       releases:<%= ' []' if boshrelease['releases']&.empty? %>
   <% boshrelease['releases']&.sort&.each do |release, info| %>
     <% if config['offline-mode'] && config['offline-mode']['boshreleases'] %>
@@ -831,7 +845,11 @@ jobs:
           echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from version:((<%= name %>-version))" >> result-dir/flight-plan
     <% end %>
   <% end %>
-          echo "check-resource -r $BUILD_PIPELINE_NAME/bosh-stemcell --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan
+    <% if offline_stemcells_enabled %>
+          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan
+    <% else %>
+          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan
+    <% end %>
       params:
         BUILD_PIPELINE_NAME: <%= depls %>-generated
   - task: fly-into-concourse

docs/reference_dataset/config_repository/private-config.yml

@@ -2,6 +2,7 @@
 # you can override values from template_repository/shared-config.yml
 #offline-mode:
 #   boshreleases: false # Default: false
+#   stemcells: false # Default: true
 #
 #default:
 #  stemcell:

docs/reference_dataset/template_repository/shared-config.yml

@@ -1,6 +1,8 @@
 ---
+# When offline-mode is enabled, bytes are retrieved from an S3 compliant location, otherwise download from bosh-io
 offline-mode:
-   boshreleases: false # Default: false
+   boshreleases: false # Choose where boshreleases are downloaded from. Default: false
+   stemcells: false # Choose where stemcells are downloaded from. Default: true
 
 default:
   stemcell:

scripts/concourse-manual-pipelines-update.rb

@@ -1,37 +1,14 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-# set -e
-
-#echo "Deploy on ${FLY_TARGET} using secrets in $SECRET_DIR"
-#for depls in ${DEPLS_LIST};do
-#    cd ${SCRIPT_DIR}/concourse
-#    ./generate-depls.rb -d ${depls} -p ${SECRET_DIR} -o ${OUTPUT_DIR} --no-dump
-#    PIPELINE="${depls}-init-generated"
-#    cd ${SCRIPT_DIR}
-#    echo "Load ${PIPELINE} on ${FLY_TARGET}"
-#    set +e
-#    fly -t ${FLY_TARGET} set-pipeline -p ${PIPELINE} -c ${OUTPUT_DIR}/pipelines/${PIPELINE}.yml  \
-#                -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-auto-init.yml \
-#                -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-mattermost-certs.yml \
-#                -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-git-config.yml
-#    set -e
-#    fly -t ${FLY_TARGET} unpause-pipeline -p ${PIPELINE}
-#    if [ "$SKIP_TRIGGER" != "true" ]
-#    then
-#        fly -t ${FLY_TARGET} trigger-job -j "${PIPELINE}/update-pipeline-${depls}"
-#    fi
-#done
-
 
 require 'yaml'
 require 'optparse'
 
 # Argument parsing
 OPTIONS = {
-  :depls => 'ops-depls',
-  :no_interactive => false,
-  :fail_fast => false,
-  :fail_on_error => false
+  depls: 'ops-depls',
+  no_interactive: false,
+  fail_fast: false,
+  fail_on_error: false
 }
 opt_parser = OptionParser.new do |opts|
   opts.banner = 'Usage: ./scripts/concourse-manual-pipelines-update.sh [options]
@@ -40,8 +17,7 @@
     PAAS_TEMPLATES: paas-templates to use - Default: ../paas-templates
     DEBUG: enable debug message - Default: false
     PIPELINES_DIR: pipelines ready to be uploaded directory- Default: boostrap-generated/pipelines
-    TARGET_NAME - Default: cw-pp-micro
-
+    TARGET_NAME - Default: fe-int
 '
 
   opts.on("--without=WITHOUT", "-wWITHOUT", "Don't update matched pipelines") do |without_string|
@@ -71,20 +47,15 @@
   opts.on('--fail-on-error', 'Fail on loading error') do |_|
     OPTIONS[:fail_on_error] = true
   end
-
 end
 opt_parser.parse!
 
-SECRETS = ENV['SECRETS'] ||"../preprod-secrets"
-PAAS_TEMPLATES = ENV['PAAS_TEMPLATES'] ||'../paas-templates'
+SECRETS = ENV['SECRETS'] || "../preprod-secrets"
+PAAS_TEMPLATES = ENV['PAAS_TEMPLATES'] || '../paas-templates'
 DEBUG = ENV['DEBUG'] || false
 PIPELINES_DIR = ENV['PIPELINES_DIR'] || 'bootstrap-generated/pipelines'
 
-
-flyrc = YAML.load_file(File.expand_path('~/.flyrc'))
-target_name = ENV['TARGET_NAME'] || 'cw-pp-micro'
-target = flyrc['targets'][target_name]
-concourse_url= target['api']
+target_name = ENV['TARGET_NAME'] || 'fe-int'
 
 PIPELINE_PREFIX = ENV['PIPELINE_PREFIX'] || ''
 
@@ -98,20 +69,20 @@ def get_pipeline_name(name)
 end
 
 def set_pipeline(target_name:, name:, config:, load: [], options: [])
-  return if OPTIONS.has_key?(:match) && !name.include?(OPTIONS[:match])
-  return if OPTIONS.has_key?(:without) && name.include?(OPTIONS[:without])
-  puts "   #{name} pipeline"
+  return if OPTIONS.key?(:match) && !name.include?(OPTIONS[:match])
+  return if OPTIONS.key?(:without) && name.include?(OPTIONS[:without])
+  puts "   Setting #{name} pipeline"
 
-  fly_cmd=(%{bash -c "fly -t #{target_name} set-pipeline \
+  fly_cmd = %{bash -c "fly -t #{target_name} set-pipeline \
     -p #{get_pipeline_name(name)} \
     -c #{config} \
   #{load.collect { |l| "-l #{l}" }.join(' ')} \
-  #{options.collect { |opt| "#{opt}" }.join(' ')}
-  "})
+  #{options.collect(&:to_s).join(' ')}
+  "}
 
   puts "Executing: #{fly_cmd}"
 
-  pipeline_successfully_loaded=system(fly_cmd)
+  pipeline_successfully_loaded = system(fly_cmd)
   puts "Pipeline successfully loaded: #{pipeline_successfully_loaded}"
   if OPTIONS[:fail_fast] && !pipeline_successfully_loaded
     raise "Failed to load pipeline #{get_pipeline_name(name)} from template #{name}"
@@ -122,11 +93,11 @@ def set_pipeline(target_name:, name:, config:, load: [], options: [])
 def generate_full_path_for_concourse_vars_files(vars_files)
   vars_files_with_path = []
   vars_files.each do |var_file|
-    if var_file =~ /versions.yml/
-      vars_files_with_path << "#{PAAS_TEMPLATES}/#{var_file}"
-    else
-      vars_files_with_path << "#{SECRETS}/#{var_file}"
-    end
+    vars_files_with_path << if var_file =~ /versions.yml/
+                              "#{PAAS_TEMPLATES}/#{var_file}"
+                            else
+                              "#{SECRETS}/#{var_file}"
+                            end
   end
   vars_files_with_path
 end
@@ -139,11 +110,11 @@ def update_pipelines(target_name)
     # puts "Found #{filename}"
     next if OPTIONS.key?(:depls) && !filename.include?(OPTIONS[:depls])
     next if OPTIONS.key?(:template) && !filename.include?(OPTIONS[:template])
-    puts "Processing only #{filename}"
+    puts "Starting processing of #{filename}"
     deployment_name = File.basename(filename, '.yml')
     ci_deployment_overview = YAML.load_file("#{SECRETS}/#{depls}/ci-deployment-overview.yml")
 
-    pipelines=ci_deployment_overview['ci-deployment'][depls]['pipelines']
+    pipelines = ci_deployment_overview['ci-deployment'][depls]['pipelines']
     current_pipeline = pipelines[deployment_name]
     if current_pipeline.nil?
       puts "invalid config #{SECRETS}/#{depls}/ci-deployment-overview.yml should contains a key ...[pipelines][#{deployment_name}][vars_files]"
@@ -164,11 +135,10 @@ def update_pipelines(target_name)
     )
     loaded_pipelines_status[deployment_name] = result
   end
+
   if OPTIONS[:fail_on_error]
-    raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select {|name, status| !status.nil? && !status}.empty?
+    raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select { |_, status| !status.nil? && !status }.empty?
   end
 end
 
-
 update_pipelines target_name
-

spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb

@@ -299,7 +299,7 @@
         boshrelease_get_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] }
           .flat_map { |plan| plan['aggregate'] }
           .compact
-          .select { |resource| expected_boshreleases.keys.include?(resource['get']) }
+          .select { |resource| expected_boshreleases.key?(resource['get']) }
           .flat_map { |resource| { resource['get'] => resource['version']['path'] } }
         expect(boshrelease_get_version).to include(*expected_boshrelease_get_version)
       end
@@ -322,6 +322,61 @@
         expect(init_args[1]).to include(*expected_init_version)
       end
     end
+
+    context 'when stemcell offline mode is disabled' do
+      let(:loaded_config) do
+        my_config_yaml = <<~YAML
+          offline-mode:
+            boshreleases: false
+            stemcells: false
+            docker-images: false
+        YAML
+        YAML.safe_load(my_config_yaml)
+      end
+      let(:expected_bosh_io_stemcell) do
+        expected_yaml = <<~YAML
+          - name: ((stemcell-main-name))
+            type: bosh-io-stemcell
+            source:
+              name: ((stemcell-name-prefix))((stemcell-main-name))
+        YAML
+        YAML.safe_load expected_yaml
+      end
+      let(:expected_stemcell_deploy_get) { ['((stemcell-version))'] * 2 }
+      let(:expected_stemcell_deploy_put) { ['((stemcell-main-name))/stemcell.tgz'] * 2 }
+      let(:expected_stemcell_init) { 'echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan' }
+
+      it 'generates bosh-io stemcell' do
+        bosh_io_stemcell = generated_pipeline['resources'].select { |resource| resource['type'] == 'bosh-io-stemcell' }
+        expect(bosh_io_stemcell).to include(*expected_bosh_io_stemcell)
+      end
+
+      it 'generates bosh_io version using path on get' do
+        stemcell_get_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] }
+          .flat_map { |plan| plan['aggregate'] }
+          .compact
+          .select { |resource| resource['get'] == '((stemcell-main-name))' }
+          .flat_map { |resource| resource['version']['version'] }
+        expect(stemcell_get_version).to match(expected_stemcell_deploy_get)
+      end
+
+      it 'generates bosh-io stemcell on deployment put' do
+        deployment_put_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] }
+          .select { |resource| resource['params']&.has_key?('stemcells') }
+          .flat_map { |resource| resource['params']['stemcells'] }
+        expect(deployment_put_version).to match(expected_stemcell_deploy_put)
+      end
+
+      it 'generates init-concourse-boshrelease-and-stemcell-for-ops-depls' do
+        init_args = generated_pipeline['jobs']
+          .select { |job| job['name'] == "init-concourse-boshrelease-and-stemcell-for-#{root_deployment_name}" }
+          .flat_map { |job| job['plan'] }
+          .select { |step| step['task'] && step['task'] == "generate-#{root_deployment_name}-flight-plan" }
+          .flat_map { |task| task['config']['run']['args'] }
+        expect(init_args[1]).to include(*expected_stemcell_init)
+      end
+    end
+
     context 'with ci deployment overview without terraform' do
       let(:all_ci_deployments) do
         ci_deployments_yaml = <<~YAML

spec/scripts/generate-depls/fixtures/references/simple-depls-ref.yml

@@ -80,7 +80,7 @@ resources:
     tag_filter: ((cf-ops-automation-tag-filter))
     skip_ssl_verification: true
 
-- name: bosh-stemcell
+- name: ((stemcell-main-name))
   type: s3
   source:
     bucket: ((s3-stemcell-bucket))
@@ -350,7 +350,7 @@ jobs:
     - aggregate:
       - get: secrets-full-writer
         params: { submodules: none}
-      - get: bosh-stemcell
+      - get: ((stemcell-main-name))
         version: { path: "((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" }
         trigger: true
         attempts: 2
@@ -469,7 +469,7 @@ jobs:
 #        source_file: bosh-generated-config/bosh_config.json
 
         stemcells:
-        - bosh-stemcell/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz
+        - ((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz
         releases: 
 
         - ntp_boshrelease/release.tgz
@@ -649,7 +649,7 @@ jobs:
         - -exc
         - |
           echo "check-resource -r $BUILD_PIPELINE_NAME/ntp_boshrelease --from version:((ntp_boshrelease-version))" >> result-dir/flight-plan
-          echo "check-resource -r $BUILD_PIPELINE_NAME/bosh-stemcell --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan
+          echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan
       params:
         BUILD_PIPELINE_NAME: simple-depls-generated
   - task: fly-into-concourse