diff --git a/concourse/pipelines/template/depls-pipeline.yml.erb b/concourse/pipelines/template/depls-pipeline.yml.erb index 1e7224c60..15941a88e 100755 --- a/concourse/pipelines/template/depls-pipeline.yml.erb +++ b/concourse/pipelines/template/depls-pipeline.yml.erb @@ -14,13 +14,14 @@ deployment_info['status'] == 'enabled' && deployment_info['bosh-deployment'] end - uniq_releases= {} + uniq_releases = {} enabled_deployments&.sort&.each do |name, boshrelease| boshrelease['releases']&.each do |release, info| - uniq_releases[release]= info + uniq_releases[release] = info end end + offline_stemcells_enabled = config['offline-mode'] && config['offline-mode']['stemcells'] && true jobs = Hash.new {|h,k| h[k]=[]} %> --- @@ -105,8 +106,8 @@ resources: skip_ssl_verification: true <% unless enabled_deployments.empty? %> -- name: bosh-stemcell -# name: ((stemcell-name)) +- name: ((stemcell-main-name)) +<% if offline_stemcells_enabled %> type: s3 source: bucket: ((s3-stemcell-bucket)) @@ -117,6 +118,11 @@ resources: secret_access_key: ((s3-stemcell-secret-key)) endpoint: ((s3-stemcell-endpoint)) skip_ssl_verification: ((s3-stemcell-skip-ssl-verification)) +<% else %> + type: bosh-io-stemcell + source: + name: ((stemcell-name-prefix))((stemcell-main-name)) +<% end %> <% end %> <% enabled_deployments.sort.each do |name, boshrelease| %> @@ -451,8 +457,12 @@ jobs: - aggregate: - get: secrets-full-writer params: { submodules: none} - - get: bosh-stemcell + - get: ((stemcell-main-name)) + <% if offline_stemcells_enabled %> version: { path: ((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz } + <% else %> + version: { version: ((stemcell-version)) } + <% end %> trigger: true attempts: 2 - get: cf-ops-automation @@ -600,7 +610,11 @@ jobs: target_file: bosh-generated-config/bosh_target <% end %> stemcells: - - bosh-stemcell/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz + <% if offline_stemcells_enabled %> + - ((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz + <% else %> + - ((stemcell-main-name))/stemcell.tgz + <% end %> releases:<%= ' []' if boshrelease['releases']&.empty? %> <% boshrelease['releases']&.sort&.each do |release, info| %> <% if config['offline-mode'] && config['offline-mode']['boshreleases'] %> @@ -831,7 +845,11 @@ jobs: echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from version:((<%= name %>-version))" >> result-dir/flight-plan <% end %> <% end %> - echo "check-resource -r $BUILD_PIPELINE_NAME/bosh-stemcell --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan + <% if offline_stemcells_enabled %> + echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan + <% else %> + echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan + <% end %> params: BUILD_PIPELINE_NAME: <%= depls %>-generated - task: fly-into-concourse diff --git a/docs/reference_dataset/config_repository/private-config.yml b/docs/reference_dataset/config_repository/private-config.yml index 9937ffefc..86672ec5d 100644 --- a/docs/reference_dataset/config_repository/private-config.yml +++ b/docs/reference_dataset/config_repository/private-config.yml @@ -2,6 +2,7 @@ # you can override values from template_repository/shared-config.yml #offline-mode: # boshreleases: false # Default: false +# stemcells: false # Default: true # #default: # stemcell: diff --git a/docs/reference_dataset/template_repository/shared-config.yml b/docs/reference_dataset/template_repository/shared-config.yml index e58d624fd..d3deda372 100644 --- a/docs/reference_dataset/template_repository/shared-config.yml +++ b/docs/reference_dataset/template_repository/shared-config.yml @@ -1,6 +1,8 @@ --- +# When offline-mode is enabled, bytes are retrieved from an S3 compliant location, otherwise download from bosh-io offline-mode: - boshreleases: false # Default: false + boshreleases: false # Choose where boshreleases are downloaded from. Default: false + stemcells: false # Choose where stemcells are downloaded from. Default: true default: stemcell: diff --git a/scripts/concourse-manual-pipelines-update.rb b/scripts/concourse-manual-pipelines-update.rb index c75a845ce..7d96b8b85 100755 --- a/scripts/concourse-manual-pipelines-update.rb +++ b/scripts/concourse-manual-pipelines-update.rb @@ -1,37 +1,14 @@ #!/usr/bin/env ruby -# encoding: utf-8 -# set -e - -#echo "Deploy on ${FLY_TARGET} using secrets in $SECRET_DIR" -#for depls in ${DEPLS_LIST};do -# cd ${SCRIPT_DIR}/concourse -# ./generate-depls.rb -d ${depls} -p ${SECRET_DIR} -o ${OUTPUT_DIR} --no-dump -# PIPELINE="${depls}-init-generated" -# cd ${SCRIPT_DIR} -# echo "Load ${PIPELINE} on ${FLY_TARGET}" -# set +e -# fly -t ${FLY_TARGET} set-pipeline -p ${PIPELINE} -c ${OUTPUT_DIR}/pipelines/${PIPELINE}.yml \ -# -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-auto-init.yml \ -# -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-mattermost-certs.yml \ -# -l ${SECRET_DIR}/micro-depls/concourse-micro/pipelines/credentials-git-config.yml -# set -e -# fly -t ${FLY_TARGET} unpause-pipeline -p ${PIPELINE} -# if [ "$SKIP_TRIGGER" != "true" ] -# then -# fly -t ${FLY_TARGET} trigger-job -j "${PIPELINE}/update-pipeline-${depls}" -# fi -#done - require 'yaml' require 'optparse' # Argument parsing OPTIONS = { - :depls => 'ops-depls', - :no_interactive => false, - :fail_fast => false, - :fail_on_error => false + depls: 'ops-depls', + no_interactive: false, + fail_fast: false, + fail_on_error: false } opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: ./scripts/concourse-manual-pipelines-update.sh [options] @@ -40,8 +17,7 @@ PAAS_TEMPLATES: paas-templates to use - Default: ../paas-templates DEBUG: enable debug message - Default: false PIPELINES_DIR: pipelines ready to be uploaded directory- Default: boostrap-generated/pipelines - TARGET_NAME - Default: cw-pp-micro - + TARGET_NAME - Default: fe-int ' opts.on("--without=WITHOUT", "-wWITHOUT", "Don't update matched pipelines") do |without_string| @@ -71,20 +47,15 @@ opts.on('--fail-on-error', 'Fail on loading error') do |_| OPTIONS[:fail_on_error] = true end - end opt_parser.parse! -SECRETS = ENV['SECRETS'] ||"../preprod-secrets" -PAAS_TEMPLATES = ENV['PAAS_TEMPLATES'] ||'../paas-templates' +SECRETS = ENV['SECRETS'] || "../preprod-secrets" +PAAS_TEMPLATES = ENV['PAAS_TEMPLATES'] || '../paas-templates' DEBUG = ENV['DEBUG'] || false PIPELINES_DIR = ENV['PIPELINES_DIR'] || 'bootstrap-generated/pipelines' - -flyrc = YAML.load_file(File.expand_path('~/.flyrc')) -target_name = ENV['TARGET_NAME'] || 'cw-pp-micro' -target = flyrc['targets'][target_name] -concourse_url= target['api'] +target_name = ENV['TARGET_NAME'] || 'fe-int' PIPELINE_PREFIX = ENV['PIPELINE_PREFIX'] || '' @@ -98,20 +69,20 @@ def get_pipeline_name(name) end def set_pipeline(target_name:, name:, config:, load: [], options: []) - return if OPTIONS.has_key?(:match) && !name.include?(OPTIONS[:match]) - return if OPTIONS.has_key?(:without) && name.include?(OPTIONS[:without]) - puts " #{name} pipeline" + return if OPTIONS.key?(:match) && !name.include?(OPTIONS[:match]) + return if OPTIONS.key?(:without) && name.include?(OPTIONS[:without]) + puts " Setting #{name} pipeline" - fly_cmd=(%{bash -c "fly -t #{target_name} set-pipeline \ + fly_cmd = %{bash -c "fly -t #{target_name} set-pipeline \ -p #{get_pipeline_name(name)} \ -c #{config} \ #{load.collect { |l| "-l #{l}" }.join(' ')} \ - #{options.collect { |opt| "#{opt}" }.join(' ')} - "}) + #{options.collect(&:to_s).join(' ')} + "} puts "Executing: #{fly_cmd}" - pipeline_successfully_loaded=system(fly_cmd) + pipeline_successfully_loaded = system(fly_cmd) puts "Pipeline successfully loaded: #{pipeline_successfully_loaded}" if OPTIONS[:fail_fast] && !pipeline_successfully_loaded raise "Failed to load pipeline #{get_pipeline_name(name)} from template #{name}" @@ -122,11 +93,11 @@ def set_pipeline(target_name:, name:, config:, load: [], options: []) def generate_full_path_for_concourse_vars_files(vars_files) vars_files_with_path = [] vars_files.each do |var_file| - if var_file =~ /versions.yml/ - vars_files_with_path << "#{PAAS_TEMPLATES}/#{var_file}" - else - vars_files_with_path << "#{SECRETS}/#{var_file}" - end + vars_files_with_path << if var_file =~ /versions.yml/ + "#{PAAS_TEMPLATES}/#{var_file}" + else + "#{SECRETS}/#{var_file}" + end end vars_files_with_path end @@ -139,11 +110,11 @@ def update_pipelines(target_name) # puts "Found #{filename}" next if OPTIONS.key?(:depls) && !filename.include?(OPTIONS[:depls]) next if OPTIONS.key?(:template) && !filename.include?(OPTIONS[:template]) - puts "Processing only #{filename}" + puts "Starting processing of #{filename}" deployment_name = File.basename(filename, '.yml') ci_deployment_overview = YAML.load_file("#{SECRETS}/#{depls}/ci-deployment-overview.yml") - pipelines=ci_deployment_overview['ci-deployment'][depls]['pipelines'] + pipelines = ci_deployment_overview['ci-deployment'][depls]['pipelines'] current_pipeline = pipelines[deployment_name] if current_pipeline.nil? puts "invalid config #{SECRETS}/#{depls}/ci-deployment-overview.yml should contains a key ...[pipelines][#{deployment_name}][vars_files]" @@ -164,11 +135,10 @@ def update_pipelines(target_name) ) loaded_pipelines_status[deployment_name] = result end + if OPTIONS[:fail_on_error] - raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select {|name, status| !status.nil? && !status}.empty? + raise "pipeline loading error. Summary #{loaded_pipelines_status}" unless loaded_pipelines_status.select { |_, status| !status.nil? && !status }.empty? end end - update_pipelines target_name - diff --git a/spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb b/spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb index 3374aed47..f86a9e809 100644 --- a/spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb +++ b/spec/lib/template_processor/template_processor_for_depls_pipeline_spec.rb @@ -299,7 +299,7 @@ boshrelease_get_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] } .flat_map { |plan| plan['aggregate'] } .compact - .select { |resource| expected_boshreleases.keys.include?(resource['get']) } + .select { |resource| expected_boshreleases.key?(resource['get']) } .flat_map { |resource| { resource['get'] => resource['version']['path'] } } expect(boshrelease_get_version).to include(*expected_boshrelease_get_version) end @@ -322,6 +322,61 @@ expect(init_args[1]).to include(*expected_init_version) end end + + context 'when stemcell offline mode is disabled' do + let(:loaded_config) do + my_config_yaml = <<~YAML + offline-mode: + boshreleases: false + stemcells: false + docker-images: false + YAML + YAML.safe_load(my_config_yaml) + end + let(:expected_bosh_io_stemcell) do + expected_yaml = <<~YAML + - name: ((stemcell-main-name)) + type: bosh-io-stemcell + source: + name: ((stemcell-name-prefix))((stemcell-main-name)) + YAML + YAML.safe_load expected_yaml + end + let(:expected_stemcell_deploy_get) { ['((stemcell-version))'] * 2 } + let(:expected_stemcell_deploy_put) { ['((stemcell-main-name))/stemcell.tgz'] * 2 } + let(:expected_stemcell_init) { 'echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from version:((stemcell-version))" >> result-dir/flight-plan' } + + it 'generates bosh-io stemcell' do + bosh_io_stemcell = generated_pipeline['resources'].select { |resource| resource['type'] == 'bosh-io-stemcell' } + expect(bosh_io_stemcell).to include(*expected_bosh_io_stemcell) + end + + it 'generates bosh_io version using path on get' do + stemcell_get_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] } + .flat_map { |plan| plan['aggregate'] } + .compact + .select { |resource| resource['get'] == '((stemcell-main-name))' } + .flat_map { |resource| resource['version']['version'] } + expect(stemcell_get_version).to match(expected_stemcell_deploy_get) + end + + it 'generates bosh-io stemcell on deployment put' do + deployment_put_version = generated_pipeline['jobs'].flat_map { |job| job['plan'] } + .select { |resource| resource['params']&.has_key?('stemcells') } + .flat_map { |resource| resource['params']['stemcells'] } + expect(deployment_put_version).to match(expected_stemcell_deploy_put) + end + + it 'generates init-concourse-boshrelease-and-stemcell-for-ops-depls' do + init_args = generated_pipeline['jobs'] + .select { |job| job['name'] == "init-concourse-boshrelease-and-stemcell-for-#{root_deployment_name}" } + .flat_map { |job| job['plan'] } + .select { |step| step['task'] && step['task'] == "generate-#{root_deployment_name}-flight-plan" } + .flat_map { |task| task['config']['run']['args'] } + expect(init_args[1]).to include(*expected_stemcell_init) + end + end + context 'with ci deployment overview without terraform' do let(:all_ci_deployments) do ci_deployments_yaml = <<~YAML diff --git a/spec/scripts/generate-depls/fixtures/references/simple-depls-ref.yml b/spec/scripts/generate-depls/fixtures/references/simple-depls-ref.yml index c591dbbd7..85356d473 100644 --- a/spec/scripts/generate-depls/fixtures/references/simple-depls-ref.yml +++ b/spec/scripts/generate-depls/fixtures/references/simple-depls-ref.yml @@ -80,7 +80,7 @@ resources: tag_filter: ((cf-ops-automation-tag-filter)) skip_ssl_verification: true -- name: bosh-stemcell +- name: ((stemcell-main-name)) type: s3 source: bucket: ((s3-stemcell-bucket)) @@ -350,7 +350,7 @@ jobs: - aggregate: - get: secrets-full-writer params: { submodules: none} - - get: bosh-stemcell + - get: ((stemcell-main-name)) version: { path: "((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" } trigger: true attempts: 2 @@ -469,7 +469,7 @@ jobs: # source_file: bosh-generated-config/bosh_config.json stemcells: - - bosh-stemcell/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz + - ((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz releases: - ntp_boshrelease/release.tgz @@ -649,7 +649,7 @@ jobs: - -exc - | echo "check-resource -r $BUILD_PIPELINE_NAME/ntp_boshrelease --from version:((ntp_boshrelease-version))" >> result-dir/flight-plan - echo "check-resource -r $BUILD_PIPELINE_NAME/bosh-stemcell --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan + echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell-version))-((stemcell-main-name)).tgz" >> result-dir/flight-plan params: BUILD_PIPELINE_NAME: simple-depls-generated - task: fly-into-concourse