From dff7e798a10264313dbd9e160275bb103ec13016 Mon Sep 17 00:00:00 2001 From: Himani Deshpande Date: Wed, 11 Oct 2023 12:38:20 -0400 Subject: [PATCH] Changing the default Spack Root directory from /home to /opt --- .../attributes/environment.rb | 2 + .../recipes/config.rb | 3 + .../resources/spack/partial/_spack_common.rb | 102 +++++++++--------- .../test/controls/spack_spec.rb | 20 ++-- 4 files changed, 69 insertions(+), 58 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/attributes/environment.rb b/cookbooks/aws-parallelcluster-environment/attributes/environment.rb index c312a103c2..a97195cf62 100644 --- a/cookbooks/aws-parallelcluster-environment/attributes/environment.rb +++ b/cookbooks/aws-parallelcluster-environment/attributes/environment.rb @@ -60,3 +60,5 @@ default['cluster']['internal_initial_shared_dir'] = "#{node['cluster']['base_dir']}/init_shared" default['cluster']['head_node_private_ip'] = nil + +default['cluster']['spack_shared_dir'] = "#{node['cluster']['shared_dir']}/spack" diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config.rb b/cookbooks/aws-parallelcluster-environment/recipes/config.rb index cdff07bc91..15b1f1f260 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/config.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/config.rb @@ -34,3 +34,6 @@ include_recipe 'aws-parallelcluster-environment::raid' include_recipe 'aws-parallelcluster-environment::efs' include_recipe 'aws-parallelcluster-environment::fsx' +spack 'Configure Spack Packages' do + action :configure +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/spack/partial/_spack_common.rb b/cookbooks/aws-parallelcluster-environment/resources/spack/partial/_spack_common.rb index a6f016046c..725a5110de 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/spack/partial/_spack_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/spack/partial/_spack_common.rb @@ -14,11 +14,14 @@ unified_mode true default_action :setup -property :spack_user, String, required: false, - default: node['cluster']['cluster_user'] - property :spack_root, String, required: false, - default: "/home/#{node['cluster']['cluster_user']}/spack" + default: node['cluster']['spack_shared_dir'] + +property :spack, String, required: false, + default: "#{node['cluster']['spack_shared_dir']}/bin/spack" + +property :spack_configs_dir, String, required: false, + default: "#{node['cluster']['spack_shared_dir']}/etc/spack" action :install_spack do return if on_docker? @@ -32,8 +35,8 @@ git new_resource.spack_root do repository 'https://github.com/spack/spack' - user new_resource.spack_user - group new_resource.spack_user + user 'root' + group 'root' end template '/etc/profile.d/spack.sh' do @@ -45,61 +48,33 @@ variables(spack_root: new_resource.spack_root) end - spack = "#{new_resource.spack_root}/bin/spack" - spack_configs_dir = "#{new_resource.spack_root}/etc/spack" - arch_target = `#{spack} arch -t`.strip - - # Find libfabric version to be used in package configs - libfabric_version = nil - ::File.open(libfabric_path).each do |line| - if line.include?('Version:') - libfabric_version = line.split[1].strip - break - end - end - - # Pull architecture dependent package config - begin - template "#{spack_configs_dir}/packages.yaml" do - cookbook 'aws-parallelcluster-environment' - source "spack/packages-#{arch_target}.yaml.erb" - owner new_resource.spack_user - group new_resource.spack_user - variables(libfabric_version: libfabric_version) - end - rescue Chef::Exceptions::FileNotFound - Chef::Log.warn "Could not find template for #{arch_target}" - end - - cookbook_file "#{spack_configs_dir}/modules.yaml" do + cookbook_file "#{new_resource.spack_configs_dir}/modules.yaml" do cookbook 'aws-parallelcluster-environment' source 'spack/modules.yaml' - owner new_resource.spack_user - group new_resource.spack_user + owner 'root' + group 'root' mode '0755' end bash 'setup Spack' do - user new_resource.spack_user - group new_resource.spack_user + user 'root' + group 'root' code <<-SPACK source #{new_resource.spack_root}/share/spack/setup-env.sh # Generate Spack user's compilers.yaml config with preinstalled compilers - #{spack} compiler add --scope site + #{new_resource.spack} compiler add --scope site # Add external packages to Spack user's packages.yaml config - #{spack} external find --scope site + #{new_resource.spack} external find --scope site # Remove all autotools/buildtools packages. These versions need to be managed by spack or it will # eventually end up in a version mismatch (e.g. when compiling gmp). - #{spack} tags build-tools | xargs -I {} #{spack} config --scope site rm packages:{} + #{new_resource.spack} tags build-tools | xargs -I {} #{new_resource.spack} config --scope site rm packages:{} SPACK end - node.default['cluster']['spack']['user'] = new_resource.spack_user node.default['cluster']['spack']['root'] = new_resource.spack_root - node.default['cluster']['libfabric_version'] = libfabric_version node_attributes 'dump node attributes' end @@ -111,18 +86,49 @@ return end - spack = "#{new_resource.spack_root}/bin/spack" - bash 'add binaries' do - user new_resource.spack_user - group new_resource.spack_user + user 'root' + group 'root' code <<-SPACK - [ -z "${CI_PROJECT_DIR}" ] && #{spack} mirror add --scope site "aws-pcluster" "https://binaries.spack.io/develop/aws-pcluster-$(spack arch -t | sed -e 's?_avx512??1')" || true - #{spack} buildcache keys --install --trust + [ -z "${CI_PROJECT_DIR}" ] && #{new_resource.spack} mirror add --scope site "aws-pcluster" "https://binaries.spack.io/develop/aws-pcluster-$(spack arch -t | sed -e 's?_avx512??1')" || true + #{new_resource.spack} buildcache keys --install --trust SPACK end end +action :configure do + return if on_docker? + + case node['cluster']['node_type'] + when 'HeadNode' + + # Find libfabric version to be used in package configs + libfabric_version = nil + ::File.open(libfabric_path).each do |line| + if line.include?('Version:') + libfabric_version = line.split[1].strip + break + end + end + + arch_target = `#{new_resource.spack} arch -t`.strip + + # Pull architecture dependent package config + template "#{new_resource.spack_configs_dir}/packages.yaml" do + cookbook 'aws-parallelcluster-environment' + source "spack/packages-#{arch_target}.yaml.erb" + owner 'root' + group 'root' + variables(libfabric_version: libfabric_version) + only_if { ::File.exist?("spack/packages-#{arch_target}.yaml.erb") && spack_installed? } + end + + node.default['cluster']['libfabric_version'] = libfabric_version + node_attributes 'dump node attributes' + + end +end + action_class do def spack_installed? ::Dir.exist?(new_resource.spack_root) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/spack_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/spack_spec.rb index 3c1003f17f..18b4b1ea6e 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/spack_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/spack_spec.rb @@ -16,14 +16,14 @@ describe directory(node['cluster']['spack']['root']) do it { should exist } - it { should be_owned_by (node['cluster']['spack']['user']).to_s } - it { should be_grouped_into (node['cluster']['spack']['user']).to_s } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } end describe directory("#{node['cluster']['spack']['root']}/share/spack") do it { should exist } - it { should be_owned_by (node['cluster']['spack']['user']).to_s } - it { should be_grouped_into (node['cluster']['spack']['user']).to_s } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } end describe file('/etc/profile.d/spack.sh') do @@ -34,13 +34,13 @@ describe file("#{node['cluster']['spack']['root']}/etc/spack/compilers.yaml") do it { should exist } - it { should be_owned_by (node['cluster']['spack']['user']).to_s } - it { should be_grouped_into (node['cluster']['spack']['user']).to_s } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } end spack = "#{node['cluster']['spack']['root']}/bin/spack" - describe "spack commands can run as cluster default user #{node['cluster']['spack']['user']}" do - subject { bash("su - #{node['cluster']['spack']['user']} -c '#{spack} find'") } + describe "spack commands can run as cluster default user #{node['cluster']['cluster_user']}" do + subject { bash("su - #{node['cluster']['cluster_user']} -c 'sudo #{spack} find'") } its('exit_status') { should eq(0) } end @@ -54,8 +54,8 @@ only_if { !os_properties.on_docker? } spack = "#{node['cluster']['spack']['root']}/bin/spack" - describe "spack can install packages as cluster default user #{node['cluster']['spack']['user']}" do - subject { bash("sudo su - #{node['cluster']['spack']['user']} -c '#{spack} install xz'") } + describe "spack can install packages as cluster default user #{node['cluster']['cluster_user']}" do + subject { bash("sudo su - #{node['cluster']['cluster_user']} -c 'sudo #{spack} install xz'") } its('exit_status') { should eq(0) } end