[Spack] Changing the default Spack Root directory from /home to /opt

cookbooks/aws-parallelcluster-environment/attributes/environment.rb

@@ -60,3 +60,5 @@
 default['cluster']['internal_initial_shared_dir'] = "#{node['cluster']['base_dir']}/init_shared"
 
 default['cluster']['head_node_private_ip'] = nil
+
+default['cluster']['spack_shared_dir'] = "#{node['cluster']['shared_dir']}/spack"

cookbooks/aws-parallelcluster-environment/kitchen.environment-config.yml

@@ -684,3 +684,18 @@ suites:
         efs_shared_dirs: '/opt/parallelcluster/init_shared'
         efs_fs_ids: 'fs-03ad31942a4205839' # Existing FS, needs to be set when running the test
         ebs_shared_dirs: ''
+  - name: spack
+    run_list:
+      - recipe[aws-parallelcluster-tests::setup]
+      - recipe[aws-parallelcluster-tests::test_resource]
+    verifier:
+      controls:
+        - /tag:config_spack/
+    attributes:
+      resource: spack:configure
+      cluster:
+        enable_efa: efa
+        node_type: HeadNode
+      dependencies:
+        - recipe:aws-parallelcluster-platform::directories
+        - resource:spack:setup

cookbooks/aws-parallelcluster-environment/recipes/config.rb

@@ -34,3 +34,6 @@
 include_recipe 'aws-parallelcluster-environment::raid'
 include_recipe 'aws-parallelcluster-environment::efs'
 include_recipe 'aws-parallelcluster-environment::fsx'
+spack 'Configure Spack Packages' do
+  action :configure
+end

cookbooks/aws-parallelcluster-environment/resources/spack/partial/_spack_common.rb

@@ -14,11 +14,8 @@
 unified_mode true
 default_action :setup
 
-property :spack_user, String, required: false,
-         default: node['cluster']['cluster_user']
-
 property :spack_root, String, required: false,
-         default: "/home/#{node['cluster']['cluster_user']}/spack"
+         default: node['cluster']['spack_shared_dir']
 
 action :install_spack do
   return if on_docker?
@@ -32,8 +29,8 @@
 
   git new_resource.spack_root do
     repository 'https://github.com/spack/spack'
-    user new_resource.spack_user
-    group new_resource.spack_user
+    user 'root'
+    group 'root'
   end
 
   template '/etc/profile.d/spack.sh' do
@@ -47,41 +44,18 @@
 
   spack = "#{new_resource.spack_root}/bin/spack"
   spack_configs_dir = "#{new_resource.spack_root}/etc/spack"
-  arch_target = `#{spack} arch -t`.strip
-
-  # Find libfabric version to be used in package configs
-  libfabric_version = nil
-  ::File.open(libfabric_path).each do |line|
-    if line.include?('Version:')
-      libfabric_version = line.split[1].strip
-      break
-    end
-  end
-
-  # Pull architecture dependent package config
-  begin
-    template "#{spack_configs_dir}/packages.yaml" do
-      cookbook 'aws-parallelcluster-environment'
-      source "spack/packages-#{arch_target}.yaml.erb"
-      owner new_resource.spack_user
-      group new_resource.spack_user
-      variables(libfabric_version: libfabric_version)
-    end
-  rescue Chef::Exceptions::FileNotFound
-    Chef::Log.warn "Could not find template for #{arch_target}"
-  end
 
   cookbook_file "#{spack_configs_dir}/modules.yaml" do
     cookbook 'aws-parallelcluster-environment'
     source 'spack/modules.yaml'
-    owner new_resource.spack_user
-    group new_resource.spack_user
+    owner 'root'
+    group 'root'
     mode '0755'
   end
 
   bash 'setup Spack' do
-    user new_resource.spack_user
-    group new_resource.spack_user
+    user 'root'
+    group 'root'
     code <<-SPACK
       source #{new_resource.spack_root}/share/spack/setup-env.sh
 
@@ -97,9 +71,7 @@
     SPACK
   end
 
-  node.default['cluster']['spack']['user'] = new_resource.spack_user
   node.default['cluster']['spack']['root'] = new_resource.spack_root
-  node.default['cluster']['libfabric_version'] = libfabric_version
   node_attributes 'dump node attributes'
 end
 
@@ -114,15 +86,51 @@
   spack = "#{new_resource.spack_root}/bin/spack"
 
   bash 'add binaries' do
-    user new_resource.spack_user
-    group new_resource.spack_user
+    user 'root'
+    group 'root'
     code <<-SPACK
       [ -z "${CI_PROJECT_DIR}" ] && #{spack} mirror add --scope site "aws-pcluster" "https://binaries.spack.io/develop/aws-pcluster-$(spack arch -t | sed -e 's?_avx512??1')" || true
       #{spack} buildcache keys --install --trust
     SPACK
   end
 end
 
+action :configure do
+  return if on_docker?
+
+  case node['cluster']['node_type']
+  when 'HeadNode'
+
+    # Find libfabric version to be used in package configs
+    libfabric_version = nil
+    ::File.open(libfabric_path).each do |line|
+      if line.include?('Version:')
+        libfabric_version = line.split[1].strip
+        break
+      end
+    end
+
+    spack = "#{new_resource.spack_root}/bin/spack"
+    spack_configs_dir = "#{new_resource.spack_root}/etc/spack"
+
+    arch_target = `#{spack} arch -t`.strip
+    Chef::Log.info "The processor family is #{arch_target}"
+    # Pull architecture dependent package config
+    template "#{spack_configs_dir}/packages.yaml" do
+      cookbook 'aws-parallelcluster-environment'
+      source "spack/packages-#{arch_target}.yaml.erb"
+      owner 'root'
+      group 'root'
+      variables(libfabric_version: libfabric_version)
+      only_if { run_context.has_template_in_cookbook?('aws-parallelcluster-environment', "spack/packages-#{arch_target}.yaml.erb") && spack_installed? }
+    end
+
+    node.default['cluster']['libfabric_version'] = libfabric_version
+    node_attributes 'dump node attributes'
+
+  end
+end
+
 action_class do
   def spack_installed?
     ::Dir.exist?(new_resource.spack_root)

cookbooks/aws-parallelcluster-environment/test/controls/spack_spec.rb

@@ -16,14 +16,14 @@
 
   describe directory(node['cluster']['spack']['root']) do
     it { should exist }
-    it { should be_owned_by (node['cluster']['spack']['user']).to_s }
-    it { should be_grouped_into (node['cluster']['spack']['user']).to_s }
+    it { should be_owned_by 'root' }
+    it { should be_grouped_into 'root' }
   end
 
   describe directory("#{node['cluster']['spack']['root']}/share/spack") do
     it { should exist }
-    it { should be_owned_by (node['cluster']['spack']['user']).to_s }
-    it { should be_grouped_into (node['cluster']['spack']['user']).to_s }
+    it { should be_owned_by 'root' }
+    it { should be_grouped_into 'root' }
   end
 
   describe file('/etc/profile.d/spack.sh') do
@@ -34,13 +34,13 @@
 
   describe file("#{node['cluster']['spack']['root']}/etc/spack/compilers.yaml") do
     it { should exist }
-    it { should be_owned_by (node['cluster']['spack']['user']).to_s }
-    it { should be_grouped_into (node['cluster']['spack']['user']).to_s }
+    it { should be_owned_by 'root' }
+    it { should be_grouped_into 'root' }
   end
 
   spack = "#{node['cluster']['spack']['root']}/bin/spack"
-  describe "spack commands can run as cluster default user #{node['cluster']['spack']['user']}" do
-    subject { bash("su - #{node['cluster']['spack']['user']} -c '#{spack} find'") }
+  describe "spack commands can run as cluster default user #{node['cluster']['cluster_user']}" do
+    subject { bash("su - #{node['cluster']['cluster_user']} -c '#{spack} find'") }
     its('exit_status') { should eq(0) }
   end
 
@@ -54,8 +54,8 @@
   only_if { !os_properties.on_docker? }
   spack = "#{node['cluster']['spack']['root']}/bin/spack"
 
-  describe "spack can install packages as cluster default user #{node['cluster']['spack']['user']}" do
-    subject { bash("sudo su - #{node['cluster']['spack']['user']} -c '#{spack} install xz'") }
+  describe "spack can install packages as cluster default user #{node['cluster']['cluster_user']}" do
+    subject { bash("sudo su - #{node['cluster']['cluster_user']} -c 'sudo #{spack} install xz'") }
     its('exit_status') { should eq(0) }
   end
 
@@ -64,3 +64,15 @@
     its('exit_status') { should eq(0) }
   end
 end
+
+control 'tag:config_spack_packages_config_exist' do
+  title 'Check that spack has packages.yaml'
+
+  only_if { !os_properties.on_docker? && instance.head_node? }
+
+  describe file("#{node['cluster']['spack']['root']}/etc/spack/packages.yaml") do
+    it { should exist }
+    it { should be_owned_by 'root' }
+    it { should be_grouped_into 'root' }
+  end
+end