From bfd5aa82f6af9972f7dbcc18b9b07ece60ff194a Mon Sep 17 00:00:00 2001
From: treydock <tdockendorf@osc.edu>
Date: Tue, 19 Sep 2023 10:16:27 -0400
Subject: [PATCH] Refactor how cluster YAML is generated to stop using ERB
 template (#137)

* Refactor how cluster YAML is generated to stop using ERB template

* Ensure ACLs are optional

* Improve grafana config handling to avoid unnecessary values showing up
---
 manifests/cluster.pp           | 178 +++++++++++++++++++++-
 spec/defines/cluster_spec.rb   |  14 +-
 templates/cluster/main.yml.erb | 266 ---------------------------------
 3 files changed, 184 insertions(+), 274 deletions(-)
 delete mode 100644 templates/cluster/main.yml.erb

diff --git a/manifests/cluster.pp b/manifests/cluster.pp
index 040c58d..c78448e 100644
--- a/manifests/cluster.pp
+++ b/manifests/cluster.pp
@@ -76,7 +76,7 @@
   Stdlib::Filemode $mode = '0644',
   Optional[Variant[Stdlib::HTTPSUrl, Stdlib::HTTPUrl]] $url = undef,
   Boolean $hidden = false,
-  Array[Openondemand::Acl] $acls = [],
+  Optional[Array[Openondemand::Acl]] $acls = undef,
   Optional[Stdlib::Host] $login_host = undef,
   Optional[Enum['torque','slurm','lsf','pbspro','sge','linux_host','kubernetes']] $job_adapter = undef,
   Optional[String] $job_cluster = undef,
@@ -92,9 +92,9 @@
   Optional[Stdlib::Absolutepath] $sge_root = undef,
   Optional[Stdlib::Absolutepath] $libdrmaa_path = undef,
   Optional[String] $job_version = undef,
-  Hash[String, Stdlib::Absolutepath] $job_bin_overrides = {},
+  Optional[Hash[String, Stdlib::Absolutepath]] $job_bin_overrides = undef,
   Optional[Stdlib::Host] $job_submit_host = undef,
-  Array[Stdlib::Host] $job_ssh_hosts = [],
+  Optional[Array[Stdlib::Host]] $job_ssh_hosts = undef,
   Optional[Integer] $job_site_timeout = undef,
   Optional[Boolean] $job_debug = undef,
   Optional[Stdlib::Absolutepath] $job_singularity_bin = undef,
@@ -109,7 +109,7 @@
   Boolean $job_all_namespaces = false,
   Boolean $job_auto_supplemental_groups = false,
   Optional[Openondemand::K8_server] $job_server = undef,
-  Array[Openondemand::K8_mount] $job_mounts = [],
+  Optional[Array[Openondemand::K8_mount]] $job_mounts = undef,
   Optional[Openondemand::K8_auth] $job_auth = undef,
   # END Kubernetes
   Optional[Enum['moab']] $scheduler_type = undef,
@@ -117,7 +117,7 @@
   Optional[Stdlib::Absolutepath] $scheduler_bin = undef,
   Optional[String] $scheduler_version = undef,
   Hash $scheduler_params = {},
-  Array[Openondemand::Acl] $rsv_query_acls = [],
+  Optional[Array[Openondemand::Acl]] $rsv_query_acls = undef,
   Optional[Stdlib::Host] $ganglia_host = undef,
   String $ganglia_scheme = 'https://',
   Array $ganglia_segments = ['gweb', 'graph.php'],
@@ -141,7 +141,7 @@
   Optional[String] $grafana_cluster_override = undef,
   Optional[Integer] $xdmod_resource_id = undef,
   Hash $custom_config = {},
-  Openondemand::Batch_connect $batch_connect = {},
+  Optional[Openondemand::Batch_connect] $batch_connect = undef,
 ) {
   include openondemand
 
@@ -160,12 +160,176 @@
     $_job_bin = $job_bin
   }
 
+  if $job_singularity_bindpath =~ Array {
+    $_job_singularity_bindpath = join($job_singularity_bindpath, ',')
+  } else {
+    $_job_singularity_bindpath = $job_singularity_bindpath
+  }
+
+  $metadata = {
+    'title' => $cluster_title,
+    'url'   => $url,
+    'hidden' => $hidden,
+  }.filter |$k,$v| { $v =~ NotUndef }
+
+  if $login_host {
+    $login = {
+      'host' => $login_host,
+    }
+  } else {
+    $login = undef
+  }
+
+  if $job_adapter {
+    if $job_adapter == 'kubernetes' {
+      $job_kubernetes = {
+        'config_file' => $job_config_file,
+        'username_prefix' => $job_username_prefix,
+        'namespace_prefix'  => $job_namespace_prefix,
+        'all_namespaces'    => $job_all_namespaces,
+        'auto_supplemental_groups'  => $job_auto_supplemental_groups,
+        'server'                    => $job_server,
+        'mounts'                    => $job_mounts,
+        'auth'                      => $job_auth,
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $job_kubernetes = {}
+    }
+    $job_base = {
+      'adapter'               => $job_adapter,
+      'cluster'               => $job_cluster,
+      'host'                  => $job_host,
+      'lib'                   => $job_lib,
+      'libdir'                => $job_libdir,
+      'bin'                   => $_job_bin,
+      'bindir'                => $job_bindir,
+      'conf'                  => $job_conf,
+      'envdir'                => $job_envdir,
+      'serverdir'             => $job_serverdir,
+      'exec'                  => $job_exec,
+      'sge_root'              => $sge_root,
+      'libdrmaa_path'         => $libdrmaa_path,
+      'version'               => $job_version,
+      'submit_host'           => $job_submit_host,
+      'ssh_hosts'             => $job_ssh_hosts,
+      'site_timeout'          => $job_site_timeout,
+      'debug'                 => $job_debug,
+      'singularity_bin'       => $job_singularity_bin,
+      'singularity_bindpath'  => $_job_singularity_bindpath,
+      'singularity_image'     => $job_singularity_image,
+      'strict_host_checking'  => $job_strict_host_checking,
+      'tmux_bin'              => $job_tmux_bin,
+      'bin_overrides'         => $job_bin_overrides,
+    }.filter |$k,$v| { $v =~ NotUndef }
+    $job = $job_base + $job_kubernetes
+  } else {
+    $job = undef
+  }
+
+  if $job_adapter == 'torque' or $scheduler_type or $ganglia_host or $grafana_host or $xdmod_resource_id or ! empty($custom_config) {
+    if $job_adapter == 'torque' {
+      $pbs = {
+        'host'    => $job_host,
+        'lib'     => $job_lib,
+        'bin'     => $job_bin,
+        'version' => $job_version,
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $pbs = undef
+    }
+    if $scheduler_type == 'moab' {
+      $moab = {
+        'host'    => $scheduler_host,
+        'bin'     => $scheduler_bin,
+        'version' => $scheduler_version,
+        'homedir' => $scheduler_params['moabhomedir'],
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $moab = undef
+    }
+    if $job_adapter == 'torque' and $scheduler_type == 'moab' {
+      $rsv_query = {
+        'torque_host'    => $job_host,
+        'torque_lib'     => $job_lib,
+        'torque_bin'     => $job_bin,
+        'torque_version' => $job_version,
+        'moab_host'      => $scheduler_host,
+        'moab_bin'       => $scheduler_bin,
+        'moab_version'   => $scheduler_version,
+        'moab_homedir'   => $scheduler_params['moabhomedir'],
+        'acls'           => $rsv_query_acls,
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $rsv_query = undef
+    }
+    if $ganglia_host {
+      $ganglia = {
+        'host'      => $ganglia_host,
+        'scheme'    => $ganglia_scheme,
+        'segments'  => $ganglia_segments,
+        'req_query' => $ganglia_req_query,
+        'opt_query' => $ganglia_opt_query,
+        'version'   => $ganglia_version,
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $ganglia = undef
+    }
+    if $grafana_host {
+      $grafana_dashboard = {
+        'name'    => $grafana_dashboard_name,
+        'uid'     => $grafana_dashboard_uid,
+        'panels'  => $grafana_dashboard_panels,
+      }.filter |$k,$v| { $v =~ NotUndef }
+      $grafana = {
+        'host'              => $grafana_host,
+        'orgId'             => $grafana_org_id,
+        'theme'             => $grafana_theme,
+        'dashboard'         => $grafana_dashboard,
+        'labels'            => $grafana_labels,
+        'cluster_override'  => $grafana_cluster_override,
+      }.filter |$k,$v| { $v =~ NotUndef }
+    } else {
+      $grafana = undef
+    }
+    if $xdmod_resource_id {
+      $xdmod = {
+        'resource_id' => $xdmod_resource_id,
+      }
+    } else {
+      $xdmod = undef
+    }
+    $custom_base = {
+      'pbs'       => $pbs,
+      'moab'      => $moab,
+      'rsv_query' => $rsv_query,
+      'ganglia'   => $ganglia,
+      'grafana'   => $grafana,
+      'xdmod'     => $xdmod,
+    }.filter |$k,$v| { $v =~ NotUndef }
+    $custom = $custom_base + $custom_config
+  } else {
+    $custom = undef
+  }
+
+  $v2 = {
+    'metadata'      => $metadata,
+    'acls'          => $acls,
+    'login'         => $login,
+    'job'           => $job,
+    'custom'        => $custom,
+    'batch_connect' => $batch_connect,
+  }.filter |$k,$v| { $v =~ NotUndef }
+
+  $config = {
+    'v2' => $v2,
+  }
+
   file { "/etc/ood/config/clusters.d/${name}.yml":
     ensure  => 'file',
     owner   => $owner,
     group   => $group,
     mode    => $mode,
-    content => template('openondemand/cluster/main.yml.erb'),
+    content => to_yaml($config),
     notify  => Class['openondemand::service'],
   }
 }
diff --git a/spec/defines/cluster_spec.rb b/spec/defines/cluster_spec.rb
index 0914191..a4abeb5 100644
--- a/spec/defines/cluster_spec.rb
+++ b/spec/defines/cluster_spec.rb
@@ -87,7 +87,15 @@
               foo_string: 'bar',
               foo_bool: false,
               foo_array: ['1', '2', 3],
-              foo_hash: { 'foo' => 'bar', 'bar' => 'baz', 'baz' => false }
+              foo_hash: { 'foo' => 'bar', 'bar' => 'baz', 'baz' => false },
+              classrooms: {
+                jupyter: {
+                  AI_BOOTCAMP_OSC: {
+                    hours: 3,
+                    project: 'FOO'
+                  }
+                }
+              }
             },
           )
         end
@@ -96,11 +104,15 @@
 
         it 'hash valid custom config' do
           content = catalogue.resource('file', '/etc/ood/config/clusters.d/test.yml').send(:parameters)[:content]
+          puts content
           data = YAML.safe_load(content)
           expect(data['v2']['custom']['foo_string']).to eq('bar')
           expect(data['v2']['custom']['foo_bool']).to eq(false)
           expect(data['v2']['custom']['foo_array']).to eq(['1', '2', 3])
           expect(data['v2']['custom']['foo_hash']).to eq('foo' => 'bar', 'bar' => 'baz', 'baz' => false)
+          expect(data['v2']['custom'].key?('classrooms')).to eq(true)
+          expect(data['v2']['custom']['classrooms'].key?('jupyter')).to eq(true)
+          expect(data['v2']['custom'].dig('classrooms', 'jupyter', 'AI_BOOTCAMP_OSC', 'hours')).to eq(3)
         end
       end
 
diff --git a/templates/cluster/main.yml.erb b/templates/cluster/main.yml.erb
deleted file mode 100644
index c8c4543..0000000
--- a/templates/cluster/main.yml.erb
+++ /dev/null
@@ -1,266 +0,0 @@
----
-v2:
-  metadata:
-    title: "<%= @cluster_title %>"
-<%- if @url -%>
-    url: "<%= @url %>"
-<%- end -%>
-    hidden: <%= @hidden %>
-<%- if ! @acls.empty? -%>
-  acls:
-  <%- @acls.each do |acl| -%>
-    - adapter: "<%= acl['adapter'] %>"
-    <%- if acl['groups'] -%>
-      groups:
-      <%- acl["groups"].each do |g| -%>
-        - "<%= g %>"
-      <%- end -%>
-    <%- end -%>
-      type: "<%= acl['type'] %>"
-  <%- end -%>
-<%- end -%>
-<% if @login_host -%>
-  login:
-    host: "<%= @login_host %>"
-<% end -%>
-<% if @job_adapter -%>
-  job:
-    adapter: "<%= @job_adapter %>"
-  <%- if @job_cluster -%>
-    cluster: "<%= @job_cluster %>"
-  <%- end -%>
-  <%- if @job_host -%>
-    host: "<%= @job_host %>"
-  <%- end -%>
-  <%- if @job_lib -%>
-    lib: "<%= @job_lib %>"
-  <%- end -%>
-  <%- if @job_libdir -%>
-    libdir: "<%= @job_libdir %>"
-  <%- end -%>
-  <%- if @_job_bin -%>
-    bin: "<%= @_job_bin %>"
-  <%- end -%>
-  <%- if @job_bindir -%>
-    bindir: "<%= @job_bindir %>"
-  <%- end -%>
-  <%- if @job_conf -%>
-    conf: "<%= @job_conf %>"
-  <%- end -%>
-  <%- if @job_envdir -%>
-    envdir: "<%= @job_envdir %>"
-  <%- end -%>
-  <%- if @job_serverdir -%>
-    serverdir: "<%= @job_serverdir %>"
-  <%- end -%>
-  <%- if @job_exec -%>
-    exec: "<%= @job_exec %>"
-  <%- end -%>
-  <%- if @sge_root -%>
-    sge_root: "<%= @sge_root %>"
-  <%- end -%>
-  <%- if @libdrmaa_path -%>
-    libdrmaa_path: "<%= @libdrmaa_path %>"
-  <%- end -%>
-  <%- if @job_version -%>
-    version: "<%= @job_version %>"
-  <%- end -%>
-  <%- if @job_submit_host -%>
-    submit_host: "<%= @job_submit_host %>"
-  <%- end -%>
-  <%- if ! @job_ssh_hosts.empty? -%>
-    ssh_hosts:
-    <%- @job_ssh_hosts.each do |ssh_host| -%>
-      - "<%= ssh_host %>"
-    <%- end -%>
-  <%- end -%>
-  <%- if @job_site_timeout -%>
-    site_timeout: <%= @job_site_timeout %>
-  <%- end -%>
-  <%- if @job_debug -%>
-    debug: <%= @job_debug %>
-  <%- end -%>
-  <%- if @job_singularity_bin -%>
-    singularity_bin: "<%= @job_singularity_bin %>"
-  <%- end -%>
-  <%- if @job_singularity_bindpath -%>
-    <%- if @job_singularity_bindpath.is_a?(Array) %>
-    singularity_bindpath: "<%= @job_singularity_bindpath.join(',') %>"
-    <%- else -%>
-    singularity_bindpath: "<%= @job_singularity_bindpath %>"
-    <%- end -%>
-  <%- end -%>
-  <%- if @job_singularity_image -%>
-    singularity_image: "<%= @job_singularity_image %>"
-  <%- end -%>
-  <%- if !@job_strict_host_checking.nil? -%>
-    strict_host_checking: <%= @job_strict_host_checking %>
-  <%- end -%>
-  <%- if @job_tmux_bin -%>
-    tmux_bin: "<%= @job_tmux_bin %>"
-  <%- end -%>
-<%- if @job_adapter == 'kubernetes' -%>
-  <%- if @job_config_file -%>
-    config_file: "<%= @job_config_file %>"
-  <%- end -%>
-  <%- if @job_username_prefix -%>
-    username_prefix: "<%= @job_username_prefix %>"
-  <%- end -%>
-  <%- if @job_namespace_prefix -%>
-    namespace_prefix: "<%= @job_namespace_prefix %>"
-  <%- end -%>
-    all_namespaces: <%= @job_all_namespaces %>
-    auto_supplemental_groups: <%= @job_auto_supplemental_groups %>
-  <%- if @job_server -%>
-    server:
-    <%- @job_server.each_pair do |key, value| -%>
-      <%= key %>: "<%= value %>"
-    <%- end -%>
-  <%- end # if job_server -%>
-  <%- if ! @job_mounts.empty? -%>
-    mounts:
-    <%- @job_mounts.each do |mount| -%>
-      - name: "<%= mount['name'] %>"
-      <%- mount.each_pair do |key, value| -%>
-        <%- next if key == 'name' -%>
-        <%= key %>: "<%= value %>"
-      <%- end -%>
-    <%- end -%>
-  <%- end # if @job_mounts empty? -%>
-  <%- if @job_auth -%>
-    auth:
-    <%- @job_auth.each_pair do |key, value| -%>
-      <%= key %>: "<%= value %>"
-    <%- end -%>
-  <%- end # if job_auth -%>
-<%- end # if kubernetes -%>
-  <%- if ! @job_bin_overrides.empty? -%>
-    bin_overrides:
-    <%- @job_bin_overrides.each_pair do |command, path| -%>
-      <%= command %>: "<%= path %>"
-    <%- end -%>
-  <%- end -%>
-<% end -%>
-<% if @job_adapter == "torque" || @scheduler_type || @ganglia_host || @grafana_host || @xdmod_resource_id || !@custom_config.empty? -%>
-  custom:
-  <%- if @job_adapter == "torque" -%>
-    pbs:
-      host: "<%= @job_host %>"
-      lib: "<%= @job_lib %>"
-      bin: "<%= @job_bin %>"
-      version: "<%= @job_version %>"
-  <%- end -%>
-  <%- if @scheduler_type == "moab" -%>
-    moab:
-      host: "<%= @scheduler_host %>"
-      bin: "<%= @scheduler_bin %>"
-      version: "<%= @scheduler_version %>"
-      homedir: "<%= @scheduler_params["moabhomedir"] %>"
-  <%- end -%>
-  <%- if @job_adapter == "torque" && @scheduler_type == "moab" -%>
-    rsv_query:
-      torque_host: "<%= @job_host %>"
-      torque_lib: "<%= @job_lib %>"
-      torque_bin: "<%= @job_bin %>"
-      torque_version: "<%= @job_version %>"
-      moab_host: "<%= @scheduler_host %>"
-      moab_bin: "<%= @scheduler_bin %>"
-      moab_version: "<%= @scheduler_version %>"
-      moab_homedir: "<%= @scheduler_params["moabhomedir"] %>"
-    <%- if ! @rsv_query_acls.empty? -%>
-      acls:
-      <%- @rsv_query_acls.each do |rsv_query_acl| -%>
-        - adapter: "<%= rsv_query_acl['adapter'] %>"
-        <%- if rsv_query_acl['groups'] -%>
-          groups:
-          <%- rsv_query_acl["groups"].each do |g| -%>
-            - "<%= g %>"
-          <%- end -%>
-        <%- end -%>
-          type: "<%= rsv_query_acl['type'] %>"
-      <%- end -%>
-    <%- end -%>
-  <%- end -%>
-  <%- if @ganglia_host -%>
-    ganglia:
-      host: "<%= @ganglia_host %>"
-      scheme: "<%= @ganglia_scheme %>"
-      segments:
-      <%- @ganglia_segments.each do |gs| -%>
-        - "<%= gs %>"
-      <%- end -%>
-      req_query:
-      <%- @ganglia_req_query.each do |k, v| -%>
-        <%= k %>: "<%= v %>"
-      <%- end -%>
-      opt_query:
-      <%- @ganglia_opt_query.each do |k, v| -%>
-        <%= k %>: "<%= v %>"
-      <%- end -%>
-      version: "<%= @ganglia_version %>"
-  <%- end -%>
-  <%- if @grafana_host -%>
-    grafana:
-      host: "<%= @grafana_host %>"
-      orgId: <%= @grafana_org_id %>
-      <%- if @grafana_theme -%>
-      theme: "<%= @grafana_theme %>"
-      <%- end -%>
-      dashboard:
-        name: "<%= @grafana_dashboard_name %>"
-        uid: "<%= @grafana_dashboard_uid %>"
-        panels:
-          cpu: <%= @grafana_dashboard_panels['cpu'] %>
-          memory: <%= @grafana_dashboard_panels['memory'] %>
-      labels:
-      <%- @grafana_labels.each_pair do |k, v| -%>
-        <%= k %>: "<%= v %>"
-      <%- end -%>
-      <%- if @grafana_cluster_override -%>
-      cluster_override: "<%= @grafana_cluster_override %>"
-      <%- end -%>
-  <%- end -%>
-  <%- if @xdmod_resource_id -%>
-    xdmod:
-      resource_id: <%= @xdmod_resource_id %>
-  <%- end -%>
-  <%- @custom_config.each_pair do |key, value| -%>
-    <%- if value.is_a?(Hash) -%>
-    <%= key %>:
-    <%- value.each_pair do |k,v| -%>
-      <%= k %>: <% if v.is_a?(String) %>"<%= v %>"<% else %><%= v %><% end %>
-    <%- end # end value.each_pair -%>
-    <%- elsif value.is_a?(Array) -%>
-    <%= key %>:
-    <%- value.each do |v| -%>
-      - <% if v.is_a?(String) %>"<%= v %>"<% else %><%= v %><% end %>
-    <%- end # end value.each -%>
-    <%- else -%>
-    <%= key %>: <% if value.is_a?(String) %>"<%= value %>"<% else %><%= value %><% end %>
-    <%- end -%>
-  <%- end -%>
-<% end -%>
-<% if ! @batch_connect.empty? -%>
-  batch_connect:
-  <%- if @batch_connect.key?('basic') and @batch_connect['basic'].size > 0 -%>
-    basic:
-    <%- if @batch_connect['basic'].key?('script_wrapper') -%>
-      script_wrapper: "<%= @batch_connect['basic']['script_wrapper'] %>"
-    <%- end -%>
-    <%- if @batch_connect['basic'].key?('set_host') -%>
-      set_host: "<%= @batch_connect['basic']['set_host'] %>"
-    <%- end -%>
-  <%- end -%>
-  <%- if @batch_connect.key?('vnc') and @batch_connect['vnc'].size > 0 -%>
-    vnc:
-    <%- if @batch_connect['vnc'].key?('script_wrapper') -%>
-      script_wrapper: "<%= @batch_connect['vnc']['script_wrapper'] %>"
-    <%- end -%>
-    <%- if @batch_connect['vnc'].key?('set_host') -%>
-      set_host: "<%= @batch_connect['vnc']['set_host'] %>"
-    <%- end -%>
-  <%- end -%>
-  <%- if @batch_connect.key?('ssh_allow') -%>
-    ssh_allow: <%= @batch_connect['ssh_allow'] %>
-  <%- end -%>
-<% end -%>