Formainpr (#93)

* From the terraform folder, can use ./login to log in the head node. * CORA ADCIRC added * New EFS software has bugs, using older version * adcirc-cora README * squash merged some updates: hindcast testing, debugged patch for adcirc, saving liveocean * Minor fixes * LiveOcean tests and fixes * LO hindcast workflow updates and testing * LO and CORA ADCIRC work on NOS Sandbox * tested deployment * cora and liveocean work from user chosen directories * Removed NAME tag from cluster/config files
ioos · Jun 13, 2024 · 1871055 · 1871055
1 parent 3153da3
commit 1871055
Show file tree

Hide file tree

Showing 67 changed files with 9,335 additions and 200 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,18 +2,16 @@
 .DS_Store
 tmp
 *.egg-info/
+*.swp
 .ipynb_checkpoints
-worflow/*.ipynb
+**/*.ipynb
 .pytest_cache/
-workflow/dist
-workflow/*.out
+cloudflow/*.out
+cloudflow/*.log
 **/worker-*.dirlock
 **/purge.lock
 **/global.lock
-workflow/workflows/lo.out
 .idea/
-workflow/.idea/
-workflow/venv/
 .eggs/
 dist/
 docs/build/
@@ -33,3 +31,4 @@ my.tfvars
 my.plan
 build/
 cloudflow/build/
+.nfs*
diff --git a/cloudflow/cluster/AWSCluster.py b/cloudflow/cluster/AWSCluster.py
@@ -28,7 +28,7 @@
 timelog.setLevel(logging.DEBUG)
 timelog.propagate = False
 
-fh = logging.FileHandler(f"{homedir}/qops_forecast.log")
+fh = logging.FileHandler(f"{homedir}/cluster_runtime.log")
 fh.setLevel(logging.DEBUG)
 formatter = logging.Formatter(' %(asctime)s  %(levelname)s | %(message)s')
 fh.setFormatter(formatter)

diff --git a/cloudflow/cluster/configs/NOS/nos.cora.cfg b/cloudflow/cluster/configs/NOS/nos.cora.cfg
@@ -0,0 +1,15 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "hpc6a.48xlarge",
+"nodeCount" : 1,
+"tags"      : [ 
+                { "Key": "Name", "Value": "CORA-node" },
+                { "Key": "Project", "Value": "NOS-cloud-sandbox" }
+              ],
+"image_id"  : "ami-082e2e4e7ef1c6d77",
+"key_name"  : "rsa_sandbox_key",
+"sg_ids"    : ["sg-05c044182398b2a27", "sg-0e5148638d9196f69", "sg-06ca6bc5d4b377dad"],
+"subnet_id" : "subnet-00075cfbfcbc8f2cf",
+"placement_group" : "ioos_cloud_sandbox_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/NOS/nos.liveocean.cfg b/cloudflow/cluster/configs/NOS/nos.liveocean.cfg
@@ -0,0 +1,15 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "r7iz.32xlarge",
+"nodeCount" : 4,
+"tags"      : [ 
+                { "Key": "Name", "Value": "LiveOcean-node" },
+                { "Key": "Project", "Value": "NOS-cloud-sandbox" }
+              ],
+"image_id"  : "ami-082e2e4e7ef1c6d77",
+"key_name"  : "rsa_sandbox_key",
+"sg_ids"    : ["sg-05c044182398b2a27", "sg-0e5148638d9196f69", "sg-06ca6bc5d4b377dad"],
+"subnet_id" : "subnet-00075cfbfcbc8f2cf",
+"placement_group" : "ioos_cloud_sandbox_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/ioos.amd.config → ...dflow/cluster/configs/RPS/ioos.amd.config b/cloudflow/cluster/configs/ioos.amd.config → ...dflow/cluster/configs/RPS/ioos.amd.config
@@ -5,8 +5,7 @@
 "nodeCount" : 2,
 "tags"      : [ 
                 { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "Project", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "NAME", "Value": "nosofs-fcst" }
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
               ],
 "image_id"  : "ami-01c437ac2b5beddda",
 "key_name"  : "ioos-sandbox",

diff --git a/cloudflow/cluster/configs/RPS/ioos.cora.cfg b/cloudflow/cluster/configs/RPS/ioos.cora.cfg
@@ -0,0 +1,19 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "hpc6a.48xlarge",
+"nodeCount" : 1,
+"tags"      : [ 
+                { "Key": "Name", "Value": "CORA-Cloud-Sandbox-node" },
+                { "Key": "Project", "Value": "IOOS-Cloud-Sandbox" }
+              ],
+"image_id"  : "ami-0d97c0e5cea6aa80b",
+"key_name"  : "ioos-sandbox",
+"sg_ids"    : [
+  "sg-070faae1c63b4b0a2",
+  "sg-0804c2befa81516aa",
+  "sg-0f6684b52d2e9e107"
+  ],
+"subnet_id" : "subnet-05764d14e38acec51",
+"placement_group" : "cora-cloud-sandbox-us-east-2b_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/ioos.intel.config → ...low/cluster/configs/RPS/ioos.intel.config b/cloudflow/cluster/configs/ioos.intel.config → ...low/cluster/configs/RPS/ioos.intel.config
@@ -5,8 +5,7 @@
 "nodeCount" : 1,
 "tags"      : [ 
                 { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "Project", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "NAME", "Value": "nosofs-fcst" }
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
               ],
 "image_id"  : "ami-052928467e84be78c",
 "key_name"  : "ioos-sandbox",

diff --git a/cloudflow/cluster/configs/liveocean.rps → ...ow/cluster/configs/RPS/ioos.liveocean.cfg b/cloudflow/cluster/configs/liveocean.rps → ...ow/cluster/configs/RPS/ioos.liveocean.cfg
@@ -2,11 +2,10 @@
 "platform"  : "AWS",
 "region"    : "us-east-2",
 "nodeType"  : "hpc6a.48xlarge",
-"nodeCount" : 2,
+"nodeCount" : 8,
 "tags"      : [ 
                 { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "Project", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "NAME", "Value": "liveocean-fcst" }
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
               ],
 "image_id"  : "ami-07d0aa6a6ac23d33a",
 "key_name"  : "ioos-sandbox",

diff --git a/cloudflow/cluster/configs/RPS/ioos.nosofs.cfg b/cloudflow/cluster/configs/RPS/ioos.nosofs.cfg
@@ -0,0 +1,15 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "hpc6a.48xlarge",
+"nodeCount" : 8,
+"tags"      : [ 
+                { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
+              ],
+"image_id"  : "ami-07d0aa6a6ac23d33a",
+"key_name"  : "ioos-sandbox",
+"sg_ids"    :  [ "sg-054f4248a8fc54750", "sg-0c9bb98732e936e66", "sg-0ce1abfe87363d80b" ],
+"subnet_id" : "subnet-04b0eef9cb8015d31",
+"placement_group" : "ioos_cloud_sandbox-us-east-2b_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/nosofs.config → cloudflow/cluster/configs/RPS/nosofs.config b/cloudflow/cluster/configs/nosofs.config → cloudflow/cluster/configs/RPS/nosofs.config
@@ -5,8 +5,7 @@
 "nodeCount" : 1,
 "tags"      : [ 
                 { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "Project", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "NAME", "Value": "nosofs-fcst" }
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
               ],
 "image_id"  : "ami-076b2467dfd17d05e",
 "key_name"  : "ioos-sandbox",

diff --git a/cloudflow/cluster/configs/RPS/test.cora.cfg b/cloudflow/cluster/configs/RPS/test.cora.cfg
@@ -0,0 +1,18 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "hpc6a.48xlarge",
+"nodeCount" : 1,
+"tags"      : [ 
+                { "Key": "Name", "Value": "CORA-Cloud-Sandbox-node" },
+                { "Key": "Project", "Value": "IOOS-Cloud-Sandbox" }
+              ],
+"image_id"  : "ami-049a762c1343a9d98",
+"key_name"  : "ioos-sandbox",
+"sg_ids"    : [ "sg-0feaeb0d419b7ddda",
+  "sg-046f683021585a25f",
+  "sg-0416cbb8c060a1779"
+  ],
+"subnet_id" : "subnet-0be869ddbf3096968",
+"placement_group" : "forPR-cloud-sandbox-us-east-2b_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/RPS/test.liveocean.cfg b/cloudflow/cluster/configs/RPS/test.liveocean.cfg
@@ -0,0 +1,18 @@
+{
+"platform"  : "AWS",
+"region"    : "us-east-2",
+"nodeType"  : "hpc6a.48xlarge",
+"nodeCount" : 8,
+"tags"      : [ 
+                { "Key": "Name", "Value": "forPR-Cloud-Sandbox-node" },
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
+              ],
+"image_id"  : "ami-049a762c1343a9d98",
+"key_name"  : "ioos-sandbox",
+"sg_ids"    : [ "sg-0feaeb0d419b7ddda",
+  "sg-046f683021585a25f",
+  "sg-0416cbb8c060a1779"
+  ],
+"subnet_id" : "subnet-0be869ddbf3096968",
+"placement_group" : "forPR-cloud-sandbox-us-east-2b_Terraform_Placement_Group"
+}
diff --git a/cloudflow/cluster/configs/get_aws_ec2meta.sh b/cloudflow/cluster/configs/get_aws_ec2meta.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#set -x
+
+TOKEN=`curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`
+META="http://169.254.169.254/latest/meta-data"
+
+INSTANCEID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/instance-id)
+IFACE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/network/interfaces/macs)
+SUBNET_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/network/interfaces/macs/${IFACE}/subnet-id)
+VPC_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/network/interfaces/macs/${IFACE}/vpc-id)
+REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/placement/availability-zone | sed 's/[a-z]$//')
+MAC=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/network/interfaces/macs | head -1)
+SGIDS=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" $META/network/interfaces/macs/${MAC}/security-group-ids)
+
+#AMIID=`tail /tmp/setup.log | grep ImageId`
+
+#PLACEMENT_GROUP=`aws ec2 describe-placement-groups | grep GroupId | awk -F\" '{print $4}'`
+#PLACEMENT_GROUP=`aws ec2 describe-placement-groups | grep GroupId`
+
+echo "instanceid: $INSTANCEID"
+#echo "iface: $IFACE"
+echo "region: $REGION"
+echo "vpc_id: $VPC_ID"
+echo "image_id: $AMIID"
+echo "sg_ids: "
+echo "$SGIDS"
+echo "subnet_id: $SUBNET_ID"
+echo "placement_group: $PLACEMENT_GROUP"
+
+#tail /tmp/setup.log | grep ImageId
+
+# Can manually create an AMI here
+# ami_name="IOOS Cloud Sandbox AMI"
+# project="IOOS-cloud-sandbox"
+
+# /usr/local/bin/aws --region ${REGION} ec2 create-image --instance-id $INSTANCEID --name "${ami_name}" \
+#  --tag-specification "ResourceType=image,Tags=[{Key=\"Name\",Value=\"${ami_name}\"},{Key=\"Project\",Value=\"${project}\"}]"
diff --git a/cloudflow/cluster/configs/ioos.config b/cloudflow/cluster/configs/ioos.config
diff --git a/cloudflow/cluster/configs/liveocean.config b/cloudflow/cluster/configs/liveocean.config
diff --git a/cloudflow/cluster/configs/liveocean.ioos b/cloudflow/cluster/configs/liveocean.ioos
@@ -0,0 +1 @@
+NOS/nos.liveocean.cfg
diff --git a/cloudflow/cluster/configs/noaa.cora.config b/cloudflow/cluster/configs/noaa.cora.config
@@ -0,0 +1 @@
+NOS/nos.cora.cfg
diff --git a/cloudflow/cluster/configs/post.config b/cloudflow/cluster/configs/post.config
@@ -5,8 +5,7 @@
 "nodeCount" : 1,
 "tags"      : [ 
                 { "Key": "Name", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "Project", "Value": "IOOS-cloud-sandbox" },
-                { "Key": "NAME", "Value": "nosofs plotting" }
+                { "Key": "Project", "Value": "IOOS-cloud-sandbox" }
               ],
 "image_id"  : "ami-052f3e9220b9961dc",
 "key_name"  : "patrick-ioos-cloud-sandbox",

diff --git a/cloudflow/cluster/nodeInfo.py b/cloudflow/cluster/nodeInfo.py
@@ -5,16 +5,13 @@
 __license__ = "BSD 3-Clause"
 
 
-
 awsTypes = {'c5.large': 1, 'c5.xlarge': 2, 'c5.2xlarge': 4, 'c5.4xlarge': 8, 'c5.9xlarge': 18, 'c5.12xlarge': 24,
             'c5.18xlarge': 36, 'c5.24xlarge': 48, 'c5.metal': 36,
             'c5n.large': 1, 'c5n.xlarge': 2, 'c5n.2xlarge': 4, 'c5n.4xlarge': 8, 'c5n.9xlarge': 18,
             'c5n.18xlarge': 36, 'c5n.24xlarge': 48, 'c5n.metal': 36,
             't3.large': 1, 't3.xlarge': 2, 't3.2xlarge': 4,
             'c5a.2xlarge': 4, 'c5a.4xlarge': 8, 'c5a.24xlarge': 48,
-            'hpc6a.48xlarge': 96 }
-
-efaTypes = ['c5n.18xlarge', 'hpc6a.48xlarge']
+            'hpc6a.48xlarge': 96, 'hpc7a.96xlarge': 192, 'hpc7a.48xlarge': 96, 'c7i.48xlarge': 96, 'r7iz.32xlarge': 64 }
 
 
 def getPPN(instance_type):
@@ -33,16 +30,6 @@ def getPPN(instance_type):
     Exception if instance_type is not found
     """
 
-    # awsTypes = {'c5.large': 1, 'c5.xlarge': 2, 'c5.2xlarge': 4, 'c5.4xlarge': 8, 'c5.9xlarge': 18, 'c5.12xlarge': 24,
-    #             'c5.18xlarge': 36, 'c5.24xlarge': 48, 'c5.metal': 36, 
-    #             'c5n.large': 1, 'c5n.xlarge': 2, 'c5n.2xlarge': 4, 'c5n.4xlarge': 8, 'c5n.9xlarge': 18, 
-    #             'c5n.18xlarge': 36, 'c5n.24xlarge': 48, 'c5n.metal': 36,
-    #             't3.large': 1, 't3.xlarge': 2, 't3.2xlarge': 4, 
-    #             'c5a.2xlarge': 4, 'c5a.4xlarge': 8, 'c5a.24xlarge': 48,
-    #             'hpc6a.48xlarge': 96 }
-
-    # efaTypes = ['c5n.18xlarge', 'hpc6a.48xlarge']
-
     try:
         ppn = awsTypes[instance_type]
     except Exception as e: