Merge pull request #753 from at88mph/add-project-fix

Add project fix

deployment/k8s-config/add-project/README.md

@@ -1,3 +1,54 @@
-To add a project, put the project dir, owner, group, and access level as arguments in the yaml file and run either the add-project-keel-dev.sh or add-project-keel-prod.sh script.
+# add-project script
 
-The configuration map only has to be created once.
+To add a new project that shows up under the `projects` folder in Cavern.
+
+## Obtain UID/GID
+
+Projects are simply POSIX folders under the base project folder (see [`./config/projectdir`](./config/projectdir)).  As such, they need the owner's unique user id (UID) and a unique group ID (GID).  These are avaiable from the POSIX Mapper.
+
+### CANFAR (AC)
+
+Use a certificate or cookie to authenticate with AC:
+
+```sh
+curl -SsL -o cadccert.pem --netrc-file ~/.netrc "https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/cred/generate?daysValid=30"
+
+curl -E cadccert.pem "https://ws-cadc.canfar.net/ac/uidmap?user=<username-to-find>"
+
+# Results in standard POSIX output:
+<username-to-find>:x:uid:uid::
+
+curl -E cadccert.pem "https://ws-cadc.canfar.net/ac/gidmap?group=<group-uri-to-find>"
+# Example Group URI - ivo://cadc.nrc.ca/gms?mygroupname
+# Results in standard POSIX output:
+mygroupname:x:gid:
+```
+
+### SRCNet (OpenID Connect)
+
+Use an access token to authenticate with the POSIX Mapper.
+
+```sh
+eval $(oidc-agent-service use) > /dev/null
+
+# token-context-name is how the token was registered.
+# See https://confluence.skatelescope.org/pages/viewpage.action?spaceKey=SRCSC&title=RED-10+Using+oidc-agent+to+authenticate+to+OpenCADC+services
+export TOKEN=$(oidc-token token-context-name)
+
+curl --header "authorization: bearer ${TOKEN}" "https://src.canfar.net/posix-mapper/uid?user=<username-to-find>"
+# Results in standard POSIX output:
+<username-to-find>:x:uid:uid::
+
+curl --header "authorization: bearer ${TOKEN}" "https://src.canfar.net/posix-mapper/uid?group=<group-uri-to-find>"
+# Example Group URI - ivo://canfar.net/gms?mygroupname
+# Results in standard POSIX output:
+mygroupname:x:gid:
+
+```
+
+Then update the appropriate Kubernetes Job file (`skaha-add-project-keel-[dev|prod].yaml`), then run it with `kubectl -n skaha-system apply -f <job-file.yaml>`.
+
+Don't forget to clean up afterward:
+```sh
+kubectl -n skaha-system delete job skaha-add-project
+```

deployment/k8s-config/add-project/skaha-add-project-keel-dev.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -8,42 +9,34 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: "skaha-add-project"
-        image: images.canfar.net/skaha-system/add-project:1.2
+        image: images.canfar.net/skaha-system/add-project:1.3
         imagePullPolicy: Always
         # TODO: automate the setting of this in the calling script
         command: ["/usr/bin/add-project"]
-        # args: project-dir-name, owner-userid, project-group-name, read-only or read-write, quota-in-gb
-        args: ["test-project", "majorb", "skaha-users", "read-write", "1000"]
+        # args: project-name, owner-uid, project-group-gid, read-only or read-write, quota-in-gb, project-base-dir
+        args: ["project-name", "owner-uid", "project-gid", "project-permission", "project-quota-gb", "project-base-dir"]
         volumeMounts:
-        - mountPath: "/config"
-          name: add-project-config
-        - mountPath: "/arc"
+        - mountPath: "/cavern"
           name: cavern-volume
           subPath: cavern
-        - mountPath: /var/lib/sss/pipes
-          name: sssd-dir
-          readOnly: true
-      securityContext:
-        runAsUser: 0
+        securityContext:
+          runAsUser: 0
+          allowPrivilegeEscalation: false
       serviceAccountName: skaha
       volumes:
-      - name: add-project-config
-        configMap:
-          name: add-project-config
       - name: cavern-volume
         cephfs:
           monitors:
           - 10.30.201.3:6789
           - 10.30.202.3:6789
           - 10.30.203.3:6789
           path: /volumes/_nogroup/dcd994bc-c0d4-4557-9fbf-28fc4ef5969e
-          user: kanfarnetes_dev
+          user: keel-dev-admin
           secretRef:
             name: cephfs-cephx-admin-key
           readOnly: false
-      - name: sssd-dir
-        hostPath:
-          path: /var/lib/ubernetes
-          type: Directory

deployment/k8s-config/add-project/skaha-add-project-keel-prod.yaml

@@ -9,42 +9,34 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: "skaha-add-project"
-        image: images.canfar.net/skaha-system/add-project:1.2
+        image: images.canfar.net/skaha-system/add-project:1.3
         imagePullPolicy: Always
         # TODO: automate the setting of this in the calling script
         command: ["/usr/bin/add-project"]
-        # args: project-dir-name, owner-userid, project-group-name, read-only or read-write, quota-in-gb
-        args: ["myproject", "majorb", "mygroup", "read-write", "1000"]
+        # args: project-name, owner-uid, project-group-gid, read-only or read-write, quota-in-gb, project-base-dir
+        args: ["project-name", "owner-uid", "project-gid", "project-permission", "project-quota-gb", "project-base-dir"]
         volumeMounts:
-        - mountPath: "/config"
-          name: add-project-config
-        - mountPath: "/arc"
+        - mountPath: "/cavern"
           name: cavern-volume
           subPath: cavern
-        - mountPath: /var/lib/sss/pipes
-          name: sssd-dir
-          readOnly: true
-      securityContext:
-        runAsUser: 0
+        securityContext:
+          runAsUser: 0
+          allowPrivilegeEscalation: false
       serviceAccountName: skaha
       volumes:
-      - name: add-project-config
-        configMap:
-          name: add-project-config
       - name: cavern-volume
         cephfs:
           monitors:
           - 10.30.201.3:6789
           - 10.30.202.3:6789
           - 10.30.203.3:6789
           path: /volumes/_nogroup/054e398e-a08e-425e-9f7c-fc394362e38e
-          user: keel_prod
+          user: keel-prod-admin
           secretRef:
             name: cephfs-cephx-admin-key
           readOnly: false
-      - name: sssd-dir
-        hostPath:
-          path: /var/lib/ubernetes
-          type: Directory

deployment/ops-containers/add-project/Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:30 
+FROM fedora:40 
 
 # add often used tools
 RUN dnf -y install which

deployment/ops-containers/add-project/VERSION

@@ -1,4 +1,4 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor
 # build version tag: timestamp
-TAGS="1.2 $(date -u +"%Y%m%dT%H%M%S")"
+TAGS="1.3 $(date -u +"%Y%m%dT%H%M%S")"

deployment/ops-containers/add-project/src/add-project

@@ -6,21 +6,22 @@ set -e
 sleep 10
 
 SELF=add-project
-CONFDIR=/config
+USAGE_MESSAGE="Usage: add-project <project-dir-name> <project-dir-owner-uid> <group-gid> <read-only | read-write> <quota in GB> <project-dir-base>"
 
 TS=$(date)
 echo "$TS $SELF START"
 
-if [ -z "$5" ]
+if [ -z "$6" ]
   then
-    echo "Usage: add-project <project-dir-name> <project-dir-owner> <group-name> <read-only | read-write> <quota in GB>"
+    echo "${USAGE_MESSAGE}"
     exit 2
 fi
 PROJECT=$1
-OWNER=$2
-GROUP=$3
+OWNER_UID=$2
+GRANT_GID=$3
 ACCESS_ARG=$4
 QUOTA=$5
+PROJECTS_BASE_DIR=$6
 ACCESS=""
 MODE=""
 
@@ -33,28 +34,33 @@ elif [ $ACCESS_ARG == "read-write" ]
     ACCESS="rwx"
     MODE="770"
 else
-  echo "Usage: add-project <project-dir-name> <project-dir-owner> <group-name> <read-only | read-write>"
+  echo "${USAGE_MESSAGE}"
   exit 2
 fi
 
-if [ ! -f $CONFDIR/projectdir ]
+if [ ! -d $PROJECTS_BASE_DIR ]
   then
-    echo "No file projectdir found in $CONFDIR"
-    exit  2
+  echo "${PROJECTS_BASE_DIR} does not exist."
+  exit 2
 fi
 
-PROJECTBASE=`cat $CONFDIR/projectdir`
-PROJECTDIR="$PROJECTBASE/$PROJECT"
+PROJECTDIR="$PROJECTS_BASE_DIR/$PROJECT"
+
+if [ -d $PROJECTDIR ]
+  then
+  echo "Project $PROJECT already exists."
+  exit 2
+fi
 
 echo "Creating project $PROJECT"
 echo -n "  Creating project dir $PROJECTDIR..."
 mkdir $PROJECTDIR
 echo " Done"
-echo -n "  Setting permissions to $ACCESS for group $GROUP"
-chown $OWNER:$OWNER $PROJECTDIR
+echo -n "  Setting permissions to $ACCESS for group $GRANT_GID"
+chown $OWNER_UID:$OWNER_UID $PROJECTDIR
 chmod $MODE $PROJECTDIR
-setfacl -d -m group:$GROUP:$ACCESS $PROJECTDIR
-setfacl -m group:$GROUP:$ACCESS $PROJECTDIR
+setfacl -d -m group:$GRANT_GID:$ACCESS $PROJECTDIR
+setfacl -m group:$GRANT_GID:$ACCESS $PROJECTDIR
 echo -n "  Setting quota to ${QUOTA}G"
 setfattr -n ceph.quota.max_bytes -v ${QUOTA}000000000 $PROJECTDIR
 setfattr -n user.ivo://ivoa.net/vospace/core#quota -v ${QUOTA}000000000 $PROJECTDIR