From f74bcf0a1362ae8371d0d86d6d480847a720633f Mon Sep 17 00:00:00 2001 From: jooho lee Date: Mon, 2 Dec 2024 15:32:21 -0500 Subject: [PATCH] remove public ca and generate ca by operator Signed-off-by: jooho lee --- config/overlays/odh/kustomization.yaml | 2 +- config/overlays/odh/ray_tls_resources.yaml | 83 --------- config/overlays/odh/ray_tls_script.yaml | 69 +++++++ config/runtimes/vllm-multinode-template.yaml | 14 +- controllers/constants/constants.go | 4 +- controllers/kserve_ray_tls_controller.go | 50 ++--- controllers/kserve_ray_tls_controller_test.go | 102 ++++------- ...dated.yaml => ray-tls-script-updated.yaml} | 4 +- ...y-tls-scripts.yaml => ray-tls-script.yaml} | 4 +- .../deploy/vllm-multinode-servingruntime.yaml | 8 +- .../testdata/secrets/ray-ca-cert-updated.yaml | 9 +- controllers/testdata/secrets/ray-ca-cert.yaml | 13 -- controllers/utils/cert.go | 171 ++++++++++++++++++ 13 files changed, 332 insertions(+), 201 deletions(-) delete mode 100644 config/overlays/odh/ray_tls_resources.yaml create mode 100644 config/overlays/odh/ray_tls_script.yaml rename controllers/testdata/configmaps/{ray-tls-scripts-updated.yaml => ray-tls-script-updated.yaml} (94%) rename controllers/testdata/configmaps/{ray-tls-scripts.yaml => ray-tls-script.yaml} (94%) delete mode 100644 controllers/testdata/secrets/ray-ca-cert.yaml create mode 100644 controllers/utils/cert.go diff --git a/config/overlays/odh/kustomization.yaml b/config/overlays/odh/kustomization.yaml index a7ee7169..20826ef7 100644 --- a/config/overlays/odh/kustomization.yaml +++ b/config/overlays/odh/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../default - - ./ray_tls_resources.yaml + - ./ray_tls_script.yaml patches: - path: odh_model_controller_manager_patch.yaml diff --git a/config/overlays/odh/ray_tls_resources.yaml b/config/overlays/odh/ray_tls_resources.yaml deleted file mode 100644 index 02b6c261..00000000 --- a/config/overlays/odh/ray_tls_resources.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ray-ca-cert - labels: - opendatahub.io/managed: 'true' -data: - # output from cat ca.crt | base64 - ca.crt: | - LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIekNDQXdlZ0F3SUJBZ0lVSlAwL1FCY0xTMFFFV1ZiRDE4NndyUnZjLzdFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0hqRWNNQm9HQTFVRUF3d1RjMlZzWmkxemFXZHVaV1F0WTJFdFkyVnlkREFnRncweU5ERXhNak13TXpReApOVEphR0E4eU1USTBNVEF6TURBek5ERTFNbG93SGpFY01Cb0dBMVVFQXd3VGMyVnNaaTF6YVdkdVpXUXRZMkV0ClkyVnlkRENDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQkFML0lpWVdabUc0Q05qOS8KMzV6RUJheU1tL2djeGhJSFArZ1M4S04wdm9YV2krdk1Kd1hEUWR3T0JOdEZaa1l0elpJc29ESHVHenRnN0RWNApyZXZONE5JOHRKTmc2b2Jma0tVcGQ3eHdvNHNIdXMwd0h6NWVwZGt1MDhNVzVzZFZOUFNRMkhpVnhialpXSnBRClpJYWYyQWRKa1psUFdtVDBaS1pPdFFEN2oySTRtM3VCeG1jYzhTNWhiNkpaYW9NbGNVVXhkNDFscG43T21iMGEKVjRBUGZiWS9vYytwZmVDczN1cG5xamxZamVGQjR2RTV4WU1ZV0FNeitJRGh4RTRxRGVSaXNMQnhhN1kvcFRScQo2OWVhVXN6Qjl5eEQ3R0FySTJsSDhyUCtVeGpGYUl1K2tBVjVtbjc4OXdlejh0TDVGNEErWlE5cGM5TVI2UXBuCmRkanlaRXcvcFpkdTcwVmo3WUE1MU91S2owcTF2dGw3d1BPcDBUc3lwUDhadW04ZkZSNG5KbmNPaFhMTWV3bGEKTWxBeFZaUWRiMEF5dEE2TUl0dFdXSjA5L3BEOWZ6SkdjRnZOL245ZzZWZ0o5NjNhbmRoTEwyYlJMc0VKRmxUTQpEdTJIeW1CNkErb0ZlcDdjZXNxOUpJRFhkVmFqR3NxMmgrZVpPNGdxWW5nWGNmQVl5ZUloYzlYNnFoT2QvVmlZCmY2eUZoOTNuUnRYTFFNdUJRN2E1WTFzRVN3RHp3WWJKdEtuK3NrcGg5SEtCMTdVRUVOU3BNNHJSNHdxekRQd3AKSmZZeWt2a2Iyd2w0TkNCb2pjaU9icDYwV2ZDQytRcTFsNEo4VXpSOEpvWmFiQ0IzOWxVcHBKa09qNVFxYnEwMApKaUFzWENQQlp3OCtnQnV0b2JBVUs0RklqMGVQQWdNQkFBR2pVekJSTUIwR0ExVWREZ1FXQkJTcmNlRWNhMjNxCjBUQm14VmtZTWtMQTNSeHpKekFmQmdOVkhTTUVHREFXZ0JTcmNlRWNhMjNxMFRCbXhWa1lNa0xBM1J4ekp6QVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUNBUUJseW5IM2doVG9ObDQrTlQ1MgpNTTA1V1A3UCszVXJkQ0tGNEJCa0VzN0VueldSUjZ4bkVoVUY5VWhGZ0ZhTFBiQ1pacnlCS2krT1hrUHUva3JCCk12aE1LVGl0WnNWbVBzRktEWDYyVG9zMEJZV0VzanZ0VDM4WFhSZXA3T3BWR0lPQi85V09YVGl3VkpaT2tSZ2MKbHd3U2U1dnBQQXRpMzhUZ3BhM0FVSk5haG00bDhHNWF5WktRQWFnUDg1NHBFTjhPOW54Nk9odytWN1hzSGlNdQpwUmpvc0VTN0JJY1lXVGJxR05yNFR3eXo1cVUwOE9LOEUySFNFSnE5THA4YzZ6UTZnZzBhV1dLYWJyTUpNeSt2CkpIbjM5TEI0U3dONzJjVXJkRWYvQVVrWktNYTRKVFRjMTJnaGpKN1JvUENYUFdPOUJGZ09aZEdoUlpBYkZRMXgKcnB6b3BLZllkT0hWZ0tncG9MOVJIRm40TzZRaTBjbnBML0NZZEFGd0pXNmZYcGEyekhobEJqWXlWdHk5T1Y4TQppV01IVUNXZnl4anVaSno5NFFWZGxLRGVrY2YzUFJzU0RBRGZ4TXlBYVdJQ1NnYXNHTVNPSnRoRTlGM0JhaXNvCnNYM1NLYzRFSEc4Sk1VM1hoeWYwbkhDY2hQdWVRblU5akFBVVBDMHRrVlZtOWhmMXVkdjlOTUk4bktncWRFMkMKK2ExNnR3RVBpZzhkS1pkaFRMOFdXMGwxS3FRcCs4SnBGdTdNWUM1SGNaa0F0NEE3QXlXOHRsYmlQQ1B4RVYwZwpsYkc0eXFyV2lIOG5rWE9tNFBKb0FEMDhzNjA5Y3lFY2Iyblgvck92KzBSdFdOOWFqZGNxYlM1Z0JJaEhRSVUwCko1N0cyTVFYZ0hHbU9QL1ZZTi8vMTlsaXd3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - # output from cat ca.key | base64 - ca.key: | - LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRQy95SW1GbVpodUFqWS8KZjkrY3hBV3NqSnY0SE1ZU0J6L29FdkNqZEw2RjFvdnJ6Q2NGdzBIY0RnVGJSV1pHTGMyU0xLQXg3aHM3WU93MQplSzNyemVEU1BMU1RZT3FHMzVDbEtYZThjS09MQjdyTk1COCtYcVhaTHRQREZ1YkhWVFQwa05oNGxjVzQyVmlhClVHU0duOWdIU1pHWlQxcGs5R1NtVHJVQSs0OWlPSnQ3Z2NabkhQRXVZVytpV1dxREpYRkZNWGVOWmFaK3pwbTkKR2xlQUQzMjJQNkhQcVgzZ3JON3FaNm81V0kzaFFlTHhPY1dER0ZnRE0vaUE0Y1JPS2cza1lyQ3djV3UyUDZVMAphdXZYbWxMTXdmY3NRK3hnS3lOcFIvS3ovbE1ZeFdpTHZwQUZlWnArL1BjSHMvTFMrUmVBUG1VUGFYUFRFZWtLClozWFk4bVJNUDZXWGJ1OUZZKzJBT2RUcmlvOUt0YjdaZThEenFkRTdNcVQvR2Jwdkh4VWVKeVozRG9WeXpIc0oKV2pKUU1WV1VIVzlBTXJRT2pDTGJWbGlkUGY2US9YOHlSbkJiemY1L1lPbFlDZmV0MnAzWVN5OW0wUzdCQ1JaVQp6QTd0aDhwZ2VnUHFCWHFlM0hyS3ZTU0ExM1ZXb3hyS3RvZm5tVHVJS21KNEYzSHdHTW5pSVhQVitxb1RuZjFZCm1IK3NoWWZkNTBiVnkwRExnVU8ydVdOYkJFc0E4OEdHeWJTcC9ySktZZlJ5Z2RlMUJCRFVxVE9LMGVNS3N3ejgKS1NYMk1wTDVHOXNKZURRZ2FJM0lqbTZldEZud2d2a0t0WmVDZkZNMGZDYUdXbXdnZC9aVkthU1pEbytVS202dApOQ1lnTEZ3andXY1BQb0FicmFHd0ZDdUJTSTlIandJREFRQUJBb0lDQUFyYzkwaG9ud3VIWGI3ZmNtU0IxU3JZClZPWWt1WDl6aHQvRWxIb1E5cDNFSSswNWhWaFdCTmpMNjBvYXRuRlhtenk3emZtTWMyRTcyemlPam1OdmpvOGcKY1l4eDlMYmQycG5RWUlBWEJ0eDV5UUxJWUFaSUwySys3NjloRUlLYksvVzQxZG9wN05vekFMQm9MMW1FenlSZgpWS0hFU0ZDMHptS3hNOUpMYllYeWowMm9QbUhBY0NHdGJHdjFrZGZ4RkdjNldrZy80c0tnY05ld3NueUdTb0lICm8zd21ZSnkvSjUxTDF5QlhPL2J2Y1hobHNMd3djamNCQ0FNUUU0aE42UjJKUUwrdDBEWGt2SjBQcnZzRE9wa0kKakdzTlEzMWVPcEpERmdwL21zNlFNWnpObHhwdXNGQTVnNUNkaUpRMHNkSGpOdUtqTXhyeUxKRk1HY0l0OExEQwpRVzF2akxLR0l1UWtraGwxOWU1S1N2SDdjUjJja0pDME5vTzhnekpudzd0dTRGaHJaK0xQeXF2R3VSYU55a2RmCi9BKzNEOUE2RW1PNWRldFU2RzJkK0l2TmprdG91Z05UalZIUklDbk9oL01zRmlFQXdycHltVVNISzhKTjVpSjIKUm1rNFljNWlXUjhOUWs0Wkh6aVFGSHJSWkh0TW9DcEkvR2ZGcnYyRVE0bFpOOG5tZHdDWDR4a3JObUJ2ZnlIdgpLWW0yMU5VWDc5U3lRbHd5VS9lNUR6eTA3Si9zcTdoVU8xN3hVcXNzTVpZTGZCSFF0VFU0VVAwbnBOZmtxUFM1CjdJRUtIVWwyRlZudXR0THoxc0ZVVHhJTS90aE9lczRtWElrOExiYzI0Yk45VWlteXplVEN1bE83a0hZSDhTVkEKZDJqZFBTZXhZSTdMeWFVNnFHRzVBb0lCQVFEbHlVQk5CaTRNekdxVnh5NjNCY1dyZC9rdVYrYTFLQ3MyYVhzagpLbVlMT0xrSkhUSjI0YU9EWkJBVGxEMllwclZEOUM1UThTeDdPQVdlQ3FqWHd1MndsOTlabXNIVkxiQTZMRUZ6CnBoYTNQVHhkaWFpMElwZVY2ZFpIQnQrdjVDVGsvSnpxSVpjc1J2REFnNTFHYzgxbERxTzFNbnVqbldBcGpSMmMKd05ZVXd6a3hicHVTc1ZHYzFBZ09tVHBHN1MrdWVVQ2FGU0NVaGkyVWVoblM5dkNrU3Y0QTZMRlpiaXhFeWp6aApycU9mN1d1TTVUWkFoTGM2RTJUQnVOeWJlWW9DblFMdHF3dnUxaFhDOGU4TGlQWFRlMVJ4U2x5dXA5RDhiWEZBCjVPVmFUZjAzcFFweURiOXNKeGhLN3FMbUgrSjlUeU5JamhTTUZQT2pKNEJFRTlPOUFvSUJBUURWcVhDMGdCVzUKYlNUWmUzc3l1QVltRi9hVDg1ZFh1NGFTMFBJR09MakE1M2h0RVdLUkJxd1JlU1prSFdtR05uNUIyOXVXTHg2UgpPZjFNOFJkY2NYSnlxMnp1TlBiWkpabllwS0x5N0FjeDBpc1RvMjdpUy9xRS85YndsNUo3QVU5UmZ2K2ZMK2RPCmxqUndRTGUvQ1dSVHVlTlNOSWpPUC96NWRra2J5Z1kvWHZHbmI0RUJheDY4K3J2a0NYbStGdFpXV3VoblM2Uy8KZHh3Ulo2VGRMd09RZTZQSzNzN3F4c2xWNmQ2dmwrSUpwa1VVZmRvWDNyWFlTeGx2cFlQYWJpWEpaVzdQWkZwRQpVQXc0VTFpSzVLMUt5d1ZjaHlhN2tQSlpRNUplS1pUT3lPL1d5ODZLak0vcUd3NUhDR2NOL2VMbDJKUUViUkwvClJiR0pGSmhUalpjN0FvSUJBQXlyNG0zYzcyRXBUSjloMG9PcFA5TksxR1RuMkFNWmFmaWdMSGd0K0Y2YURDb2kKZ0F2cU9YZ2ZabnVONnkrbDBjMGpoQUpXcWx0SkpaWW5oRlFSbmNYbE9oM1kyT09HbDNjOXhZWTVISHVTVnVmWgpsWUlKZms1NERLYnlEQmZJL3ZmWnJsV0M4TEV5WUVoZGVhak83ZjZxcGdCeC9qdHhqRUgrVkNtMndKZDRoSWpqClRwVHlUa3ZWclhRUW94UVNORlRzdnRGQVpRR0x2S3U1Wi84b092RDBhYmxuRzVDUThNUUNXd1VlK2tyeGJzTGcKU1BPWjNmakg1UUNCenppTHBUNnJwZU94VVFFa3NTS0U4T2V6NzhwdnZLSmF0VzIwTjJRVUxQQ2xMcmlpSUZxWApNVkpFeTgrTkFGdnhlTzR6eCt1ZEY1Y0Nyc05pekdTczR2ZmVHQWtDZ2dFQkFMdFRnbWdPd0gxQlR3U0t1Ym4vCkZBMEVCNEV5R2FlbTExY1RjSTY1M21ucXgyL0F4VTFucnlibXRCMGttR2Mra2JYR1FDRE5rUnc4M25NK0VZQlEKU3NwMHQ5MmxmQ05vVHhsZFJ5eDZlZGhaYnNFYUVsYS96SllkQk9NTjBUU2RNbUMrV3ZuRGN5WTRsU014NnFmSQpZVGp6Q25ZQmIweDlWNXVUOUljenVnU0hocEdKTm03Njd3azdQODZ2N0JnWVI3V1FvS0FuOXZxVFFIMldCRHFVClJLakJiaHFvL0h0azdCS3lLRGFGa0gxclZMZWhtN3cvMitrVjl1Z25FcEpJN2tKRDkwSkh0c2liOGdyVU1CWWUKWmp6a0FRQmQwanl5MlhnZndVMWpZWDluTnJoNUdjM3BwVVNZa2d6L05mTlRmRUtPZnovZUxjQzM1dTdMcXIzZQpydzhDZ2dFQkFMT2tsTkJNRVBmM20yTXBjaVRjRmNKb08vZzBMUUpHaTJtWkN6S1g3eDJFS0N2N1ZvVWVtRkk0CjRkRFVmSlBJWlBFTUpkTHRSUy9qUDEyZWkxek9lWHIrVGlUTklpUUVoemRtL0RZWUdjd2hyb0xLNDZVTFJKY0YKYzdxZ2xNQ1Z1MW9DTmtDdTJvZ08renczRm9makJzK1pqcE1BS2kyOTZ2ZDk2YVlYNThYR0RKekdmdjhuZEF1dwpEUmU1ZE5oQU5iaHZqSlM1VXJwNnhoMVMycTNYOHorTlFGWW9CNDM1Q2NXNW50WWMzemIxYVdzY0NxMWJsUGJGCjc0QTFLTHJNNlpvU0ZlcUVWZzhvajhpWjlDaitiTTJXYm9BREIvRTROM0kyNmFDK1dDRWxtdTd3ZDdQaExQT2IKN3RrTXh2Zm10dDE5T2dYbTRKZm9SZWlkMTNYbHFoZz0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ray-tls-scripts - labels: - opendatahub.io/managed: 'true' -data: - gencert_ray.sh: | - #!/bin/sh - ## Create tls.key - openssl genrsa -out /etc/ray/tls/tls.key 2048 - - ## Write CSR Config - cat > /etc/ray/tls/csr.conf < /etc/ray/tls/cert.conf < /tmp/ca.srl - - ## Generate tls.cert - openssl x509 -req \ - -in /etc/ray/tls/ca.csr \ - -CA /etc/ca/tls/ca.crt -CAkey /etc/ca/tls/ca.key \ - -CAserial /tmp/ca.srl -out /etc/ray/tls/tls.crt \ - -days 36500 \ - -sha256 -extfile /etc/ray/tls/cert.conf diff --git a/config/overlays/odh/ray_tls_script.yaml b/config/overlays/odh/ray_tls_script.yaml new file mode 100644 index 00000000..e439097b --- /dev/null +++ b/config/overlays/odh/ray_tls_script.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-tls-script + labels: + opendatahub.io/managed: 'true' +data: + gencert_ray.sh: | + #!/bin/sh + ## Create tls.key + openssl genrsa -out /etc/ray/tls/tls.key 2048 + + ## Write CSR Config + cat > /etc/ray/tls/csr.conf < /etc/ray/tls/cert.conf < /tmp/ca.srl + + ## Generate tls.cert + openssl x509 -req \ + -in /etc/ray/tls/ca.csr \ + -CA /etc/ca/tls/tls.crt -CAkey /etc/ca/tls/tls.key \ + -CAserial /tmp/ca.srl -out /etc/ray/tls/tls.crt \ + -days 3650 \ + -sha256 -extfile /etc/ray/tls/cert.conf diff --git a/config/runtimes/vllm-multinode-template.yaml b/config/runtimes/vllm-multinode-template.yaml index eb744173..ac269006 100644 --- a/config/runtimes/vllm-multinode-template.yaml +++ b/config/runtimes/vllm-multinode-template.yaml @@ -41,7 +41,8 @@ objects: - | # Generate self signed certificate if [[ $RAY_USE_TLS == "1" ]]; then - /etc/gen/tls/gencert_ray.sh + echo "Generating Self Signed Certificate for Ray nodes" + /etc/gen/tls/gencert_ray.sh > /dev/null 2>&1 fi ray start --head --disable-usage-stats --include-dashboard false @@ -64,7 +65,7 @@ objects: - name: RAY_TLS_SERVER_KEY value: '/etc/ray/tls/tls.key' - name: RAY_TLS_CA_CERT - value: '/etc/ca/tls/ca.crt' + value: '/etc/ca/tls/tls.crt' - name: RAY_PORT value: '6379' - name: RAY_ADDRESS @@ -209,7 +210,7 @@ objects: # The gencert_ray.sh can be prebaked into the docker container so the configMap is optional - name: gen-tls-script configMap: - name: ray-tls-scripts + name: ray-tls-script defaultMode: 0777 # An array of keys from the ConfigMap to create as files items: @@ -226,7 +227,8 @@ objects: - | # Generate self signed certificate if [[ $RAY_USE_TLS == "1" ]]; then - /etc/gen/tls/gencert_ray.sh + echo "Generating Self Signed Certificate for Ray nodes" + /etc/gen/tls/gencert_ray.sh > /dev/null 2>&1 fi SECONDS=0 @@ -260,7 +262,7 @@ objects: - name: RAY_TLS_SERVER_KEY value: '/etc/ray/tls/tls.key' - name: RAY_TLS_CA_CERT - value: '/etc/ca/tls/ca.crt' + value: '/etc/ca/tls/tls.crt' - name: POD_NAME valueFrom: fieldRef: @@ -346,7 +348,7 @@ objects: # The gencert_ray.sh can be prebaked into the docker container so the configMap is optional - name: gen-tls-script configMap: - name: ray-tls-scripts + name: ray-tls-script defaultMode: 0777 # An array of keys from the ConfigMap to create as files items: diff --git a/controllers/constants/constants.go b/controllers/constants/constants.go index 46c2c324..c454dbe1 100644 --- a/controllers/constants/constants.go +++ b/controllers/constants/constants.go @@ -82,6 +82,6 @@ const ( // Ray const ( - RayCATlsSecretName = "ray-ca-cert" - RayTlsScriptConfigMapName = "ray-tls-scripts" + RayCASecretName = "ray-ca-cert" + RayTlsScriptConfigMapName = "ray-tls-script" ) diff --git a/controllers/kserve_ray_tls_controller.go b/controllers/kserve_ray_tls_controller.go index f8b4bade..a06b5226 100644 --- a/controllers/kserve_ray_tls_controller.go +++ b/controllers/kserve_ray_tls_controller.go @@ -40,8 +40,8 @@ func NewKServeRayTlsReconciler(client client.Client, log logr.Logger) *KServeRay // - On deletion: The ray-tls-script ConfigMap and ray-ca-cert Secret are deleted only when multinode ServingRuntimes are deleted from the target namespace. // ConfigMap: -// - When the original ConfigMap is updated in the control namespace: The ray-tls-scripts ConfigMap is deleted and recreated in the namespace where multinode ServingRuntimes exist. -// - When the ConfigMap is deleted in the target namespace: The ray-tls-scripts ConfigMap will be recreated. +// - When the original ConfigMap is updated in the control namespace: The ray-tls-script ConfigMap is deleted and recreated in the namespace where multinode ServingRuntimes exist. +// - When the ConfigMap is deleted in the target namespace: The ray-tls-script ConfigMap will be recreated. // Secret: // - When the original Secret is updated in the control namespace: The ray-ca-cert Secret is deleted and recreated in the namespace where multinode ServingRuntimes exist. @@ -49,6 +49,19 @@ func NewKServeRayTlsReconciler(client client.Client, log logr.Logger) *KServeRay func (r *KServeRayTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.log controllerNs := os.Getenv("POD_NAMESPACE") + + srcSecret := &corev1.Secret{} + err := r.client.Get(ctx, types.NamespacedName{Name: constants.RayCASecretName, Namespace: controllerNs}, srcSecret) + if err != nil { + if apierrs.IsNotFound(err) { + createErr := utils.CreateSelfSignedCertificate(ctx, r.client, constants.RayCASecretName, "Ray Self Signed Certs", controllerNs) + if createErr != nil { + return ctrl.Result{}, createErr + } + } else { + return ctrl.Result{}, err + } + } var servingRuntimeList kservev1alpha1.ServingRuntimeList if err := r.client.List(ctx, &servingRuntimeList); err != nil { return ctrl.Result{}, err @@ -69,9 +82,9 @@ func (r *KServeRayTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request if err := r.reconcileRayTlsScriptsConfigMap(ctx, log, controllerNs, req.Namespace, noMultiNodeSrExistInNs); err != nil { return ctrl.Result{}, err } - } else if req.Name == constants.RayCATlsSecretName { + } else if req.Name == constants.RayCASecretName { if req.Namespace == controllerNs { - log.Info("Original Ray CA Cert Secret is updated", "name", constants.RayCATlsSecretName, "namespace", req.Namespace) + log.Info("Original Ray CA Cert Secret is updated", "name", constants.RayCASecretName, "namespace", req.Namespace) for _, sr := range servingRuntimeList.Items { if isMultiNodeServingRuntime(sr) { if err := r.cleanupRayResourcesByKind(ctx, log, sr.Namespace, "Secret"); err != nil { @@ -80,7 +93,7 @@ func (r *KServeRayTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request } } } - if err := r.reconcileRayCACertSecret(ctx, log, controllerNs, req.Namespace, noMultiNodeSrExistInNs); err != nil { + if err := r.reconcileRayCACertSecret(ctx, log, srcSecret, controllerNs, req.Namespace, noMultiNodeSrExistInNs); err != nil { return ctrl.Result{}, err } } else { @@ -88,7 +101,7 @@ func (r *KServeRayTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request if err != nil { return ctrl.Result{}, err } - err = r.reconcileRayCACertSecret(ctx, log, controllerNs, req.Namespace, !existMultiNodeServingRuntimeInNs(req.Namespace, servingRuntimeList)) + err = r.reconcileRayCACertSecret(ctx, log, srcSecret, controllerNs, req.Namespace, !existMultiNodeServingRuntimeInNs(req.Namespace, servingRuntimeList)) if err != nil { return ctrl.Result{}, err } @@ -97,10 +110,10 @@ func (r *KServeRayTlsReconciler) Reconcile(ctx context.Context, req ctrl.Request } func checkRayTLSResource(objectName string) bool { - return objectName == constants.RayCATlsSecretName || objectName == constants.RayTlsScriptConfigMapName + return objectName == constants.RayCASecretName || objectName == constants.RayTlsScriptConfigMapName } -// reconcileRayTLSResource filters out ConfigMaps and Secrets that do not match the predefined constants: RayCATlsSecretName or RayTlsScriptConfigMapName. +// reconcileRayTLSResource filters out ConfigMaps and Secrets that do not match the predefined constants: RayCASecretName or RayTlsScriptConfigMapName. // This ensures that only the relevant ConfigMaps and Secrets for Ray TLS configuration are captured and processed for the servingRuntime. func reconcileRayTLSResource() predicate.Predicate { return predicate.Funcs{ @@ -140,8 +153,8 @@ func (r *KServeRayTlsReconciler) SetupWithManager(mgr ctrl.Manager) error { return builder.Complete(r) } -// reconcileRayTlsScriptsConfigMap watch ray-tls-scripts configmap in the cluster -// and it will create/update/delete ray-tls-scripts configmap in the namespace where multinode ServingRuntime created +// reconcileRayTlsScriptsConfigMap watch ray-tls-script configmap in the cluster +// and it will create/update/delete ray-tls-script configmap in the namespace where multinode ServingRuntime created func (r *KServeRayTlsReconciler) reconcileRayTlsScriptsConfigMap(ctx context.Context, log logr.Logger, ctrlNs string, targetNs string, noMultiNodeSrExistInNs bool) error { // When original configmap is updated, it does not need to reconcile if ctrlNs == targetNs { @@ -245,18 +258,13 @@ func shouldDeleteRayConfigMap(existingConfigMap *corev1.ConfigMap, noMultiNodeSr // reconcileRayCACertSecret watch ray-ca-cert secret in the cluster // and it will create/update/delete ray-ca-cert secret in the namespace where multinode ServingRuntime created -func (r *KServeRayTlsReconciler) reconcileRayCACertSecret(ctx context.Context, log logr.Logger, ctrlNs string, targetNs string, noMultiNodeSrExistInNs bool) error { +func (r *KServeRayTlsReconciler) reconcileRayCACertSecret(ctx context.Context, log logr.Logger, srcSecret *corev1.Secret, ctrlNs string, targetNs string, noMultiNodeSrExistInNs bool) error { // When original secret is updated, it does not need to reconcile if ctrlNs == targetNs { return nil } - log.Info("Reconciling Ray CA Cert Secret", "name", constants.RayCATlsSecretName, "namespace", targetNs) - srcSecret := &corev1.Secret{} - err := r.client.Get(ctx, types.NamespacedName{Name: constants.RayCATlsSecretName, Namespace: ctrlNs}, srcSecret) - if err != nil { - return err - } + log.Info("Reconciling Ray CA Cert Secret", "name", constants.RayCASecretName, "namespace", targetNs) // Create Desired resource desiredSecretResource, err := r.createDesiredSecretResource(targetNs, srcSecret) if err != nil { @@ -265,7 +273,7 @@ func (r *KServeRayTlsReconciler) reconcileRayCACertSecret(ctx context.Context, l // Get Existing resource existingSecretResource := &corev1.Secret{} - err = r.client.Get(ctx, types.NamespacedName{Name: constants.RayCATlsSecretName, Namespace: targetNs}, existingSecretResource) + err = r.client.Get(ctx, types.NamespacedName{Name: constants.RayCASecretName, Namespace: targetNs}, existingSecretResource) if err != nil { if apierrs.IsNotFound(err) { existingSecretResource = nil @@ -368,17 +376,17 @@ func (r *KServeRayTlsReconciler) cleanupRayResourcesByKind(ctx context.Context, if kind == "Secret" { secret := &corev1.Secret{} err := r.client.Get(ctx, types.NamespacedName{ - Name: constants.RayCATlsSecretName, + Name: constants.RayCASecretName, Namespace: targetNs, }, secret) if err != nil { if apierrs.IsNotFound(err) { - log.Info("Secret not found, skipping", "name", constants.RayCATlsSecretName, "namespace", targetNs) + log.Info("Secret not found, skipping", "name", constants.RayCASecretName, "namespace", targetNs) } return err } - log.Info("Deleting Secret", "name", constants.RayCATlsSecretName, "namespace", targetNs) + log.Info("Deleting Secret", "name", constants.RayCASecretName, "namespace", targetNs) err = r.client.Delete(ctx, secret) if err != nil { return err diff --git a/controllers/kserve_ray_tls_controller_test.go b/controllers/kserve_ray_tls_controller_test.go index 78a44f58..14950254 100644 --- a/controllers/kserve_ray_tls_controller_test.go +++ b/controllers/kserve_ray_tls_controller_test.go @@ -17,9 +17,10 @@ package controllers import ( "context" - "k8s.io/apimachinery/pkg/types" "time" + "k8s.io/apimachinery/pkg/types" + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,9 +30,8 @@ import ( const ( multinodeServingRuntimePath = "./testdata/deploy/vllm-multinode-servingruntime.yaml" - rayTlsScriptsPath = "./testdata/configmaps/ray-tls-scripts.yaml" - rayTlsScriptsUpdatedPath = "./testdata/configmaps/ray-tls-scripts-updated.yaml" - rayCaCertPath = "./testdata/secrets/ray-ca-cert.yaml" + rayTlsScriptsPath = "./testdata/configmaps/ray-tls-script.yaml" + rayTlsScriptsUpdatedPath = "./testdata/configmaps/ray-tls-script-updated.yaml" rayCaCertUpdatedPath = "./testdata/secrets/ray-ca-cert-updated.yaml" ) @@ -39,23 +39,17 @@ var _ = Describe("KServe Ray TLS controller", func() { ctx := context.Background() Context("when a non-multinode ServingRuntime created", func() { - It("should not create a 'ray-ca-cert' Secret and 'ray-tls-scripts' ConfigMap in the testNs", func() { + It("should not create a 'ray-ca-cert' Secret and 'ray-tls-script' ConfigMap in the testNs", func() { testNamespace := Namespaces.Create(cli) testNs := testNamespace.Name - // Create ray tls resources in the ctrl namespace + // Create ray tls script in the ctrl namespace rayTlsScriptsConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsPath, rayTlsScriptsConfigMap) Expect(err).NotTo(HaveOccurred()) rayTlsScriptsConfigMap.SetNamespace(WorkingNamespace) Expect(cli.Create(ctx, rayTlsScriptsConfigMap)).Should(Succeed()) - rayCaCertSecret := &corev1.Secret{} - err = convertToStructuredResource(rayCaCertPath, rayCaCertSecret) - Expect(err).NotTo(HaveOccurred()) - rayCaCertSecret.SetNamespace(WorkingNamespace) - Expect(cli.Create(ctx, rayCaCertSecret)).Should(Succeed()) - By("creating non-multinode ServingRuntime") nonMultinodeServingRuntime := &kservev1alpha1.ServingRuntime{} err = convertToStructuredResource(ServingRuntimePath1, nonMultinodeServingRuntime) @@ -63,35 +57,29 @@ var _ = Describe("KServe Ray TLS controller", func() { nonMultinodeServingRuntime.SetNamespace(testNs) Expect(cli.Create(ctx, nonMultinodeServingRuntime)).Should(Succeed()) - // Check if all ray tls resources are NOT created in the testNs + // Check if all ray tls script are NOT created in the testNs configmap, err := waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 3, 1*time.Second) Expect(err).To(HaveOccurred()) Expect(configmap).To(BeNil()) - secret, err := waitForSecret(cli, testNs, constants.RayCATlsSecretName, 3, 1*time.Second) + secret, err := waitForSecret(cli, testNs, constants.RayCASecretName, 3, 1*time.Second) Expect(err).To(HaveOccurred()) Expect(secret).To(BeNil()) }) }) Context("when a multinode ServingRuntime created", func() { - It("should create a 'ray-ca-cert' Secret and 'ray-tls-scripts' ConfigMap in the testNs where the SR exist", func() { + It("should create a 'ray-ca-cert' Secret and 'ray-tls-script' ConfigMap in the testNs where the SR exist", func() { testNamespace := Namespaces.Create(cli) testNs := testNamespace.Name - // Create ray tls resources in the ctrl namespace + // Create ray tls script in the ctrl namespace rayTlsScriptsConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsPath, rayTlsScriptsConfigMap) Expect(err).NotTo(HaveOccurred()) rayTlsScriptsConfigMap.SetNamespace(WorkingNamespace) Expect(cli.Create(ctx, rayTlsScriptsConfigMap)).Should(Succeed()) - rayCaCertSecret := &corev1.Secret{} - err = convertToStructuredResource(rayCaCertPath, rayCaCertSecret) - Expect(err).NotTo(HaveOccurred()) - rayCaCertSecret.SetNamespace(WorkingNamespace) - Expect(cli.Create(ctx, rayCaCertSecret)).Should(Succeed()) - By("creating multinode ServingRuntime") multinodeServingRuntime := &kservev1alpha1.ServingRuntime{} err = convertToStructuredResource(multinodeServingRuntimePath, multinodeServingRuntime) @@ -99,10 +87,10 @@ var _ = Describe("KServe Ray TLS controller", func() { multinodeServingRuntime.SetNamespace(testNs) Expect(cli.Create(ctx, multinodeServingRuntime)).Should(Succeed()) - // Check if all ray tls resources are created in the testNs + // Check if all ray tls script are created in the testNs _, err = waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, testNs, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, testNs, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) }) @@ -115,19 +103,13 @@ var _ = Describe("KServe Ray TLS controller", func() { testNamespace := Namespaces.Create(cli) testNs = testNamespace.Name - // Create ray tls resources in the ctrl namespace + // Create ray tls script in the ctrl namespace rayTlsScriptsConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsPath, rayTlsScriptsConfigMap) Expect(err).NotTo(HaveOccurred()) rayTlsScriptsConfigMap.SetNamespace(WorkingNamespace) Expect(cli.Create(ctx, rayTlsScriptsConfigMap)).Should(Succeed()) - rayCaCertSecret := &corev1.Secret{} - err = convertToStructuredResource(rayCaCertPath, rayCaCertSecret) - Expect(err).NotTo(HaveOccurred()) - rayCaCertSecret.SetNamespace(WorkingNamespace) - Expect(cli.Create(ctx, rayCaCertSecret)).Should(Succeed()) - // Create a multinode ServingRuntime multinodeServingRuntime := &kservev1alpha1.ServingRuntime{} err = convertToStructuredResource(multinodeServingRuntimePath, multinodeServingRuntime) @@ -135,23 +117,23 @@ var _ = Describe("KServe Ray TLS controller", func() { multinodeServingRuntime.SetNamespace(testNs) Expect(cli.Create(ctx, multinodeServingRuntime)).Should(Succeed()) - // Check if all ray tls resources are created in the testNs + // Check if all ray tls script are created in the testNs _, err = waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, testNs, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, testNs, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) It("should recreate a 'ray-ca-cert' Secret when it is removed manually", func() { secret := &corev1.Secret{} - err := cli.Get(ctx, types.NamespacedName{Name: constants.RayCATlsSecretName, Namespace: testNs}, secret) + err := cli.Get(ctx, types.NamespacedName{Name: constants.RayCASecretName, Namespace: testNs}, secret) Expect(err).NotTo(HaveOccurred()) By("deleting a 'ray-ca-cert' Secret in the testNs") Expect(cli.Delete(ctx, secret)).To(Succeed()) // Check if 'ray-ca-cert' Secret is recreated - _, err = waitForSecret(cli, testNs, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, testNs, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) It("should rollback 'ray-ca-cert' Secret in the target ns when it is changed", func() { @@ -163,35 +145,35 @@ var _ = Describe("KServe Ray TLS controller", func() { Expect(cli.Update(ctx, rayCACertUpdatedSecret)).Should(Succeed()) // Check if 'ray-ca-cert' Secret is rollback - originalRayCaCertSecret, err := waitForSecret(cli, WorkingNamespace, constants.RayCATlsSecretName, 30, 1*time.Second) + originalRayCaCertSecret, err := waitForSecret(cli, WorkingNamespace, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { - updatedSecretFromTestNs, err := waitForSecret(cli, testNs, constants.RayCATlsSecretName, 1, 1*time.Second) + updatedSecretFromTestNs, err := waitForSecret(cli, testNs, constants.RayCASecretName, 1, 1*time.Second) Expect(err).NotTo(HaveOccurred()) return compareSecrets(originalRayCaCertSecret, updatedSecretFromTestNs) }, timeout, interval).Should(BeTrue()) }) - It("should create a 'ray-tls-scripts' ConfigMap when it is removed manually", func() { + It("should create a 'ray-tls-script' ConfigMap when it is removed manually", func() { configMap := &corev1.ConfigMap{} err := cli.Get(ctx, types.NamespacedName{Name: constants.RayTlsScriptConfigMapName, Namespace: testNs}, configMap) Expect(err).NotTo(HaveOccurred()) - By("deleting a 'ray-tls-scripts' configMap in the testNs") + By("deleting a 'ray-tls-script' configMap in the testNs") Expect(cli.Delete(ctx, configMap)).To(Succeed()) - // Check if 'ray-tls-scripts' ConfigMap is recreated + // Check if 'ray-tls-script' ConfigMap is recreated _, err = waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) - It("should rollback 'ray-tls-scripts' ConfigMap in the target ns when it is changed", func() { - By("updating existing 'ray-tls-scripts' ConfigMap in the testNs") + It("should rollback 'ray-tls-script' ConfigMap in the target ns when it is changed", func() { + By("updating existing 'ray-tls-script' ConfigMap in the testNs") rayTlsScriptsUpdatedConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsUpdatedPath, rayTlsScriptsUpdatedConfigMap) Expect(err).NotTo(HaveOccurred()) rayTlsScriptsUpdatedConfigMap.SetNamespace(testNs) Expect(cli.Update(ctx, rayTlsScriptsUpdatedConfigMap)).Should(Succeed()) - // Check if 'ray-tls-scripts' ConfigMap is rollback + // Check if 'ray-tls-script' ConfigMap is rollback originalRayTlsScriptsConfigMap, err := waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) @@ -201,8 +183,8 @@ var _ = Describe("KServe Ray TLS controller", func() { return compareConfigMap(originalRayTlsScriptsConfigMap, updatedConfigMapFromTestNs) }, timeout, interval).Should(BeTrue()) }) - It("should 'ray-tls-scripts' ConfigMap in the testNs when original one in the ctrlNs updated", func() { - By("updating original 'ray-tls-scripts' ConfigMap in the ctrlNs") + It("should 'ray-tls-script' ConfigMap in the testNs when original one in the ctrlNs updated", func() { + By("updating original 'ray-tls-script' ConfigMap in the ctrlNs") rayTlsScriptsUpdatedConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsUpdatedPath, rayTlsScriptsUpdatedConfigMap) Expect(err).NotTo(HaveOccurred()) @@ -212,7 +194,7 @@ var _ = Describe("KServe Ray TLS controller", func() { _, err = waitForConfigMap(cli, WorkingNamespace, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - // Check if 'ray-tls-scripts' ConfigMap in the testNs is updated. + // Check if 'ray-tls-script' ConfigMap in the testNs is updated. Eventually(func() bool { updatedConfigMapFromTestNs, err := waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 1, 1*time.Second) Expect(err).NotTo(HaveOccurred()) @@ -220,19 +202,19 @@ var _ = Describe("KServe Ray TLS controller", func() { }, timeout, interval).Should(BeTrue()) }) It("should update a 'ray-ca-cert' Secret in the testNs when original one in the ctrlNs updated", func() { - By("updating original 'ray-ca-cert Secret in the ctrlNs") + By("updating original 'ray-ca-cert' Secret in the ctrlNs") rayCaCertUpdatedSecret := &corev1.Secret{} err := convertToStructuredResource(rayCaCertUpdatedPath, rayCaCertUpdatedSecret) Expect(err).NotTo(HaveOccurred()) rayCaCertUpdatedSecret.SetNamespace(WorkingNamespace) Expect(cli.Update(ctx, rayCaCertUpdatedSecret)).Should(Succeed()) - _, err = waitForSecret(cli, WorkingNamespace, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, WorkingNamespace, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) // Check if 'ray-ca-cert' Secret in the testNs is updated. Eventually(func() bool { - updatedSecretFromTestNs, err := waitForSecret(cli, testNs, constants.RayCATlsSecretName, 1, 1*time.Second) + updatedSecretFromTestNs, err := waitForSecret(cli, testNs, constants.RayCASecretName, 1, 1*time.Second) Expect(err).NotTo(HaveOccurred()) return compareSecrets(rayCaCertUpdatedSecret, updatedSecretFromTestNs) }, timeout, interval).Should(BeTrue()) @@ -244,19 +226,13 @@ var _ = Describe("KServe Ray TLS controller", func() { testNamespace := Namespaces.Create(cli) testNs = testNamespace.Name - // Create ray tls resources in the ctrl namespace + // Create ray tls script in the ctrl namespace rayTlsScriptsConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(rayTlsScriptsPath, rayTlsScriptsConfigMap) Expect(err).NotTo(HaveOccurred()) rayTlsScriptsConfigMap.SetNamespace(WorkingNamespace) Expect(cli.Create(ctx, rayTlsScriptsConfigMap)).Should(Succeed()) - rayCaCertSecret := &corev1.Secret{} - err = convertToStructuredResource(rayCaCertPath, rayCaCertSecret) - Expect(err).NotTo(HaveOccurred()) - rayCaCertSecret.SetNamespace(WorkingNamespace) - Expect(cli.Create(ctx, rayCaCertSecret)).Should(Succeed()) - // Create a multinode ServingRuntime in the testNs multinodeServingRuntime := &kservev1alpha1.ServingRuntime{} err = convertToStructuredResource(multinodeServingRuntimePath, multinodeServingRuntime) @@ -266,10 +242,10 @@ var _ = Describe("KServe Ray TLS controller", func() { _, err = waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, testNs, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, testNs, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) - It("ray tls resources should not be removed if there is a multinode ServingRuntime in the testNs", func() { + It("ray tls script should not be removed if there is a multinode ServingRuntime in the testNs", func() { By("creating another multinode ServingRuntime in the testNs") // Create another multinode ServingRuntime multinodeServingRuntime := &kservev1alpha1.ServingRuntime{} @@ -282,13 +258,13 @@ var _ = Describe("KServe Ray TLS controller", func() { By("deleting one multinode ServingRuntime in the testNs") Expect(cli.Delete(ctx, multinodeServingRuntime)).Should(Succeed()) - // Check if all ray tls resources are NOT removed + // Check if all ray tls script are NOT removed _, err = waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, testNs, constants.RayCATlsSecretName, 30, 1*time.Second) + _, err = waitForSecret(cli, testNs, constants.RayCASecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) - It("ray tls resources should be removed if there is no multinode ServingRuntime in the testNs", func() { + It("ray tls script should be removed if there is no multinode ServingRuntime in the testNs", func() { By("deleting a multinode ServingRuntime") multinodeServingRuntime := &kservev1alpha1.ServingRuntime{} err := convertToStructuredResource(multinodeServingRuntimePath, multinodeServingRuntime) @@ -296,12 +272,12 @@ var _ = Describe("KServe Ray TLS controller", func() { multinodeServingRuntime.SetNamespace(testNs) Expect(cli.Delete(ctx, multinodeServingRuntime)).Should(Succeed()) - // Check if all ray tls resources are removed + // Check if all ray tls script are removed configmap, err := waitForConfigMap(cli, testNs, constants.RayTlsScriptConfigMapName, 3, 1*time.Second) Expect(err).To(HaveOccurred()) Expect(configmap).To(BeNil()) - secret, err := waitForSecret(cli, testNs, constants.RayCATlsSecretName, 3, 1*time.Second) + secret, err := waitForSecret(cli, testNs, constants.RayCASecretName, 3, 1*time.Second) Expect(err).To(HaveOccurred()) Expect(secret).To(BeNil()) }) diff --git a/controllers/testdata/configmaps/ray-tls-scripts-updated.yaml b/controllers/testdata/configmaps/ray-tls-script-updated.yaml similarity index 94% rename from controllers/testdata/configmaps/ray-tls-scripts-updated.yaml rename to controllers/testdata/configmaps/ray-tls-script-updated.yaml index 77491ceb..a6271639 100644 --- a/controllers/testdata/configmaps/ray-tls-scripts-updated.yaml +++ b/controllers/testdata/configmaps/ray-tls-script-updated.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ray-tls-scripts + name: ray-tls-script labels: opendatahub.io/managed: 'true' data: @@ -64,7 +64,7 @@ data: ## Generate tls.cert openssl x509 -req \ -in /etc/ray/tls/ca.csr \ - -CA /etc/ca/tls/ca.crt -CAkey /etc/ca/tls/ca.key \ + -CA /etc/ca/tls/tls.crt -CAkey /etc/ca/tls/tls.key \ -CAserial /tmp/ca.srl -out /etc/ray/tls/tls.crt \ -days 36500 \ -sha256 -extfile /etc/ray/tls/cert.conf diff --git a/controllers/testdata/configmaps/ray-tls-scripts.yaml b/controllers/testdata/configmaps/ray-tls-script.yaml similarity index 94% rename from controllers/testdata/configmaps/ray-tls-scripts.yaml rename to controllers/testdata/configmaps/ray-tls-script.yaml index df4c00ce..7192a5b5 100644 --- a/controllers/testdata/configmaps/ray-tls-scripts.yaml +++ b/controllers/testdata/configmaps/ray-tls-script.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ray-tls-scripts + name: ray-tls-script labels: opendatahub.io/managed: 'true' data: @@ -63,7 +63,7 @@ data: ## Generate tls.cert openssl x509 -req \ -in /etc/ray/tls/ca.csr \ - -CA /etc/ca/tls/ca.crt -CAkey /etc/ca/tls/ca.key \ + -CA /etc/ca/tls/tls.crt -CAkey /etc/ca/tls/ca.key \ -CAserial /tmp/ca.srl -out /etc/ray/tls/tls.crt \ -days 36500 \ -sha256 -extfile /etc/ray/tls/cert.conf diff --git a/controllers/testdata/deploy/vllm-multinode-servingruntime.yaml b/controllers/testdata/deploy/vllm-multinode-servingruntime.yaml index acfeb4fc..cc95baab 100644 --- a/controllers/testdata/deploy/vllm-multinode-servingruntime.yaml +++ b/controllers/testdata/deploy/vllm-multinode-servingruntime.yaml @@ -41,7 +41,7 @@ spec: - name: RAY_TLS_SERVER_KEY value: '/etc/ray/tls/tls.key' - name: RAY_TLS_CA_CERT - value: '/etc/ca/tls/ca.crt' + value: '/etc/ca/tls/tls.crt' - name: RAY_PORT value: '6379' - name: RAY_ADDRESS @@ -179,7 +179,7 @@ spec: # The gencert_ray.sh can be prebaked into the docker container so the configMap is optional - name: gen-tls-script configMap: - name: ray-tls-scripts + name: ray-tls-script defaultMode: 0777 # An array of keys from the ConfigMap to create as files items: @@ -227,7 +227,7 @@ spec: - name: RAY_TLS_SERVER_KEY value: '/etc/ray/tls/tls.key' - name: RAY_TLS_CA_CERT - value: '/etc/ca/tls/ca.crt' + value: '/etc/ca/tls/tls.crt' - name: POD_NAME valueFrom: fieldRef: @@ -312,7 +312,7 @@ spec: # The gencert_ray.sh can be prebaked into the docker container so the configMap is optional - name: gen-tls-script configMap: - name: ray-tls-scripts + name: ray-tls-script defaultMode: 0777 # An array of keys from the ConfigMap to create as files items: diff --git a/controllers/testdata/secrets/ray-ca-cert-updated.yaml b/controllers/testdata/secrets/ray-ca-cert-updated.yaml index b590c941..ef71b289 100644 --- a/controllers/testdata/secrets/ray-ca-cert-updated.yaml +++ b/controllers/testdata/secrets/ray-ca-cert-updated.yaml @@ -1,13 +1,14 @@ apiVersion: v1 kind: Secret +type: kubernetes.io/tls metadata: name: ray-ca-cert labels: opendatahub.io/managed: 'true' data: - # output from cat ca.crt | base64 - ca.crt: | + # output from cat tls.crt | base64 + tls.crt: | LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIekNDQXdlZ0F3SUJBZ0lVSlAwL1FCY0xTMFFFV1ZiRDE4NndyUnZjLzdFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0hqRWNNQm9HQTFVRUF3d1RjMlZzWmkxemFXZHVaV1F0WTJFdFkyVnlkREFnRncweU5ERXhNak13TXpReApOVEphR0E4eU1USTBNVEF6TURBek5ERTFNbG93SGpFY01Cb0dBMVVFQXd3VGMyVnNaaTF6YVdkdVpXUXRZMkV0ClkyVnlkRENDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQkFML0lpWVdabUc0Q05qOS8KMzV6RUJheU1tL2djeGhJSFArZ1M4S04wdm9YV2krdk1Kd1hEUWR3T0JOdEZaa1l0elpJc29ESHVHenRnN0RWNApyZXZONE5JOHRKTmc2b2Jma0tVcGQ3eHdvNHNIdXMwd0h6NWVwZGt1MDhNVzVzZFZOUFNRMkhpVnhialpXSnBRClpJYWYyQWRKa1psUFdtVDBaS1pPdFFEN2oySTRtM3VCeG1jYzhTNWhiNkpaYW9NbGNVVXhkNDFscG43T21iMGEKVjRBUGZiWS9vYytwZmVDczN1cG5xamxZamVGQjR2RTV4WU1ZV0FNeitJRGh4RTRxRGVSaXNMQnhhN1kvcFRScQo2OWVhVXN6Qjl5eEQ3R0FySTJsSDhyUCtVeGpGYUl1K2tBVjVtbjc4OXdlejh0TDVGNEErWlE5cGM5TVI2UXBuCmRkanlaRXcvcFpkdTcwVmo3WUE1MU91S2owcTF2dGw3d1BPcDBUc3lwUDhadW04ZkZSNG5KbmNPaFhMTWV3bGEKTWxBeFZaUWRiMEF5dEE2TUl0dFdXSjA5L3BEOWZ6SkdjRnZOL245ZzZWZ0o5NjNhbmRoTEwyYlJMc0VKRmxUTQpEdTJIeW1CNkErb0ZlcDdjZXNxOUpJRFhkVmFqR3NxMmgrZVpPNGdxWW5nWGNmQVl5ZUloYzlYNnFoT2QvVmlZCmY2eUZoOTNuUnRYTFFNdUJRN2E1WTFzRVN3RHp3WWJKdEtuK3NrcGg5SEtCMTdVRUVOU3BNNHJSNHdxekRQd3AKSmZZeWt2a2Iyd2w0TkNCb2pjaU9icDYwV2ZDQytRcTFsNEo4VXpSOEpvWmFiQ0IzOWxVcHBKa09qNVFxYnEwMApKaUFzWENQQlp3OCtnQnV0b2JBVUs0RklqMGVQQWdNQkFBR2pVekJSTUIwR0ExVWREZ1FXQkJTcmNlRWNhMjNxCjBUQm14VmtZTWtMQTNSeHpKekFmQmdOVkhTTUVHREFXZ0JTcmNlRWNhMjNxMFRCbXhWa1lNa0xBM1J4ekp6QVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUNBUUJseW5IM2doVG9ObDQrTlQ1MgpNTTA1V1A3UCszVXJkQ0tGNEJCa0VzN0VueldSUjZ4bkVoVUY5VWhGZ0ZhTFBiQ1pacnlCS2krT1hrUHUva3JCCk12aE1LVGl0WnNWbVBzRktEWDYyVG9zMEJZV0VzanZ0VDM4WFhSZXA3T3BWR0lPQi85V09YVGl3VkpaT2tSZ2MKbHd3U2U1dnBQQXRpMzhUZ3BhM0FVSk5haG00bDhHNWF5WktRQWFnUDg1NHBFTjhPOW54Nk9odytWN1hzSGlNdQpwUmpvc0VTN0JJY1lXVGJxR05yNFR3eXo1cVUwOE9LOEUySFNFSnE5THA4YzZ6UTZnZzBhV1dLYWJyTUpNeSt2CkpIbjM5TEI0U3dONzJjVXJkRWYvQVVrWktNYTRKVFRjMTJnaGpKN1JvUENYUFdPOUJGZ09aZEdoUlpBYkZRMXgKcnB6b3BLZllkT0hWZ0tncG9MOVJIRm40TzZRaTBjbnBML0NZZEFGd0pXNmZYcGEyekhobEJqWXlWdHk5T1Y4TQppV01IVUNXZnl4anVaSno5NFFWZGxLRGVrY2YzUFJzU0RBRGZ4TXlBYVdJQ1NnYXNHTVNPSnRoRTlGM0JhaXNvCnNYM1NLYzRFSEc4Sk1VM1hoeWYwbkhDY2hQdWVRblU5akFBVVBDMHRrVlZtOWhmMXVkdjlOTUk4bktncWRFMkMKK2ExNnR3RVBpZzhkS1pkaFRMOFdXMGwxS3FRcCs4SnBGdTdNWUM1SGNaa0F0NEE3QXlXOHRsYmlQQ1B4RVYwZwpsYkc0eXFyV2lIOG5rWE9tNFBKb0FEMDhzNjA5Y3lFY2Iyblgvck92KzBSdFdOOWFqZGNxYlM1Z0JJaEhRSVUwCko1N0cyTVFYZ0hHbU9QL1ZZTi8vMTlsaXd3PT1URVNUCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - # output from cat ca.key | base64 - ca.key: | + # output from cat tls.key | base64 + tls.key: | LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIekNDQXdlZ0F3SUJBZ0lVSlAwL1FCY0xTMFFFV1ZiRDE4NndyUnZjLzdFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0hqRWNNQm9HQTFVRUF3d1RjMlZzWmkxemFXZHVaV1F0WTJFdFkyVnlkREFnRncweU5ERXhNak13TXpReApOVEphR0E4eU1USTBNVEF6TURBek5ERTFNbG93SGpFY01Cb0dBMVVFQXd3VGMyVnNaaTF6YVdkdVpXUXRZMkV0ClkyVnlkRENDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQkFML0lpWVdabUc0Q05qOS8KMzV6RUJheU1tL2djeGhJSFArZ1M4S04wdm9YV2krdk1Kd1hEUWR3T0JOdEZaa1l0elpJc29ESHVHenRnN0RWNApyZXZONE5JOHRKTmc2b2Jma0tVcGQ3eHdvNHNIdXMwd0h6NWVwZGt1MDhNVzVzZFZOUFNRMkhpVnhialpXSnBRClpJYWYyQWRKa1psUFdtVDBaS1pPdFFEN2oySTRtM3VCeG1jYzhTNWhiNkpaYW9NbGNVVXhkNDFscG43T21iMGEKVjRBUGZiWS9vYytwZmVDczN1cG5xamxZamVGQjR2RTV4WU1ZV0FNeitJRGh4RTRxRGVSaXNMQnhhN1kvcFRScQo2OWVhVXN6Qjl5eEQ3R0FySTJsSDhyUCtVeGpGYUl1K2tBVjVtbjc4OXdlejh0TDVGNEErWlE5cGM5TVI2UXBuCmRkanlaRXcvcFpkdTcwVmo3WUE1MU91S2owcTF2dGw3d1BPcDBUc3lwUDhadW04ZkZSNG5KbmNPaFhMTWV3bGEKTWxBeFZaUWRiMEF5dEE2TUl0dFdXSjA5L3BEOWZ6SkdjRnZOL245ZzZWZ0o5NjNhbmRoTEwyYlJMc0VKRmxUTQpEdTJIeW1CNkErb0ZlcDdjZXNxOUpJRFhkVmFqR3NxMmgrZVpPNGdxWW5nWGNmQVl5ZUloYzlYNnFoT2QvVmlZCmY2eUZoOTNuUnRYTFFNdUJRN2E1WTFzRVN3RHp3WWJKdEtuK3NrcGg5SEtCMTdVRUVOU3BNNHJSNHdxekRQd3AKSmZZeWt2a2Iyd2w0TkNCb2pjaU9icDYwV2ZDQytRcTFsNEo4VXpSOEpvWmFiQ0IzOWxVcHBKa09qNVFxYnEwMApKaUFzWENQQlp3OCtnQnV0b2JBVUs0RklqMGVQQWdNQkFBR2pVekJSTUIwR0ExVWREZ1FXQkJTcmNlRWNhMjNxCjBUQm14VmtZTWtMQTNSeHpKekFmQmdOVkhTTUVHREFXZ0JTcmNlRWNhMjNxMFRCbXhWa1lNa0xBM1J4ekp6QVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUNBUUJseW5IM2doVG9ObDQrTlQ1MgpNTTA1V1A3UCszVXJkQ0tGNEJCa0VzN0VueldSUjZ4bkVoVUY5VWhGZ0ZhTFBiQ1pacnlCS2krT1hrUHUva3JCCk12aE1LVGl0WnNWbVBzRktEWDYyVG9zMEJZV0VzanZ0VDM4WFhSZXA3T3BWR0lPQi85V09YVGl3VkpaT2tSZ2MKbHd3U2U1dnBQQXRpMzhUZ3BhM0FVSk5haG00bDhHNWF5WktRQWFnUDg1NHBFTjhPOW54Nk9odytWN1hzSGlNdQpwUmpvc0VTN0JJY1lXVGJxR05yNFR3eXo1cVUwOE9LOEUySFNFSnE5THA4YzZ6UTZnZzBhV1dLYWJyTUpNeSt2CkpIbjM5TEI0U3dONzJjVXJkRWYvQVVrWktNYTRKVFRjMTJnaGpKN1JvUENYUFdPOUJGZ09aZEdoUlpBYkZRMXgKcnB6b3BLZllkT0hWZ0tncG9MOVJIRm40TzZRaTBjbnBML0NZZEFGd0pXNmZYcGEyekhobEJqWXlWdHk5T1Y4TQppV01IVUNXZnl4anVaSno5NFFWZGxLRGVrY2YzUFJzU0RBRGZ4TXlBYVdJQ1NnYXNHTVNPSnRoRTlGM0JhaXNvCnNYM1NLYzRFSEc4Sk1VM1hoeWYwbkhDY2hQdWVRblU5akFBVVBDMHRrVlZtOWhmMXVkdjlOTUk4bktncWRFMkMKK2ExNnR3RVBpZzhkS1pkaFRMOFdXMGwxS3FRcCs4SnBGdTdNWUM1SGNaa0F0NEE3QXlXOHRsYmlQQ1B4RVYwZwpsYkc0eXFyV2lIOG5rWE9tNFBKb0FEMDhzNjA5Y3lFY2Iyblgvck92KzBSdFdOOWFqZGNxYlM1Z0JJaEhRSVUwCko1N0cyTVFYZ0hHbU9QL1ZZTi8vMTlsaXd3PT1URVNUCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K diff --git a/controllers/testdata/secrets/ray-ca-cert.yaml b/controllers/testdata/secrets/ray-ca-cert.yaml deleted file mode 100644 index 77335ebf..00000000 --- a/controllers/testdata/secrets/ray-ca-cert.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ray-ca-cert - labels: - opendatahub.io/managed: 'true' -data: - # output from cat ca.crt | base64 - ca.crt: | - LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZIekNDQXdlZ0F3SUJBZ0lVSlAwL1FCY0xTMFFFV1ZiRDE4NndyUnZjLzdFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0hqRWNNQm9HQTFVRUF3d1RjMlZzWmkxemFXZHVaV1F0WTJFdFkyVnlkREFnRncweU5ERXhNak13TXpReApOVEphR0E4eU1USTBNVEF6TURBek5ERTFNbG93SGpFY01Cb0dBMVVFQXd3VGMyVnNaaTF6YVdkdVpXUXRZMkV0ClkyVnlkRENDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQkFML0lpWVdabUc0Q05qOS8KMzV6RUJheU1tL2djeGhJSFArZ1M4S04wdm9YV2krdk1Kd1hEUWR3T0JOdEZaa1l0elpJc29ESHVHenRnN0RWNApyZXZONE5JOHRKTmc2b2Jma0tVcGQ3eHdvNHNIdXMwd0h6NWVwZGt1MDhNVzVzZFZOUFNRMkhpVnhialpXSnBRClpJYWYyQWRKa1psUFdtVDBaS1pPdFFEN2oySTRtM3VCeG1jYzhTNWhiNkpaYW9NbGNVVXhkNDFscG43T21iMGEKVjRBUGZiWS9vYytwZmVDczN1cG5xamxZamVGQjR2RTV4WU1ZV0FNeitJRGh4RTRxRGVSaXNMQnhhN1kvcFRScQo2OWVhVXN6Qjl5eEQ3R0FySTJsSDhyUCtVeGpGYUl1K2tBVjVtbjc4OXdlejh0TDVGNEErWlE5cGM5TVI2UXBuCmRkanlaRXcvcFpkdTcwVmo3WUE1MU91S2owcTF2dGw3d1BPcDBUc3lwUDhadW04ZkZSNG5KbmNPaFhMTWV3bGEKTWxBeFZaUWRiMEF5dEE2TUl0dFdXSjA5L3BEOWZ6SkdjRnZOL245ZzZWZ0o5NjNhbmRoTEwyYlJMc0VKRmxUTQpEdTJIeW1CNkErb0ZlcDdjZXNxOUpJRFhkVmFqR3NxMmgrZVpPNGdxWW5nWGNmQVl5ZUloYzlYNnFoT2QvVmlZCmY2eUZoOTNuUnRYTFFNdUJRN2E1WTFzRVN3RHp3WWJKdEtuK3NrcGg5SEtCMTdVRUVOU3BNNHJSNHdxekRQd3AKSmZZeWt2a2Iyd2w0TkNCb2pjaU9icDYwV2ZDQytRcTFsNEo4VXpSOEpvWmFiQ0IzOWxVcHBKa09qNVFxYnEwMApKaUFzWENQQlp3OCtnQnV0b2JBVUs0RklqMGVQQWdNQkFBR2pVekJSTUIwR0ExVWREZ1FXQkJTcmNlRWNhMjNxCjBUQm14VmtZTWtMQTNSeHpKekFmQmdOVkhTTUVHREFXZ0JTcmNlRWNhMjNxMFRCbXhWa1lNa0xBM1J4ekp6QVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUNBUUJseW5IM2doVG9ObDQrTlQ1MgpNTTA1V1A3UCszVXJkQ0tGNEJCa0VzN0VueldSUjZ4bkVoVUY5VWhGZ0ZhTFBiQ1pacnlCS2krT1hrUHUva3JCCk12aE1LVGl0WnNWbVBzRktEWDYyVG9zMEJZV0VzanZ0VDM4WFhSZXA3T3BWR0lPQi85V09YVGl3VkpaT2tSZ2MKbHd3U2U1dnBQQXRpMzhUZ3BhM0FVSk5haG00bDhHNWF5WktRQWFnUDg1NHBFTjhPOW54Nk9odytWN1hzSGlNdQpwUmpvc0VTN0JJY1lXVGJxR05yNFR3eXo1cVUwOE9LOEUySFNFSnE5THA4YzZ6UTZnZzBhV1dLYWJyTUpNeSt2CkpIbjM5TEI0U3dONzJjVXJkRWYvQVVrWktNYTRKVFRjMTJnaGpKN1JvUENYUFdPOUJGZ09aZEdoUlpBYkZRMXgKcnB6b3BLZllkT0hWZ0tncG9MOVJIRm40TzZRaTBjbnBML0NZZEFGd0pXNmZYcGEyekhobEJqWXlWdHk5T1Y4TQppV01IVUNXZnl4anVaSno5NFFWZGxLRGVrY2YzUFJzU0RBRGZ4TXlBYVdJQ1NnYXNHTVNPSnRoRTlGM0JhaXNvCnNYM1NLYzRFSEc4Sk1VM1hoeWYwbkhDY2hQdWVRblU5akFBVVBDMHRrVlZtOWhmMXVkdjlOTUk4bktncWRFMkMKK2ExNnR3RVBpZzhkS1pkaFRMOFdXMGwxS3FRcCs4SnBGdTdNWUM1SGNaa0F0NEE3QXlXOHRsYmlQQ1B4RVYwZwpsYkc0eXFyV2lIOG5rWE9tNFBKb0FEMDhzNjA5Y3lFY2Iyblgvck92KzBSdFdOOWFqZGNxYlM1Z0JJaEhRSVUwCko1N0cyTVFYZ0hHbU9QL1ZZTi8vMTlsaXd3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - # output from cat ca.key | base64 - ca.key: | - LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRQy95SW1GbVpodUFqWS8KZjkrY3hBV3NqSnY0SE1ZU0J6L29FdkNqZEw2RjFvdnJ6Q2NGdzBIY0RnVGJSV1pHTGMyU0xLQXg3aHM3WU93MQplSzNyemVEU1BMU1RZT3FHMzVDbEtYZThjS09MQjdyTk1COCtYcVhaTHRQREZ1YkhWVFQwa05oNGxjVzQyVmlhClVHU0duOWdIU1pHWlQxcGs5R1NtVHJVQSs0OWlPSnQ3Z2NabkhQRXVZVytpV1dxREpYRkZNWGVOWmFaK3pwbTkKR2xlQUQzMjJQNkhQcVgzZ3JON3FaNm81V0kzaFFlTHhPY1dER0ZnRE0vaUE0Y1JPS2cza1lyQ3djV3UyUDZVMAphdXZYbWxMTXdmY3NRK3hnS3lOcFIvS3ovbE1ZeFdpTHZwQUZlWnArL1BjSHMvTFMrUmVBUG1VUGFYUFRFZWtLClozWFk4bVJNUDZXWGJ1OUZZKzJBT2RUcmlvOUt0YjdaZThEenFkRTdNcVQvR2Jwdkh4VWVKeVozRG9WeXpIc0oKV2pKUU1WV1VIVzlBTXJRT2pDTGJWbGlkUGY2US9YOHlSbkJiemY1L1lPbFlDZmV0MnAzWVN5OW0wUzdCQ1JaVQp6QTd0aDhwZ2VnUHFCWHFlM0hyS3ZTU0ExM1ZXb3hyS3RvZm5tVHVJS21KNEYzSHdHTW5pSVhQVitxb1RuZjFZCm1IK3NoWWZkNTBiVnkwRExnVU8ydVdOYkJFc0E4OEdHeWJTcC9ySktZZlJ5Z2RlMUJCRFVxVE9LMGVNS3N3ejgKS1NYMk1wTDVHOXNKZURRZ2FJM0lqbTZldEZud2d2a0t0WmVDZkZNMGZDYUdXbXdnZC9aVkthU1pEbytVS202dApOQ1lnTEZ3andXY1BQb0FicmFHd0ZDdUJTSTlIandJREFRQUJBb0lDQUFyYzkwaG9ud3VIWGI3ZmNtU0IxU3JZClZPWWt1WDl6aHQvRWxIb1E5cDNFSSswNWhWaFdCTmpMNjBvYXRuRlhtenk3emZtTWMyRTcyemlPam1OdmpvOGcKY1l4eDlMYmQycG5RWUlBWEJ0eDV5UUxJWUFaSUwySys3NjloRUlLYksvVzQxZG9wN05vekFMQm9MMW1FenlSZgpWS0hFU0ZDMHptS3hNOUpMYllYeWowMm9QbUhBY0NHdGJHdjFrZGZ4RkdjNldrZy80c0tnY05ld3NueUdTb0lICm8zd21ZSnkvSjUxTDF5QlhPL2J2Y1hobHNMd3djamNCQ0FNUUU0aE42UjJKUUwrdDBEWGt2SjBQcnZzRE9wa0kKakdzTlEzMWVPcEpERmdwL21zNlFNWnpObHhwdXNGQTVnNUNkaUpRMHNkSGpOdUtqTXhyeUxKRk1HY0l0OExEQwpRVzF2akxLR0l1UWtraGwxOWU1S1N2SDdjUjJja0pDME5vTzhnekpudzd0dTRGaHJaK0xQeXF2R3VSYU55a2RmCi9BKzNEOUE2RW1PNWRldFU2RzJkK0l2TmprdG91Z05UalZIUklDbk9oL01zRmlFQXdycHltVVNISzhKTjVpSjIKUm1rNFljNWlXUjhOUWs0Wkh6aVFGSHJSWkh0TW9DcEkvR2ZGcnYyRVE0bFpOOG5tZHdDWDR4a3JObUJ2ZnlIdgpLWW0yMU5VWDc5U3lRbHd5VS9lNUR6eTA3Si9zcTdoVU8xN3hVcXNzTVpZTGZCSFF0VFU0VVAwbnBOZmtxUFM1CjdJRUtIVWwyRlZudXR0THoxc0ZVVHhJTS90aE9lczRtWElrOExiYzI0Yk45VWlteXplVEN1bE83a0hZSDhTVkEKZDJqZFBTZXhZSTdMeWFVNnFHRzVBb0lCQVFEbHlVQk5CaTRNekdxVnh5NjNCY1dyZC9rdVYrYTFLQ3MyYVhzagpLbVlMT0xrSkhUSjI0YU9EWkJBVGxEMllwclZEOUM1UThTeDdPQVdlQ3FqWHd1MndsOTlabXNIVkxiQTZMRUZ6CnBoYTNQVHhkaWFpMElwZVY2ZFpIQnQrdjVDVGsvSnpxSVpjc1J2REFnNTFHYzgxbERxTzFNbnVqbldBcGpSMmMKd05ZVXd6a3hicHVTc1ZHYzFBZ09tVHBHN1MrdWVVQ2FGU0NVaGkyVWVoblM5dkNrU3Y0QTZMRlpiaXhFeWp6aApycU9mN1d1TTVUWkFoTGM2RTJUQnVOeWJlWW9DblFMdHF3dnUxaFhDOGU4TGlQWFRlMVJ4U2x5dXA5RDhiWEZBCjVPVmFUZjAzcFFweURiOXNKeGhLN3FMbUgrSjlUeU5JamhTTUZQT2pKNEJFRTlPOUFvSUJBUURWcVhDMGdCVzUKYlNUWmUzc3l1QVltRi9hVDg1ZFh1NGFTMFBJR09MakE1M2h0RVdLUkJxd1JlU1prSFdtR05uNUIyOXVXTHg2UgpPZjFNOFJkY2NYSnlxMnp1TlBiWkpabllwS0x5N0FjeDBpc1RvMjdpUy9xRS85YndsNUo3QVU5UmZ2K2ZMK2RPCmxqUndRTGUvQ1dSVHVlTlNOSWpPUC96NWRra2J5Z1kvWHZHbmI0RUJheDY4K3J2a0NYbStGdFpXV3VoblM2Uy8KZHh3Ulo2VGRMd09RZTZQSzNzN3F4c2xWNmQ2dmwrSUpwa1VVZmRvWDNyWFlTeGx2cFlQYWJpWEpaVzdQWkZwRQpVQXc0VTFpSzVLMUt5d1ZjaHlhN2tQSlpRNUplS1pUT3lPL1d5ODZLak0vcUd3NUhDR2NOL2VMbDJKUUViUkwvClJiR0pGSmhUalpjN0FvSUJBQXlyNG0zYzcyRXBUSjloMG9PcFA5TksxR1RuMkFNWmFmaWdMSGd0K0Y2YURDb2kKZ0F2cU9YZ2ZabnVONnkrbDBjMGpoQUpXcWx0SkpaWW5oRlFSbmNYbE9oM1kyT09HbDNjOXhZWTVISHVTVnVmWgpsWUlKZms1NERLYnlEQmZJL3ZmWnJsV0M4TEV5WUVoZGVhak83ZjZxcGdCeC9qdHhqRUgrVkNtMndKZDRoSWpqClRwVHlUa3ZWclhRUW94UVNORlRzdnRGQVpRR0x2S3U1Wi84b092RDBhYmxuRzVDUThNUUNXd1VlK2tyeGJzTGcKU1BPWjNmakg1UUNCenppTHBUNnJwZU94VVFFa3NTS0U4T2V6NzhwdnZLSmF0VzIwTjJRVUxQQ2xMcmlpSUZxWApNVkpFeTgrTkFGdnhlTzR6eCt1ZEY1Y0Nyc05pekdTczR2ZmVHQWtDZ2dFQkFMdFRnbWdPd0gxQlR3U0t1Ym4vCkZBMEVCNEV5R2FlbTExY1RjSTY1M21ucXgyL0F4VTFucnlibXRCMGttR2Mra2JYR1FDRE5rUnc4M25NK0VZQlEKU3NwMHQ5MmxmQ05vVHhsZFJ5eDZlZGhaYnNFYUVsYS96SllkQk9NTjBUU2RNbUMrV3ZuRGN5WTRsU014NnFmSQpZVGp6Q25ZQmIweDlWNXVUOUljenVnU0hocEdKTm03Njd3azdQODZ2N0JnWVI3V1FvS0FuOXZxVFFIMldCRHFVClJLakJiaHFvL0h0azdCS3lLRGFGa0gxclZMZWhtN3cvMitrVjl1Z25FcEpJN2tKRDkwSkh0c2liOGdyVU1CWWUKWmp6a0FRQmQwanl5MlhnZndVMWpZWDluTnJoNUdjM3BwVVNZa2d6L05mTlRmRUtPZnovZUxjQzM1dTdMcXIzZQpydzhDZ2dFQkFMT2tsTkJNRVBmM20yTXBjaVRjRmNKb08vZzBMUUpHaTJtWkN6S1g3eDJFS0N2N1ZvVWVtRkk0CjRkRFVmSlBJWlBFTUpkTHRSUy9qUDEyZWkxek9lWHIrVGlUTklpUUVoemRtL0RZWUdjd2hyb0xLNDZVTFJKY0YKYzdxZ2xNQ1Z1MW9DTmtDdTJvZ08renczRm9makJzK1pqcE1BS2kyOTZ2ZDk2YVlYNThYR0RKekdmdjhuZEF1dwpEUmU1ZE5oQU5iaHZqSlM1VXJwNnhoMVMycTNYOHorTlFGWW9CNDM1Q2NXNW50WWMzemIxYVdzY0NxMWJsUGJGCjc0QTFLTHJNNlpvU0ZlcUVWZzhvajhpWjlDaitiTTJXYm9BREIvRTROM0kyNmFDK1dDRWxtdTd3ZDdQaExQT2IKN3RrTXh2Zm10dDE5T2dYbTRKZm9SZWlkMTNYbHFoZz0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= diff --git a/controllers/utils/cert.go b/controllers/utils/cert.go new file mode 100644 index 00000000..ee799ee2 --- /dev/null +++ b/controllers/utils/cert.go @@ -0,0 +1,171 @@ +package utils + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CreateSelfSignedCertificate(ctx context.Context, c client.Client, secretName, domain, namespace string) error { + certSecret, err := GenerateSelfSignedCertificateAsSecret(secretName, domain, namespace) + if err != nil { + return fmt.Errorf("failed generating self-signed certificate: %w", err) + } + + if errGen := generateCertSecret(ctx, c, certSecret); errGen != nil { + return fmt.Errorf("failed update self-signed certificate secret: %w", errGen) + } + + return nil +} + +func GenerateSelfSignedCertificateAsSecret(name, addr, namespace string) (*corev1.Secret, error) { + cert, key, err := generateCertificate(addr) + if err != nil { + return nil, errors.WithStack(err) + } + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "opendatahub.io/managed": "true", + "app.kubernetes.io/name": "odh-model-controller", + "app.kubernetes.io/component": "kserve", + "app.kubernetes.io/part-of": "odh-model-serving", + "app.kubernetes.io/managed-by": "odh-model-controller", + }, + }, + Data: map[string][]byte{ + corev1.TLSCertKey: cert, + corev1.TLSPrivateKeyKey: key, + }, + Type: corev1.SecretTypeTLS, + }, nil +} + +func generateCertificate(addr string) ([]byte, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + seededRand, cryptErr := rand.Int(rand.Reader, big.NewInt(time.Now().UnixNano())) + if cryptErr != nil { + return nil, nil, errors.WithStack(cryptErr) + } + + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: seededRand, + Subject: pkix.Name{ + CommonName: addr, + Organization: []string{"serving-self-signed"}, + }, + NotBefore: now.UTC(), + NotAfter: now.Add(time.Second * 60 * 60 * 24 * 365 * 10).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + certDERBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, key.Public(), key) + if err != nil { + return nil, nil, errors.WithStack(err) + } + certificate, err := x509.ParseCertificate(certDERBytes) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certificate.Raw, + }); err != nil { + return nil, nil, errors.WithStack(err) + } + + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }); err != nil { + return nil, nil, errors.WithStack(err) + } + + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} + +// recreateSecret deletes the existing secret and creates a new one. +func recreateSecret(ctx context.Context, c client.Client, existingSecret, newSecret *corev1.Secret) error { + if err := c.Delete(ctx, existingSecret); err != nil { + return fmt.Errorf("failed to delete existing secret before recreating new one: %w", err) + } + if err := c.Create(ctx, newSecret); err != nil { + return fmt.Errorf("failed to create new secret after existing one has been deleted: %w", err) + } + return nil +} + +// generateCertSecret creates a secret if it does not exist; recreate this secret if type not match; update data if outdated. +func generateCertSecret(ctx context.Context, c client.Client, certSecret *corev1.Secret) error { + existingSecret := &corev1.Secret{} + errGet := c.Get(ctx, client.ObjectKeyFromObject(certSecret), existingSecret) + switch { + case errGet == nil: + // Secret exists but with a different type, delete and create it again + if existingSecret.Type != certSecret.Type { + return recreateSecret(ctx, c, existingSecret, certSecret) + } + // update data if found with same type but outdated content + if isSecretOutdated(existingSecret.Data, certSecret.Data) { + if errUpdate := c.Update(ctx, certSecret); errUpdate != nil { + return fmt.Errorf("failed to update existing secret: %w", errUpdate) + } + } + case k8serr.IsNotFound(errGet): + // Secret does not exist, create it + if errCreate := c.Create(ctx, certSecret); errCreate != nil { + return fmt.Errorf("failed creating new certificate secret: %w", errCreate) + } + default: + return fmt.Errorf("failed getting certificate secret: %w", errGet) + } + + return nil +} + +// isSecretOutdated compares two secret data of type map[string][]byte and returns true if they are not equal. +func isSecretOutdated(existingSecretData, newSecretData map[string][]byte) bool { + if len(existingSecretData) != len(newSecretData) { + return true + } + + for key, value1 := range existingSecretData { + value2, ok := newSecretData[key] + if !ok { + return true + } + if !bytes.Equal(value1, value2) { + return true + } + } + + return false +}