Actually upload the generated sitemaps (and perform logging)

deployment/clouddeploy/gke-workers/base/generate-sitemap.yaml

@@ -13,7 +13,7 @@ spec:
           - name: generate-sitemap-cron
             image: cron
             imagePullPolicy: Always
-            command: ["/usr/local/bin/generate_sitemap/generate_sitemap.py", "--base_url", "$BASE_URL"]
+            command: ["/usr/local/bin/generate_sitemap/generate_and_upload.sh"]
             resources:
               requests:
                 cpu: 1

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/generate-sitemap.yaml

@@ -14,3 +14,5 @@ spec:
               value: "https://test.osv.dev"
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb-test
+            - name: OUTPUT_GCS_BUCKET
+              value: test-osv-dev-sitemap

deployment/clouddeploy/gke-workers/environments/oss-vdb/generate-sitemap.yaml

@@ -14,3 +14,5 @@ spec:
               value: "https://osv.dev"
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb
+            - name: OUTPUT_GCS_BUCKET
+              value: osv-dev-sitemap

docker/cron/generate_sitemap/generate_and_upload.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+SITEMAP_OUTPUT="sitemap_output/"
+OUTPUT_BUCKET="${OUTPUT_GCS_BUCKET:=test-osv-dev-sitemap}"
+BASE_URL_PATH="${BASE_URL:=https://test.osv.dev}"
+
+echo "Begin sitemap generation for $BASE_URL_PATH"
+
+./generate_sitemap.py --base_url $BASE_URL_PATH
+
+echo "Begin Syncing with cloud to $OUTPUT_BUCKET"
+
+gsutil -m rsync -c -d $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/"

docker/cron/generate_sitemap/generate_sitemap.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Generate sitemap."""
+import logging
 import sys
 import os
 import osv
@@ -59,6 +60,7 @@ def get_sitemap_url_for_ecosystem(ecosystem: str, base_url: str) -> str:
 
 def generate_sitemap_for_ecosystem(ecosystem: str, base_url: str) -> None:
   """Generate a sitemap for the give n ecosystem."""
+  logging.info("Generating sitemap for ecosystem '%s'.", ecosystem)
   vulnerability_ids = fetch_vulnerability_ids(ecosystem)
   filename = get_sitemap_filename_for_ecosystem(ecosystem)
   urlset = Element(
@@ -79,6 +81,7 @@ def generate_sitemap_for_ecosystem(ecosystem: str, base_url: str) -> None:
 
 def generate_sitemap_index(ecosystems: set[str], base_url: str) -> None:
   """Generate a sitemap index."""
+  logging.info("Generating sitemap index.")
   sitemapindex = Element(
       'sitemapindex', xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")