Merge branch 'master' into correct-ecosystem-count-cache

another-rex · Jun 4, 2024 · 2b13b45 · 2b13b45
2 parents fa40590 + 9a8b7d6
commit 2b13b45
Show file tree

Hide file tree

Showing 10 changed files with 92 additions and 39 deletions.
diff --git a/deployment/clouddeploy/gke-workers/base/debian-cve-convert.yaml b/deployment/clouddeploy/gke-workers/base/debian-cve-convert.yaml
@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: debian-cve-convert
+spec:
+  schedule: "0 */1 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 3600
+      template:
+        spec:
+          containers:
+          - name: debian-cve-convert
+            image: debian-cve-convert
+            imagePullPolicy: Always
+            securityContext:
+              privileged: true
+            resources:
+              requests:
+                cpu: 1
+                memory: "1G"
+              limits:
+                cpu: 1
+                memory: "2G"
+          restartPolicy: OnFailure
+          volumes:
+            - name: "ssd"
+              hostPath:
+                path: "/mnt/disks/ssd0"
diff --git a/deployment/clouddeploy/gke-workers/base/kustomization.yaml b/deployment/clouddeploy/gke-workers/base/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
 - importer-deleter.yaml
 - exporter.yaml
 - alpine-cve-convert.yaml
+- debian-cve-convert.yaml
 - combine-to-osv.yaml
 - debian-convert.yaml
 - debian-first-version.yaml

diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-cve-convert.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-cve-convert.yaml
@@ -3,33 +3,14 @@ kind: CronJob
 metadata:
   name: debian-cve-convert
 spec:
-  schedule: "0 */1 * * *"
-  concurrencyPolicy: Forbid
   jobTemplate:
     spec:
-      activeDeadlineSeconds: 3600
       template:
         spec:
           containers:
           - name: debian-cve-convert
-            image: debian-cve-convert
-            imagePullPolicy: Always
             env:
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb-test
             - name: OUTPUT_GCS_BUCKET
               value: osv-test-cve-osv-conversion
-            securityContext:
-              privileged: true
-            resources:
-              requests:
-                cpu: 1
-                memory: "1G"
-              limits:
-                cpu: 1
-                memory: "2G"
-          restartPolicy: OnFailure
-          volumes:
-            - name: "ssd"
-              hostPath:
-                path: "/mnt/disks/ssd0"
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml
@@ -1,13 +1,13 @@
 resources:
 - ../../base
-- debian-cve-convert.yaml
 patches:
 - path: workers.yaml
 - path: scaler.yaml
 - path: importer.yaml
 - path: importer-deleter.yaml
 - path: exporter.yaml
 - path: alpine-cve-convert.yaml
+- path: debian-cve-convert.yaml
 - path: combine-to-osv.yaml
 - path: debian-convert.yaml
 - path: debian-first-version.yaml

diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/debian-cve-convert.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/debian-cve-convert.yaml
@@ -0,0 +1,16 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: debian-cve-convert
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: debian-cve-convert
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb
+            - name: OUTPUT_GCS_BUCKET
+              value: cve-osv-conversion
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml
@@ -8,6 +8,7 @@ patches:
 - path: importer-deleter.yaml
 - path: exporter.yaml
 - path: alpine-cve-convert.yaml
+- path: debian-cve-convert.yaml
 - path: combine-to-osv.yaml
 - path: debian-convert.yaml
 - path: debian-first-version.yaml

diff --git a/deployment/terraform/environments/oss-vdb-test/main.tf b/deployment/terraform/environments/oss-vdb-test/main.tf
@@ -18,6 +18,11 @@ module "osv_test" {
   esp_version    = "2.47.0"
 }
 
+import {
+  to = module.osv_test.google_firestore_database.datastore
+  id = "oss-vdb-test/(default)"
+}
+
 output "website_dns_records" {
   description = "DNS records that need to be created for the osv.dev website"
   value       = module.osv_test.website_dns_records

diff --git a/deployment/terraform/environments/oss-vdb/main.tf b/deployment/terraform/environments/oss-vdb/main.tf
@@ -18,6 +18,11 @@ module "osv" {
   esp_version    = "2.47.0"
 }
 
+import {
+  to = module.osv.google_firestore_database.datastore
+  id = "oss-vdb/(default)"
+}
+
 output "website_dns_records" {
   description = "DNS records that need to be created for the osv.dev website"
   value       = module.osv.website_dns_records

diff --git a/deployment/terraform/modules/osv/main.tf b/deployment/terraform/modules/osv/main.tf
@@ -8,6 +8,18 @@ resource "google_app_engine_application" "app" {
 
 }
 
+# Datastore
+resource "google_firestore_database" "datastore" {
+  project     = var.project_id
+  name        = "(default)"
+  location_id = "us-west2"
+  type        = "DATASTORE_MODE"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
 # MemoryStore
 resource "google_redis_instance" "frontend" {
   project            = var.project_id

diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go
@@ -479,15 +479,16 @@ func TestExtractGitCommit(t *testing.T) {
 			inputCommitType: Fixed,
 			expectFailure:   true,
 		},
-		{
-			description:     "Valid GitLab commit URL",
-			inputLink:       "https://gitlab.freedesktop.org/virgl/virglrenderer/-/commit/b05bb61f454eeb8a85164c8a31510aeb9d79129c",
-			inputCommitType: Fixed,
-			expectedAffectedCommit: AffectedCommit{
-				Repo:  "https://gitlab.freedesktop.org/virgl/virglrenderer",
-				Fixed: "b05bb61f454eeb8a85164c8a31510aeb9d79129c",
-			},
-		},
+		// Skips this test as "https://gitlab.freedesktop.org" is currently under maintenance.
+		// {
+		// 	description:     "Valid GitLab commit URL",
+		// 	inputLink:       "https://gitlab.freedesktop.org/virgl/virglrenderer/-/commit/b05bb61f454eeb8a85164c8a31510aeb9d79129c",
+		// 	inputCommitType: Fixed,
+		// 	expectedAffectedCommit: AffectedCommit{
+		// 		Repo:  "https://gitlab.freedesktop.org/virgl/virglrenderer",
+		// 		Fixed: "b05bb61f454eeb8a85164c8a31510aeb9d79129c",
+		// 	},
+		// },
 		{
 			description:     "Valid GitLab commit URL with .patch extension",
 			inputLink:       "https://gitlab.com/muttmua/mutt/-/commit/452ee330e094bfc7c9a68555e5152b1826534555.patch",
@@ -862,16 +863,17 @@ func TestExtractVersionInfo(t *testing.T) {
 			},
 			expectedNotes: []string{},
 		},
-		{
-			description:        "A CVE with a weird GitLab reference that breaks version enumeration in the worker",
-			inputCVEItem:       loadTestData2("CVE-2022-46285"),
-			inputValidVersions: []string{},
-			expectedVersionInfo: VersionInfo{
-				AffectedCommits:  []AffectedCommit{{Repo: "https://gitlab.freedesktop.org/xorg/lib/libxpm", Fixed: "a3a7c6dcc3b629d7650148"}},
-				AffectedVersions: []AffectedVersion{{Fixed: "3.5.15"}},
-			},
-			expectedNotes: []string{},
-		},
+		// Skips this test as "https://gitlab.freedesktop.org" is currently under maintenance.
+		// {
+		// 	description:        "A CVE with a weird GitLab reference that breaks version enumeration in the worker",
+		// 	inputCVEItem:       loadTestData2("CVE-2022-46285"),
+		// 	inputValidVersions: []string{},
+		// 	expectedVersionInfo: VersionInfo{
+		// 		AffectedCommits:  []AffectedCommit{{Repo: "https://gitlab.freedesktop.org/xorg/lib/libxpm", Fixed: "a3a7c6dcc3b629d7650148"}},
+		// 		AffectedVersions: []AffectedVersion{{Fixed: "3.5.15"}},
+		// 	},
+		// 	expectedNotes: []string{},
+		// },
 		{
 			description:  "A CVE with a different GitWeb reference URL that was not previously being extracted successfully",
 			inputCVEItem: loadTestData2("CVE-2021-28429"),