refactor workflows to split

Signed-off-by: linus-sun <[email protected]>

.github/workflows/identity_monitor.yml

@@ -0,0 +1,61 @@
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This GHA should not be run concurrently.
+# TODO: Once the workflows for consistency check and identity monitor are fully split, 
+# the consistency check workflow should no longer require a checkpoint file.
+
+name: Rekor Identity Monitor Template
+
+on:
+  workflow_call:
+    inputs:
+      once:
+        default: true
+        required: false
+        type: boolean
+      config:
+        description: 'multiline yaml of configuration settings for identity monitor run'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  detect-workflow:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # Needed to detect the current reusable repository and ref.
+    outputs:
+      repository: ${{ steps.detect.outputs.repository }}
+      ref: ${{ steps.detect.outputs.ref }}
+    steps:
+      - name: Detect the repository and ref
+        id: detect
+        uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@5a775b367a56d5bd118a224a811bba288150a563 # v2.0.0
+
+  monitor:
+    runs-on: ubuntu-latest
+    needs: [detect-workflow]
+    steps:
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          repository: ${{ needs.detect-workflow.outputs.repository }}
+          ref: "${{ needs.detect-workflow.outputs.ref }}"
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version: '1.23'
+      - name: Run identity monitor
+        run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --once

.github/workflows/main.yml

@@ -88,8 +88,8 @@ jobs:
         with:
           go-version-file: './go.mod'
           check-latest: true
-      - name: run e2e test
-        run: ./pkg/test/e2e/e2e_test.sh
+      - name: run consistency check test
+        run: ./pkg/test/consistency_check/consistency_check_e2e_test.sh
       - name: run identity monitor test
         run: ./pkg/test/identity_workflow/identity_workflow_e2e_test.sh
 

README.md

@@ -3,9 +3,13 @@
 Rekor Log Monitor provides an easy-to-use monitor to verify log consistency,
 that the log is immutability and append-only. Monitoring is critical to
 the transparency log ecosystem, as logs are tamper-evident but not tamper-proof.
+Rekor Log Monitor also provides a monitor to search for identities within a log,
+and send a list of found identities via various notification platforms.
+
+## Consistency check
 
 To run, create a GitHub Actions workflow that uses the
-[reusable monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/reusable_monitoring.yml).
+[consistency check workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/consistency_check.yml).
 It is recommended to run the log monitor every hour for optimal performance.
 
 Example workflow:
@@ -46,6 +50,10 @@ Please read [this](https://github.com/google/re2/wiki/Syntax) for syntax referen
 Note: The log monitor only starts monitoring from the latest checkpoint. If you want to search previous
 entries, you will need to query the log.
 
+To run, create a GitHub Actions workflow that uses the
+[identity monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/identity_monitor.yml).
+It is recommended to run the log monitor every hour for optimal performance.
+
 Example workflow below:
 
 ```
@@ -66,26 +74,27 @@ jobs:
     with:
       file_issue: true # Strongly recommended: Files an issue on monitoring failure
       artifact_retention_days: 14 # Optional, default is 14: Must be longer than the cron job frequency
-      identities: |
-        certIdentities:
-          - certSubject: user@domain\.com
-          - certSubject: otheruser@domain\.com
-            issuers:
-              - https://accounts\.google\.com
-              - https://github\.com/login
-          - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.*
-            issuers:
-              - https://token\.actions\.githubusercontent\.com
-        subjects:
-          - subject@domain\.com
-        fingerprints:
-          - A0B1C2D3E4F5
-        fulcioExtensions:
-          build-config-uri:
-            - https://example.com/owner/repository/build-config.yml
-        customExtensions:
-          - objectIdentifier: 1.3.6.1.4.1.57264.1.9
-            extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/[email protected]
+      config: |
+        monitoredValues: |
+          certIdentities:
+            - certSubject: user@domain\.com
+            - certSubject: otheruser@domain\.com
+              issuers:
+                - https://accounts\.google\.com
+                - https://github\.com/login
+            - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.*
+              issuers:
+                - https://token\.actions\.githubusercontent\.com
+          subjects:
+            - subject@domain\.com
+          fingerprints:
+            - A0B1C2D3E4F5
+          fulcioExtensions:
+            build-config-uri:
+              - https://example.com/owner/repository/build-config.yml
+          customExtensions:
+            - objectIdentifier: 1.3.6.1.4.1.57264.1.9
+              extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/[email protected]
 ```
 
 In this example, the monitor will log:

cmd/rekor_consistency/main.go

@@ -51,8 +51,6 @@ func main() {
 	once := flag.Bool("once", false, "Perform consistency check once and exit")
 	monitoredValsInput := flag.String("monitored-values", "", "yaml of certificate subjects and issuers, key subjects, "+
 		"and fingerprints. For certificates, if no issuers are specified, match any OIDC provider.")
-	outputIdentitiesFile := flag.String("output-identities", outputIdentitiesFileName,
-		"Name of the file containing indices and identities found in the log. Format is \"subject issuer index uuid\"")
 	userAgentString := flag.String("user-agent", "", "details to include in the user agent string")
 	flag.Parse()
 
@@ -85,7 +83,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once)
+	err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, once)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -100,7 +98,7 @@ func main() {
 
 	// To get an immediate first tick
 	for ; ; <-ticker.C {
-		err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile)
+		err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "error running consistency check: %v", err)
 			return

cmd/monitor/main.go → cmd/rekor_monitor/main.go

@@ -30,15 +30,13 @@ import (
 	"github.com/sigstore/rekor-monitor/pkg/rekor"
 	"github.com/sigstore/rekor-monitor/pkg/util/file"
 	"github.com/sigstore/rekor/pkg/client"
-	"github.com/sigstore/rekor/pkg/util"
 	"gopkg.in/yaml.v2"
 	"sigs.k8s.io/release-utils/version"
 )
 
 // Default values for monitoring job parameters
 const (
 	publicRekorServerURL     = "https://rekor.sigstore.dev"
-	logInfoFileName          = "logInfo.txt"
 	outputIdentitiesFileName = "identities.txt"
 )
 
@@ -48,35 +46,41 @@ const (
 func main() {
 	// Command-line flags that are parameters to the verifier job
 	configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings")
+	configYamlInput := flag.String("config", "", "path to yaml configuration file containing identity monitor settings")
 	once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once")
 	flag.Parse()
 
-	if configFilePath == nil {
-		log.Fatalf("empty configuration file path")
+	if *configFilePath == "" && *configYamlInput == "" {
+		log.Fatalf("empty configuration input")
 	}
 
-	readConfig, err := os.ReadFile(*configFilePath)
-	if err != nil {
-		log.Fatalf("error reading from identity monitor configuration file: %v", err)
+	if *configFilePath != "" && *configYamlInput != "" {
+		log.Fatalf("only input one of configuration file path or yaml input")
 	}
 
-	configString := string(readConfig)
 	var config notifications.IdentityMonitorConfiguration
-	if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
-		log.Fatalf("error parsing identities: %v", err)
+
+	if *configFilePath != "" {
+		readConfig, err := os.ReadFile(*configFilePath)
+		if err != nil {
+			log.Fatalf("error reading from identity monitor configuration file: %v", err)
+		}
+
+		configString := string(readConfig)
+		if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
 	}
 
-	rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH))))
-	if err != nil {
-		log.Fatalf("getting Rekor client: %v", err)
+	if *configYamlInput != "" {
+		if err := yaml.Unmarshal([]byte(*configYamlInput), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
 	}
 
 	if config.ServerURL == "" {
 		config.ServerURL = publicRekorServerURL
 	}
-	if config.LogInfoFile == "" {
-		config.LogInfoFile = logInfoFileName
-	}
 	if config.OutputIdentitiesFile == "" {
 		config.OutputIdentitiesFile = outputIdentitiesFileName
 	}
@@ -85,13 +89,36 @@ func main() {
 		config.Interval = &defaultInterval
 	}
 
+	rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH))))
+	if err != nil {
+		log.Fatalf("getting Rekor client: %v", err)
+	}
+
 	ticker := time.NewTicker(*config.Interval)
 	defer ticker.Stop()
 
 	// To get an immediate first tick
 	for ; ; <-ticker.C {
 		inputEndIndex := config.EndIndex
-		if config.StartIndex == nil || config.EndIndex == nil {
+
+		if config.StartIndex == nil {
+			if config.IdentityMetadataFile != nil {
+				idMetadata, err := file.ReadIdentityMetadata(*config.IdentityMetadataFile)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "error reading from identity metadata file: %v", err)
+					return
+				}
+
+				if config.StartIndex == nil {
+					config.StartIndex = &idMetadata.LatestIndex
+				}
+			} else {
+				defaultStartIndex := 0
+				config.StartIndex = &defaultStartIndex
+			}
+		}
+
+		if config.EndIndex == nil {
 			logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "error getting log info: %v", err)
@@ -104,20 +131,8 @@ func main() {
 				return
 			}
 
-			var prevCheckpoint *util.SignedCheckpoint
-			prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "reading checkpoint log: %v", err)
-				return
-			}
-
-			checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
-			if config.StartIndex == nil {
-				config.StartIndex = &checkpointStartIndex
-			}
-			if config.EndIndex == nil {
-				config.EndIndex = &checkpointEndIndex
-			}
+			checkpointEndIndex := rekor.GetCheckpointIndex(logInfo, checkpoint)
+			config.EndIndex = &checkpointEndIndex
 		}
 
 		if *config.StartIndex >= *config.EndIndex {

pkg/rekor/identity.go

@@ -334,37 +334,16 @@ func oidMatchesPolicy(cert *x509.Certificate, oid asn1.ObjectIdentifier, extensi
 	return false, nil, "", nil
 }
 
-func GetPrevCurrentCheckpoints(client *client.Rekor, logInfoFile string) (*util.SignedCheckpoint, *util.SignedCheckpoint, error) {
-	logInfo, err := GetLogInfo(context.Background(), client)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error getting log info: %v", err)
-	}
-
-	checkpoint, err := ReadLatestCheckpoint(logInfo)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error reading checkpoint: %v", err)
-	}
-
-	var prevCheckpoint *util.SignedCheckpoint
-	prevCheckpoint, err = file.ReadLatestCheckpoint(logInfoFile)
-	if err != nil {
-		return nil, nil, fmt.Errorf("reading checkpoint log: %v", err)
-	}
-
-	return prevCheckpoint, checkpoint, nil
-}
-
-// GetCheckpointIndices fetches the start and end indexes between two checkpoints and returns them.
-func GetCheckpointIndices(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint) (int, int) {
+// GetCheckpointIndex fetches the index of a checkpoint and returns it.
+func GetCheckpointIndex(logInfo *models.LogInfo, checkpoint *util.SignedCheckpoint) int {
 	// Get log size of inactive shards
 	totalSize := 0
 	for _, s := range logInfo.InactiveShards {
 		totalSize += int(*s.TreeSize)
 	}
-	startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow
-	endIndex := int(checkpoint.Size) + totalSize - 1       //nolint: gosec // G115
+	index := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115
 
-	return startIndex, endIndex
+	return index
 }
 
 func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, monitoredValues identity.MonitoredValues, outputIdentitiesFile string, idMetadataFile *string) ([]identity.MonitoredIdentity, error) {
@@ -404,18 +383,3 @@ func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, mon
 	monitoredIdentities := identity.CreateMonitoredIdentities(idEntries, identities)
 	return monitoredIdentities, nil
 }
-
-// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file.
-func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error {
-	// Get log size of inactive shards
-	startIndex, endIndex := GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
-
-	// Search for identities in the log range
-	if identity.MonitoredValuesExist(monitoredValues) {
-		_, err := IdentitySearch(startIndex, endIndex, rekorClient, monitoredValues, outputIdentitiesFile, nil)
-		if err != nil {
-			return fmt.Errorf("error monitoring for identities: %v", err)
-		}
-	}
-	return nil
-}