refactor verifier workflow (#513)

Signed-off-by: linus-sun <[email protected]>
Signed-off-by: Linus Sun <[email protected]>

.github/workflows/reusable_monitoring.yml

@@ -76,7 +76,7 @@ jobs:
         run: cat ${{ env.LOG_FILE }}
         # Skip on first run
         continue-on-error: true
-      - run: go run ./cmd/verifier --file ${{ env.LOG_FILE }} --once --monitored-values "${{ inputs.identities }}" --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}"
+      - run: go run ./cmd/rekor_consistency --file ${{ env.LOG_FILE }} --once --monitored-values "${{ inputs.identities }}" --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}"
       - name: Upload checkpoint
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:

cmd/verifier/main.go → cmd/rekor_consistency/main.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"os"
 	"runtime"
 	"strings"
 	"time"
@@ -59,6 +60,7 @@ func main() {
 	if err := yaml.Unmarshal([]byte(*monitoredValsInput), &monitoredVals); err != nil {
 		log.Fatalf("error parsing identities: %v", err)
 	}
+
 	for _, certID := range monitoredVals.CertificateIdentities {
 		if len(certID.Issuers) == 0 {
 			fmt.Printf("Monitoring certificate subject %s\n", certID.CertSubject)
@@ -88,8 +90,24 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = rekor.RunConsistencyCheck(*interval, rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile, *once)
-	if err != nil {
-		log.Fatalf("%v", err)
+	ticker := time.NewTicker(*interval)
+	defer ticker.Stop()
+
+	// Loop will:
+	// 1. Fetch latest checkpoint and verify
+	// 2. If old checkpoint is present, verify consistency proof
+	// 3. Write latest checkpoint to file
+
+	// To get an immediate first tick
+	for ; ; <-ticker.C {
+		err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error running consistency check: %v", err)
+			return
+		}
+
+		if *once {
+			return
+		}
 	}
 }

pkg/rekor/verifier.go

@@ -111,62 +111,49 @@ func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string,
 }
 
 // RunConsistencyCheck periodically verifies the root hash consistency of a Rekor log.
-func RunConsistencyCheck(interval time.Duration, rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string, mvs identity.MonitoredValues, outputIdentitiesFile string, once bool) error {
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-
-	// Loop will:
-	// 1. Fetch latest checkpoint and verify
-	// 2. If old checkpoint is present, verify consistency proof
-	// 3. Write latest checkpoint to file
+func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string, mvs identity.MonitoredValues, outputIdentitiesFile string) error {
+	logInfo, err := GetLogInfo(context.Background(), rekorClient)
+	if err != nil {
+		return fmt.Errorf("failed to get log info: %v", err)
+	}
+	checkpoint, err := verifyLatestCheckpointSignature(logInfo, verifier)
+	if err != nil {
+		return fmt.Errorf("failed to verify signature of latest checkpoint: %v", err)
+	}
 
-	// To get an immediate first tick
-	for ; ; <-ticker.C {
-		logInfo, err := GetLogInfo(context.Background(), rekorClient)
-		if err != nil {
-			return fmt.Errorf("failed to get log info: %v", err)
-		}
-		checkpoint, err := verifyLatestCheckpointSignature(logInfo, verifier)
+	fi, err := os.Stat(logInfoFile)
+	// File containing previous checkpoints exists
+	var prevCheckpoint *util.SignedCheckpoint
+	if err == nil && fi.Size() != 0 {
+		prevCheckpoint, err = verifyCheckpointConsistency(logInfoFile, checkpoint, *logInfo.TreeID, rekorClient, verifier)
 		if err != nil {
-			return fmt.Errorf("failed to verify signature of latest checkpoint: %v", err)
+			return fmt.Errorf("failed to verify previous checkpoint: %v", err)
 		}
 
-		fi, err := os.Stat(logInfoFile)
-		// File containing previous checkpoints exists
-		var prevCheckpoint *util.SignedCheckpoint
-		if err == nil && fi.Size() != 0 {
-			prevCheckpoint, err = verifyCheckpointConsistency(logInfoFile, checkpoint, *logInfo.TreeID, rekorClient, verifier)
-			if err != nil {
-				return fmt.Errorf("failed to verify previous checkpoint: %v", err)
-			}
-
-		}
-
-		// Write if there was no stored checkpoint or the sizes differ
-		if prevCheckpoint == nil || prevCheckpoint.Size != checkpoint.Size {
-			if err := file.WriteCheckpoint(checkpoint, logInfoFile); err != nil {
-				// TODO: Once the consistency check and identity search are split into separate tasks, this should hard fail.
-				// Temporarily skipping this to allow this job to succeed, remediating the issue noted here: https://github.com/sigstore/rekor-monitor/issues/271
-				fmt.Fprintf(os.Stderr, "failed to write checkpoint: %v", err)
-			}
-		}
+	}
 
-		if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size {
-			err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, mvs, rekorClient, outputIdentitiesFile)
-			if err != nil {
-				return fmt.Errorf("failed to monitor identities: %v", err)
-			}
+	// Write if there was no stored checkpoint or the sizes differ
+	if prevCheckpoint == nil || prevCheckpoint.Size != checkpoint.Size {
+		if err := file.WriteCheckpoint(checkpoint, logInfoFile); err != nil {
+			// TODO: Once the consistency check and identity search are split into separate tasks, this should hard fail.
+			// Temporarily skipping this to allow this job to succeed, remediating the issue noted here: https://github.com/sigstore/rekor-monitor/issues/271
+			fmt.Fprintf(os.Stderr, "failed to write checkpoint: %v", err)
 		}
+	}
 
-		// TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need
-		// to persist the last checkpoint.
-		// Delete old checkpoints to avoid the log growing indefinitely
-		if err := file.DeleteOldCheckpoints(logInfoFile); err != nil {
-			return fmt.Errorf("failed to delete old checkpoints: %v", err)
+	if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size {
+		err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, mvs, rekorClient, outputIdentitiesFile)
+		if err != nil {
+			return fmt.Errorf("failed to monitor identities: %v", err)
 		}
+	}
 
-		if once {
-			return nil
-		}
+	// TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need
+	// to persist the last checkpoint.
+	// Delete old checkpoints to avoid the log growing indefinitely
+	if err := file.DeleteOldCheckpoints(logInfoFile); err != nil {
+		return fmt.Errorf("failed to delete old checkpoints: %v", err)
 	}
+
+	return nil
 }

pkg/test/e2e/e2e_test.go

@@ -31,7 +31,6 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/sigstore/rekor-monitor/pkg/fulcio/extensions"
 	"github.com/sigstore/rekor-monitor/pkg/identity"
@@ -148,8 +147,6 @@ func TestRunConsistencyCheck(t *testing.T) {
 	tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name()
 	defer os.Remove(tempOutputIdentitiesFileName)
 
-	interval := time.Minute
-
 	monitoredVals := identity.MonitoredValues{
 		Subjects: []string{subject},
 		CertificateIdentities: []identity.CertificateIdentity{
@@ -168,9 +165,8 @@ func TestRunConsistencyCheck(t *testing.T) {
 			certFingerprint,
 		},
 	}
-	once := true
 
-	err = rekor.RunConsistencyCheck(interval, rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName, once)
+	err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName)
 	if err != nil {
 		t.Errorf("first consistency check failed: %v", err)
 	}
@@ -210,7 +206,7 @@ func TestRunConsistencyCheck(t *testing.T) {
 		t.Errorf("expected checkpoint size of 2, received size %d", checkpoint.Size)
 	}
 
-	err = rekor.RunConsistencyCheck(interval, rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName, once)
+	err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName)
 	if err != nil {
 		t.Errorf("second consistency check failed: %v", err)
 	}