Implement certificate transparency monitoring workflow

cmd/ct_monitor/main.go

@@ -0,0 +1,156 @@
+//
+// Copyright 2024 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	ctgo "github.com/google/certificate-transparency-go"
+	ctclient "github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"github.com/sigstore/rekor-monitor/pkg/ct"
+	"github.com/sigstore/rekor-monitor/pkg/identity"
+	"github.com/sigstore/rekor-monitor/pkg/notifications"
+	"gopkg.in/yaml.v2"
+)
+
+// Default values for monitoring job parameters
+const (
+	publicCTServerURL        = "https://ctfe.sigstore.dev/2022"
+	logInfoFileName          = "ctLogInfo.txt"
+	outputIdentitiesFileName = "ctIdentities.txt"
+)
+
+// This main function performs a periodic identity search.
+// Upon starting, any existing latest snapshot data is loaded and the function runs
+// indefinitely to perform identity search for every time interval that was specified.
+func main() {
+	configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings")
+	configYamlInput := flag.String("config", "", "path to yaml configuration file containing identity monitor settings")
+	once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once")
+	logInfoFile := flag.String("file", logInfoFileName, "path to the initial log info checkpoint file to be read from")
+	serverURL := flag.String("url", publicCTServerURL, "URL to the rekor server that is to be monitored")
+	interval := flag.Duration("interval", 5*time.Minute, "Length of interval between each periodical consistency check")
+	flag.Parse()
+
+	var config notifications.IdentityMonitorConfiguration
+
+	if *configFilePath != "" {
+		readConfig, err := os.ReadFile(*configFilePath)
+		if err != nil {
+			log.Fatalf("error reading from identity monitor configuration file: %v", err)
+		}
+
+		configString := string(readConfig)
+		if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
+	}
+
+	if *configYamlInput != "" {
+		if err := yaml.Unmarshal([]byte(*configYamlInput), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
+	}
+
+	var fulcioClient *ctclient.LogClient
+	fulcioClient, err := ctclient.New(*serverURL, http.DefaultClient, jsonclient.Options{})
+	if err != nil {
+		log.Fatalf("getting Fulcio client: %v", err)
+	}
+
+	allOIDMatchers, err := config.MonitoredValues.OIDMatchers.RenderOIDMatchers()
+	if err != nil {
+		fmt.Printf("error parsing OID matchers: %v", err)
+	}
+
+	monitoredValues := identity.MonitoredValues{
+		CertificateIdentities: config.MonitoredValues.CertificateIdentities,
+		OIDMatchers:           allOIDMatchers,
+	}
+
+	for _, certID := range monitoredValues.CertificateIdentities {
+		if len(certID.Issuers) == 0 {
+			fmt.Printf("Monitoring certificate subject %s\n", certID.CertSubject)
+		} else {
+			fmt.Printf("Monitoring certificate subject %s for issuer(s) %s\n", certID.CertSubject, strings.Join(certID.Issuers, ","))
+		}
+	}
+	for _, fp := range monitoredValues.Fingerprints {
+		fmt.Printf("Monitoring fingerprint %s\n", fp)
+	}
+	for _, sub := range monitoredValues.Subjects {
+		fmt.Printf("Monitoring subject %s\n", sub)
+	}
+
+	ticker := time.NewTicker(*interval)
+	defer ticker.Stop()
+
+	// To get an immediate first tick
+	for ; ; <-ticker.C {
+		inputEndIndex := config.EndIndex
+
+		// TODO: Handle Rekor sharding
+		// https://github.com/sigstore/rekor-monitor/issues/57
+		var prevSTH *ctgo.SignedTreeHead
+		prevSTH, currentSTH, err := ct.RunConsistencyCheck(fulcioClient, *logInfoFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to successfully complete consistency check: %v", err)
+			return
+		}
+
+		if config.StartIndex == nil {
+			if prevSTH != nil {
+				checkpointStartIndex := int(prevSTH.TreeSize) //nolint: gosec // G115, log will never be large enough to overflow
+				config.StartIndex = &checkpointStartIndex
+			} else {
+				defaultStartIndex := 0
+				config.StartIndex = &defaultStartIndex
+			}
+		}
+
+		if config.EndIndex == nil {
+			checkpointEndIndex := int(currentSTH.TreeSize) //nolint: gosec // G115
+			config.EndIndex = &checkpointEndIndex
+		}
+
+		if *config.StartIndex >= *config.EndIndex {
+			fmt.Fprintf(os.Stderr, "start index %d must be strictly less than end index %d", *config.StartIndex, *config.EndIndex)
+		}
+
+		if identity.MonitoredValuesExist(monitoredValues) {
+			_, err = ct.IdentitySearch(fulcioClient, *config.StartIndex, *config.EndIndex, monitoredValues)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "failed to successfully complete identity search: %v", err)
+				return
+			}
+		}
+
+		if *once || inputEndIndex != nil {
+			return
+		}
+
+		config.StartIndex = config.EndIndex
+		config.EndIndex = nil
+	}
+
+}

cmd/rekor_monitor/main.go

@@ -123,6 +123,8 @@ func main() {
 	for ; ; <-ticker.C {
 		inputEndIndex := config.EndIndex
 
+		// TODO: Handle Rekor sharding
+		// https://github.com/sigstore/rekor-monitor/issues/57
 		var logInfo *models.LogInfo
 		var prevCheckpoint *util.SignedCheckpoint
 		prevCheckpoint, logInfo, err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile)

pkg/ct/consistency.go

@@ -17,6 +17,7 @@ package ct
 import (
 	"context"
 	"fmt"
+	"os"
 
 	ct "github.com/google/certificate-transparency-go"
 	ctclient "github.com/google/certificate-transparency-go/client"
@@ -33,18 +34,18 @@ AaYalIJmBZ8yyezPjTqhxrKBpMnaocVtLJBI1eM3uXnQzQGAJdJ4gs9Fyw==
 -----END PUBLIC KEY-----`
 )
 
-func verifyCertificateTransparencyConsistency(logInfoFile string, logClient *ctclient.LogClient, signedTreeHead *ct.SignedTreeHead) error {
+func verifyCertificateTransparencyConsistency(logInfoFile string, logClient *ctclient.LogClient, signedTreeHead *ct.SignedTreeHead) (*ct.SignedTreeHead, error) {
 	prevSTH, err := file.ReadLatestCTSignedTreeHead(logInfoFile)
 	if err != nil {
-		return fmt.Errorf("error reading checkpoint: %v", err)
+		return nil, fmt.Errorf("error reading checkpoint: %v", err)
 	}
 
 	if logClient.Verifier == nil {
 		// TODO: this public key is currently hardcoded- should be fetched from TUF repository instead
 		pubKey, err := cryptoutils.UnmarshalPEMToPublicKey([]byte(ctfe2022PubKey))
 
 		if err != nil {
-			return fmt.Errorf("error loading public key: %v", err)
+			return nil, fmt.Errorf("error loading public key: %v", err)
 		}
 		logClient.Verifier = &ct.SignatureVerifier{
 			PubKey: pubKey,
@@ -53,23 +54,49 @@ func verifyCertificateTransparencyConsistency(logInfoFile string, logClient *ctc
 
 	err = logClient.VerifySTHSignature(*prevSTH)
 	if err != nil {
-		return fmt.Errorf("error verifying previous STH signature: %v", err)
+		return nil, fmt.Errorf("error verifying previous STH signature: %v", err)
 	}
 	err = logClient.VerifySTHSignature(*signedTreeHead)
 	if err != nil {
-		return fmt.Errorf("error verifying current STH signature: %v", err)
+		return nil, fmt.Errorf("error verifying current STH signature: %v", err)
 	}
 
 	first := prevSTH.TreeSize
 	second := signedTreeHead.TreeSize
 	pf, err := logClient.GetSTHConsistency(context.Background(), first, second)
 	if err != nil {
-		return fmt.Errorf("error getting consistency proof: %v", err)
+		return nil, fmt.Errorf("error getting consistency proof: %v", err)
 	}
 
 	if err := proof.VerifyConsistency(rfc6962.DefaultHasher, first, second, pf, prevSTH.SHA256RootHash[:], signedTreeHead.SHA256RootHash[:]); err != nil {
-		return fmt.Errorf("error verifying consistency: %v", err)
+		return nil, fmt.Errorf("error verifying consistency: %v", err)
 	}
 
-	return nil
+	return prevSTH, nil
+}
+
+// RunConsistencyCheck periodically verifies the root hash consistency of a certificate transparency log.
+func RunConsistencyCheck(logClient *ctclient.LogClient, logInfoFile string) (*ct.SignedTreeHead, *ct.SignedTreeHead, error) {
+	currentSTH, err := logClient.GetSTH(context.Background())
+	if err != nil {
+		return nil, nil, fmt.Errorf("error fetching latest STH: %v", err)
+	}
+
+	fi, err := os.Stat(logInfoFile)
+	// File containing previous checkpoints exists
+	var prevSTH *ct.SignedTreeHead
+	if err == nil && fi.Size() != 0 {
+		prevSTH, err = verifyCertificateTransparencyConsistency(logInfoFile, logClient, currentSTH)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error verifying consistency between previous and current STHs: %v", err)
+		}
+	}
+
+	if prevSTH == nil || prevSTH.TreeSize != currentSTH.TreeSize {
+		if err := file.WriteCTSignedTreeHead(currentSTH, logInfoFile); err != nil {
+			return nil, nil, fmt.Errorf("failed to write checkpoint: %v", err)
+		}
+	}
+
+	return prevSTH, currentSTH, nil
 }

pkg/ct/consistency_test.go

@@ -74,8 +74,11 @@ func TestVerifyCertificateTransparencyConsistency(t *testing.T) {
 		t.Errorf("error creating log client: %v", err)
 	}
 
-	err = verifyCertificateTransparencyConsistency(tempLogInfoFileName, logClient, sth)
+	prevSTH, err := verifyCertificateTransparencyConsistency(tempLogInfoFileName, logClient, sth)
 	if err == nil {
 		t.Errorf("expected error verifying ct consistency, received nil")
 	}
+	if prevSTH != nil {
+		t.Errorf("expected nil, received %v", prevSTH)
+	}
 }