From d035f531258f785775fc35d9dab6a9ed53418ef1 Mon Sep 17 00:00:00 2001 From: linus-sun Date: Tue, 5 Nov 2024 23:43:33 +0000 Subject: [PATCH] refactor workflows to split Signed-off-by: linus-sun --- ...e_monitoring.yml => consistency_check.yml} | 0 .github/workflows/identity_monitor.yml | 61 ++++++ .github/workflows/main.yml | 4 +- README.md | 51 +++-- cmd/rekor_consistency/main.go | 6 +- cmd/{monitor => rekor_monitor}/main.go | 77 ++++--- pkg/rekor/identity.go | 44 +--- pkg/rekor/identity_test.go | 197 ++---------------- pkg/rekor/verifier.go | 15 +- pkg/rekor/verifier_test.go | 65 +++--- .../consistency_check_e2e_test.go} | 77 +------ .../consistency_check_e2e_test.sh} | 2 +- .../identity_workflow_e2e_test.go | 27 ++- 13 files changed, 210 insertions(+), 416 deletions(-) rename .github/workflows/{reusable_monitoring.yml => consistency_check.yml} (100%) create mode 100644 .github/workflows/identity_monitor.yml rename cmd/{monitor => rekor_monitor}/main.go (73%) rename pkg/test/{e2e/e2e_test.go => consistency_check/consistency_check_e2e_test.go} (67%) rename pkg/test/{e2e/e2e_test.sh => consistency_check/consistency_check_e2e_test.sh} (96%) diff --git a/.github/workflows/reusable_monitoring.yml b/.github/workflows/consistency_check.yml similarity index 100% rename from .github/workflows/reusable_monitoring.yml rename to .github/workflows/consistency_check.yml diff --git a/.github/workflows/identity_monitor.yml b/.github/workflows/identity_monitor.yml new file mode 100644 index 00000000..b8690a4b --- /dev/null +++ b/.github/workflows/identity_monitor.yml @@ -0,0 +1,61 @@ +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: This GHA should not be run concurrently. +# TODO: Once the workflows for consistency check and identity monitor are fully split, +# the consistency check workflow should no longer require a checkpoint file. + +name: Rekor Identity Monitor Template + +on: + workflow_call: + inputs: + once: + default: true + required: false + type: boolean + config: + description: 'multiline yaml of configuration settings for identity monitor run' + required: true + type: string + +permissions: + contents: read + +jobs: + detect-workflow: + runs-on: ubuntu-latest + permissions: + id-token: write # Needed to detect the current reusable repository and ref. + outputs: + repository: ${{ steps.detect.outputs.repository }} + ref: ${{ steps.detect.outputs.ref }} + steps: + - name: Detect the repository and ref + id: detect + uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@5a775b367a56d5bd118a224a811bba288150a563 # v2.0.0 + + monitor: + runs-on: ubuntu-latest + needs: [detect-workflow] + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + repository: ${{ needs.detect-workflow.outputs.repository }} + ref: "${{ needs.detect-workflow.outputs.ref }}" + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: '1.23' + - name: Run identity monitor + run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --once \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21c25129..4d66c7f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,8 +88,8 @@ jobs: with: go-version-file: './go.mod' check-latest: true - - name: run e2e test - run: ./pkg/test/e2e/e2e_test.sh + - name: run consistency check test + run: ./pkg/test/consistency_check/consistency_check_e2e_test.sh - name: run identity monitor test run: ./pkg/test/identity_workflow/identity_workflow_e2e_test.sh diff --git a/README.md b/README.md index de0e8ec3..f70cd8cc 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,13 @@ Rekor Log Monitor provides an easy-to-use monitor to verify log consistency, that the log is immutability and append-only. Monitoring is critical to the transparency log ecosystem, as logs are tamper-evident but not tamper-proof. +Rekor Log Monitor also provides a monitor to search for identities within a log, +and send a list of found identities via various notification platforms. + +## Consistency check To run, create a GitHub Actions workflow that uses the -[reusable monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/reusable_monitoring.yml). +[consistency check workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/consistency_check.yml). It is recommended to run the log monitor every hour for optimal performance. Example workflow: @@ -46,6 +50,10 @@ Please read [this](https://github.com/google/re2/wiki/Syntax) for syntax referen Note: The log monitor only starts monitoring from the latest checkpoint. If you want to search previous entries, you will need to query the log. +To run, create a GitHub Actions workflow that uses the +[identity monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/identity_monitor.yml). +It is recommended to run the log monitor every hour for optimal performance. + Example workflow below: ``` @@ -66,26 +74,27 @@ jobs: with: file_issue: true # Strongly recommended: Files an issue on monitoring failure artifact_retention_days: 14 # Optional, default is 14: Must be longer than the cron job frequency - identities: | - certIdentities: - - certSubject: user@domain\.com - - certSubject: otheruser@domain\.com - issuers: - - https://accounts\.google\.com - - https://github\.com/login - - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.* - issuers: - - https://token\.actions\.githubusercontent\.com - subjects: - - subject@domain\.com - fingerprints: - - A0B1C2D3E4F5 - fulcioExtensions: - build-config-uri: - - https://example.com/owner/repository/build-config.yml - customExtensions: - - objectIdentifier: 1.3.6.1.4.1.57264.1.9 - extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0 + config: | + monitoredValues: | + certIdentities: + - certSubject: user@domain\.com + - certSubject: otheruser@domain\.com + issuers: + - https://accounts\.google\.com + - https://github\.com/login + - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.* + issuers: + - https://token\.actions\.githubusercontent\.com + subjects: + - subject@domain\.com + fingerprints: + - A0B1C2D3E4F5 + fulcioExtensions: + build-config-uri: + - https://example.com/owner/repository/build-config.yml + customExtensions: + - objectIdentifier: 1.3.6.1.4.1.57264.1.9 + extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0 ``` In this example, the monitor will log: diff --git a/cmd/rekor_consistency/main.go b/cmd/rekor_consistency/main.go index 073e45d5..679b6dd6 100644 --- a/cmd/rekor_consistency/main.go +++ b/cmd/rekor_consistency/main.go @@ -51,8 +51,6 @@ func main() { once := flag.Bool("once", false, "Perform consistency check once and exit") monitoredValsInput := flag.String("monitored-values", "", "yaml of certificate subjects and issuers, key subjects, "+ "and fingerprints. For certificates, if no issuers are specified, match any OIDC provider.") - outputIdentitiesFile := flag.String("output-identities", outputIdentitiesFileName, - "Name of the file containing indices and identities found in the log. Format is \"subject issuer index uuid\"") userAgentString := flag.String("user-agent", "", "details to include in the user agent string") flag.Parse() @@ -85,7 +83,7 @@ func main() { log.Fatal(err) } - err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once) + err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, once) if err != nil { log.Fatal(err) } @@ -100,7 +98,7 @@ func main() { // To get an immediate first tick for ; ; <-ticker.C { - err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile) + err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile) if err != nil { fmt.Fprintf(os.Stderr, "error running consistency check: %v", err) return diff --git a/cmd/monitor/main.go b/cmd/rekor_monitor/main.go similarity index 73% rename from cmd/monitor/main.go rename to cmd/rekor_monitor/main.go index a2c5bb19..29428067 100644 --- a/cmd/monitor/main.go +++ b/cmd/rekor_monitor/main.go @@ -30,7 +30,6 @@ import ( "github.com/sigstore/rekor-monitor/pkg/rekor" "github.com/sigstore/rekor-monitor/pkg/util/file" "github.com/sigstore/rekor/pkg/client" - "github.com/sigstore/rekor/pkg/util" "gopkg.in/yaml.v2" "sigs.k8s.io/release-utils/version" ) @@ -38,7 +37,6 @@ import ( // Default values for monitoring job parameters const ( publicRekorServerURL = "https://rekor.sigstore.dev" - logInfoFileName = "logInfo.txt" outputIdentitiesFileName = "identities.txt" ) @@ -48,35 +46,41 @@ const ( func main() { // Command-line flags that are parameters to the verifier job configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings") + configYamlInput := flag.String("config", "", "path to yaml configuration file containing identity monitor settings") once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once") flag.Parse() - if configFilePath == nil { - log.Fatalf("empty configuration file path") + if *configFilePath == "" && *configYamlInput == "" { + log.Fatalf("empty configuration input") } - readConfig, err := os.ReadFile(*configFilePath) - if err != nil { - log.Fatalf("error reading from identity monitor configuration file: %v", err) + if *configFilePath != "" && *configYamlInput != "" { + log.Fatalf("only input one of configuration file path or yaml input") } - configString := string(readConfig) var config notifications.IdentityMonitorConfiguration - if err := yaml.Unmarshal([]byte(configString), &config); err != nil { - log.Fatalf("error parsing identities: %v", err) + + if *configFilePath != "" { + readConfig, err := os.ReadFile(*configFilePath) + if err != nil { + log.Fatalf("error reading from identity monitor configuration file: %v", err) + } + + configString := string(readConfig) + if err := yaml.Unmarshal([]byte(configString), &config); err != nil { + log.Fatalf("error parsing identities: %v", err) + } } - rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) - if err != nil { - log.Fatalf("getting Rekor client: %v", err) + if *configYamlInput != "" { + if err := yaml.Unmarshal([]byte(*configYamlInput), &config); err != nil { + log.Fatalf("error parsing identities: %v", err) + } } if config.ServerURL == "" { config.ServerURL = publicRekorServerURL } - if config.LogInfoFile == "" { - config.LogInfoFile = logInfoFileName - } if config.OutputIdentitiesFile == "" { config.OutputIdentitiesFile = outputIdentitiesFileName } @@ -85,13 +89,36 @@ func main() { config.Interval = &defaultInterval } + rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) + if err != nil { + log.Fatalf("getting Rekor client: %v", err) + } + ticker := time.NewTicker(*config.Interval) defer ticker.Stop() // To get an immediate first tick for ; ; <-ticker.C { inputEndIndex := config.EndIndex - if config.StartIndex == nil || config.EndIndex == nil { + + if config.StartIndex == nil { + if config.IdentityMetadataFile != nil { + idMetadata, err := file.ReadIdentityMetadata(*config.IdentityMetadataFile) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading from identity metadata file: %v", err) + return + } + + if config.StartIndex == nil { + config.StartIndex = &idMetadata.LatestIndex + } + } else { + defaultStartIndex := 0 + config.StartIndex = &defaultStartIndex + } + } + + if config.EndIndex == nil { logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient) if err != nil { fmt.Fprintf(os.Stderr, "error getting log info: %v", err) @@ -104,20 +131,8 @@ func main() { return } - var prevCheckpoint *util.SignedCheckpoint - prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile) - if err != nil { - fmt.Fprintf(os.Stderr, "reading checkpoint log: %v", err) - return - } - - checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint) - if config.StartIndex == nil { - config.StartIndex = &checkpointStartIndex - } - if config.EndIndex == nil { - config.EndIndex = &checkpointEndIndex - } + checkpointEndIndex := rekor.GetCheckpointIndex(logInfo, checkpoint) + config.EndIndex = &checkpointEndIndex } if *config.StartIndex >= *config.EndIndex { diff --git a/pkg/rekor/identity.go b/pkg/rekor/identity.go index fdad6d57..7f081643 100644 --- a/pkg/rekor/identity.go +++ b/pkg/rekor/identity.go @@ -334,37 +334,16 @@ func oidMatchesPolicy(cert *x509.Certificate, oid asn1.ObjectIdentifier, extensi return false, nil, "", nil } -func GetPrevCurrentCheckpoints(client *client.Rekor, logInfoFile string) (*util.SignedCheckpoint, *util.SignedCheckpoint, error) { - logInfo, err := GetLogInfo(context.Background(), client) - if err != nil { - return nil, nil, fmt.Errorf("error getting log info: %v", err) - } - - checkpoint, err := ReadLatestCheckpoint(logInfo) - if err != nil { - return nil, nil, fmt.Errorf("error reading checkpoint: %v", err) - } - - var prevCheckpoint *util.SignedCheckpoint - prevCheckpoint, err = file.ReadLatestCheckpoint(logInfoFile) - if err != nil { - return nil, nil, fmt.Errorf("reading checkpoint log: %v", err) - } - - return prevCheckpoint, checkpoint, nil -} - -// GetCheckpointIndices fetches the start and end indexes between two checkpoints and returns them. -func GetCheckpointIndices(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint) (int, int) { +// GetCheckpointIndex fetches the index of a checkpoint and returns it. +func GetCheckpointIndex(logInfo *models.LogInfo, checkpoint *util.SignedCheckpoint) int { // Get log size of inactive shards totalSize := 0 for _, s := range logInfo.InactiveShards { totalSize += int(*s.TreeSize) } - startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow - endIndex := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 + index := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 - return startIndex, endIndex + return index } func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, monitoredValues identity.MonitoredValues, outputIdentitiesFile string, idMetadataFile *string) ([]identity.MonitoredIdentity, error) { @@ -404,18 +383,3 @@ func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, mon monitoredIdentities := identity.CreateMonitoredIdentities(idEntries, identities) return monitoredIdentities, nil } - -// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file. -func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error { - // Get log size of inactive shards - startIndex, endIndex := GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint) - - // Search for identities in the log range - if identity.MonitoredValuesExist(monitoredValues) { - _, err := IdentitySearch(startIndex, endIndex, rekorClient, monitoredValues, outputIdentitiesFile, nil) - if err != nil { - return fmt.Errorf("error monitoring for identities: %v", err) - } - } - return nil -} diff --git a/pkg/rekor/identity_test.go b/pkg/rekor/identity_test.go index d63a07e4..bae8c606 100644 --- a/pkg/rekor/identity_test.go +++ b/pkg/rekor/identity_test.go @@ -27,8 +27,6 @@ import ( "encoding/asn1" "encoding/base64" "encoding/hex" - "fmt" - "os" "strings" "testing" "time" @@ -36,10 +34,7 @@ import ( "github.com/go-openapi/swag" "github.com/sigstore/rekor-monitor/pkg/fulcio/extensions" "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/rekor/mock" "github.com/sigstore/rekor-monitor/pkg/test" - "github.com/sigstore/rekor-monitor/pkg/util/file" - "github.com/sigstore/rekor/pkg/generated/client" "github.com/sigstore/rekor/pkg/generated/models" "github.com/sigstore/rekor/pkg/types" hashedrekord_v001 "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1" @@ -47,7 +42,6 @@ import ( "github.com/sigstore/rekor/pkg/util" "github.com/sigstore/sigstore/pkg/cryptoutils" "github.com/sigstore/sigstore/pkg/signature" - "golang.org/x/mod/sumdb/note" ) const ( @@ -866,7 +860,7 @@ func TestMatchedIndicesFailures(t *testing.T) { } } -func TestGetCheckpointIndices(t *testing.T) { +func TestGetCheckpointIndex(t *testing.T) { shardTreeSize := int64(1) inactiveShard := models.InactiveShardLogInfo{ TreeSize: &shardTreeSize, @@ -879,99 +873,36 @@ func TestGetCheckpointIndices(t *testing.T) { emptyLogInfo := &models.LogInfo{ InactiveShards: emptyInactiveShards, } - prevCheckpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 1, - }, - } checkpoint := &util.SignedCheckpoint{ Checkpoint: util.Checkpoint{ Size: 2, }, } getCheckpointIndicesTests := map[string]struct { - inputLogInfo *models.LogInfo - inputPrevCheckpoint *util.SignedCheckpoint - inputCheckpoint *util.SignedCheckpoint - expectedStartIndex int - expectedEndIndex int + inputLogInfo *models.LogInfo + inputCheckpoint *util.SignedCheckpoint + expectedEndIndex int }{ "populated inactive shards": { - inputLogInfo: logInfo, - inputPrevCheckpoint: prevCheckpoint, - inputCheckpoint: checkpoint, - expectedStartIndex: 1, - expectedEndIndex: 2, + inputLogInfo: logInfo, + inputCheckpoint: checkpoint, + expectedEndIndex: 2, }, "empty inactive shards": { - inputLogInfo: emptyLogInfo, - inputPrevCheckpoint: prevCheckpoint, - inputCheckpoint: checkpoint, - expectedStartIndex: 0, - expectedEndIndex: 1, + inputLogInfo: emptyLogInfo, + inputCheckpoint: checkpoint, + expectedEndIndex: 1, }, } for testCaseName, testCase := range getCheckpointIndicesTests { - expectedStartIndex := testCase.expectedStartIndex expectedEndIndex := testCase.expectedEndIndex - resultStartIndex, resultEndIndex := GetCheckpointIndices(testCase.inputLogInfo, testCase.inputPrevCheckpoint, testCase.inputCheckpoint) - if resultStartIndex != expectedStartIndex || resultEndIndex != expectedEndIndex { - t.Errorf("%s failed: expected %d, %d indices, received %d, %d indices", testCaseName, expectedStartIndex, expectedEndIndex, resultStartIndex, resultEndIndex) + resultEndIndex := GetCheckpointIndex(testCase.inputLogInfo, testCase.inputCheckpoint) + if resultEndIndex != expectedEndIndex { + t.Errorf("%s failed: expected %d index, received %d", testCaseName, expectedEndIndex, resultEndIndex) } } } -func TestGetPrevCurrentCheckpoints(t *testing.T) { - // generate checkpoint - root, _ := hex.DecodeString("1a341bc342ff4e567387de9789ab14000b147124317841489172419874198147") - expectedCheckpoint, err := util.CreateSignedCheckpoint(util.Checkpoint{ - Origin: "origin", - Size: uint64(123), - Hash: root, - }) - expectedCheckpoint.Signatures = []note.Signature{{Name: "name", Hash: 1, Base64: "adbadbadb"}} - if err != nil { - t.Fatal(err) - } - checkpointBytes, err := expectedCheckpoint.MarshalText() - if err != nil { - t.Fatal(err) - } - if err != nil { - t.Errorf("error marshalling checkpoint: %v", err) - } - checkpointString := string(checkpointBytes) - - var mClient client.Rekor - mClient.Tlog = &mock.TlogClient{ - LogInfo: &models.LogInfo{ - SignedTreeHead: &checkpointString, - }, - } - - tempDir := t.TempDir() - tempLogInfoFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp log file: %v", err) - } - tempLogInfoFileName := tempLogInfoFile.Name() - defer os.Remove(tempLogInfoFileName) - file.WriteCheckpoint(expectedCheckpoint, tempLogInfoFileName) - - prevCheckpoint, currentCheckpoint, err := GetPrevCurrentCheckpoints(&mClient, tempLogInfoFileName) - if err != nil { - t.Errorf("expected nil, received %v", err) - } - - if prevCheckpoint.Checkpoint.String() != expectedCheckpoint.Checkpoint.String() { - t.Errorf("expected prev checkpoint %s, received %s", expectedCheckpoint.Checkpoint, prevCheckpoint.Checkpoint) - } - - if currentCheckpoint.Checkpoint.String() != expectedCheckpoint.Checkpoint.String() { - t.Errorf("expected current checkpoint %s, received %s", expectedCheckpoint.Checkpoint, currentCheckpoint.Checkpoint) - } -} - func mockCertificateWithExtension(oid asn1.ObjectIdentifier, value string) (*x509.Certificate, error) { extValue, err := asn1.Marshal(value) if err != nil { @@ -1037,105 +968,3 @@ func TestOIDMatchesValue(t *testing.T) { t.Errorf("Expected string to equal 'test cert value', got %s", extValue) } } - -// Test monitoring for identities using mock Rekor client -func TestWriteIdentitiesBetweenCheckpoints(t *testing.T) { - maxIndex := 2 - var logEntries []*models.LogEntry - - rootCert, rootKey, _ := test.GenerateRootCA() - leafCert, leafKey, _ := test.GenerateLeafCert(subject, issuer, rootCert, rootKey) - - signer, err := signature.LoadECDSASignerVerifier(leafKey, crypto.SHA256) - if err != nil { - t.Fatal(err) - } - pemCert, _ := cryptoutils.MarshalCertificateToPEM(leafCert) - - payload := []byte{1, 2, 3, 4} - sig, err := signer.SignMessage(bytes.NewReader(payload)) - if err != nil { - t.Fatal(err) - } - - hashedrekord := &hashedrekord_v001.V001Entry{} - hash := sha256.Sum256(payload) - pe, err := hashedrekord.CreateFromArtifactProperties(context.Background(), types.ArtifactProperties{ - ArtifactHash: hex.EncodeToString(hash[:]), - SignatureBytes: sig, - PublicKeyBytes: [][]byte{pemCert}, - PKIFormat: "x509", - }) - if err != nil { - t.Fatal(err) - } - entry, err := types.UnmarshalEntry(pe) - if err != nil { - t.Fatal(err) - } - leaf, err := entry.Canonicalize(context.Background()) - if err != nil { - t.Fatal(err) - } - - for i := 0; i <= maxIndex; i++ { - lea := models.LogEntryAnon{ - Body: base64.StdEncoding.EncodeToString(leaf), - LogIndex: swag.Int64(int64(i)), - } - data := models.LogEntry{ - fmt.Sprint(i): lea, - } - logEntries = append(logEntries, &data) - } - - var mClient client.Rekor - mClient.Entries = &mock.EntriesClient{ - Entries: logEntries, - } - testLogInfo := &models.LogInfo{} - mClient.Tlog = &mock.TlogClient{ - LogInfo: testLogInfo, - } - - logInfo, err := GetLogInfo(context.Background(), &mClient) - if err != nil { - t.Errorf("failed fetching log info: %v", err) - } - prevCheckpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 0, - }, - } - checkpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 1, - }, - } - monitoredValues := identity.MonitoredValues{ - Subjects: []string{ - subject, - }, - } - tempDir := t.TempDir() - tempOutputIdentitiesFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp output identities file: %v", err) - } - tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name() - defer os.Remove(tempOutputIdentitiesFileName) - - err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, monitoredValues, &mClient, tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("failed write identities between checkpoints: %v", err) - } - - tempOutputIdentities, err := os.ReadFile(tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("error reading from output identities file: %v", err) - } - tempOutputIdentitiesString := string(tempOutputIdentities) - if !strings.Contains(tempOutputIdentitiesString, subject) { - t.Errorf("expected to find subject %s, did not", subject) - } -} diff --git a/pkg/rekor/verifier.go b/pkg/rekor/verifier.go index fbb54114..7f69e79f 100644 --- a/pkg/rekor/verifier.go +++ b/pkg/rekor/verifier.go @@ -23,7 +23,6 @@ import ( "os" "time" - "github.com/sigstore/rekor-monitor/pkg/identity" "github.com/sigstore/rekor-monitor/pkg/util/file" "github.com/sigstore/rekor/pkg/generated/client" "github.com/sigstore/rekor/pkg/generated/models" @@ -94,16 +93,13 @@ func verifyCheckpointConsistency(logInfoFile string, checkpoint *util.SignedChec } // VerifyConsistencyCheckInputs verifies that the input flag values to the rekor-monitor workflow are not nil. -func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, outputIdentitiesFile *string, once *bool) error { +func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, once *bool) error { if interval == nil { return errors.New("--interval flag equal to nil") } if logInfoFile == nil { return errors.New("--file flag equal to nil") } - if outputIdentitiesFile == nil { - return errors.New("--output-identities flag equal to nil") - } if once == nil { return errors.New("--once flag equal to nil") } @@ -111,7 +107,7 @@ func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, } // RunConsistencyCheck periodically verifies the root hash consistency of a Rekor log. -func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string, mvs identity.MonitoredValues, outputIdentitiesFile string) error { +func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string) error { logInfo, err := GetLogInfo(context.Background(), rekorClient) if err != nil { return fmt.Errorf("failed to get log info: %v", err) @@ -141,13 +137,6 @@ func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, } } - if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size { - err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, mvs, rekorClient, outputIdentitiesFile) - if err != nil { - return fmt.Errorf("failed to monitor identities: %v", err) - } - } - // TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need // to persist the last checkpoint. // Delete old checkpoints to avoid the log growing indefinitely diff --git a/pkg/rekor/verifier_test.go b/pkg/rekor/verifier_test.go index 989c1121..6c0e0edd 100644 --- a/pkg/rekor/verifier_test.go +++ b/pkg/rekor/verifier_test.go @@ -53,66 +53,51 @@ func TestGetLogVerifier(t *testing.T) { func TestVerifyConsistencyCheckInputs(t *testing.T) { interval := 5 * time.Minute logInfoFile := "./test/example_log_info_file_path.txt" - outputIdentitiesFile := "./test/example_output_identities_file.txt" once := true verifyConsistencyCheckInputTests := map[string]struct { - interval *time.Duration - logInfoFile *string - outputIdentitiesFile *string - once *bool - expectedError error + interval *time.Duration + logInfoFile *string + once *bool + expectedError error }{ "successful verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: nil, + interval: &interval, + logInfoFile: &logInfoFile, + once: &once, + expectedError: nil, }, "fail --interval verification": { - interval: nil, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: errors.New("--interval flag equal to nil"), + interval: nil, + logInfoFile: &logInfoFile, + once: &once, + expectedError: errors.New("--interval flag equal to nil"), }, "fail --file verification": { - interval: &interval, - logInfoFile: nil, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: errors.New("--file flag equal to nil"), - }, - "fail --output-identities verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: nil, - once: &once, - expectedError: errors.New("--output-identities flag equal to nil"), + interval: &interval, + logInfoFile: nil, + once: &once, + expectedError: errors.New("--file flag equal to nil"), }, "fail --once verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: nil, - expectedError: errors.New("--once flag equal to nil"), + interval: &interval, + logInfoFile: &logInfoFile, + once: nil, + expectedError: errors.New("--once flag equal to nil"), }, "empty case": { - interval: nil, - logInfoFile: nil, - outputIdentitiesFile: nil, - once: nil, - expectedError: errors.New("--interval flag equal to nil"), + interval: nil, + logInfoFile: nil, + once: nil, + expectedError: errors.New("--interval flag equal to nil"), }, } for verifyConsistencyCheckInputTestCaseName, verifyConsistencyCheckInputTestCase := range verifyConsistencyCheckInputTests { interval := verifyConsistencyCheckInputTestCase.interval logInfoFile := verifyConsistencyCheckInputTestCase.logInfoFile - outputIdentitiesFile := verifyConsistencyCheckInputTestCase.outputIdentitiesFile once := verifyConsistencyCheckInputTestCase.once expectedError := verifyConsistencyCheckInputTestCase.expectedError - err := VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once) + err := VerifyConsistencyCheckInputs(interval, logInfoFile, once) if (err == nil && expectedError != nil) || (err != nil && expectedError != nil && err.Error() != expectedError.Error()) { t.Errorf("%s: expected error %v, received error %v", verifyConsistencyCheckInputTestCaseName, expectedError, err) } diff --git a/pkg/test/e2e/e2e_test.go b/pkg/test/consistency_check/consistency_check_e2e_test.go similarity index 67% rename from pkg/test/e2e/e2e_test.go rename to pkg/test/consistency_check/consistency_check_e2e_test.go index aee29ca3..ff8b5456 100755 --- a/pkg/test/e2e/e2e_test.go +++ b/pkg/test/consistency_check/consistency_check_e2e_test.go @@ -32,9 +32,6 @@ import ( "strings" "testing" - "github.com/sigstore/rekor-monitor/pkg/fulcio/extensions" - "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/notifications" "github.com/sigstore/rekor-monitor/pkg/rekor" "github.com/sigstore/rekor-monitor/pkg/test" "github.com/sigstore/rekor/pkg/client" @@ -107,13 +104,6 @@ func TestRunConsistencyCheck(t *testing.T) { t.Fatalf("error creating hashed rekord entry: %v", err) } - x509Cert, err := cryptoutils.UnmarshalCertificatesFromPEM(pemCert) - if err != nil { - t.Fatal(err) - } - digest := sha256.Sum256(x509Cert[0].Raw) - certFingerprint := hex.EncodeToString(digest[:]) - params := entries.NewCreateLogEntryParams() params.SetProposedEntry(pe) resp, err := rekorClient.Entries.CreateLogEntry(params) @@ -141,49 +131,7 @@ func TestRunConsistencyCheck(t *testing.T) { tempLogInfoFileName := tempLogInfoFile.Name() defer os.Remove(tempLogInfoFileName) - tempOutputIdentitiesFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp output identities file: %v", err) - } - tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name() - defer os.Remove(tempOutputIdentitiesFileName) - - configMonitoredValues := notifications.ConfigMonitoredValues{ - Subjects: []string{subject}, - CertificateIdentities: []identity.CertificateIdentity{ - { - CertSubject: ".*ubje.*", - Issuers: []string{".+@domain.com"}, - }, - }, - OIDMatchers: extensions.OIDMatchers{ - OIDExtensions: []extensions.OIDExtension{ - { - ObjectIdentifier: oid, - ExtensionValues: []string{extValueString}, - }, - }, - FulcioExtensions: extensions.FulcioExtensions{}, - CustomExtensions: []extensions.CustomExtension{}, - }, - Fingerprints: []string{ - certFingerprint, - }, - } - - configRenderedOIDMatchers, err := configMonitoredValues.OIDMatchers.RenderOIDMatchers() - if err != nil { - t.Errorf("error rendering OID matchers: %v", err) - } - - monitoredVals := identity.MonitoredValues{ - Fingerprints: configMonitoredValues.Fingerprints, - Subjects: configMonitoredValues.Subjects, - OIDMatchers: configRenderedOIDMatchers, - CertificateIdentities: configMonitoredValues.CertificateIdentities, - } - - err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName) + err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName) if err != nil { t.Errorf("first consistency check failed: %v", err) } @@ -223,29 +171,8 @@ func TestRunConsistencyCheck(t *testing.T) { t.Errorf("expected checkpoint size of 2, received size %d", checkpoint.Size) } - err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName) + err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName) if err != nil { t.Errorf("second consistency check failed: %v", err) } - - tempOutputIdentities, err := os.ReadFile(tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("error reading from output identities file: %v", err) - } - tempOutputIdentitiesString := string(tempOutputIdentities) - if !strings.Contains(tempOutputIdentitiesString, subject) { - t.Errorf("expected to find subject %s, did not", subject) - } - if !strings.Contains(tempOutputIdentitiesString, issuer) { - t.Errorf("expected to find issuer %s, did not", issuer) - } - if !strings.Contains(tempOutputIdentitiesString, oid.String()) { - t.Errorf("expected to find oid %s, did not", oid.String()) - } - if !strings.Contains(tempOutputIdentitiesString, oid.String()) { - t.Errorf("expected to find oid value %s, did not", extValueString) - } - if !strings.Contains(tempOutputIdentitiesString, certFingerprint) { - t.Errorf("expected to find fingerprint %s, did not", certFingerprint) - } } diff --git a/pkg/test/e2e/e2e_test.sh b/pkg/test/consistency_check/consistency_check_e2e_test.sh similarity index 96% rename from pkg/test/e2e/e2e_test.sh rename to pkg/test/consistency_check/consistency_check_e2e_test.sh index 46e86b13..76731cfe 100755 --- a/pkg/test/e2e/e2e_test.sh +++ b/pkg/test/consistency_check/consistency_check_e2e_test.sh @@ -66,4 +66,4 @@ echo echo "running tests" popd -go test -tags=e2e -v -race ./pkg/test/e2e/... \ No newline at end of file +go test -tags=e2e -v -race ./pkg/test/consistency_check/... \ No newline at end of file diff --git a/pkg/test/identity_workflow/identity_workflow_e2e_test.go b/pkg/test/identity_workflow/identity_workflow_e2e_test.go index 93e1e5de..052d4471 100644 --- a/pkg/test/identity_workflow/identity_workflow_e2e_test.go +++ b/pkg/test/identity_workflow/identity_workflow_e2e_test.go @@ -34,6 +34,7 @@ import ( "github.com/sigstore/rekor-monitor/pkg/fulcio/extensions" "github.com/sigstore/rekor-monitor/pkg/identity" + "github.com/sigstore/rekor-monitor/pkg/notifications" "github.com/sigstore/rekor-monitor/pkg/rekor" "github.com/sigstore/rekor-monitor/pkg/test" "github.com/sigstore/rekor/pkg/client" @@ -149,7 +150,7 @@ func TestIdentitySearch(t *testing.T) { tempMetadataFileName := tempMetadataFile.Name() defer os.Remove(tempMetadataFileName) - monitoredVals := identity.MonitoredValues{ + configMonitoredValues := notifications.ConfigMonitoredValues{ Subjects: []string{subject}, CertificateIdentities: []identity.CertificateIdentity{ { @@ -157,17 +158,33 @@ func TestIdentitySearch(t *testing.T) { Issuers: []string{".+@domain.com"}, }, }, - OIDMatchers: []extensions.OIDExtension{ - { - ObjectIdentifier: oid, - ExtensionValues: []string{extValueString}, + OIDMatchers: extensions.OIDMatchers{ + OIDExtensions: []extensions.OIDExtension{ + { + ObjectIdentifier: oid, + ExtensionValues: []string{extValueString}, + }, }, + FulcioExtensions: extensions.FulcioExtensions{}, + CustomExtensions: []extensions.CustomExtension{}, }, Fingerprints: []string{ certFingerprint, }, } + configRenderedOIDMatchers, err := configMonitoredValues.OIDMatchers.RenderOIDMatchers() + if err != nil { + t.Errorf("error rendering OID matchers: %v", err) + } + + monitoredVals := identity.MonitoredValues{ + Subjects: configMonitoredValues.Subjects, + Fingerprints: configMonitoredValues.Fingerprints, + OIDMatchers: configRenderedOIDMatchers, + CertificateIdentities: configMonitoredValues.CertificateIdentities, + } + payload = []byte{1, 2, 3, 4, 5, 6} sig, err = signer.SignMessage(bytes.NewReader(payload)) if err != nil {