From cb917ddbd614b6b515347c086fcd0a6f1468805b Mon Sep 17 00:00:00 2001 From: linus-sun Date: Tue, 5 Nov 2024 23:43:33 +0000 Subject: [PATCH 1/2] refactor workflows to split Signed-off-by: linus-sun --- .github/workflows/main.yml | 6 +- .github/workflows/reusable_monitoring.yml | 21 +- README.md | 51 ++-- cmd/monitor/main.go | 153 ----------- cmd/rekor_consistency/main.go | 113 --------- cmd/rekor_monitor/main.go | 209 +++++++++++++++ pkg/rekor/identity.go | 44 +--- pkg/rekor/identity_test.go | 197 +------------- pkg/rekor/verifier.go | 15 +- pkg/rekor/verifier_test.go | 65 ++--- pkg/test/e2e/e2e_test.sh | 69 ----- .../identity_workflow_e2e_test.go | 240 ------------------ .../rekor_monitor_e2e_test.go} | 53 ++-- .../rekor_monitor_e2e_test.sh} | 0 14 files changed, 336 insertions(+), 900 deletions(-) delete mode 100644 cmd/monitor/main.go delete mode 100644 cmd/rekor_consistency/main.go create mode 100644 cmd/rekor_monitor/main.go delete mode 100755 pkg/test/e2e/e2e_test.sh delete mode 100644 pkg/test/identity_workflow/identity_workflow_e2e_test.go rename pkg/test/{e2e/e2e_test.go => rekor_e2e/rekor_monitor_e2e_test.go} (89%) mode change 100755 => 100644 rename pkg/test/{identity_workflow/identity_workflow_e2e_test.sh => rekor_e2e/rekor_monitor_e2e_test.sh} (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21c25129..d1c56ba8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,8 +88,6 @@ jobs: with: go-version-file: './go.mod' check-latest: true - - name: run e2e test - run: ./pkg/test/e2e/e2e_test.sh - - name: run identity monitor test - run: ./pkg/test/identity_workflow/identity_workflow_e2e_test.sh + - name: run Rekor end-to-end test + run: ./pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh diff --git a/.github/workflows/reusable_monitoring.yml b/.github/workflows/reusable_monitoring.yml index 339caca3..0ba709e6 100644 --- a/.github/workflows/reusable_monitoring.yml +++ b/.github/workflows/reusable_monitoring.yml @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: This GHA should not be run concurrently. - -# TODO: Write identities to issues when found - name: Rekor Monitoring Template on: @@ -30,9 +26,14 @@ on: required: false type: number default: 14 - identities: - description: 'multiline yaml of certificate subjects and issuers, key subjects, and fingerprints. For certificates, if no issuers are specified, match any OIDC provider' + once: + description: 'whether to run the identity monitor once or periodically' + default: true required: false + type: boolean + config: + description: 'multiline yaml of configuration settings for identity monitor run' + required: true type: string permissions: @@ -54,6 +55,10 @@ jobs: - name: Detect the repository and ref id: detect uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@5a775b367a56d5bd118a224a811bba288150a563 # v2.0.0 + # NOTE: This GHA should not be run concurrently. + concurrency: + group: rekor-consistency-check + cancel-in-progress: true monitor: runs-on: ubuntu-latest @@ -76,7 +81,7 @@ jobs: run: cat ${{ env.LOG_FILE }} # Skip on first run continue-on-error: true - - run: go run ./cmd/rekor_consistency --file ${{ env.LOG_FILE }} --once --monitored-values "${{ inputs.identities }}" --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}" + - run: go run ./cmd/rekor_consistency --file ${{ env.LOG_FILE }} --once=${{ inputs.once }} --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}" - name: Upload checkpoint uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: @@ -85,6 +90,8 @@ jobs: retention-days: ${{ inputs.artifact_retention_days }} - name: Log new checkpoints run: cat ${{ env.LOG_FILE }} + - name: Run identity monitor + run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --once=${{ inputs.once }} if-succeeded: runs-on: ubuntu-latest diff --git a/README.md b/README.md index de0e8ec3..f70cd8cc 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,13 @@ Rekor Log Monitor provides an easy-to-use monitor to verify log consistency, that the log is immutability and append-only. Monitoring is critical to the transparency log ecosystem, as logs are tamper-evident but not tamper-proof. +Rekor Log Monitor also provides a monitor to search for identities within a log, +and send a list of found identities via various notification platforms. + +## Consistency check To run, create a GitHub Actions workflow that uses the -[reusable monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/reusable_monitoring.yml). +[consistency check workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/consistency_check.yml). It is recommended to run the log monitor every hour for optimal performance. Example workflow: @@ -46,6 +50,10 @@ Please read [this](https://github.com/google/re2/wiki/Syntax) for syntax referen Note: The log monitor only starts monitoring from the latest checkpoint. If you want to search previous entries, you will need to query the log. +To run, create a GitHub Actions workflow that uses the +[identity monitoring workflow](https://github.com/sigstore/rekor-monitor/blob/main/.github/workflows/identity_monitor.yml). +It is recommended to run the log monitor every hour for optimal performance. + Example workflow below: ``` @@ -66,26 +74,27 @@ jobs: with: file_issue: true # Strongly recommended: Files an issue on monitoring failure artifact_retention_days: 14 # Optional, default is 14: Must be longer than the cron job frequency - identities: | - certIdentities: - - certSubject: user@domain\.com - - certSubject: otheruser@domain\.com - issuers: - - https://accounts\.google\.com - - https://github\.com/login - - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.* - issuers: - - https://token\.actions\.githubusercontent\.com - subjects: - - subject@domain\.com - fingerprints: - - A0B1C2D3E4F5 - fulcioExtensions: - build-config-uri: - - https://example.com/owner/repository/build-config.yml - customExtensions: - - objectIdentifier: 1.3.6.1.4.1.57264.1.9 - extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0 + config: | + monitoredValues: | + certIdentities: + - certSubject: user@domain\.com + - certSubject: otheruser@domain\.com + issuers: + - https://accounts\.google\.com + - https://github\.com/login + - certSubject: https://github\.com/actions/starter-workflows/blob/main/\.github/workflows/lint\.yaml@.* + issuers: + - https://token\.actions\.githubusercontent\.com + subjects: + - subject@domain\.com + fingerprints: + - A0B1C2D3E4F5 + fulcioExtensions: + build-config-uri: + - https://example.com/owner/repository/build-config.yml + customExtensions: + - objectIdentifier: 1.3.6.1.4.1.57264.1.9 + extensionValues: https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0 ``` In this example, the monitor will log: diff --git a/cmd/monitor/main.go b/cmd/monitor/main.go deleted file mode 100644 index a2c5bb19..00000000 --- a/cmd/monitor/main.go +++ /dev/null @@ -1,153 +0,0 @@ -// -// Copyright 2021 The Sigstore Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "flag" - "fmt" - "log" - "os" - "runtime" - "strings" - "time" - - "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/notifications" - "github.com/sigstore/rekor-monitor/pkg/rekor" - "github.com/sigstore/rekor-monitor/pkg/util/file" - "github.com/sigstore/rekor/pkg/client" - "github.com/sigstore/rekor/pkg/util" - "gopkg.in/yaml.v2" - "sigs.k8s.io/release-utils/version" -) - -// Default values for monitoring job parameters -const ( - publicRekorServerURL = "https://rekor.sigstore.dev" - logInfoFileName = "logInfo.txt" - outputIdentitiesFileName = "identities.txt" -) - -// This main function performs a periodic identity search. -// Upon starting, any existing latest snapshot data is loaded and the function runs -// indefinitely to perform identity search for every time interval that was specified. -func main() { - // Command-line flags that are parameters to the verifier job - configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings") - once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once") - flag.Parse() - - if configFilePath == nil { - log.Fatalf("empty configuration file path") - } - - readConfig, err := os.ReadFile(*configFilePath) - if err != nil { - log.Fatalf("error reading from identity monitor configuration file: %v", err) - } - - configString := string(readConfig) - var config notifications.IdentityMonitorConfiguration - if err := yaml.Unmarshal([]byte(configString), &config); err != nil { - log.Fatalf("error parsing identities: %v", err) - } - - rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) - if err != nil { - log.Fatalf("getting Rekor client: %v", err) - } - - if config.ServerURL == "" { - config.ServerURL = publicRekorServerURL - } - if config.LogInfoFile == "" { - config.LogInfoFile = logInfoFileName - } - if config.OutputIdentitiesFile == "" { - config.OutputIdentitiesFile = outputIdentitiesFileName - } - if config.Interval == nil { - defaultInterval := time.Hour - config.Interval = &defaultInterval - } - - ticker := time.NewTicker(*config.Interval) - defer ticker.Stop() - - // To get an immediate first tick - for ; ; <-ticker.C { - inputEndIndex := config.EndIndex - if config.StartIndex == nil || config.EndIndex == nil { - logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient) - if err != nil { - fmt.Fprintf(os.Stderr, "error getting log info: %v", err) - return - } - - checkpoint, err := rekor.ReadLatestCheckpoint(logInfo) - if err != nil { - fmt.Fprintf(os.Stderr, "error reading checkpoint: %v", err) - return - } - - var prevCheckpoint *util.SignedCheckpoint - prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile) - if err != nil { - fmt.Fprintf(os.Stderr, "reading checkpoint log: %v", err) - return - } - - checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint) - if config.StartIndex == nil { - config.StartIndex = &checkpointStartIndex - } - if config.EndIndex == nil { - config.EndIndex = &checkpointEndIndex - } - } - - if *config.StartIndex >= *config.EndIndex { - fmt.Fprintf(os.Stderr, "start index %d must be strictly less than end index %d", *config.StartIndex, *config.EndIndex) - } - - allOIDMatchers, err := config.MonitoredValues.OIDMatchers.RenderOIDMatchers() - if err != nil { - fmt.Printf("error parsing OID matchers: %v", err) - } - - monitoredValues := identity.MonitoredValues{ - CertificateIdentities: config.MonitoredValues.CertificateIdentities, - Subjects: config.MonitoredValues.Subjects, - Fingerprints: config.MonitoredValues.Fingerprints, - OIDMatchers: allOIDMatchers, - } - - // TODO: This should subsequently read from the identity metadata file to fetch the latest index. - _, err = rekor.IdentitySearch(*config.StartIndex, *config.EndIndex, rekorClient, monitoredValues, config.OutputIdentitiesFile, config.IdentityMetadataFile) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to successfully complete identity search: %v", err) - return - } - - if *once || inputEndIndex != nil { - return - } - - config.StartIndex = config.EndIndex - config.EndIndex = nil - } -} diff --git a/cmd/rekor_consistency/main.go b/cmd/rekor_consistency/main.go deleted file mode 100644 index 073e45d5..00000000 --- a/cmd/rekor_consistency/main.go +++ /dev/null @@ -1,113 +0,0 @@ -// -// Copyright 2021 The Sigstore Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "flag" - "fmt" - "log" - "os" - "runtime" - "strings" - "time" - - "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/rekor" - "github.com/sigstore/rekor/pkg/client" - "gopkg.in/yaml.v3" - - "sigs.k8s.io/release-utils/version" -) - -// Default values for monitoring job parameters -const ( - publicRekorServerURL = "https://rekor.sigstore.dev" - logInfoFileName = "logInfo.txt" - outputIdentitiesFileName = "identities.txt" -) - -// This main function performs a periodic root hash consistency check. -// Upon starting, any existing latest snapshot data is loaded and the function runs -// indefinitely to perform consistency check for every time interval that was specified. -func main() { - // Command-line flags that are parameters to the verifier job - serverURL := flag.String("url", publicRekorServerURL, "URL to the rekor server that is to be monitored") - interval := flag.Duration("interval", 5*time.Minute, "Length of interval between each periodical consistency check") - logInfoFile := flag.String("file", logInfoFileName, "Name of the file containing initial merkle tree information") - once := flag.Bool("once", false, "Perform consistency check once and exit") - monitoredValsInput := flag.String("monitored-values", "", "yaml of certificate subjects and issuers, key subjects, "+ - "and fingerprints. For certificates, if no issuers are specified, match any OIDC provider.") - outputIdentitiesFile := flag.String("output-identities", outputIdentitiesFileName, - "Name of the file containing indices and identities found in the log. Format is \"subject issuer index uuid\"") - userAgentString := flag.String("user-agent", "", "details to include in the user agent string") - flag.Parse() - - var monitoredVals identity.MonitoredValues - if err := yaml.Unmarshal([]byte(*monitoredValsInput), &monitoredVals); err != nil { - log.Fatalf("error parsing identities: %v", err) - } - - for _, certID := range monitoredVals.CertificateIdentities { - if len(certID.Issuers) == 0 { - fmt.Printf("Monitoring certificate subject %s\n", certID.CertSubject) - } else { - fmt.Printf("Monitoring certificate subject %s for issuer(s) %s\n", certID.CertSubject, strings.Join(certID.Issuers, ",")) - } - } - for _, fp := range monitoredVals.Fingerprints { - fmt.Printf("Monitoring fingerprint %s\n", fp) - } - for _, sub := range monitoredVals.Subjects { - fmt.Printf("Monitoring subject %s\n", sub) - } - - rekorClient, err := client.GetRekorClient(*serverURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s) %s", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH, *userAgentString)))) - if err != nil { - log.Fatalf("getting Rekor client: %v", err) - } - - verifier, err := rekor.GetLogVerifier(context.Background(), rekorClient) - if err != nil { - log.Fatal(err) - } - - err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once) - if err != nil { - log.Fatal(err) - } - - ticker := time.NewTicker(*interval) - defer ticker.Stop() - - // Loop will: - // 1. Fetch latest checkpoint and verify - // 2. If old checkpoint is present, verify consistency proof - // 3. Write latest checkpoint to file - - // To get an immediate first tick - for ; ; <-ticker.C { - err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile) - if err != nil { - fmt.Fprintf(os.Stderr, "error running consistency check: %v", err) - return - } - - if *once { - return - } - } -} diff --git a/cmd/rekor_monitor/main.go b/cmd/rekor_monitor/main.go new file mode 100644 index 00000000..77d4ca1b --- /dev/null +++ b/cmd/rekor_monitor/main.go @@ -0,0 +1,209 @@ +// +// Copyright 2021 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "runtime" + "strings" + "time" + + "github.com/sigstore/rekor-monitor/pkg/identity" + "github.com/sigstore/rekor-monitor/pkg/notifications" + "github.com/sigstore/rekor-monitor/pkg/rekor" + "github.com/sigstore/rekor-monitor/pkg/util/file" + "github.com/sigstore/rekor/pkg/client" + "github.com/sigstore/rekor/pkg/generated/models" + "github.com/sigstore/rekor/pkg/util" + "gopkg.in/yaml.v2" + "sigs.k8s.io/release-utils/version" +) + +// Default values for monitoring job parameters +const ( + publicRekorServerURL = "https://rekor.sigstore.dev" + outputIdentitiesFileName = "identities.txt" + logInfoFileName = "logInfo.txt" +) + +// This main function performs a periodic identity search. +// Upon starting, any existing latest snapshot data is loaded and the function runs +// indefinitely to perform identity search for every time interval that was specified. +func main() { + // Command-line flags that are parameters to the verifier job + configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings") + configYamlInput := flag.String("config", "", "path to yaml configuration file containing identity monitor settings") + once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once") + serverURL := flag.String("url", publicRekorServerURL, "URL to the rekor server that is to be monitored") + interval := flag.Duration("interval", 5*time.Minute, "Length of interval between each periodical consistency check") + userAgentString := flag.String("user-agent", "", "details to include in the user agent string") + flag.Parse() + flag.Parse() + + if *configFilePath == "" && *configYamlInput == "" { + log.Fatalf("empty configuration input") + } + + if *configFilePath != "" && *configYamlInput != "" { + log.Fatalf("only input one of configuration file path or yaml input") + } + + var config notifications.IdentityMonitorConfiguration + + if *configFilePath != "" { + readConfig, err := os.ReadFile(*configFilePath) + if err != nil { + log.Fatalf("error reading from identity monitor configuration file: %v", err) + } + + configString := string(readConfig) + if err := yaml.Unmarshal([]byte(configString), &config); err != nil { + log.Fatalf("error parsing identities: %v", err) + } + } + + if *configYamlInput != "" { + if err := yaml.Unmarshal([]byte(*configYamlInput), &config); err != nil { + log.Fatalf("error parsing identities: %v", err) + } + } + + if config.ServerURL == "" { + config.ServerURL = publicRekorServerURL + } + if config.OutputIdentitiesFile == "" { + config.OutputIdentitiesFile = outputIdentitiesFileName + } + if config.Interval == nil { + defaultInterval := time.Hour + config.Interval = &defaultInterval + } + + err := rekor.VerifyConsistencyCheckInputs(interval, &config.LogInfoFile, once) + if err != nil { + log.Fatal(err) + } + + rekorClient, err := client.GetRekorClient(*serverURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s) %s", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH, *userAgentString)))) + if err != nil { + log.Fatalf("getting Rekor client: %v", err) + } + + verifier, err := rekor.GetLogVerifier(context.Background(), rekorClient) + if err != nil { + log.Fatal(err) + } + + allOIDMatchers, err := config.MonitoredValues.OIDMatchers.RenderOIDMatchers() + if err != nil { + fmt.Printf("error parsing OID matchers: %v", err) + } + + monitoredValues := identity.MonitoredValues{ + CertificateIdentities: config.MonitoredValues.CertificateIdentities, + Subjects: config.MonitoredValues.Subjects, + Fingerprints: config.MonitoredValues.Fingerprints, + OIDMatchers: allOIDMatchers, + } + + for _, certID := range monitoredValues.CertificateIdentities { + if len(certID.Issuers) == 0 { + fmt.Printf("Monitoring certificate subject %s\n", certID.CertSubject) + } else { + fmt.Printf("Monitoring certificate subject %s for issuer(s) %s\n", certID.CertSubject, strings.Join(certID.Issuers, ",")) + } + } + for _, fp := range monitoredValues.Fingerprints { + fmt.Printf("Monitoring fingerprint %s\n", fp) + } + for _, sub := range monitoredValues.Subjects { + fmt.Printf("Monitoring subject %s\n", sub) + } + + ticker := time.NewTicker(*config.Interval) + defer ticker.Stop() + + // To get an immediate first tick + for ; ; <-ticker.C { + inputEndIndex := config.EndIndex + + var logInfo *models.LogInfo + if config.StartIndex == nil || config.EndIndex == nil { + logInfo, err = rekor.GetLogInfo(context.Background(), rekorClient) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting log info: %v", err) + return + } + } + + if config.StartIndex == nil { + if config.LogInfoFile != "" { + var prevCheckpoint *util.SignedCheckpoint + prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile) + if err != nil { + fmt.Fprintf(os.Stderr, "reading checkpoint log: %v", err) + return + } + + checkpointStartIndex := rekor.GetCheckpointIndex(logInfo, prevCheckpoint) + config.StartIndex = &checkpointStartIndex + } else { + defaultStartIndex := 0 + config.StartIndex = &defaultStartIndex + } + } + + if config.EndIndex == nil { + + checkpoint, err := rekor.ReadLatestCheckpoint(logInfo) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading checkpoint: %v", err) + return + } + + checkpointEndIndex := rekor.GetCheckpointIndex(logInfo, checkpoint) + config.EndIndex = &checkpointEndIndex + } + + if *config.StartIndex >= *config.EndIndex { + fmt.Fprintf(os.Stderr, "start index %d must be strictly less than end index %d", *config.StartIndex, *config.EndIndex) + } + + err = rekor.RunConsistencyCheck(rekorClient, verifier, config.LogInfoFile) + if err != nil { + fmt.Fprintf(os.Stderr, "error running consistency check: %v", err) + return + } + + // TODO: This should subsequently read from the identity metadata file to fetch the latest index. + _, err = rekor.IdentitySearch(*config.StartIndex, *config.EndIndex, rekorClient, monitoredValues, config.OutputIdentitiesFile, config.IdentityMetadataFile) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to successfully complete identity search: %v", err) + return + } + + if *once || inputEndIndex != nil { + return + } + + config.StartIndex = config.EndIndex + config.EndIndex = nil + } +} diff --git a/pkg/rekor/identity.go b/pkg/rekor/identity.go index fdad6d57..7f081643 100644 --- a/pkg/rekor/identity.go +++ b/pkg/rekor/identity.go @@ -334,37 +334,16 @@ func oidMatchesPolicy(cert *x509.Certificate, oid asn1.ObjectIdentifier, extensi return false, nil, "", nil } -func GetPrevCurrentCheckpoints(client *client.Rekor, logInfoFile string) (*util.SignedCheckpoint, *util.SignedCheckpoint, error) { - logInfo, err := GetLogInfo(context.Background(), client) - if err != nil { - return nil, nil, fmt.Errorf("error getting log info: %v", err) - } - - checkpoint, err := ReadLatestCheckpoint(logInfo) - if err != nil { - return nil, nil, fmt.Errorf("error reading checkpoint: %v", err) - } - - var prevCheckpoint *util.SignedCheckpoint - prevCheckpoint, err = file.ReadLatestCheckpoint(logInfoFile) - if err != nil { - return nil, nil, fmt.Errorf("reading checkpoint log: %v", err) - } - - return prevCheckpoint, checkpoint, nil -} - -// GetCheckpointIndices fetches the start and end indexes between two checkpoints and returns them. -func GetCheckpointIndices(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint) (int, int) { +// GetCheckpointIndex fetches the index of a checkpoint and returns it. +func GetCheckpointIndex(logInfo *models.LogInfo, checkpoint *util.SignedCheckpoint) int { // Get log size of inactive shards totalSize := 0 for _, s := range logInfo.InactiveShards { totalSize += int(*s.TreeSize) } - startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow - endIndex := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 + index := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115 - return startIndex, endIndex + return index } func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, monitoredValues identity.MonitoredValues, outputIdentitiesFile string, idMetadataFile *string) ([]identity.MonitoredIdentity, error) { @@ -404,18 +383,3 @@ func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, mon monitoredIdentities := identity.CreateMonitoredIdentities(idEntries, identities) return monitoredIdentities, nil } - -// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file. -func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error { - // Get log size of inactive shards - startIndex, endIndex := GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint) - - // Search for identities in the log range - if identity.MonitoredValuesExist(monitoredValues) { - _, err := IdentitySearch(startIndex, endIndex, rekorClient, monitoredValues, outputIdentitiesFile, nil) - if err != nil { - return fmt.Errorf("error monitoring for identities: %v", err) - } - } - return nil -} diff --git a/pkg/rekor/identity_test.go b/pkg/rekor/identity_test.go index d63a07e4..bae8c606 100644 --- a/pkg/rekor/identity_test.go +++ b/pkg/rekor/identity_test.go @@ -27,8 +27,6 @@ import ( "encoding/asn1" "encoding/base64" "encoding/hex" - "fmt" - "os" "strings" "testing" "time" @@ -36,10 +34,7 @@ import ( "github.com/go-openapi/swag" "github.com/sigstore/rekor-monitor/pkg/fulcio/extensions" "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/rekor/mock" "github.com/sigstore/rekor-monitor/pkg/test" - "github.com/sigstore/rekor-monitor/pkg/util/file" - "github.com/sigstore/rekor/pkg/generated/client" "github.com/sigstore/rekor/pkg/generated/models" "github.com/sigstore/rekor/pkg/types" hashedrekord_v001 "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1" @@ -47,7 +42,6 @@ import ( "github.com/sigstore/rekor/pkg/util" "github.com/sigstore/sigstore/pkg/cryptoutils" "github.com/sigstore/sigstore/pkg/signature" - "golang.org/x/mod/sumdb/note" ) const ( @@ -866,7 +860,7 @@ func TestMatchedIndicesFailures(t *testing.T) { } } -func TestGetCheckpointIndices(t *testing.T) { +func TestGetCheckpointIndex(t *testing.T) { shardTreeSize := int64(1) inactiveShard := models.InactiveShardLogInfo{ TreeSize: &shardTreeSize, @@ -879,99 +873,36 @@ func TestGetCheckpointIndices(t *testing.T) { emptyLogInfo := &models.LogInfo{ InactiveShards: emptyInactiveShards, } - prevCheckpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 1, - }, - } checkpoint := &util.SignedCheckpoint{ Checkpoint: util.Checkpoint{ Size: 2, }, } getCheckpointIndicesTests := map[string]struct { - inputLogInfo *models.LogInfo - inputPrevCheckpoint *util.SignedCheckpoint - inputCheckpoint *util.SignedCheckpoint - expectedStartIndex int - expectedEndIndex int + inputLogInfo *models.LogInfo + inputCheckpoint *util.SignedCheckpoint + expectedEndIndex int }{ "populated inactive shards": { - inputLogInfo: logInfo, - inputPrevCheckpoint: prevCheckpoint, - inputCheckpoint: checkpoint, - expectedStartIndex: 1, - expectedEndIndex: 2, + inputLogInfo: logInfo, + inputCheckpoint: checkpoint, + expectedEndIndex: 2, }, "empty inactive shards": { - inputLogInfo: emptyLogInfo, - inputPrevCheckpoint: prevCheckpoint, - inputCheckpoint: checkpoint, - expectedStartIndex: 0, - expectedEndIndex: 1, + inputLogInfo: emptyLogInfo, + inputCheckpoint: checkpoint, + expectedEndIndex: 1, }, } for testCaseName, testCase := range getCheckpointIndicesTests { - expectedStartIndex := testCase.expectedStartIndex expectedEndIndex := testCase.expectedEndIndex - resultStartIndex, resultEndIndex := GetCheckpointIndices(testCase.inputLogInfo, testCase.inputPrevCheckpoint, testCase.inputCheckpoint) - if resultStartIndex != expectedStartIndex || resultEndIndex != expectedEndIndex { - t.Errorf("%s failed: expected %d, %d indices, received %d, %d indices", testCaseName, expectedStartIndex, expectedEndIndex, resultStartIndex, resultEndIndex) + resultEndIndex := GetCheckpointIndex(testCase.inputLogInfo, testCase.inputCheckpoint) + if resultEndIndex != expectedEndIndex { + t.Errorf("%s failed: expected %d index, received %d", testCaseName, expectedEndIndex, resultEndIndex) } } } -func TestGetPrevCurrentCheckpoints(t *testing.T) { - // generate checkpoint - root, _ := hex.DecodeString("1a341bc342ff4e567387de9789ab14000b147124317841489172419874198147") - expectedCheckpoint, err := util.CreateSignedCheckpoint(util.Checkpoint{ - Origin: "origin", - Size: uint64(123), - Hash: root, - }) - expectedCheckpoint.Signatures = []note.Signature{{Name: "name", Hash: 1, Base64: "adbadbadb"}} - if err != nil { - t.Fatal(err) - } - checkpointBytes, err := expectedCheckpoint.MarshalText() - if err != nil { - t.Fatal(err) - } - if err != nil { - t.Errorf("error marshalling checkpoint: %v", err) - } - checkpointString := string(checkpointBytes) - - var mClient client.Rekor - mClient.Tlog = &mock.TlogClient{ - LogInfo: &models.LogInfo{ - SignedTreeHead: &checkpointString, - }, - } - - tempDir := t.TempDir() - tempLogInfoFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp log file: %v", err) - } - tempLogInfoFileName := tempLogInfoFile.Name() - defer os.Remove(tempLogInfoFileName) - file.WriteCheckpoint(expectedCheckpoint, tempLogInfoFileName) - - prevCheckpoint, currentCheckpoint, err := GetPrevCurrentCheckpoints(&mClient, tempLogInfoFileName) - if err != nil { - t.Errorf("expected nil, received %v", err) - } - - if prevCheckpoint.Checkpoint.String() != expectedCheckpoint.Checkpoint.String() { - t.Errorf("expected prev checkpoint %s, received %s", expectedCheckpoint.Checkpoint, prevCheckpoint.Checkpoint) - } - - if currentCheckpoint.Checkpoint.String() != expectedCheckpoint.Checkpoint.String() { - t.Errorf("expected current checkpoint %s, received %s", expectedCheckpoint.Checkpoint, currentCheckpoint.Checkpoint) - } -} - func mockCertificateWithExtension(oid asn1.ObjectIdentifier, value string) (*x509.Certificate, error) { extValue, err := asn1.Marshal(value) if err != nil { @@ -1037,105 +968,3 @@ func TestOIDMatchesValue(t *testing.T) { t.Errorf("Expected string to equal 'test cert value', got %s", extValue) } } - -// Test monitoring for identities using mock Rekor client -func TestWriteIdentitiesBetweenCheckpoints(t *testing.T) { - maxIndex := 2 - var logEntries []*models.LogEntry - - rootCert, rootKey, _ := test.GenerateRootCA() - leafCert, leafKey, _ := test.GenerateLeafCert(subject, issuer, rootCert, rootKey) - - signer, err := signature.LoadECDSASignerVerifier(leafKey, crypto.SHA256) - if err != nil { - t.Fatal(err) - } - pemCert, _ := cryptoutils.MarshalCertificateToPEM(leafCert) - - payload := []byte{1, 2, 3, 4} - sig, err := signer.SignMessage(bytes.NewReader(payload)) - if err != nil { - t.Fatal(err) - } - - hashedrekord := &hashedrekord_v001.V001Entry{} - hash := sha256.Sum256(payload) - pe, err := hashedrekord.CreateFromArtifactProperties(context.Background(), types.ArtifactProperties{ - ArtifactHash: hex.EncodeToString(hash[:]), - SignatureBytes: sig, - PublicKeyBytes: [][]byte{pemCert}, - PKIFormat: "x509", - }) - if err != nil { - t.Fatal(err) - } - entry, err := types.UnmarshalEntry(pe) - if err != nil { - t.Fatal(err) - } - leaf, err := entry.Canonicalize(context.Background()) - if err != nil { - t.Fatal(err) - } - - for i := 0; i <= maxIndex; i++ { - lea := models.LogEntryAnon{ - Body: base64.StdEncoding.EncodeToString(leaf), - LogIndex: swag.Int64(int64(i)), - } - data := models.LogEntry{ - fmt.Sprint(i): lea, - } - logEntries = append(logEntries, &data) - } - - var mClient client.Rekor - mClient.Entries = &mock.EntriesClient{ - Entries: logEntries, - } - testLogInfo := &models.LogInfo{} - mClient.Tlog = &mock.TlogClient{ - LogInfo: testLogInfo, - } - - logInfo, err := GetLogInfo(context.Background(), &mClient) - if err != nil { - t.Errorf("failed fetching log info: %v", err) - } - prevCheckpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 0, - }, - } - checkpoint := &util.SignedCheckpoint{ - Checkpoint: util.Checkpoint{ - Size: 1, - }, - } - monitoredValues := identity.MonitoredValues{ - Subjects: []string{ - subject, - }, - } - tempDir := t.TempDir() - tempOutputIdentitiesFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp output identities file: %v", err) - } - tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name() - defer os.Remove(tempOutputIdentitiesFileName) - - err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, monitoredValues, &mClient, tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("failed write identities between checkpoints: %v", err) - } - - tempOutputIdentities, err := os.ReadFile(tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("error reading from output identities file: %v", err) - } - tempOutputIdentitiesString := string(tempOutputIdentities) - if !strings.Contains(tempOutputIdentitiesString, subject) { - t.Errorf("expected to find subject %s, did not", subject) - } -} diff --git a/pkg/rekor/verifier.go b/pkg/rekor/verifier.go index fbb54114..7f69e79f 100644 --- a/pkg/rekor/verifier.go +++ b/pkg/rekor/verifier.go @@ -23,7 +23,6 @@ import ( "os" "time" - "github.com/sigstore/rekor-monitor/pkg/identity" "github.com/sigstore/rekor-monitor/pkg/util/file" "github.com/sigstore/rekor/pkg/generated/client" "github.com/sigstore/rekor/pkg/generated/models" @@ -94,16 +93,13 @@ func verifyCheckpointConsistency(logInfoFile string, checkpoint *util.SignedChec } // VerifyConsistencyCheckInputs verifies that the input flag values to the rekor-monitor workflow are not nil. -func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, outputIdentitiesFile *string, once *bool) error { +func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, once *bool) error { if interval == nil { return errors.New("--interval flag equal to nil") } if logInfoFile == nil { return errors.New("--file flag equal to nil") } - if outputIdentitiesFile == nil { - return errors.New("--output-identities flag equal to nil") - } if once == nil { return errors.New("--once flag equal to nil") } @@ -111,7 +107,7 @@ func VerifyConsistencyCheckInputs(interval *time.Duration, logInfoFile *string, } // RunConsistencyCheck periodically verifies the root hash consistency of a Rekor log. -func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string, mvs identity.MonitoredValues, outputIdentitiesFile string) error { +func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, logInfoFile string) error { logInfo, err := GetLogInfo(context.Background(), rekorClient) if err != nil { return fmt.Errorf("failed to get log info: %v", err) @@ -141,13 +137,6 @@ func RunConsistencyCheck(rekorClient *client.Rekor, verifier signature.Verifier, } } - if prevCheckpoint != nil && prevCheckpoint.Size != checkpoint.Size { - err = writeIdentitiesBetweenCheckpoints(logInfo, prevCheckpoint, checkpoint, mvs, rekorClient, outputIdentitiesFile) - if err != nil { - return fmt.Errorf("failed to monitor identities: %v", err) - } - } - // TODO: Switch to writing checkpoints to GitHub so that the history is preserved. Then we only need // to persist the last checkpoint. // Delete old checkpoints to avoid the log growing indefinitely diff --git a/pkg/rekor/verifier_test.go b/pkg/rekor/verifier_test.go index 989c1121..6c0e0edd 100644 --- a/pkg/rekor/verifier_test.go +++ b/pkg/rekor/verifier_test.go @@ -53,66 +53,51 @@ func TestGetLogVerifier(t *testing.T) { func TestVerifyConsistencyCheckInputs(t *testing.T) { interval := 5 * time.Minute logInfoFile := "./test/example_log_info_file_path.txt" - outputIdentitiesFile := "./test/example_output_identities_file.txt" once := true verifyConsistencyCheckInputTests := map[string]struct { - interval *time.Duration - logInfoFile *string - outputIdentitiesFile *string - once *bool - expectedError error + interval *time.Duration + logInfoFile *string + once *bool + expectedError error }{ "successful verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: nil, + interval: &interval, + logInfoFile: &logInfoFile, + once: &once, + expectedError: nil, }, "fail --interval verification": { - interval: nil, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: errors.New("--interval flag equal to nil"), + interval: nil, + logInfoFile: &logInfoFile, + once: &once, + expectedError: errors.New("--interval flag equal to nil"), }, "fail --file verification": { - interval: &interval, - logInfoFile: nil, - outputIdentitiesFile: &outputIdentitiesFile, - once: &once, - expectedError: errors.New("--file flag equal to nil"), - }, - "fail --output-identities verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: nil, - once: &once, - expectedError: errors.New("--output-identities flag equal to nil"), + interval: &interval, + logInfoFile: nil, + once: &once, + expectedError: errors.New("--file flag equal to nil"), }, "fail --once verification": { - interval: &interval, - logInfoFile: &logInfoFile, - outputIdentitiesFile: &outputIdentitiesFile, - once: nil, - expectedError: errors.New("--once flag equal to nil"), + interval: &interval, + logInfoFile: &logInfoFile, + once: nil, + expectedError: errors.New("--once flag equal to nil"), }, "empty case": { - interval: nil, - logInfoFile: nil, - outputIdentitiesFile: nil, - once: nil, - expectedError: errors.New("--interval flag equal to nil"), + interval: nil, + logInfoFile: nil, + once: nil, + expectedError: errors.New("--interval flag equal to nil"), }, } for verifyConsistencyCheckInputTestCaseName, verifyConsistencyCheckInputTestCase := range verifyConsistencyCheckInputTests { interval := verifyConsistencyCheckInputTestCase.interval logInfoFile := verifyConsistencyCheckInputTestCase.logInfoFile - outputIdentitiesFile := verifyConsistencyCheckInputTestCase.outputIdentitiesFile once := verifyConsistencyCheckInputTestCase.once expectedError := verifyConsistencyCheckInputTestCase.expectedError - err := VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once) + err := VerifyConsistencyCheckInputs(interval, logInfoFile, once) if (err == nil && expectedError != nil) || (err != nil && expectedError != nil && err.Error() != expectedError.Error()) { t.Errorf("%s: expected error %v, received error %v", verifyConsistencyCheckInputTestCaseName, expectedError, err) } diff --git a/pkg/test/e2e/e2e_test.sh b/pkg/test/e2e/e2e_test.sh deleted file mode 100755 index 46e86b13..00000000 --- a/pkg/test/e2e/e2e_test.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2024 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -pushd $HOME - -echo "downloading service repos" -for repo in rekor ; do - if [[ ! -d $repo ]]; then - git clone https://github.com/sigstore/${repo}.git - else - pushd $repo - git pull - popd - fi -done - -docker_compose="docker compose" - - -echo "starting services" -for repo in rekor ; do - pushd $repo - ${docker_compose} up -d - echo -n "waiting up to 60 sec for system to start" - count=0 - until [ $(${docker_compose} ps | grep -c "(healthy)") == 3 ]; - do - if [ $count -eq 6 ]; then - echo "! timeout reached" - exit 1 - else - echo -n "." - sleep 10 - let 'count+=1' - fi - done - popd -done - -function cleanup_services() { - echo "cleaning up" - for repo in rekor; do - pushd $HOME/$repo - ${docker_compose} down - popd - done -} -trap cleanup_services EXIT - -echo -echo "running tests" - -popd -go test -tags=e2e -v -race ./pkg/test/e2e/... \ No newline at end of file diff --git a/pkg/test/identity_workflow/identity_workflow_e2e_test.go b/pkg/test/identity_workflow/identity_workflow_e2e_test.go deleted file mode 100644 index 93e1e5de..00000000 --- a/pkg/test/identity_workflow/identity_workflow_e2e_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2024 The Sigstore Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build e2e -// +build e2e - -package e2e - -import ( - "bytes" - "context" - "crypto" - "crypto/sha256" - "crypto/x509/pkix" - "encoding/asn1" - "encoding/hex" - "fmt" - "log" - "os" - "runtime" - "strings" - "testing" - - "github.com/sigstore/rekor-monitor/pkg/fulcio/extensions" - "github.com/sigstore/rekor-monitor/pkg/identity" - "github.com/sigstore/rekor-monitor/pkg/rekor" - "github.com/sigstore/rekor-monitor/pkg/test" - "github.com/sigstore/rekor/pkg/client" - "github.com/sigstore/rekor/pkg/generated/client/entries" - "github.com/sigstore/rekor/pkg/types" - "github.com/sigstore/rekor/pkg/util" - "github.com/sigstore/sigstore/pkg/cryptoutils" - "github.com/sigstore/sigstore/pkg/signature" - "sigs.k8s.io/release-utils/version" - - hashedrekord_v001 "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1" -) - -const ( - rekorURL = "http://127.0.0.1:3000" - subject = "subject@example.com" - issuer = "oidc-issuer@domain.com" - extValueString = "test cert value" -) - -// Test IdentitySearch: -// Check that Rekor-monitor reusable identity search workflow successfully -// finds a monitored identity within the checkpoint indices and writes it to file. -func TestIdentitySearch(t *testing.T) { - rekorClient, err := client.GetRekorClient(rekorURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) - if err != nil { - log.Fatalf("getting Rekor client: %v", err) - } - - oid := asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 57264, 1, 9} - extValue, err := asn1.Marshal(extValueString) - if err != nil { - t.Fatal(err) - } - extension := pkix.Extension{ - Id: oid, - Critical: false, - Value: extValue, - } - - rootCert, rootKey, _ := test.GenerateRootCA() - leafCert, leafKey, _ := test.GenerateLeafCert(subject, issuer, rootCert, rootKey, extension) - - signer, err := signature.LoadECDSASignerVerifier(leafKey, crypto.SHA256) - if err != nil { - t.Fatal(err) - } - pemCert, _ := cryptoutils.MarshalCertificateToPEM(leafCert) - - payload := []byte{1, 2, 3, 4} - sig, err := signer.SignMessage(bytes.NewReader(payload)) - if err != nil { - t.Fatal(err) - } - - hashedrekord := &hashedrekord_v001.V001Entry{} - hash := sha256.Sum256(payload) - pe, err := hashedrekord.CreateFromArtifactProperties(context.Background(), types.ArtifactProperties{ - ArtifactHash: hex.EncodeToString(hash[:]), - SignatureBytes: sig, - PublicKeyBytes: [][]byte{pemCert}, - PKIFormat: "x509", - }) - if err != nil { - t.Fatalf("error creating hashed rekord entry: %v", err) - } - - x509Cert, err := cryptoutils.UnmarshalCertificatesFromPEM(pemCert) - if err != nil { - t.Fatal(err) - } - digest := sha256.Sum256(x509Cert[0].Raw) - certFingerprint := hex.EncodeToString(digest[:]) - - params := entries.NewCreateLogEntryParams() - params.SetProposedEntry(pe) - resp, err := rekorClient.Entries.CreateLogEntry(params) - if !resp.IsSuccess() || err != nil { - t.Errorf("error creating log entry: %v", err) - } - - logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient) - if err != nil { - t.Errorf("error getting log info: %v", err) - } - checkpoint := &util.SignedCheckpoint{} - if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil { - t.Errorf("%v", err) - } - if checkpoint.Size != 1 { - t.Errorf("expected checkpoint size of 1, received size %d", checkpoint.Size) - } - - tempDir := t.TempDir() - tempLogInfoFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp log file: %v", err) - } - tempLogInfoFileName := tempLogInfoFile.Name() - defer os.Remove(tempLogInfoFileName) - - tempOutputIdentitiesFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp output identities file: %v", err) - } - tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name() - defer os.Remove(tempOutputIdentitiesFileName) - - tempMetadataFile, err := os.CreateTemp(tempDir, "") - if err != nil { - t.Errorf("failed to create temp output identities file: %v", err) - } - tempMetadataFileName := tempMetadataFile.Name() - defer os.Remove(tempMetadataFileName) - - monitoredVals := identity.MonitoredValues{ - Subjects: []string{subject}, - CertificateIdentities: []identity.CertificateIdentity{ - { - CertSubject: ".*ubje.*", - Issuers: []string{".+@domain.com"}, - }, - }, - OIDMatchers: []extensions.OIDExtension{ - { - ObjectIdentifier: oid, - ExtensionValues: []string{extValueString}, - }, - }, - Fingerprints: []string{ - certFingerprint, - }, - } - - payload = []byte{1, 2, 3, 4, 5, 6} - sig, err = signer.SignMessage(bytes.NewReader(payload)) - if err != nil { - t.Fatalf("error signing message: %v", err) - } - hashedrekord = &hashedrekord_v001.V001Entry{} - hash = sha256.Sum256(payload) - pe, err = hashedrekord.CreateFromArtifactProperties(context.Background(), types.ArtifactProperties{ - ArtifactHash: hex.EncodeToString(hash[:]), - SignatureBytes: sig, - PublicKeyBytes: [][]byte{pemCert}, - PKIFormat: "x509", - }) - if err != nil { - t.Fatalf("error creating hashed rekord log entry: %v", err) - } - params = entries.NewCreateLogEntryParams() - params.SetProposedEntry(pe) - resp, err = rekorClient.Entries.CreateLogEntry(params) - if !resp.IsSuccess() || err != nil { - t.Errorf("error creating log entry: %v", err) - } - - logInfo, err = rekor.GetLogInfo(context.Background(), rekorClient) - if err != nil { - t.Errorf("error getting log info: %v", err) - } - checkpoint = &util.SignedCheckpoint{} - if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil { - t.Errorf("%v", err) - } - if checkpoint.Size != 2 { - t.Errorf("expected checkpoint size of 2, received size %d", checkpoint.Size) - } - - _, err = rekor.IdentitySearch(0, 1, rekorClient, monitoredVals, tempOutputIdentitiesFileName, nil) - if err != nil { - log.Fatal(err.Error()) - } - - tempOutputIdentities, err := os.ReadFile(tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("error reading from output identities file: %v", err) - } - tempOutputIdentitiesString := string(tempOutputIdentities) - if !strings.Contains(tempOutputIdentitiesString, subject) { - t.Errorf("expected to find subject %s, did not", subject) - } - if !strings.Contains(tempOutputIdentitiesString, issuer) { - t.Errorf("expected to find issuer %s, did not", issuer) - } - if !strings.Contains(tempOutputIdentitiesString, oid.String()) { - t.Errorf("expected to find oid %s, did not", oid.String()) - } - if !strings.Contains(tempOutputIdentitiesString, oid.String()) { - t.Errorf("expected to find oid value %s, did not", extValueString) - } - if !strings.Contains(tempOutputIdentitiesString, certFingerprint) { - t.Errorf("expected to find fingerprint %s, did not", certFingerprint) - } - - tempMetadata, err := os.ReadFile(tempMetadataFileName) - if err != nil { - t.Errorf("error reading from output identities file: %v", err) - } - tempMetadataString := string(tempMetadata) - if !strings.Contains(tempOutputIdentitiesString, "2") { - t.Errorf("expected to find latest index 2 in %s, did not", tempMetadataString) - } -} diff --git a/pkg/test/e2e/e2e_test.go b/pkg/test/rekor_e2e/rekor_monitor_e2e_test.go old mode 100755 new mode 100644 similarity index 89% rename from pkg/test/e2e/e2e_test.go rename to pkg/test/rekor_e2e/rekor_monitor_e2e_test.go index aee29ca3..dceec0ca --- a/pkg/test/e2e/e2e_test.go +++ b/pkg/test/rekor_e2e/rekor_monitor_e2e_test.go @@ -55,20 +55,15 @@ const ( extValueString = "test cert value" ) -// Test RunConsistencyCheck: -// Check that Rekor-monitor reusable monitoring workflow successfully verifies consistency of the log checkpoint -// and is able to find a monitored identity within the checkpoint indices and write it to file. -func TestRunConsistencyCheck(t *testing.T) { +// Test IdentitySearch: +// Check that Rekor-monitor reusable identity search workflow successfully +// finds a monitored identity within the checkpoint indices and writes it to file. +func TestIdentitySearch(t *testing.T) { rekorClient, err := client.GetRekorClient(rekorURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)))) if err != nil { log.Fatalf("getting Rekor client: %v", err) } - verifier, err := rekor.GetLogVerifier(context.Background(), rekorClient) - if err != nil { - t.Errorf("error getting log verifier: %v", err) - } - oid := asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 57264, 1, 9} extValue, err := asn1.Marshal(extValueString) if err != nil { @@ -148,6 +143,13 @@ func TestRunConsistencyCheck(t *testing.T) { tempOutputIdentitiesFileName := tempOutputIdentitiesFile.Name() defer os.Remove(tempOutputIdentitiesFileName) + tempMetadataFile, err := os.CreateTemp(tempDir, "") + if err != nil { + t.Errorf("failed to create temp output identities file: %v", err) + } + tempMetadataFileName := tempMetadataFile.Name() + defer os.Remove(tempMetadataFileName) + configMonitoredValues := notifications.ConfigMonitoredValues{ Subjects: []string{subject}, CertificateIdentities: []identity.CertificateIdentity{ @@ -171,23 +173,28 @@ func TestRunConsistencyCheck(t *testing.T) { }, } + verifier, err := rekor.GetLogVerifier(context.Background(), rekorClient) + if err != nil { + t.Errorf("error getting log verifier: %v", err) + } + + err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName) + if err != nil { + t.Errorf("first consistency check failed: %v", err) + } + configRenderedOIDMatchers, err := configMonitoredValues.OIDMatchers.RenderOIDMatchers() if err != nil { t.Errorf("error rendering OID matchers: %v", err) } monitoredVals := identity.MonitoredValues{ - Fingerprints: configMonitoredValues.Fingerprints, Subjects: configMonitoredValues.Subjects, + Fingerprints: configMonitoredValues.Fingerprints, OIDMatchers: configRenderedOIDMatchers, CertificateIdentities: configMonitoredValues.CertificateIdentities, } - err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName) - if err != nil { - t.Errorf("first consistency check failed: %v", err) - } - payload = []byte{1, 2, 3, 4, 5, 6} sig, err = signer.SignMessage(bytes.NewReader(payload)) if err != nil { @@ -223,11 +230,16 @@ func TestRunConsistencyCheck(t *testing.T) { t.Errorf("expected checkpoint size of 2, received size %d", checkpoint.Size) } - err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName, monitoredVals, tempOutputIdentitiesFileName) + err = rekor.RunConsistencyCheck(rekorClient, verifier, tempLogInfoFileName) if err != nil { t.Errorf("second consistency check failed: %v", err) } + _, err = rekor.IdentitySearch(0, 1, rekorClient, monitoredVals, tempOutputIdentitiesFileName, nil) + if err != nil { + log.Fatal(err.Error()) + } + tempOutputIdentities, err := os.ReadFile(tempOutputIdentitiesFileName) if err != nil { t.Errorf("error reading from output identities file: %v", err) @@ -248,4 +260,13 @@ func TestRunConsistencyCheck(t *testing.T) { if !strings.Contains(tempOutputIdentitiesString, certFingerprint) { t.Errorf("expected to find fingerprint %s, did not", certFingerprint) } + + tempMetadata, err := os.ReadFile(tempMetadataFileName) + if err != nil { + t.Errorf("error reading from output identities file: %v", err) + } + tempMetadataString := string(tempMetadata) + if !strings.Contains(tempOutputIdentitiesString, "2") { + t.Errorf("expected to find latest index 2 in %s, did not", tempMetadataString) + } } diff --git a/pkg/test/identity_workflow/identity_workflow_e2e_test.sh b/pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh similarity index 100% rename from pkg/test/identity_workflow/identity_workflow_e2e_test.sh rename to pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh From 2ea93b0f76b80f72fdec5c89eacf21fbf8aef201 Mon Sep 17 00:00:00 2001 From: linus-sun Date: Mon, 11 Nov 2024 22:36:05 +0000 Subject: [PATCH 2/2] rollback split Signed-off-by: linus-sun --- .github/workflows/reusable_monitoring.yml | 4 +--- cmd/rekor_monitor/main.go | 10 +--------- pkg/notifications/notifications.go | 2 -- pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/reusable_monitoring.yml b/.github/workflows/reusable_monitoring.yml index 0ba709e6..38eb2c1c 100644 --- a/.github/workflows/reusable_monitoring.yml +++ b/.github/workflows/reusable_monitoring.yml @@ -81,7 +81,7 @@ jobs: run: cat ${{ env.LOG_FILE }} # Skip on first run continue-on-error: true - - run: go run ./cmd/rekor_consistency --file ${{ env.LOG_FILE }} --once=${{ inputs.once }} --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}" + - run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --file ${{ env.LOG_FILE }} --once=${{ inputs.once }} --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}" - name: Upload checkpoint uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: @@ -90,8 +90,6 @@ jobs: retention-days: ${{ inputs.artifact_retention_days }} - name: Log new checkpoints run: cat ${{ env.LOG_FILE }} - - name: Run identity monitor - run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --once=${{ inputs.once }} if-succeeded: runs-on: ubuntu-latest diff --git a/cmd/rekor_monitor/main.go b/cmd/rekor_monitor/main.go index 77d4ca1b..6f774ebe 100644 --- a/cmd/rekor_monitor/main.go +++ b/cmd/rekor_monitor/main.go @@ -55,7 +55,6 @@ func main() { interval := flag.Duration("interval", 5*time.Minute, "Length of interval between each periodical consistency check") userAgentString := flag.String("user-agent", "", "details to include in the user agent string") flag.Parse() - flag.Parse() if *configFilePath == "" && *configYamlInput == "" { log.Fatalf("empty configuration input") @@ -85,16 +84,9 @@ func main() { } } - if config.ServerURL == "" { - config.ServerURL = publicRekorServerURL - } if config.OutputIdentitiesFile == "" { config.OutputIdentitiesFile = outputIdentitiesFileName } - if config.Interval == nil { - defaultInterval := time.Hour - config.Interval = &defaultInterval - } err := rekor.VerifyConsistencyCheckInputs(interval, &config.LogInfoFile, once) if err != nil { @@ -137,7 +129,7 @@ func main() { fmt.Printf("Monitoring subject %s\n", sub) } - ticker := time.NewTicker(*config.Interval) + ticker := time.NewTicker(*interval) defer ticker.Stop() // To get an immediate first tick diff --git a/pkg/notifications/notifications.go b/pkg/notifications/notifications.go index e377cc1a..8764690d 100644 --- a/pkg/notifications/notifications.go +++ b/pkg/notifications/notifications.go @@ -69,7 +69,6 @@ type IdentityMonitorConfiguration struct { StartIndex *int `yaml:"startIndex"` EndIndex *int `yaml:"endIndex"` MonitoredValues ConfigMonitoredValues `yaml:"monitoredValues"` - ServerURL string `yaml:"serverURL"` OutputIdentitiesFile string `yaml:"outputIdentities"` LogInfoFile string `yaml:"logInfoFile"` IdentityMetadataFile *string `yaml:"identityMetadataFile"` @@ -77,7 +76,6 @@ type IdentityMonitorConfiguration struct { EmailNotificationSMTP *EmailNotificationInput `yaml:"emailNotificationSMTP"` EmailNotificationMailgun *MailgunNotificationInput `yaml:"emailNotificationMailgun"` EmailNotificationSendGrid *SendGridNotificationInput `yaml:"emailNotificationSendGrid"` - Interval *time.Duration `yaml:"interval"` } func CreateNotificationPool(config IdentityMonitorConfiguration) []NotificationPlatform { diff --git a/pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh b/pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh index f4106a96..35bd126b 100755 --- a/pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh +++ b/pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh @@ -66,4 +66,4 @@ echo echo "running tests" popd -go test -tags=e2e -v -race ./pkg/test/identity_workflow/... \ No newline at end of file +go test -tags=e2e -v -race ./pkg/test/rekor_e2e/... \ No newline at end of file