refactor workflows to split

Signed-off-by: linus-sun <[email protected]>

.github/workflows/consistency_check.yml

@@ -0,0 +1,123 @@
+# Copyright 2022 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This GHA should not be run concurrently.
+
+# TODO: Write identities to issues when found
+
+name: Rekor Monitoring Template
+
+on:
+  workflow_call:
+    inputs:
+      file_issue:
+        description: 'True to file an issue on monitoring failure'
+        required: true
+        type: boolean
+      artifact_retention_days:
+        description: 'The number of days to retain an artifact (default: 14). If this workflow runs as a cron job, it must be greater than the frequency of the job'
+        required: false
+        type: number
+        default: 14
+      identities:
+        description: 'multiline yaml of certificate subjects and issuers, key subjects, and fingerprints. For certificates, if no issuers are specified, match any OIDC provider'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  UPLOADED_LOG_NAME: checkpoint
+  LOG_FILE: checkpoint_log.txt
+
+jobs:
+  detect-workflow:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # Needed to detect the current reusable repository and ref.
+    outputs:
+      repository: ${{ steps.detect.outputs.repository }}
+      ref: ${{ steps.detect.outputs.ref }}
+    steps:
+      - name: Detect the repository and ref
+        id: detect
+        uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@5a775b367a56d5bd118a224a811bba288150a563 # v2.0.0
+
+  monitor:
+    runs-on: ubuntu-latest
+    needs: [detect-workflow]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: ${{ needs.detect-workflow.outputs.repository }}
+          ref: "${{ needs.detect-workflow.outputs.ref }}"
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        with:
+          go-version: '1.23'
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6
+        with:
+          name: ${{ env.UPLOADED_LOG_NAME }}
+        # Skip on first run since there will be no checkpoint
+        continue-on-error: true
+      - name: Log current checkpoints
+        run: cat ${{ env.LOG_FILE }}
+        # Skip on first run
+        continue-on-error: true
+      - run: go run ./cmd/rekor_consistency --file ${{ env.LOG_FILE }} --once --monitored-values "${{ inputs.identities }}" --user-agent "${{ format('{0}/{1}/{2}', needs.detect-workflow.outputs.repository, needs.detect-workflow.outputs.ref, github.run_id) }}"
+      - name: Upload checkpoint
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: ${{ env.UPLOADED_LOG_NAME }}
+          path: ${{ env.LOG_FILE }}
+          retention-days: ${{ inputs.artifact_retention_days }}
+      - name: Log new checkpoints
+        run: cat ${{ env.LOG_FILE }}
+
+  if-succeeded:
+    runs-on: ubuntu-latest
+    needs: [monitor, detect-workflow]
+    permissions:
+      issues: 'write'
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      ISSUE_REPOSITORY: ${{ github.repository }}
+    if: ${{ needs.monitor.result == 'success' && inputs.file_issue }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: ${{ needs.detect-workflow.outputs.repository }}
+          ref: "${{ needs.detect-workflow.outputs.ref }}"
+      - run: |
+          set -euo pipefail
+          ./.github/workflows/scripts/report_success.sh
+
+  if-failed:
+    runs-on: ubuntu-latest
+    needs: [monitor, detect-workflow]
+    permissions:
+      issues: 'write'
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      ISSUE_REPOSITORY: ${{ github.repository }}
+    if: ${{ always() && needs.monitor.result == 'failure' && inputs.file_issue }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: ${{ needs.detect-workflow.outputs.repository }}
+          ref: "${{ needs.detect-workflow.outputs.ref }}"
+      - run: |
+          set -euo pipefail
+          ./.github/workflows/scripts/report_failure.sh

.github/workflows/identity_monitor.yml

@@ -0,0 +1,61 @@
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This GHA should not be run concurrently.
+# TODO: Once the workflows for consistency check and identity monitor are fully split, 
+# the consistency check workflow should no longer require a checkpoint file.
+
+name: Rekor Identity Monitor Template
+
+on:
+  workflow_call:
+    inputs:
+      once:
+        default: true
+        required: false
+        type: boolean
+      config:
+        description: 'multiline yaml of configuration settings for identity monitor run'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  detect-workflow:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # Needed to detect the current reusable repository and ref.
+    outputs:
+      repository: ${{ steps.detect.outputs.repository }}
+      ref: ${{ steps.detect.outputs.ref }}
+    steps:
+      - name: Detect the repository and ref
+        id: detect
+        uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@5a775b367a56d5bd118a224a811bba288150a563 # v2.0.0
+
+  monitor:
+    runs-on: ubuntu-latest
+    needs: [detect-workflow]
+    steps:
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          repository: ${{ needs.detect-workflow.outputs.repository }}
+          ref: "${{ needs.detect-workflow.outputs.ref }}"
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version: '1.23'
+      - name: Run identity monitor
+        run: go run ./cmd/rekor_monitor --config ${{ inputs.config }} --once

.github/workflows/main.yml

@@ -88,8 +88,8 @@ jobs:
         with:
           go-version-file: './go.mod'
           check-latest: true
-      - name: run e2e test
-        run: ./pkg/test/e2e/e2e_test.sh
+      - name: run consistency check test
+        run: ./pkg/test/consistency_check/consistency_check_e2e_test.sh
       - name: run identity monitor test
         run: ./pkg/test/identity_workflow/identity_workflow_e2e_test.sh
 

cmd/rekor_consistency/main.go

@@ -51,8 +51,6 @@ func main() {
 	once := flag.Bool("once", false, "Perform consistency check once and exit")
 	monitoredValsInput := flag.String("monitored-values", "", "yaml of certificate subjects and issuers, key subjects, "+
 		"and fingerprints. For certificates, if no issuers are specified, match any OIDC provider.")
-	outputIdentitiesFile := flag.String("output-identities", outputIdentitiesFileName,
-		"Name of the file containing indices and identities found in the log. Format is \"subject issuer index uuid\"")
 	userAgentString := flag.String("user-agent", "", "details to include in the user agent string")
 	flag.Parse()
 
@@ -85,7 +83,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, outputIdentitiesFile, once)
+	err = rekor.VerifyConsistencyCheckInputs(interval, logInfoFile, once)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -100,7 +98,7 @@ func main() {
 
 	// To get an immediate first tick
 	for ; ; <-ticker.C {
-		err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile, monitoredVals, *outputIdentitiesFile)
+		err = rekor.RunConsistencyCheck(rekorClient, verifier, *logInfoFile)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "error running consistency check: %v", err)
 			return

cmd/monitor/main.go → cmd/rekor_monitor/main.go

@@ -30,7 +30,6 @@ import (
 	"github.com/sigstore/rekor-monitor/pkg/rekor"
 	"github.com/sigstore/rekor-monitor/pkg/util/file"
 	"github.com/sigstore/rekor/pkg/client"
-	"github.com/sigstore/rekor/pkg/util"
 	"gopkg.in/yaml.v2"
 	"sigs.k8s.io/release-utils/version"
 )
@@ -48,22 +47,36 @@ const (
 func main() {
 	// Command-line flags that are parameters to the verifier job
 	configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings")
+	configYamlInput := flag.String("config", "", "path to yaml configuration file containing identity monitor settings")
 	once := flag.Bool("once", true, "whether to run the monitor on a repeated interval or once")
 	flag.Parse()
 
-	if configFilePath == nil {
-		log.Fatalf("empty configuration file path")
+	if *configFilePath == "" && *configYamlInput == "" {
+		log.Fatalf("empty configuration input")
 	}
 
-	readConfig, err := os.ReadFile(*configFilePath)
-	if err != nil {
-		log.Fatalf("error reading from identity monitor configuration file: %v", err)
+	if *configFilePath != "" && *configYamlInput != "" {
+		log.Fatalf("only input one of configuration file path or yaml input")
 	}
 
-	configString := string(readConfig)
 	var config notifications.IdentityMonitorConfiguration
-	if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
-		log.Fatalf("error parsing identities: %v", err)
+
+	if *configFilePath != "" {
+		readConfig, err := os.ReadFile(*configFilePath)
+		if err != nil {
+			log.Fatalf("error reading from identity monitor configuration file: %v", err)
+		}
+
+		configString := string(readConfig)
+		if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
+	}
+
+	if *configYamlInput != "" {
+		if err := yaml.Unmarshal([]byte(*configYamlInput), &config); err != nil {
+			log.Fatalf("error parsing identities: %v", err)
+		}
 	}
 
 	rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH))))
@@ -91,33 +104,39 @@ func main() {
 	// To get an immediate first tick
 	for ; ; <-ticker.C {
 		inputEndIndex := config.EndIndex
-		if config.StartIndex == nil || config.EndIndex == nil {
-			logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "error getting log info: %v", err)
+
+		if config.StartIndex == nil {
+			if config.IdentityMetadataFile == nil {
+				fmt.Fprintf(os.Stderr, "no identity metadata file")
 				return
 			}
 
-			checkpoint, err := rekor.ReadLatestCheckpoint(logInfo)
+			idMetadata, err := file.ReadIdentityMetadata(*config.IdentityMetadataFile)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "error reading checkpoint: %v", err)
+				fmt.Fprintf(os.Stderr, "error reading from identity metadata file: %v", err)
 				return
 			}
 
-			var prevCheckpoint *util.SignedCheckpoint
-			prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile)
+			if config.StartIndex == nil {
+				config.StartIndex = &idMetadata.LatestIndex
+			}
+		}
+
+		if config.EndIndex == nil {
+			logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "reading checkpoint log: %v", err)
+				fmt.Fprintf(os.Stderr, "error getting log info: %v", err)
 				return
 			}
 
-			checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
-			if config.StartIndex == nil {
-				config.StartIndex = &checkpointStartIndex
-			}
-			if config.EndIndex == nil {
-				config.EndIndex = &checkpointEndIndex
+			checkpoint, err := rekor.ReadLatestCheckpoint(logInfo)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error reading checkpoint: %v", err)
+				return
 			}
+
+			checkpointEndIndex := rekor.GetCheckpointIndex(logInfo, checkpoint)
+			config.EndIndex = &checkpointEndIndex
 		}
 
 		if *config.StartIndex >= *config.EndIndex {

pkg/rekor/identity.go

@@ -354,17 +354,16 @@ func GetPrevCurrentCheckpoints(client *client.Rekor, logInfoFile string) (*util.
 	return prevCheckpoint, checkpoint, nil
 }
 
-// GetCheckpointIndices fetches the start and end indexes between two checkpoints and returns them.
-func GetCheckpointIndices(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint) (int, int) {
+// GetCheckpointIndex fetches the index of a checkpoint and returns it.
+func GetCheckpointIndex(logInfo *models.LogInfo, checkpoint *util.SignedCheckpoint) int {
 	// Get log size of inactive shards
 	totalSize := 0
 	for _, s := range logInfo.InactiveShards {
 		totalSize += int(*s.TreeSize)
 	}
-	startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow
-	endIndex := int(checkpoint.Size) + totalSize - 1       //nolint: gosec // G115
+	index := int(checkpoint.Size) + totalSize - 1 //nolint: gosec // G115
 
-	return startIndex, endIndex
+	return index
 }
 
 func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, monitoredValues identity.MonitoredValues, outputIdentitiesFile string, idMetadataFile *string) ([]identity.MonitoredIdentity, error) {
@@ -404,18 +403,3 @@ func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, mon
 	monitoredIdentities := identity.CreateMonitoredIdentities(idEntries, identities)
 	return monitoredIdentities, nil
 }
-
-// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file.
-func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error {
-	// Get log size of inactive shards
-	startIndex, endIndex := GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
-
-	// Search for identities in the log range
-	if identity.MonitoredValuesExist(monitoredValues) {
-		_, err := IdentitySearch(startIndex, endIndex, rekorClient, monitoredValues, outputIdentitiesFile, nil)
-		if err != nil {
-			return fmt.Errorf("error monitoring for identities: %v", err)
-		}
-	}
-	return nil
-}