Merge branch 'main' of github.com:container-registry/helm-charts-oci-…

…proxy

.github/workflows/build-and-push.yaml

@@ -0,0 +1,101 @@
+name: "Build container images"
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+  push:
+    tags:
+      - 'v*.*.*'
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+jobs:
+  tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+      - name: Setup Go Cache Paths
+        id: go-cache-paths
+        run: |
+          echo "go-build=$(go env GOCACHE)" >>$GITHUB_OUTPUT
+          echo "go-mod=$(go env GOMODCACHE)" >>$GITHUB_OUTPUT
+      - name: Go Build Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+      - name: Go Mod Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+      - name: Run Unit Tests
+        id: run-tests
+        run: |
+          ./do.sh tests
+
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-latest
+    needs: [tests]
+    outputs:
+      env_name: ${{ env.ENV_NAME }}
+      tag_name:  ${{ env.TAG_NAME }}
+    steps:
+      - name: Export Variables
+        run: echo "DATETIME=$(date +%s)" >> $GITHUB_ENV
+
+      - name: Staging (merge)
+        if: github.event_name != 'pull_request' && github.ref_name == 'main'
+        run: |
+          echo "ENV_NAME=staging" >> "$GITHUB_ENV"
+          echo "TAG_NAME=staging-${{ env.DATETIME }}" >> "$GITHUB_ENV"
+
+      - name: Staging (PR)
+        if: github.event_name == 'pull_request' && github.head_ref == 'main'
+        run: |
+          echo "ENV_NAME=staging" >> "$GITHUB_ENV"
+          echo "TAG_NAME=staging-${{ env.DATETIME }}" >> "$GITHUB_ENV"
+
+      - name: Release Tag
+        if: startsWith(github.event.ref, 'refs/tags/v')
+        run: |
+          echo "ENV_NAME=production" >> "$GITHUB_ENV"
+          TAG=${GITHUB_REF#refs/*/}
+          echo "TAG_NAME=${TAG#v}" >> "$GITHUB_ENV"
+
+  build-container:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: prepare
+    environment: ${{ needs.prepare.outputs.env_name }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.REGISTRY_ADDR }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Build & push container image
+        id: build-container
+        env:
+          REPOSITORY: ${{ vars.SERVER_IMAGE_REPO }}
+          IMAGE_TAG: ${{ needs.prepare.outputs.tag_name }}
+        run: |
+          docker build --build-arg="VERSION=$IMAGE_TAG" -t $REPOSITORY:$IMAGE_TAG .
+          docker push $REPOSITORY:$IMAGE_TAG
+
+
+
+

.github/workflows/publish_chart.yml

@@ -1,5 +1,4 @@
-
-name: "Build and Publish"
+name: "Publish Chart"
 
 on:
   pull_request:
@@ -14,7 +13,7 @@ on:
       - main
 
 jobs:
-  build:
+  publish-chart:
     runs-on: ubuntu-latest
     outputs:
       CHART_VERSION: ${{ steps.version.outputs.version }}

README.md

@@ -115,7 +115,9 @@ There are not many options in configure the application except the following.
 
 * `PORT` - specifies port, default `9000`
 * `DEBUG` - enabled debug if it's `TRUE`
-* `CACHE_TTL` - for how many seconds we have to store manifest and related blobs, default value is 60
+* `MANIFEST_CACHE_TTL` - for how long we have stores manifest and its related blobs, the default value is `60` seconds.
+* `INDEX_CACHE_TTL` - for how long we store chart index file content, the default value is `14400` seconds (4h)
+* `INDEX_ERROR_CACHE_TTL` - for how long we do not try to obtain index files again if it's failed for some reason. The default value is `30` seconds.
 * `USE_TLS` - enabled HTTP over TLS
 
 

cmd/serve.go

@@ -17,8 +17,9 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"github.com/container-registry/helm-charts-oci-proxy/registry/blobs/handler"
+	"github.com/container-registry/helm-charts-oci-proxy/registry/blobs"
 	"github.com/container-registry/helm-charts-oci-proxy/registry/blobs/handler/mem"
+	"github.com/container-registry/helm-charts-oci-proxy/registry/manifest"
 	"github.com/container-registry/helm-charts-oci-proxy/registry/registry"
 	"github.com/dgraph-io/ristretto"
 	"k8s.io/utils/env"
@@ -60,7 +61,10 @@ Contents are only stored in memory, and when the process exits, pushed data is l
 			}
 
 			debug, _ := env.GetBool("DEBUG", false)
-			cacheTTL, _ := env.GetInt("CACHE_TTL", 60)
+			cacheTTL, _ := env.GetInt("MANIFEST_CACHE_TTL", 60)              // 1 minute
+			indexCacheTTL, _ := env.GetInt("INDEX_CACHE_TTL", 3600*4)        // 4 hours
+			indexErrorCacheTTL, _ := env.GetInt("INDEX_ERROR_CACHE_TTL", 30) // 30 seconds
+
 			useTLS, _ := env.GetBool("USE_TLS", false)
 			certFile := env.GetString("CERT_FILE", "certs/registry.pem")
 			keyfileFile := env.GetString("KEY_FILE", "certs/registry-key.pem")
@@ -81,30 +85,25 @@ Contents are only stored in memory, and when the process exits, pushed data is l
 				l.Fatalln(err)
 			}
 
-			//db, err := badger.Open(badger.DefaultOptions(dbLocation).
-			//	WithCompression(options.None).
-			//	WithBloomFalsePositive(0).
-			//	WithMemTableSize(1024 * 1204 * 8), //8mb instead of 64
-			//)
-			//if err != nil {
-			//	log.Fatal(err)
-			//}
-			//defer db.Close()
+			blobsHandler := mem.NewMemHandler()
 
-			var blobsHandler handler.BlobHandler
+			manifests := manifest.NewManifests(ctx, blobsHandler, manifest.Config{
+				Debug:              debug,
+				CacheTTL:           time.Duration(cacheTTL) * time.Second,
+				IndexCacheTTL:      time.Duration(indexCacheTTL) * time.Second,
+				IndexErrorCacheTTl: time.Duration(indexErrorCacheTTL) * time.Second,
+			}, indexCache, l)
 
-			//blobsHandler = badger2.NewHandler(db)
-			blobsHandler = mem.NewMemHandler()
+			blobsHttpHandler := blobs.NewBlobs(blobsHandler, l)
 			//blobsHandler = file.NewHandler(dbLocation)
 			s := &http.Server{
 				ReadHeaderTimeout: 5 * time.Second, // prevent slowloris, quiet linter
-				Handler: registry.New(ctx,
-					registry.Debug(debug),
-					registry.CacheTTL(cacheTTL),
-					registry.Logger(l),
-					registry.IndexCache(indexCache),
-					registry.BlobsHandler(blobsHandler),
-				),
+				Handler: registry.New(
+					manifests.Handle,
+					blobsHttpHandler.Handle,
+					manifests.HandleTags,
+					manifests.HandleCatalog,
+					registry.Debug(debug), registry.Logger(l)),
 			}
 
 			errCh := make(chan error)

do.sh

@@ -31,6 +31,11 @@ run() {
 tests() {
   go test -v ./...
 }
+
+list_of_actions() {
+  act -l --container-architecture linux/amd64
+}
+
 "$@" # <- execute the task
 
 [ "$#" -gt 0 ] || printf "Usage:\n\t./do.sh %s\n" "($(compgen -A function | grep '^[^_]' | paste -sd '|' -))"

registry/blobs/blobs.go

@@ -7,8 +7,8 @@ import (
 	"github.com/container-registry/helm-charts-oci-proxy/registry/blobs/handler"
 	"github.com/container-registry/helm-charts-oci-proxy/registry/errors"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/sirupsen/logrus"
 	"io"
-	"log"
 	"net/http"
 	"path"
 	"strings"
@@ -23,14 +23,14 @@ type Blobs struct {
 	// Each upload gets a unique id that writes occur to until finalized.
 	// Temporary storage
 	lock sync.Mutex
-	log  *log.Logger
+	log  logrus.StdLogger
 }
 
-func NewBlobs(blobHandler handler.BlobHandler, log *log.Logger) *Blobs {
+func NewBlobs(blobHandler handler.BlobHandler, log logrus.StdLogger) *Blobs {
 	return &Blobs{handler: blobHandler, log: log}
 }
 
-func (b *Blobs) Handle(resp http.ResponseWriter, req *http.Request) *errors.RegError {
+func (b *Blobs) Handle(resp http.ResponseWriter, req *http.Request) error {
 	ctx := req.Context()
 
 	elem := strings.Split(req.URL.Path, "/")

registry/errors/error.go

@@ -16,6 +16,7 @@ package errors
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 )
 
@@ -25,6 +26,10 @@ type RegError struct {
 	Message string
 }
 
+func (r *RegError) Error() string {
+	return fmt.Sprintf("error: status: %d; code: %s; %s", r.Status, r.Code, r.Message)
+}
+
 func (r *RegError) Write(resp http.ResponseWriter) error {
 	resp.WriteHeader(r.Status)
 

registry/manifest/cache.go

@@ -0,0 +1,8 @@
+package manifest
+
+import "time"
+
+type Cache interface {
+	SetWithTTL(key, value interface{}, cost int64, ttl time.Duration) bool
+	Get(key interface{}) (interface{}, bool)
+}

registry/manifest/charts.go

@@ -19,7 +19,6 @@ import (
 	"path/filepath"
 	"sigs.k8s.io/yaml"
 	"strings"
-	"time"
 )
 
 func (m *Manifests) prepareChart(ctx context.Context, repo string, reference string) *errors.RegError {
@@ -68,7 +67,7 @@ func (m *Manifests) prepareChart(ctx context.Context, repo string, reference str
 
 	u, err := url.Parse(chartVer.URLs[0])
 	if err != nil {
-		errors.RegErrInternal(err)
+		return errors.RegErrInternal(err)
 	}
 	if u.IsAbs() {
 		downloadUrl = u.String()
@@ -94,6 +93,7 @@ func (m *Manifests) prepareChart(ctx context.Context, repo string, reference str
 			ocispec.AnnotationTitle: "$config",
 		},
 	}
+
 	err = memStore.Push(ctx, desc, bytes.NewReader(configData))
 	if err != nil {
 		return errors.RegErrInternal(err)
@@ -159,21 +159,19 @@ func (m *Manifests) GetIndex(repoURLPath string) (*repo.IndexFile, error) {
 		err error
 	}
 
-	if m.indexCache == nil {
-		return m.downloadIndex(repoURLPath)
-	}
-
-	c, ok := m.indexCache.Get(repoURLPath)
+	c, ok := m.cache.Get(repoURLPath)
 
 	if !ok || c == nil {
 		// nothing in the cache
 		res := &cacheResp{}
 		res.c, res.err = m.downloadIndex(repoURLPath)
+
+		var ttl = m.config.IndexCacheTTL
 		if res.err != nil {
-			m.indexCache.SetWithTTL(repoURLPath, res, 10, time.Second*5)
-		} else {
-			m.indexCache.Set(repoURLPath, res, 10)
+			// cache error too to avoid external resource exhausting
+			ttl = m.config.IndexErrorCacheTTl
 		}
+		m.cache.SetWithTTL(repoURLPath, res, 1000, ttl)
 		return res.c, res.err
 	}
 
@@ -186,7 +184,7 @@ func (m *Manifests) GetIndex(repoURLPath string) (*repo.IndexFile, error) {
 
 func (m *Manifests) downloadIndex(repoURLPath string) (*repo.IndexFile, error) {
 	url := fmt.Sprintf("https://%s/index.yaml", repoURLPath)
-	if m.debug {
+	if m.config.Debug {
 		m.log.Printf("download index: %s\n", url)
 	}
 	data, err := m.getIndexBytes(url)
@@ -223,25 +221,25 @@ func (m *Manifests) downloadIndex(repoURLPath string) (*repo.IndexFile, error) {
 }
 
 func (m *Manifests) getIndexBytes(url string) ([]byte, error) {
-	if m.indexCache == nil {
-		return m.download(url)
-	}
+
 	type cacheResp struct {
 		c   []byte
 		err error
 	}
 
-	c, ok := m.indexCache.Get(url)
+	c, ok := m.cache.Get(url)
 
 	if !ok || c == nil {
 		// nothing in the cache
 		res := &cacheResp{}
 		res.c, res.err = m.download(url)
+
+		var ttl = m.config.IndexCacheTTL
 		if res.err != nil {
-			m.indexCache.SetWithTTL(url, res, 10, time.Second*5)
-		} else {
-			m.indexCache.Set(url, res, 10)
+			// cache error too to avoid external resource exhausting
+			ttl = m.config.IndexErrorCacheTTl
 		}
+		m.cache.SetWithTTL(url, res, 1000, ttl)
 		return res.c, res.err
 	}
 
@@ -254,7 +252,7 @@ func (m *Manifests) getIndexBytes(url string) ([]byte, error) {
 }
 
 func (m *Manifests) download(url string) ([]byte, error) {
-	if m.debug {
+	if m.config.Debug {
 		m.log.Printf("downloading : %s\n", url)
 	}
 	resp, err := http.Get(url)

registry/manifest/config.go

@@ -0,0 +1,10 @@
+package manifest
+
+import "time"
+
+type Config struct {
+	Debug              bool
+	CacheTTL           time.Duration // for how long store manifest
+	IndexCacheTTL      time.Duration
+	IndexErrorCacheTTl time.Duration
+}