diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 04164fb0..f37a15f7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ jobs: with: go-version-file: go.mod check-latest: true - - run: TARGET_DIRECTORY=. make build-kas + - run: PATH=$PATH:$GOPATH/bin TARGET_DIRECTORY=. make build-kas build-agentk: name: Build agentk runs-on: ubuntu-latest @@ -29,7 +29,27 @@ jobs: with: go-version-file: go.mod check-latest: true - - run: TARGET_DIRECTORY=. make build-agentk + - run: PATH=$PATH:$GOPATH/bin TARGET_DIRECTORY=. make build-agentk + image-kas: + name: Build kas image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - run: PATH=$PATH:$GOPATH/bin TARGET_DIRECTORY=. make docker-kas + image-agentk: + name: Build agentk image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + - run: PATH=$PATH:$GOPATH/bin TARGET_DIRECTORY=. make docker-agentk test: name: Unit test runs-on: ubuntu-latest @@ -39,7 +59,7 @@ jobs: with: go-version-file: go.mod check-latest: true - - run: make test + - run: PATH=$PATH:$GOPATH/bin make test lint: name: Lint runs-on: ubuntu-latest diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml deleted file mode 100644 index 935f33a4..00000000 --- a/.gitlab/.gitlab-ci.yml +++ /dev/null @@ -1,258 +0,0 @@ -# https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates -include: - - template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml - - template: Jobs/Dependency-Scanning.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml - - project: "gitlab-org/quality/pipeline-common" - file: - - "/ci/danger-review.yml" - -default: - tags: - - gitlab-org - -variables: - # Image built using https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent-ci-image - # and pushed into https://hub.docker.com/repository/docker/gitlab/gitlab-agent-ci-image - BUILD_IMAGE_NAME: "gitlab/gitlab-agent-ci-image" - # must use image digest to invalidate cache if image is updated. - BUILD_IMAGE_SHA: "latest@sha256:73ffac7d6fef297a904e76bd531ecea99e1e017f0d827f87deb43562d82412d5" - FIPS_BUILD_IMAGE_SHA: "latest-fips@sha256:128ac915e6d6a6b4027d1f41099bbee31ba9b3202560e1a6420c5f99cf150a9a" - DOCKER_VERSION: "24.0.6" # https://hub.docker.com/_/docker - DOCKER_TLS_CERTDIR: "/certs" - -stages: - - test - - push_image - - create_release - -danger-review: - allow_failure: true - variables: - GITLAB_DANGERFILES_VERSION: "3.7.0" - -.bazel_build: - image: - name: "$BUILD_IMAGE_NAME:$BUILD_IMAGE_SHA" - entrypoint: [""] - before_script: - - echo "build --verbose_failures" >> .bazelrc - - echo "build --curses=no" >> .bazelrc - - echo "build --show_timestamps" >> .bazelrc - - echo "common --http_timeout_scaling=3.0" >> .bazelrc - - | - if [[ -f "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then - echo "build --google_default_credentials" >> .bazelrc - echo "build --remote_cache=https://storage.googleapis.com/gitlab-kubernetes-test-bucket/$(cat .bazelversion)-$BUILD_IMAGE_SHA" >> .bazelrc - echo "build --remote_download_minimal" >> .bazelrc - echo "run --remote_download_outputs=toplevel" >> .bazelrc - fi - - echo "test --test_output=all" >> .bazelrc - - echo "test --test_arg=-test.v" >> .bazelrc - # - echo "build --sandbox_base=/dev/shm" >> .bazelrc # disabled because it's not big enough - -.registry_creds: ®istry_creds - - mkdir -p "$HOME/.docker" - - | - credentials=$(echo -n "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" | base64) - cat << EOF > "$HOME/.docker/config.json" - { - "auths": { - "$CI_REGISTRY": { - "auth": "$credentials" - } - } - } - EOF - -test: - stage: test - extends: .bazel_build - services: - - redis - tags: - - saas-linux-large-amd64 - script: - - echo 'test --test_env=REDIS_URL=redis://redis:6379' >> .bazelrc - - make test-ci - - *registry_creds - - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then make release-latest; fi - -test-fips: - image: "docker:${DOCKER_VERSION}" - services: - - "docker:${DOCKER_VERSION}-dind" - - redis - stage: test - parallel: - matrix: - - ARCH: arm64 - RUNNER_TAG: arm64 - RUN_IN_FORKS: 'false' - - ARCH: amd64 - RUNNER_TAG: saas-linux-large-amd64 - RUN_IN_FORKS: 'true' - tags: - - "${RUNNER_TAG}" - script: - - export REDIS_URL=redis://redis:6379 - - docker run --rm --platform "linux/${ARCH}" --entrypoint /bin/sh -v "$(pwd):/src" -w /src "${BUILD_IMAGE_NAME}:${FIPS_BUILD_IMAGE_SHA}" -c 'make test-ci-fips' - rules: - - if: '$RUN_IN_FORKS || $CI_PROJECT_PATH == "gitlab-org/cluster-integration/gitlab-agent"' - -verify: - stage: test - extends: .bazel_build - tags: - - saas-linux-large-amd64 - script: - - make verify-ci - -# You can override the included template(s) by including variable overrides -# See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables -sast: - variables: - # We already run gosec as part of the lint job, no point in running it again here. - SAST_DEFAULT_ANALYZERS: kubesec - SCAN_KUBERNETES_MANIFESTS: "true" - stage: test - -push_image_tag: - stage: push_image - extends: .bazel_build - rules: - - if: $CI_COMMIT_TAG - script: - - *registry_creds - - git fetch - - > - if [[ $(git branch --remotes --contains "tags/${CI_COMMIT_TAG}" | grep "${CI_DEFAULT_BRANCH}") ]]; then - make release-tag release-stable - else - make release-tag - fi - -push_image_tag_fips: - stage: push_image - image: - name: "${BUILD_IMAGE_NAME}:${FIPS_BUILD_IMAGE_SHA}" - entrypoint: [""] - services: - - "docker:${DOCKER_VERSION}-dind" - parallel: - matrix: - - ARCH: arm64 - RUNNER_TAG: arm64 - RUN_IN_FORKS: 'false' - - ARCH: amd64 - RUNNER_TAG: saas-linux-large-amd64 - RUN_IN_FORKS: 'true' - tags: - - "${RUNNER_TAG}" - variables: - BUILDER_IMAGE: "${BUILD_IMAGE_NAME}:${FIPS_BUILD_IMAGE_SHA}" - # Docker vars are documented here https://docs.docker.com/engine/reference/commandline/cli/ - # Vars below mimic logic from https://github.com/docker-library/docker/blob/master/20.10/cli/docker-entrypoint.sh - # Daemon logic: https://github.com/docker-library/docker/blob/master/20.10/dind/dockerd-entrypoint.sh - DOCKER_HOST: "tcp://docker:2376" - DOCKER_TLS_VERIFY: "1" - DOCKER_CERT_PATH: "${DOCKER_TLS_CERTDIR}/client" - rules: - - if: '$CI_COMMIT_TAG && ($RUN_IN_FORKS || $CI_PROJECT_PATH == "gitlab-org/cluster-integration/gitlab-agent")' - script: - - *registry_creds - - git fetch - - > - if [[ $(git branch --remotes --contains "tags/${CI_COMMIT_TAG}" | grep "${CI_DEFAULT_BRANCH}") ]]; then - make release-tag-and-stable-fips - else - make release-tag-fips - fi - -push_image_tag_fips_manifest: - stage: push_image - image: - name: "${BUILD_IMAGE_NAME}:${FIPS_BUILD_IMAGE_SHA}" - entrypoint: [""] - services: - - "docker:${DOCKER_VERSION}-dind" - variables: - BUILDER_IMAGE: "${BUILD_IMAGE_NAME}:${FIPS_BUILD_IMAGE_SHA}" - # Docker vars are documented here https://docs.docker.com/engine/reference/commandline/cli/ - # Vars below mimic logic from https://github.com/docker-library/docker/blob/master/20.10/cli/docker-entrypoint.sh - # Daemon logic: https://github.com/docker-library/docker/blob/master/20.10/dind/dockerd-entrypoint.sh - DOCKER_HOST: "tcp://docker:2376" - DOCKER_TLS_VERIFY: "1" - DOCKER_CERT_PATH: "${DOCKER_TLS_CERTDIR}/client" - needs: ["push_image_tag_fips"] - rules: - - if: $CI_COMMIT_TAG - script: - - *registry_creds - - git fetch - - > - if [[ $(git branch --remotes --contains "tags/${CI_COMMIT_TAG}" | grep "${CI_DEFAULT_BRANCH}") ]]; then - make release-tag-and-stable-fips-manifest - else - make release-tag-fips-manifest - fi - -push_image_manual: - stage: push_image - extends: .bazel_build - rules: - # Allow creating releases manually for any branch or tag. - - when: manual - allow_failure: true - script: - - *registry_creds - - make release-commit - -create_release: - stage: create_release - rules: - - if: $CI_COMMIT_TAG - image: registry.gitlab.com/gitlab-org/release-cli:latest - script: - - echo 'release job' - release: - name: "$CI_COMMIT_TAG" - tag_name: "$CI_COMMIT_TAG" - ref: "$CI_COMMIT_TAG" - description: "Release for tag $CI_COMMIT_TAG" - -build-package-and-qa: - stage: test - trigger: - project: 'gitlab-org/build/omnibus-gitlab-mirror' - branch: 'master' - inherit: - variables: false - variables: - GITLAB_KAS_VERSION: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH - TOP_UPSTREAM_SOURCE_REF: $CI_COMMIT_REF_NAME - TOP_UPSTREAM_SOURCE_JOB: $CI_JOB_URL - ee: "true" - rules: - # For MRs that change dependencies, we want to automatically ensure builds - # aren't broken. In such cases, we don't want the QA tests to be run - # automatically, but still available for developers to manually run. - - if: '$CI_MERGE_REQUEST_IID' - changes: - - go.sum - variables: - BUILD_ON_ALL_OS: "true" - MANUAL_QA_TEST: "true" - allow_failure: false - # For other MRs, we still provide this job as a manual job for developers - # to obtain a package for testing and run QA tests. - - if: '$CI_MERGE_REQUEST_IID' - when: manual - allow_failure: true - needs: [] diff --git a/Makefile b/Makefile index dad19e68..7b3610a1 100644 --- a/Makefile +++ b/Makefile @@ -132,45 +132,3 @@ lint: $(PRE) ## run linters .PHONY: fix fix: $(PRE) ## fix issues found by linters golangci-lint run --fix ./... - -################################ Old definitions - -# Build and push all FIPS docker images tagged with the tag on the current commit and as "stable". -# This only works on a linux machine -# Set ARCH to the desired CPU architecture. -# Set CI_REGISTRY_IMAGE to the desired container image name. -# Set BUILDER_IMAGE to the image to be used for building the agentk binary. -.PHONY: release-tag-and-stable-fips -release-tag-and-stable-fips: - docker buildx build --build-arg 'BUILDER_IMAGE=$(BUILDER_IMAGE)' --platform 'linux/$(ARCH)' --file build/agentk.ubi8-fips.Dockerfile --tag '$(FIPS_TAG_GIT_ARCH)' --tag '$(FIPS_TAG_STABLE_ARCH)' . - docker push '$(FIPS_TAG_GIT_ARCH)' - docker push '$(FIPS_TAG_STABLE_ARCH)' - -.PHONY: release-tag-and-stable-fips-manifest -release-tag-and-stable-fips-manifest: - docker manifest create '$(FIPS_TAG_GIT)' \ - --amend '$(FIPS_TAG_GIT)-amd64' \ - --amend '$(FIPS_TAG_GIT)-arm64' - docker manifest push '$(FIPS_TAG_GIT)' - docker manifest create '$(FIPS_TAG_STABLE)' \ - --amend '$(FIPS_TAG_STABLE)-amd64' \ - --amend '$(FIPS_TAG_STABLE)-arm64' - docker manifest push '$(FIPS_TAG_STABLE)' - -# Build and push all FIPS docker images tagged with the tag on the current commit. -# This only works on a linux machine -# Set ARCH to the desired CPU architecture. -# Set CI_REGISTRY_IMAGE to the desired container image name. -# Set BUILDER_IMAGE to the image to be used for building the agentk binary. -.PHONY: release-tag-fips -release-tag-fips: - docker buildx build --build-arg 'BUILDER_IMAGE=$(BUILDER_IMAGE)' --platform 'linux/$(ARCH)' --file build/agentk.ubi8-fips.Dockerfile --tag '$(FIPS_TAG_GIT_ARCH)' . - docker push '$(FIPS_TAG_GIT_ARCH)' - -.PHONY: release-tag-fips-manifest -release-tag-fips-manifest: - docker manifest create '$(FIPS_TAG_GIT)' \ - --amend '$(FIPS_TAG_GIT)-amd64' \ - --amend '$(FIPS_TAG_GIT)-arm64' - docker manifest push '$(FIPS_TAG_GIT)' - diff --git a/cmd/kas/kasapp/configured_app.go b/cmd/kas/kasapp/configured_app.go index f83f1215..ccd33f71 100644 --- a/cmd/kas/kasapp/configured_app.go +++ b/cmd/kas/kasapp/configured_app.go @@ -15,6 +15,7 @@ import ( "github.com/ash2k/stager" "github.com/getsentry/sentry-go" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/pluralsh/kuberentes-agent/cmd/kas/kasapp/plural" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/redis/rueidis" @@ -36,7 +37,6 @@ import ( _ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor "github.com/pluralsh/kuberentes-agent/cmd" - "github.com/pluralsh/kuberentes-agent/cmd/kas/kasapp/fake" "github.com/pluralsh/kuberentes-agent/internal/api" gapi "github.com/pluralsh/kuberentes-agent/internal/gitlab/api" agent_registrar_server "github.com/pluralsh/kuberentes-agent/internal/module/agent_registrar/server" @@ -310,7 +310,7 @@ func (a *ConfiguredApp) constructFakeRpcApiFactory(errRep errz.ErrReporter, sent log: a.Log, sentryHub: sentryHub, } - fAgent := fake.ServerAgentRpcApiFactory{ + fAgent := plural.ServerAgentRpcApiFactory{ RPCApiFactory: f.New, AgentInfoCache: cache.NewWithError[api.AgentToken, *api.AgentInfo]( aCfg.InfoCacheTtl.AsDuration(), diff --git a/cmd/kas/kasapp/plural/agent_rpc_api.go b/cmd/kas/kasapp/plural/agent_rpc_api.go new file mode 100644 index 00000000..f5fc5191 --- /dev/null +++ b/cmd/kas/kasapp/plural/agent_rpc_api.go @@ -0,0 +1,51 @@ +package plural + +import ( + "context" + + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" + "go.uber.org/zap" + + "github.com/pluralsh/kuberentes-agent/internal/api" + fake "github.com/pluralsh/kuberentes-agent/internal/fake/api" + "github.com/pluralsh/kuberentes-agent/internal/gitlab" + "github.com/pluralsh/kuberentes-agent/internal/module/modserver" + "github.com/pluralsh/kuberentes-agent/internal/tool/cache" +) + +type ServerAgentRpcApi struct { + modserver.RpcApi + Token api.AgentToken + AgentInfoCache *cache.CacheWithErr[api.AgentToken, *api.AgentInfo] +} + +func (a *ServerAgentRpcApi) AgentToken() api.AgentToken { + return a.Token +} + +func (a *ServerAgentRpcApi) AgentInfo(ctx context.Context, log *zap.Logger) (*api.AgentInfo, error) { + return a.getAgentInfoCached(ctx) +} + +func (a *ServerAgentRpcApi) getAgentInfoCached(ctx context.Context) (*api.AgentInfo, error) { + return a.AgentInfoCache.GetItem(ctx, a.Token, func() (*api.AgentInfo, error) { + return fake.GetAgentInfo(ctx, a.Token, gitlab.WithoutRetries()) + }) +} + +type ServerAgentRpcApiFactory struct { + RPCApiFactory modserver.RpcApiFactory + AgentInfoCache *cache.CacheWithErr[api.AgentToken, *api.AgentInfo] +} + +func (f *ServerAgentRpcApiFactory) New(ctx context.Context, fullMethodName string) (modserver.AgentRpcApi, error) { + token, err := grpc_auth.AuthFromMD(ctx, "bearer") + if err != nil { + return nil, err + } + return &ServerAgentRpcApi{ + RpcApi: f.RPCApiFactory(ctx, fullMethodName), + Token: api.AgentToken(token), + AgentInfoCache: f.AgentInfoCache, + }, nil +}