Skip to content

Commit

Permalink
Merge pull request #3 from pluralsh/master
Browse files Browse the repository at this point in the history
Initial changes
  • Loading branch information
zreigz authored Oct 19, 2023
2 parents 8daa08c + 88acf66 commit 9c0fc77
Show file tree
Hide file tree
Showing 13 changed files with 432 additions and 101 deletions.
44 changes: 44 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: CI
on:
push:
branches:
- master
pull_request:
branches:
- master
env:
GOPATH: /home/runner/go/
GOPROXY: "https://proxy.golang.org"
jobs:
build-kas:
name: Build kas
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version-file: go.mod
check-latest: true
- run: TARGET_DIRECTORY=. make kas
build-agentk:
name: Build agentk
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version-file: go.mod
check-latest: true
- run: TARGET_DIRECTORY=. make agentk
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version-file: go.mod
check-latest: true
- uses: golangci/golangci-lint-action@v3
with:
version: latest
36 changes: 36 additions & 0 deletions .github/workflows/trivy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Trivy
on:
workflow_dispatch:
pull_request:
branches:
- main
push:
branches:
- main
schedule:
- cron: "0 0 * * *"
jobs:
trivy-scan:
name: Trivy IaC scan
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write
actions: read
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Run Trivy vulnerability scanner in IaC mode
uses: aquasecurity/trivy-action@master
with:
scan-type: 'fs'
hide-progress: false
scan-ref: '.'
format: 'sarif'
output: 'trivy-results.sarif'
scanners: 'vuln,secret,config'
ignore-unfixed: true
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: 'trivy-results.sarif'
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,8 @@ go_build_kas*
.DS_Store

# settings for Jetbrains Goland
.idea
.idea

# Dist binaries
agentk
kas
51 changes: 0 additions & 51 deletions .gitlab/.gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ variables:

stages:
- test
- scan
- push_image
- create_release

Expand Down Expand Up @@ -112,22 +111,6 @@ verify:
script:
- make verify-ci

lint:
image: golangci/golangci-lint:v1.54.0-alpine
stage: test
tags:
- saas-linux-large-amd64
script:
# Write the code coverage report to gl-code-quality-report.json
# and print linting issues to stdout in the format: path/to/file:line description
- apk --no-cache add jq
- golangci-lint run --out-format code-climate | tee gl-code-quality-report.json | jq -r '.[] | "\(.location.path):\(.location.lines.begin) \(.description)"'
artifacts:
reports:
codequality: gl-code-quality-report.json
paths:
- gl-code-quality-report.json

# You can override the included template(s) by including variable overrides
# See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Note that environment variables can be set in several places
Expand Down Expand Up @@ -243,40 +226,6 @@ create_release:
ref: "$CI_COMMIT_TAG"
description: "Release for tag $CI_COMMIT_TAG"

# Broken, disable for now https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/jobs/2308411911
## see https://docs.gitlab.com/ee/user/project/code_intelligence.html#configuration
#code_navigation:
# image: sourcegraph/lsif-go:v1
# allow_failure: true # recommended
# script:
# - lsif-go
# artifacts:
# reports:
# lsif: dump.lsif

container_scanning:
stage: scan
variables:
CI_APPLICATION_REPOSITORY: "$CI_REGISTRY_IMAGE/agentk"
CI_APPLICATION_TAG: latest

# enable container scanning with https://gitlab.com/gitlab-com/gl-security/appsec/container-scanners
appsec_container_scanning:
stage: scan
image: registry.gitlab.com/gitlab-com/gl-security/appsec/container-scanners:latest
environment:
name: appsec_container_scanning
action: prepare
script:
- /run/appsec-container-scan "$CI_REGISTRY_IMAGE/agentk:latest,$CI_REGISTRY_IMAGE/agentk:latest-race,$CI_REGISTRY_IMAGE/agentk-fips:stable" > gl-container-scanning-report.json
artifacts:
reports:
container_scanning: gl-container-scanning-report.json
rules:
# Skip on forks, because external contributors can't run this pipeline
- if: $CONTAINER_SCAN_PIPELINE_TRIGGER_TOKEN
allow_failure: true

build-package-and-qa:
stage: test
trigger:
Expand Down
13 changes: 13 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM golang:1.21

WORKDIR /usr/src/app

# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
COPY go.mod go.sum ./
RUN go mod download && go mod verify

COPY . .
RUN go build \
-o /usr/local/bin/app ./cmd/kas

CMD ["app"]
2 changes: 1 addition & 1 deletion cmd/agentk/agentkapp/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ import (
)

const (
defaultLogLevel agentcfg.LogLevelEnum = 0 // whatever is 0 is the default value
defaultLogLevel agentcfg.LogLevelEnum = 3 // whatever is 0 is the default value
defaultGrpcLogLevel = agentcfg.LogLevelEnum_error

defaultObservabilityListenNetwork = "tcp"
Expand Down
68 changes: 49 additions & 19 deletions cmd/kas/kasapp/configured_app.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,27 @@ import (
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/redis/rueidis"
"github.com/redis/rueidis/rueidisotel"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
promexp "go.opentelemetry.io/otel/exporters/prometheus"
otelmetric "go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
metricsdk "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"golang.org/x/time/rate"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
_ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor
"google.golang.org/grpc/stats"

"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/cmd/kas/kasapp/fake"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/api"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/gitaly"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/gitaly/vendored/client"
Expand Down Expand Up @@ -56,24 +76,6 @@ import (
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/retry"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/tlstool"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/pkg/kascfg"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
promexp "go.opentelemetry.io/otel/exporters/prometheus"
otelmetric "go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
metricsdk "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"golang.org/x/time/rate"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
_ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor
"google.golang.org/grpc/stats"
)

const (
Expand Down Expand Up @@ -176,7 +178,8 @@ func (a *ConfiguredApp) Run(ctx context.Context) (retErr error) {
grpcServerErrorReporter := &serverErrorReporter{log: a.Log, errReporter: errRep}

// RPC API factory
rpcApiFactory, agentRpcApiFactory := a.constructRpcApiFactory(errRep, sentryHub, gitLabClient, redisClient, dt)
// Plural: Use fake factory
rpcApiFactory, agentRpcApiFactory := a.constructFakeRpcApiFactory(errRep, sentryHub, redisClient, dt)

// Server for handling API requests from other kas instances
privateApiSrv, err := newPrivateApiServer(a.Log, errRep, a.Configuration, tp, mp, p, csh, ssh, rpcApiFactory, // nolint: contextcheck
Expand Down Expand Up @@ -376,6 +379,33 @@ func (a *ConfiguredApp) constructRpcApiFactory(errRep errz.ErrReporter, sentryHu
return f.New, fAgent.New
}

func (a *ConfiguredApp) constructFakeRpcApiFactory(errRep errz.ErrReporter, sentryHub *sentry.Hub, redisClient rueidis.Client, dt trace.Tracer) (modserver.RpcApiFactory, modserver.AgentRpcApiFactory) {
aCfg := a.Configuration.Agent
f := serverRpcApiFactory{
log: a.Log,
sentryHub: sentryHub,
}
fAgent := fake.ServerAgentRpcApiFactory{
RPCApiFactory: f.New,
AgentInfoCache: cache.NewWithError[api.AgentToken, *api.AgentInfo](
aCfg.InfoCacheTtl.AsDuration(),
aCfg.InfoCacheErrorTtl.AsDuration(),
&redistool.ErrCacher[api.AgentToken]{
Log: a.Log,
ErrRep: errRep,
Client: redisClient,
ErrMarshaler: prototool.ProtoErrMarshaler{},
KeyToRedisKey: func(key api.AgentToken) string {
return a.Configuration.Redis.KeyPrefix + ":agent_info_errs:" + string(api.AgentToken2key(key))
},
},
dt,
gapi.IsCacheableError,
),
}
return f.New, fAgent.New
}

func (a *ConfiguredApp) constructAgentTracker(errRep errz.ErrReporter, redisClient rueidis.Client) agent_tracker.Tracker {
cfg := a.Configuration
return agent_tracker.NewRedisTracker(
Expand Down
53 changes: 53 additions & 0 deletions cmd/kas/kasapp/fake/agent_rpc_api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package fake

import (
"context"
"sync"

grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth"
"go.uber.org/zap"

"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/api"
fake "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/fake/api"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/gitlab"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/modserver"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/cache"
)

type ServerAgentRpcApi struct {
modserver.RpcApi
Token api.AgentToken
AgentInfoCache *cache.CacheWithErr[api.AgentToken, *api.AgentInfo]
agentIdAttrOnce sync.Once

Check failure on line 21 in cmd/kas/kasapp/fake/agent_rpc_api.go

View workflow job for this annotation

GitHub Actions / Lint

field `agentIdAttrOnce` is unused (unused)
}

func (a *ServerAgentRpcApi) AgentToken() api.AgentToken {
return a.Token
}

func (a *ServerAgentRpcApi) AgentInfo(ctx context.Context, log *zap.Logger) (*api.AgentInfo, error) {
return a.getAgentInfoCached(ctx)
}

func (a *ServerAgentRpcApi) getAgentInfoCached(ctx context.Context) (*api.AgentInfo, error) {
return a.AgentInfoCache.GetItem(ctx, a.Token, func() (*api.AgentInfo, error) {
return fake.GetAgentInfo(ctx, a.Token, gitlab.WithoutRetries())
})
}

type ServerAgentRpcApiFactory struct {
RPCApiFactory modserver.RpcApiFactory
AgentInfoCache *cache.CacheWithErr[api.AgentToken, *api.AgentInfo]
}

func (f *ServerAgentRpcApiFactory) New(ctx context.Context, fullMethodName string) (modserver.AgentRpcApi, error) {
token, err := grpc_auth.AuthFromMD(ctx, "bearer")
if err != nil {
return nil, err
}
return &ServerAgentRpcApi{
RpcApi: f.RPCApiFactory(ctx, fullMethodName),
Token: api.AgentToken(token),
AgentInfoCache: f.AgentInfoCache,
}, nil
}
29 changes: 29 additions & 0 deletions internal/fake/api/get_agent_info.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package api

import (
"context"

"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/pkg/entity"

"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/api"
"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/gitlab"
)

const (
AgentInfoApiPath = "/api/v4/internal/kubernetes/agent_info"
)

func GetAgentInfo(ctx context.Context, agentToken api.AgentToken, opts ...gitlab.DoOption) (*api.AgentInfo, error) {
return &api.AgentInfo{
Id: 123456,
ProjectId: 0,
Name: "fake-agent",
Repository: nil,
DefaultBranch: "",
GitalyInfo: &entity.GitalyInfo{
Address: "127.0.0.1",
Token: "123",
Features: nil,
},
}, nil
}
2 changes: 1 addition & 1 deletion internal/module/agent_configuration/server/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ type Factory struct {

func (f *Factory) New(config *modserver.Config) (modserver.Module, error) {
agentCfg := config.Config.Agent.Configuration
rpc.RegisterAgentConfigurationServer(config.AgentServer, &server{
rpc.RegisterAgentConfigurationServer(config.AgentServer, &fakeServer{
serverApi: config.Api,
gitaly: config.Gitaly,
gitLabClient: config.GitLabClient,
Expand Down
Loading

0 comments on commit 9c0fc77

Please sign in to comment.