From c33e71cebe556bf0364ab3138d78c7e3238b00fb Mon Sep 17 00:00:00 2001 From: Quentin Lemaire Date: Thu, 15 Sep 2022 19:23:18 +0200 Subject: [PATCH] Setup webhooks --- .github/workflows/main.yml | 7 - PROJECT | 4 + README.md | 5 - api/v1alpha1/feed_types.go | 38 ++-- api/v1alpha1/feed_types_test.go | 6 +- api/v1alpha1/feed_webhook.go | 123 +++++++++++ api/v1alpha1/feed_webhook_test.go | 134 ++++++++++++ api/v1alpha1/webhook_suite_test.go | 135 ++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 19 +- config/certmanager/certificate.yaml | 25 +++ config/certmanager/kustomization.yaml | 5 + config/certmanager/kustomizeconfig.yaml | 8 + config/crd/bases/putio.skynewz.dev_feeds.yaml | 24 +-- config/crd/kustomization.yaml | 4 +- config/default/kustomization.yaml | 202 +++++++++--------- config/default/manager_webhook_patch.yaml | 23 ++ config/default/webhookcainjection_patch.yaml | 15 ++ config/webhook/kustomization.yaml | 6 + config/webhook/kustomizeconfig.yaml | 22 ++ config/webhook/manifests.yaml | 54 +++++ config/webhook/service.yaml | 13 ++ controllers/feed_controller.go | 8 +- controllers/feed_controller_test.go | 22 +- internal/putio/feed.go | 18 +- internal/putio/putio.go | 21 +- internal/putio/putio_test.go | 6 +- main.go | 17 +- 27 files changed, 764 insertions(+), 200 deletions(-) create mode 100644 api/v1alpha1/feed_webhook.go create mode 100644 api/v1alpha1/feed_webhook_test.go create mode 100644 api/v1alpha1/webhook_suite_test.go create mode 100644 config/certmanager/certificate.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/service.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 01d15bb..e238f33 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,13 +57,6 @@ jobs: - name: Run tests run: make test - integration-tests: - runs-on: ubuntu-latest - name: Kubernetes tests - steps: - - name: Setup kind - run: echo "Running integration tests" - lint: runs-on: ubuntu-latest name: Go lint diff --git a/PROJECT b/PROJECT index a00b383..cce9d8b 100644 --- a/PROJECT +++ b/PROJECT @@ -13,4 +13,8 @@ resources: kind: Feed path: github.com/SkYNewZ/putio-operator/api/v1alpha1 version: v1alpha1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/README.md b/README.md index fcd3b1e..da6912a 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -## TODOs - -- [ ] Write integration tests -- [ ] Run integrations tests in CI \ No newline at end of file diff --git a/api/v1alpha1/feed_types.go b/api/v1alpha1/feed_types.go index c1ed66d..67a574e 100644 --- a/api/v1alpha1/feed_types.go +++ b/api/v1alpha1/feed_types.go @@ -31,38 +31,38 @@ type AuthSecretReference struct { // FeedSpec defines the desired state of Feed. type FeedSpec struct { // +kubebuilder:validation:MinLength:=1 - // Title of the RSS feed as will appear on the site + // Title of the RSS feed as will appear on the site. Title string `json:"title"` // +kubebuilder:validation:MinLength:=1 - // The URL of the RSS feed to be watched + // The URL of the RSS feed to be watched. RssSourceURL string `json:"rss_source_url"` - // +kubebuilder:default:=0 - // The file ID of the folder to place the RSS feed files in - ParentDirID *uint32 `json:"parent_dir_id,omitempty"` + // The file ID of the folder to place the RSS feed files in. Default to the root directory (0). + // +optional + ParentDirID *uint `json:"parent_dir_id,omitempty"` - // +kubebuilder:default:=false - // Should old files in the folder be deleted when space is low - DeleteOldFiles bool `json:"delete_old_files,omitempty"` + // Should old files in the folder be deleted when space is low. Default to false. + // +optional + DeleteOldFiles *bool `json:"delete_old_files,omitempty"` - // +kubebuilder:default:=false - // Should the current items in the feed, at creation time, be ignored - DontProcessWholeFeed bool `json:"dont_process_whole_feed,omitempty"` + // Should the current items in the feed, at creation time, be ignored. + // +optional + DontProcessWholeFeed *bool `json:"dont_process_whole_feed,omitempty"` // +kubebuilder:validation:MinLength:=1 - // Only items with titles that contain any of these words will be transferred (comma-separated list of words) + // Only items with titles that contain any of these words will be transferred (comma-separated list of words). Keyword string `json:"keyword"` + // No items with titles that contain any of these words will be transferred (comma-separated list of words). // +optional - // No items with titles that contain any of these words will be transferred (comma-separated list of words) UnwantedKeywords string `json:"unwanted_keywords,omitempty"` - // +kubebuilder:default:=false - // Should the RSS feed be created in the paused state - Paused bool `json:"paused,omitempty"` + // Should the RSS feed be created in the paused state. Default to false. + // +optional + Paused *bool `json:"paused,omitempty"` - // Authentication reference to Put.io token in a secret + // Authentication reference to Put.io token in a secret. AuthSecretRef AuthSecretReference `json:"authSecretRef"` } @@ -94,8 +94,8 @@ type Feed struct { Status FeedStatus `json:"status,omitempty"` } -func (in *Feed) AuthSecretRef() AuthSecretReference { - return in.Spec.AuthSecretRef +func (r *Feed) AuthSecretRef() AuthSecretReference { + return r.Spec.AuthSecretRef } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/feed_types_test.go b/api/v1alpha1/feed_types_test.go index 34f11e7..a1d3148 100644 --- a/api/v1alpha1/feed_types_test.go +++ b/api/v1alpha1/feed_types_test.go @@ -28,11 +28,11 @@ func TestFeed_AuthSecretRef(t *testing.T) { Title: "", RssSourceURL: "", ParentDirID: nil, - DeleteOldFiles: false, - DontProcessWholeFeed: false, + DeleteOldFiles: new(bool), + DontProcessWholeFeed: new(bool), Keyword: "", UnwantedKeywords: "", - Paused: false, + Paused: new(bool), AuthSecretRef: AuthSecretReference{ Name: "foo", Key: "bar", diff --git a/api/v1alpha1/feed_webhook.go b/api/v1alpha1/feed_webhook.go new file mode 100644 index 0000000..d84a14f --- /dev/null +++ b/api/v1alpha1/feed_webhook.go @@ -0,0 +1,123 @@ +/* +Copyright 2022 Quentin Lemaire . + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "net/url" + + "go.opentelemetry.io/otel" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + // log is for logging in this package. + feedlog = logf.Log.WithName("feed-resource") + tracer = otel.GetTracerProvider().Tracer("webhook") +) + +const defaultParentDirID uint = 0 + +func (r *Feed) SetupWebhookWithManager(mgr ctrl.Manager) error { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.SetupWebhookWithManager") + defer span.End() + + //nolint:wrapcheck + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-putio-skynewz-dev-v1alpha1-feed,mutating=true,failurePolicy=fail,sideEffects=None,groups=putio.skynewz.dev,resources=feeds,verbs=create;update,versions=v1alpha1,name=mfeed.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &Feed{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (r *Feed) Default() { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.Default") + defer span.End() + + feedlog.Info("default", "name", r.Name) + + if r.Spec.ParentDirID == nil { + r.Spec.ParentDirID = new(uint) + *r.Spec.ParentDirID = defaultParentDirID + } + + if r.Spec.DeleteOldFiles == nil { + r.Spec.DeleteOldFiles = new(bool) + } + + if r.Spec.DontProcessWholeFeed == nil { + r.Spec.DontProcessWholeFeed = new(bool) + } + + if r.Spec.Paused == nil { + r.Spec.Paused = new(bool) + } +} + +//+kubebuilder:webhook:path=/validate-putio-skynewz-dev-v1alpha1-feed,mutating=false,failurePolicy=fail,sideEffects=None,groups=putio.skynewz.dev,resources=feeds,verbs=create;update,versions=v1alpha1,name=vfeed.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &Feed{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *Feed) ValidateCreate() error { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.ValidateCreate") + defer span.End() + + feedlog.Info("validate create", "name", r.Name) + return r.validateFeedSpec() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *Feed) ValidateUpdate(_ runtime.Object) error { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.ValidateUpdate") + defer span.End() + + feedlog.Info("validate update", "name", r.Name) + return r.validateFeedSpec() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *Feed) ValidateDelete() error { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.ValidateDelete") + defer span.End() + + feedlog.Info("validate delete", "name", r.Name) + return nil // nothing to validate on deletion +} + +func (r *Feed) validateFeedSpec() error { + _, span := tracer.Start(context.Background(), "v1alpha1.Feed.validateFeedSpec") + defer span.End() + + // validate URL + return r.validateRSSSourceURL(r.Spec.RssSourceURL, field.NewPath("spec").Child("rss_source_url")) +} + +func (r *Feed) validateRSSSourceURL(u string, fldPath *field.Path) error { + if _, err := url.ParseRequestURI(u); err != nil { + return field.Invalid(fldPath, u, "invalid URL provided") + } + + return nil +} diff --git a/api/v1alpha1/feed_webhook_test.go b/api/v1alpha1/feed_webhook_test.go new file mode 100644 index 0000000..6fbe949 --- /dev/null +++ b/api/v1alpha1/feed_webhook_test.go @@ -0,0 +1,134 @@ +package v1alpha1 + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestFeed_validateRSSSourceURL(t *testing.T) { + type fields struct { + TypeMeta v1.TypeMeta + ObjectMeta v1.ObjectMeta + Spec FeedSpec + Status FeedStatus + } + type args struct { + u string + fldPath *field.Path + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "valid URL", + fields: fields{}, + args: args{ + u: "https://google.fr", + fldPath: field.NewPath("spec").Child("rss_source_url"), + }, + wantErr: false, + }, + { + name: "invalid url", + fields: fields{}, + args: args{ + u: "foo bar", + fldPath: field.NewPath("spec").Child("rss_source_url"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Feed{ + TypeMeta: tt.fields.TypeMeta, + ObjectMeta: tt.fields.ObjectMeta, + Spec: tt.fields.Spec, + Status: tt.fields.Status, + } + if err := r.validateRSSSourceURL(tt.args.u, tt.args.fldPath); (err != nil) != tt.wantErr { + t.Errorf("validateRSSSourceURL() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +var _ = Describe("Feed webhook", func() { + // Define utility constants for object names and testing timeouts/durations and intervals. + const ( + FeedName = "test-cronjob" + FeedNamespace = "default" + ) + + Context("When creating Feed", func() { + It("Should validate Feed URL", func() { + By("By giving a wrong URL") + ctx := context.Background() + feed := &Feed{ + TypeMeta: v1.TypeMeta{ + Kind: "Feed", + APIVersion: "putio.skynewz.dev/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: FeedName, + Namespace: FeedNamespace, + }, + Spec: FeedSpec{ + Title: "foo", + RssSourceURL: "foo bar", + ParentDirID: nil, + DeleteOldFiles: new(bool), + DontProcessWholeFeed: new(bool), + Keyword: "foo", + UnwantedKeywords: "", + Paused: new(bool), + AuthSecretRef: AuthSecretReference{ + Name: "putio-token", + Key: "token", + }, + }, + Status: FeedStatus{}, + } + + expectedError := "admission webhook \"vfeed.kb.io\" denied the request: spec.rss_source_url: Invalid value: \"foo bar\": invalid URL provided" + Expect(k8sClient.Create(ctx, feed)).Should(MatchError(expectedError)) + + By("By giving a valid URL") + feed = &Feed{ + TypeMeta: v1.TypeMeta{ + Kind: "Feed", + APIVersion: "putio.skynewz.dev/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: FeedName, + Namespace: FeedNamespace, + }, + Spec: FeedSpec{ + Title: "foo", + RssSourceURL: "https://www.google.fr", + ParentDirID: nil, + DeleteOldFiles: new(bool), + DontProcessWholeFeed: new(bool), + Keyword: "foo", + UnwantedKeywords: "", + Paused: new(bool), + AuthSecretRef: AuthSecretReference{ + Name: "putio-token", + Key: "token", + }, + }, + Status: FeedStatus{}, + } + + Expect(k8sClient.Create(ctx, feed)).Should(Succeed()) + }) + }) +}) diff --git a/api/v1alpha1/webhook_suite_test.go b/api/v1alpha1/webhook_suite_test.go new file mode 100644 index 0000000..f34a647 --- /dev/null +++ b/api/v1alpha1/webhook_suite_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 Quentin Lemaire . + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + //+kubebuilder:scaffold:imports + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Webhook Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := runtime.NewScheme() + err = AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + LeaderElection: false, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&Feed{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + conn.Close() + return nil + }).Should(Succeed()) + +}, 60) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9822f13..c2b7923 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -105,7 +105,22 @@ func (in *FeedSpec) DeepCopyInto(out *FeedSpec) { *out = *in if in.ParentDirID != nil { in, out := &in.ParentDirID, &out.ParentDirID - *out = new(uint32) + *out = new(uint) + **out = **in + } + if in.DeleteOldFiles != nil { + in, out := &in.DeleteOldFiles, &out.DeleteOldFiles + *out = new(bool) + **out = **in + } + if in.DontProcessWholeFeed != nil { + in, out := &in.DontProcessWholeFeed, &out.DontProcessWholeFeed + *out = new(bool) + **out = **in + } + if in.Paused != nil { + in, out := &in.Paused, &out.Paused + *out = new(bool) **out = **in } out.AuthSecretRef = in.AuthSecretRef diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 0000000..7301f1b --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..d1d835a --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/bases/putio.skynewz.dev_feeds.yaml b/config/crd/bases/putio.skynewz.dev_feeds.yaml index 8f484db..8f1856e 100644 --- a/config/crd/bases/putio.skynewz.dev_feeds.yaml +++ b/config/crd/bases/putio.skynewz.dev_feeds.yaml @@ -65,7 +65,7 @@ spec: description: FeedSpec defines the desired state of Feed. properties: authSecretRef: - description: Authentication reference to Put.io token in a secret + description: Authentication reference to Put.io token in a secret. properties: key: minLength: 1 @@ -78,41 +78,37 @@ spec: - name type: object delete_old_files: - default: false description: Should old files in the folder be deleted when space - is low + is low. Default to false. type: boolean dont_process_whole_feed: - default: false description: Should the current items in the feed, at creation time, - be ignored + be ignored. type: boolean keyword: description: Only items with titles that contain any of these words - will be transferred (comma-separated list of words) + will be transferred (comma-separated list of words). minLength: 1 type: string parent_dir_id: - default: 0 description: The file ID of the folder to place the RSS feed files - in - format: int32 + in. Default to the root directory (0). type: integer paused: - default: false - description: Should the RSS feed be created in the paused state + description: Should the RSS feed be created in the paused state. Default + to false. type: boolean rss_source_url: - description: The URL of the RSS feed to be watched + description: The URL of the RSS feed to be watched. minLength: 1 type: string title: - description: Title of the RSS feed as will appear on the site + description: Title of the RSS feed as will appear on the site. minLength: 1 type: string unwanted_keywords: description: No items with titles that contain any of these words - will be transferred (comma-separated list of words) + will be transferred (comma-separated list of words). type: string required: - authSecretRef diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 2f3b0bb..266028d 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,12 +8,12 @@ resources: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_feeds.yaml +- patches/webhook_in_feeds.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_feeds.yaml +- patches/cainjection_in_feeds.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 1649856..caf0d8a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -21,9 +21,9 @@ resources: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -42,109 +42,109 @@ patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- manager_webhook_patch.yaml +- manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml +- webhookcainjection_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +replacements: + - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.namespace # namespace of the certificate CR + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.name + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - source: # Add cert-manager annotation to the webhook Service + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.name # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + - source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.namespace # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..738de35 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 0000000..4383c27 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# CERTIFICATE_NAMESPACE and CERTIFICATE_NAME will be substituted by kustomize +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..206316e --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,22 @@ +# the following config is for teaching kustomize where to look at when substituting nameReference. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..b79785e --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-putio-skynewz-dev-v1alpha1-feed + failurePolicy: Fail + name: mfeed.kb.io + rules: + - apiGroups: + - putio.skynewz.dev + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - feeds + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-putio-skynewz-dev-v1alpha1-feed + failurePolicy: Fail + name: vfeed.kb.io + rules: + - apiGroups: + - putio.skynewz.dev + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - feeds + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..3f638bd --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,13 @@ + +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/feed_controller.go b/controllers/feed_controller.go index 32652bd..dd9e03b 100644 --- a/controllers/feed_controller.go +++ b/controllers/feed_controller.go @@ -291,7 +291,7 @@ func (r *FeedReconciler) setPauseStatus(ctx context.Context, putioClient *putio. r.Recorder.Event(feed, corev1.EventTypeNormal, eventSetPauseStatus, "setting feed pause status") var err error - switch feed.Spec.Paused { + switch *feed.Spec.Paused { case true: err = putioClient.Rss.Pause(ctx, feedID) case false: @@ -336,9 +336,9 @@ func makePutioFeedFromSpec(ctx context.Context, feed *skynewzdevv1alpha1.Feed) * return &putio.Feed{ Title: makeFeedTitleWithGenerationNumber(ctx, feed), RssSourceURL: feed.Spec.RssSourceURL, - ParentDirID: feed.Spec.ParentDirID, - DeleteOldFiles: feed.Spec.DeleteOldFiles, - DontProcessWholeFeed: feed.Spec.DontProcessWholeFeed, + ParentDirID: *feed.Spec.ParentDirID, + DeleteOldFiles: *feed.Spec.DeleteOldFiles, + DontProcessWholeFeed: *feed.Spec.DontProcessWholeFeed, Keyword: feed.Spec.Keyword, UnwantedKeywords: feed.Spec.UnwantedKeywords, } diff --git a/controllers/feed_controller_test.go b/controllers/feed_controller_test.go index e04e8fd..af3fd9a 100644 --- a/controllers/feed_controller_test.go +++ b/controllers/feed_controller_test.go @@ -11,7 +11,7 @@ import ( ) func Test_makePutioFeedFromSpec(t *testing.T) { - parentDirID := uint32(1234) + parentDirID := uint(1234) type args struct { ctx context.Context @@ -33,8 +33,8 @@ func Test_makePutioFeedFromSpec(t *testing.T) { Title: "foo", RssSourceURL: "https://www.google.com", ParentDirID: &parentDirID, - DeleteOldFiles: true, - DontProcessWholeFeed: true, + DeleteOldFiles: boolToPtr(true), + DontProcessWholeFeed: boolToPtr(true), Keyword: "foo", UnwantedKeywords: "bar", // Paused: true, // Pause is not handle during creation/update @@ -47,7 +47,7 @@ func Test_makePutioFeedFromSpec(t *testing.T) { ID: nil, Title: "foo|0|managed by Kubernetes/putio-operator", RssSourceURL: "https://www.google.com", - ParentDirID: &parentDirID, + ParentDirID: parentDirID, DeleteOldFiles: true, DontProcessWholeFeed: true, Keyword: "foo", @@ -101,11 +101,11 @@ func Test_makeFeedTitleWithGenerationNumber(t *testing.T) { Title: "foo", RssSourceURL: "", ParentDirID: nil, - DeleteOldFiles: false, - DontProcessWholeFeed: false, + DeleteOldFiles: new(bool), + DontProcessWholeFeed: new(bool), Keyword: "", UnwantedKeywords: "", - Paused: false, + Paused: new(bool), AuthSecretRef: skynewzdevv1alpha1.AuthSecretReference{}, }, Status: skynewzdevv1alpha1.FeedStatus{}, @@ -142,7 +142,7 @@ func Test_isAlreadyProcessed(t *testing.T) { ID: nil, Title: "foo|1234|managed by Kubernetes/putio-operator", RssSourceURL: "", - ParentDirID: nil, + ParentDirID: 0, DeleteOldFiles: false, DontProcessWholeFeed: false, Keyword: "", @@ -176,7 +176,7 @@ func Test_isAlreadyProcessed(t *testing.T) { ID: nil, Title: "foo|4321|managed by Kubernetes/putio-operator", RssSourceURL: "", - ParentDirID: nil, + ParentDirID: 0, DeleteOldFiles: false, DontProcessWholeFeed: false, Keyword: "", @@ -211,3 +211,7 @@ func Test_isAlreadyProcessed(t *testing.T) { }) } } + +func boolToPtr(v bool) *bool { + return &v +} diff --git a/internal/putio/feed.go b/internal/putio/feed.go index 5217c62..b8f9599 100644 --- a/internal/putio/feed.go +++ b/internal/putio/feed.go @@ -28,15 +28,15 @@ func (t *Time) GetTime() time.Time { } type Feed struct { - ID *uint `json:"id"` - Title string `json:"title"` - RssSourceURL string `json:"rss_source_url"` - ParentDirID *uint32 `json:"parent_dir_id"` - DeleteOldFiles bool `json:"delete_old_files"` - DontProcessWholeFeed bool `json:"dont_process_whole_feed"` - Keyword string `json:"keyword"` - UnwantedKeywords string `json:"unwanted_keywords"` - Paused bool `json:"paused"` + ID *uint `json:"id"` + Title string `json:"title"` + RssSourceURL string `json:"rss_source_url"` + ParentDirID uint `json:"parent_dir_id"` + DeleteOldFiles bool `json:"delete_old_files"` + DontProcessWholeFeed bool `json:"dont_process_whole_feed"` + Keyword string `json:"keyword"` + UnwantedKeywords string `json:"unwanted_keywords"` + Paused bool `json:"paused"` Extract bool `json:"extract"` FailedItemCount uint `json:"failed_item_count"` diff --git a/internal/putio/putio.go b/internal/putio/putio.go index 987fd62..0df7733 100644 --- a/internal/putio/putio.go +++ b/internal/putio/putio.go @@ -114,16 +114,11 @@ func (s *rssService) Create(ctx context.Context, feed *Feed) (*Feed, error) { params := url.Values{} params.Set("title", feed.Title) params.Set("rss_source_url", feed.RssSourceURL) - params.Set("keyword", feed.Keyword) - params.Set("unwanted_keywords", feed.UnwantedKeywords) - + params.Set("parent_dir_id", strconv.Itoa(int(feed.ParentDirID))) params.Set("delete_old_files", boolToString(feed.DeleteOldFiles)) params.Set("dont_process_whole_feed", boolToString(feed.DontProcessWholeFeed)) - params.Set("paused", boolToString(feed.Paused)) - - if v := feed.ParentDirID; v != nil { - params.Set("parent_dir_id", strconv.Itoa(int(*v))) - } + params.Set("keyword", feed.Keyword) + params.Set("unwanted_keywords", feed.UnwantedKeywords) req, err := s.client.NewRequest(ctx, http.MethodPost, "/v2/rss/create", strings.NewReader(params.Encode())) if err != nil { @@ -153,15 +148,11 @@ func (s *rssService) Update(ctx context.Context, feed *Feed, id uint) error { params := url.Values{} params.Set("title", feed.Title) params.Set("rss_source_url", feed.RssSourceURL) - params.Set("keyword", feed.Keyword) - params.Set("unwanted_keywords", feed.UnwantedKeywords) - + params.Set("parent_dir_id", strconv.Itoa(int(feed.ParentDirID))) params.Set("delete_old_files", boolToString(feed.DeleteOldFiles)) params.Set("dont_process_whole_feed", boolToString(feed.DontProcessWholeFeed)) - - if v := feed.ParentDirID; v != nil { - params.Set("parent_dir_id", strconv.Itoa(int(*v))) - } + params.Set("keyword", feed.Keyword) + params.Set("unwanted_keywords", feed.UnwantedKeywords) req, err := s.client.NewRequest(ctx, http.MethodPost, fmt.Sprintf("/v2/rss/%d", id), strings.NewReader(params.Encode())) if err != nil { diff --git a/internal/putio/putio_test.go b/internal/putio/putio_test.go index 2e074a1..702fc6e 100644 --- a/internal/putio/putio_test.go +++ b/internal/putio/putio_test.go @@ -71,13 +71,12 @@ func Test_rssService_List(t *testing.T) { args: args{context.Background()}, want: func() []*Feed { feedID := uint(125559) - parentDirID := uint32(998868232) return []*Feed{ { ID: &feedID, Title: "For all mankind", RssSourceURL: "https://rss.site.fr", - ParentDirID: &parentDirID, + ParentDirID: 998868232, DeleteOldFiles: false, DontProcessWholeFeed: false, Keyword: "FOR.ALL.MANKIND&S03&2160P&FRATERNITY", @@ -149,12 +148,11 @@ func Test_rssService_Get(t *testing.T) { }, want: func() *Feed { feedID := uint(125559) - parentDirID := uint32(998868232) return &Feed{ ID: &feedID, Title: "For all mankind", RssSourceURL: "https://rss.site.fr", - ParentDirID: &parentDirID, + ParentDirID: 998868232, DeleteOldFiles: false, DontProcessWholeFeed: false, Keyword: "FOR.ALL.MANKIND&S03&2160P&FRATERNITY", diff --git a/main.go b/main.go index 2214215..c442672 100644 --- a/main.go +++ b/main.go @@ -23,16 +23,17 @@ import ( "os" "time" - skynewzdevv1alpha1 "github.com/SkYNewZ/putio-operator/api/v1alpha1" - "github.com/SkYNewZ/putio-operator/controllers" - "github.com/SkYNewZ/putio-operator/internal/logger" - "github.com/SkYNewZ/putio-operator/internal/sentry" - "github.com/SkYNewZ/putio-operator/internal/tracing" "github.com/go-logr/zapr" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + putiov1alpha1 "github.com/SkYNewZ/putio-operator/api/v1alpha1" + "github.com/SkYNewZ/putio-operator/controllers" + "github.com/SkYNewZ/putio-operator/internal/logger" + "github.com/SkYNewZ/putio-operator/internal/sentry" + "github.com/SkYNewZ/putio-operator/internal/tracing" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -56,7 +57,7 @@ const ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(skynewzdevv1alpha1.AddToScheme(scheme)) + utilruntime.Must(putiov1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -129,6 +130,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Feed") os.Exit(1) } + if err = (&putiov1alpha1.Feed{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Feed") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {